From 1c32f6bb2e6aa6894a1b0c57a79615a1f5322615 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 24 Aug 2017 12:38:00 +0200 Subject: [PATCH 001/262] [CSE] Check the server for client side encryption Call the apps api and check if the response has the client_side_encryption key. Signed-off-by: Tomaz Canabrava --- src/libsync/connectionvalidator.cpp | 25 +++++++++++++++++++++++++ src/libsync/connectionvalidator.h | 1 + 2 files changed, 26 insertions(+) diff --git a/src/libsync/connectionvalidator.cpp b/src/libsync/connectionvalidator.cpp index 2acbf74f1..75a2ff23f 100644 --- a/src/libsync/connectionvalidator.cpp +++ b/src/libsync/connectionvalidator.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -245,12 +246,36 @@ void ConnectionValidator::slotAuthSuccess() checkServerCapabilities(); } +void ConnectionValidator::checkClientSideEncryption() +{ + JsonApiJob *job = new JsonApiJob(_account, QLatin1String("ocs/v2.php/cloud/apps"), this); + job->setTimeout(timeoutToUseMsec); + connect(job, &JsonApiJob::jsonReceived, [this](const QJsonDocument& json, int httpResponse) { + Q_UNUSED(httpResponse); + auto apps = json.object().value("ocs").toObject().value("data").toObject().value("apps").toArray().toVariantList(); + + bool hasClientSideEncryption = false; + for(const auto& app : qAsConst(apps)) { + if (app.toString() == "client_side_encryption") { + hasClientSideEncryption = true; + qCInfo(lcConnectionValidator()) << "Found Client Side Encryption"; + break; + } + } + + // _account->setHasClientSideEncryption(hasClientSideEncryption); + }); + job->start(); +} + void ConnectionValidator::checkServerCapabilities() { JsonApiJob *job = new JsonApiJob(_account, QLatin1String("ocs/v1.php/cloud/capabilities"), this); job->setTimeout(timeoutToUseMsec); QObject::connect(job, &JsonApiJob::jsonReceived, this, &ConnectionValidator::slotCapabilitiesRecieved); job->start(); + + checkClientSideEncryption(); } void ConnectionValidator::slotCapabilitiesRecieved(const QJsonDocument &json) diff --git a/src/libsync/connectionvalidator.h b/src/libsync/connectionvalidator.h index b8bca9af9..9513aef06 100644 --- a/src/libsync/connectionvalidator.h +++ b/src/libsync/connectionvalidator.h @@ -128,6 +128,7 @@ protected slots: private: void reportResult(Status status); void checkServerCapabilities(); + void checkClientSideEncryption(); void fetchUser(); /** Sets the account's server version From 9318c487b9b8c5e34a605ab218380907fa6e023d Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 24 Aug 2017 15:53:26 +0200 Subject: [PATCH 002/262] [CSE] Display menu for encrytp / decryot folders If the server supports client syde encryption, display a menu on right click that should display encrypt and decrypt. ideally it would show the encrypt if the folder is decrypted, and decrypt if the folder is encrypted but currently there's no way for the client to know that. --- src/gui/accountsettings.cpp | 26 ++++++++++++++++++++++++++ src/gui/folder.h | 2 ++ src/libsync/account.cpp | 9 +++++++++ src/libsync/account.h | 4 ++++ src/libsync/connectionvalidator.cpp | 2 +- 5 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 23b287244..696a97f0a 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -275,6 +275,23 @@ void AccountSettings::slotCustomContextMenuRequested(const QPoint &pos) ac->setEnabled(false); } + qCInfo(lcAccountSettings) << "Display Client Side Encryption Options" + << accountsState()->account()->hasClientSideEncryption(); + + if (accountsState()->account()->hasClientSideEncryption()) { + ac = menu->addAction(tr("Encrypt")); + connect(ac, &QAction::triggered, [this, &index](bool triggered) { + Q_UNUSED(triggered); + + }); + + ac = menu->addAction(tr("Decrypt")); + connect(ac, &QAction::triggered, [this, &index](bool triggered) { + Q_UNUSED(triggered); + + }); + + } menu->exec(QCursor::pos()); return; @@ -701,6 +718,15 @@ void AccountSettings::slotAccountStateChanged() _toggleSignInOutAction->setText(tr("Log out")); } } + + if (state == AccountState::State::Connected) { + /* TODO: We should probably do something better here. + * Verify if the user has a private key already uploaded to the server, + * if it has, do not offer to create one. + */ + qCInfo(lcAccountSettings) << "Accout" << accountsState()->account()->displayName() + << "Client Side Encryption" << accountsState()->account()->hasClientSideEncryption(); + } } void AccountSettings::slotLinkActivated(const QString &link) diff --git a/src/gui/folder.h b/src/gui/folder.h index 7b503a6d6..0ac050c0d 100644 --- a/src/gui/folder.h +++ b/src/gui/folder.h @@ -62,6 +62,8 @@ public: bool paused; /// whether the folder syncs hidden files bool ignoreHiddenFiles; + /// the folder has client side encryption + bool isClientSideEncrypted; /// Saves the folder definition, creating a new settings group. static void save(QSettings &settings, const FolderDefinition &folder); diff --git a/src/libsync/account.cpp b/src/libsync/account.cpp index 0c2e44dbe..dbe8b0019 100644 --- a/src/libsync/account.cpp +++ b/src/libsync/account.cpp @@ -484,5 +484,14 @@ void Account::setNonShib(bool nonShib) } } +bool Account::hasClientSideEncryption() const +{ + return _hasClientSideEncryption; +} + +void Account::setHasClientSideEncryption(bool cse) +{ + _hasClientSideEncryption = cse; +} } // namespace OCC diff --git a/src/libsync/account.h b/src/libsync/account.h index aa1009287..e33d8baac 100644 --- a/src/libsync/account.h +++ b/src/libsync/account.h @@ -225,6 +225,9 @@ public: /// Called by network jobs on credential errors, emits invalidCredentials() void handleInvalidCredentials(); + bool hasClientSideEncryption() const; + void setHasClientSideEncryption(bool cse); + public slots: /// Used when forgetting credentials void clearQNAMCache(); @@ -275,6 +278,7 @@ private: QSharedPointer _am; QScopedPointer _credentials; bool _http2Supported = false; + bool _hasClientSideEncryption; /// Certificates that were explicitly rejected by the user QList _rejectedCertificates; diff --git a/src/libsync/connectionvalidator.cpp b/src/libsync/connectionvalidator.cpp index 75a2ff23f..c0de1cb6c 100644 --- a/src/libsync/connectionvalidator.cpp +++ b/src/libsync/connectionvalidator.cpp @@ -263,7 +263,7 @@ void ConnectionValidator::checkClientSideEncryption() } } - // _account->setHasClientSideEncryption(hasClientSideEncryption); + _account->setHasClientSideEncryption(hasClientSideEncryption); }); job->start(); } From 9870f39dcbf793a7531c0d5b52a697342e049ef4 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 25 Aug 2017 13:56:13 +0200 Subject: [PATCH 003/262] [CSE] Fetch file-id for subfolders File id is a must if we want to call any API. Signed-off-by: Tomaz Canabrava --- src/gui/folderstatusmodel.cpp | 8 ++++++-- src/gui/folderstatusmodel.h | 3 +-- src/gui/selectivesyncdialog.cpp | 4 ++-- src/libsync/networkjobs.cpp | 10 ++++++---- src/libsync/networkjobs.h | 11 +++++++++-- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/gui/folderstatusmodel.cpp b/src/gui/folderstatusmodel.cpp index 0c593ebdb..cc632bdb1 100644 --- a/src/gui/folderstatusmodel.cpp +++ b/src/gui/folderstatusmodel.cpp @@ -554,7 +554,9 @@ void FolderStatusModel::fetchMore(const QModelIndex &parent) LsColJob *job = new LsColJob(_accountState->account(), path, this); job->setProperties(QList() << "resourcetype" << "http://owncloud.org/ns:size" - << "http://owncloud.org/ns:permissions"); + << "http://owncloud.org/ns:permissions" + << "http://owncloud.org/ns:fileid"); + job->setTimeout(60 * 1000); connect(job, &LsColJob::directoryListingSubfolders, this, &FolderStatusModel::slotUpdateDirectories); @@ -655,11 +657,13 @@ void FolderStatusModel::slotUpdateDirectories(const QStringList &list) newInfo._folder = parentInfo->_folder; newInfo._pathIdx = parentInfo->_pathIdx; newInfo._pathIdx << newSubs.size(); - newInfo._size = job->_sizes.value(path); newInfo._isExternal = permissionMap.value(removeTrailingSlash(path)).toString().contains("M"); newInfo._path = relativePath; newInfo._name = removeTrailingSlash(relativePath).split('/').last(); + const auto& folderInfo = job->_folderInfos.value(path); + newInfo._size = folderInfo.size; + newInfo._fileId = folderInfo.fileId; if (relativePath.isEmpty()) continue; diff --git a/src/gui/folderstatusmodel.h b/src/gui/folderstatusmodel.h index 7cdf49c7f..8fcf1058f 100644 --- a/src/gui/folderstatusmodel.h +++ b/src/gui/folderstatusmodel.h @@ -79,9 +79,9 @@ public: bool _hasError; // If the last fetching job ended in an error QString _lastErrorString; bool _fetchingLabel; // Whether a 'fetching in progress' label is shown. - // undecided folders are the big folders that the user has not accepted yet bool _isUndecided; + QByteArray _fileId; // the file id for this folder on the server. Qt::CheckState _checked; @@ -168,7 +168,6 @@ signals: // Tell the view that this item should be expanded because it has an undecided item void suggestExpand(const QModelIndex &); - friend struct SubFolderInfo; }; diff --git a/src/gui/selectivesyncdialog.cpp b/src/gui/selectivesyncdialog.cpp index 87d2bdc22..69c10e1f2 100644 --- a/src/gui/selectivesyncdialog.cpp +++ b/src/gui/selectivesyncdialog.cpp @@ -235,7 +235,7 @@ void SelectiveSyncWidget::slotUpdateDirectories(QStringList list) root->setIcon(0, Theme::instance()->applicationIcon()); root->setData(0, Qt::UserRole, QString()); root->setCheckState(0, Qt::Checked); - qint64 size = job ? job->_sizes.value(pathToRemove, -1) : -1; + qint64 size = job ? job->_folderInfos[pathToRemove].size : -1; if (size >= 0) { root->setText(1, Utility::octetsToString(size)); root->setData(1, Qt::UserRole, size); @@ -244,7 +244,7 @@ void SelectiveSyncWidget::slotUpdateDirectories(QStringList list) Utility::sortFilenames(list); foreach (QString path, list) { - auto size = job ? job->_sizes.value(path) : 0; + auto size = job ? job->_folderInfos[path].size : 0; path.remove(pathToRemove); QStringList paths = path.split('/'); if (paths.last().isEmpty()) diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index 3af0f6c21..e2f416d0a 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -185,7 +185,7 @@ LsColXMLParser::LsColXMLParser() { } -bool LsColXMLParser::parse(const QByteArray &xml, QHash *sizes, const QString &expectedPath) +bool LsColXMLParser::parse(const QByteArray &xml, QHash *fileInfo, const QString &expectedPath) { // Parse DAV response QXmlStreamReader reader(xml); @@ -241,9 +241,11 @@ bool LsColXMLParser::parse(const QByteArray &xml, QHash *sizes, } else if (name == QLatin1String("size")) { bool ok = false; auto s = propertyContent.toLongLong(&ok); - if (ok && sizes) { - sizes->insert(currentHref, s); + if (ok && fileInfo) { + (*fileInfo)[currentHref].size = s; } + } else if (name == QLatin1String("fileid")) { + (*fileInfo)[currentHref].fileId = propertyContent.toUtf8(); } currentTmpProperties.insert(reader.name().toString(), propertyContent); } @@ -372,7 +374,7 @@ bool LsColJob::finished() this, &LsColJob::finishedWithoutError); QString expectedPath = reply()->request().url().path(); // something like "/owncloud/remote.php/webdav/folder" - if (!parser.parse(reply()->readAll(), &_sizes, expectedPath)) { + if (!parser.parse(reply()->readAll(), &_folderInfos, expectedPath)) { // XML parse error emit finishedWithError(reply()); } diff --git a/src/libsync/networkjobs.h b/src/libsync/networkjobs.h index 2965d5ae3..4e2b8b36c 100644 --- a/src/libsync/networkjobs.h +++ b/src/libsync/networkjobs.h @@ -41,6 +41,11 @@ private slots: virtual bool finished() Q_DECL_OVERRIDE; }; +struct ExtraFolderInfo { + QByteArray fileId; + qint64 size = -1; +}; + /** * @brief The LsColJob class * @ingroup libsync @@ -51,7 +56,9 @@ class OWNCLOUDSYNC_EXPORT LsColXMLParser : public QObject public: explicit LsColXMLParser(); - bool parse(const QByteArray &xml, QHash *sizes, const QString &expectedPath); + bool parse(const QByteArray &xml, + QHash *sizes, + const QString &expectedPath); signals: void directoryListingSubfolders(const QStringList &items); @@ -67,7 +74,7 @@ public: explicit LsColJob(AccountPtr account, const QString &path, QObject *parent = 0); explicit LsColJob(AccountPtr account, const QUrl &url, QObject *parent = 0); void start() Q_DECL_OVERRIDE; - QHash _sizes; + QHash _folderInfos; /** * Used to specify which properties shall be retrieved. From e2091bb0a388bf7921935f1f4e324f8bb40eb56b Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 25 Aug 2017 14:12:27 +0200 Subject: [PATCH 004/262] [CSE] Add FileIdRole for the FolderStatusModel This way we can actually request the id from outside of the model. --- src/gui/folderstatusmodel.cpp | 2 ++ src/gui/folderstatusmodel.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/gui/folderstatusmodel.cpp b/src/gui/folderstatusmodel.cpp index cc632bdb1..d3a06f4f5 100644 --- a/src/gui/folderstatusmodel.cpp +++ b/src/gui/folderstatusmodel.cpp @@ -163,6 +163,8 @@ QVariant FolderStatusModel::data(const QModelIndex &index, int role) const return QColor(Qt::red); } break; + case FileIdRole: + return x._fileId; case FolderStatusDelegate::FolderPathRole: { auto f = x._folder; if (!f) diff --git a/src/gui/folderstatusmodel.h b/src/gui/folderstatusmodel.h index 8fcf1058f..4b6a82915 100644 --- a/src/gui/folderstatusmodel.h +++ b/src/gui/folderstatusmodel.h @@ -37,6 +37,8 @@ class FolderStatusModel : public QAbstractItemModel { Q_OBJECT public: + enum {FileIdRole = Qt::UserRole+1}; + FolderStatusModel(QObject *parent = 0); ~FolderStatusModel(); void setAccountState(const AccountState *accountState); From 5b51346e83b01dfc1922ab506688d29625f95baa Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 25 Aug 2017 14:59:13 +0200 Subject: [PATCH 005/262] [CSE] New Network Job: DeleteApiJob This network job does a DELETE http request on a URL. It's the second class that does basically the same, but this one returns the http return code, and it's set to do a api call. --- src/libsync/networkjobs.cpp | 33 +++++++++++++++++++++++++++++++++ src/libsync/networkjobs.h | 21 +++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index e2f416d0a..3e968d0a0 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -820,6 +820,7 @@ bool JsonApiJob::finished() return true; } + DetermineAuthTypeJob::DetermineAuthTypeJob(AccountPtr account, QObject *parent) : AbstractNetworkJob(account, QString(), parent) , _redirects(0) @@ -907,4 +908,36 @@ bool SimpleNetworkJob::finished() return true; } +DeleteApiJob::DeleteApiJob(AccountPtr account, const QString &path, QObject *parent) + : AbstractNetworkJob(account, path, parent) +{ + +} + +void DeleteApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + sendRequest("DELETE", url, req); + AbstractNetworkJob::start(); +} + +bool DeleteApiJob::finished() +{ + qCInfo(lcJsonApiJob) << "JsonApiJob of" << reply()->request().url() << "FINISHED WITH STATUS" + << reply()->error() + << (reply()->error() == QNetworkReply::NoError ? QLatin1String("") : errorString()); + + int statusCode = 0; + + if (reply()->error() != QNetworkReply::NoError) { + qCWarning(lcJsonApiJob) << "Network error: " << path() << errorString() << reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute); + emit result(statusCode); + return true; + } + + const auto replyData = QString::fromUtf8(reply()->readAll()); + qCInfo(lcJsonApiJob()) << "TMX Delete Job" << replyData; +} } // namespace OCC diff --git a/src/libsync/networkjobs.h b/src/libsync/networkjobs.h index 4e2b8b36c..eee55e144 100644 --- a/src/libsync/networkjobs.h +++ b/src/libsync/networkjobs.h @@ -41,6 +41,27 @@ private slots: virtual bool finished() Q_DECL_OVERRIDE; }; +/** + * @brief sends a DELETE http request to a url. + * + * See Nextcloud API usage for the possible DELETE requests. + * + * This does *not* delete files, it does a http request. + */ +class OWNCLOUDSYNC_EXPORT DeleteApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit DeleteApiJob(AccountPtr account, const QString &path, QObject *parent = 0); + void start() override; + +signals: + void result(int httpCode); + +private slots: + virtual bool finished() override; +}; + struct ExtraFolderInfo { QByteArray fileId; qint64 size = -1; From f7e74f520b33a3fb60c1e7d0969bbdd29b309291 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 25 Aug 2017 15:00:30 +0200 Subject: [PATCH 006/262] [CSE] Call encrypt and unencrypt from the GUI If the server allows client side encryption, show the user the menu with the encrypt and unencrupt options --- src/gui/accountsettings.cpp | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 696a97f0a..f64b584e7 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -274,21 +275,30 @@ void AccountSettings::slotCustomContextMenuRequested(const QPoint &pos) if (!QFile::exists(fileName)) { ac->setEnabled(false); } + auto fileId = _model->data(index, FolderStatusModel::FileIdRole).toByteArray(); - qCInfo(lcAccountSettings) << "Display Client Side Encryption Options" - << accountsState()->account()->hasClientSideEncryption(); - - if (accountsState()->account()->hasClientSideEncryption()) { + if (accountsState()->account()->hasClientSideEncryption()) { ac = menu->addAction(tr("Encrypt")); - connect(ac, &QAction::triggered, [this, &index](bool triggered) { + connect(ac, &QAction::triggered, [this, &fileId](bool triggered) { Q_UNUSED(triggered); + auto job = new OCC::JsonApiJob(accountsState()->account(), + "ocs/v2.php/apps/client_side_encryption/api/v1/encrypted/" + QString(fileId)); + connect(job, &OCC::JsonApiJob::jsonReceived, [this](const QJsonDocument& json, int httpResponse) { + qCInfo(lcAccountSettings) << "Encrypt Http Response" << httpResponse; + }); + job->start(); }); ac = menu->addAction(tr("Decrypt")); - connect(ac, &QAction::triggered, [this, &index](bool triggered) { + connect(ac, &QAction::triggered, [this, &fileId](bool triggered) { Q_UNUSED(triggered); - + auto job = new OCC::DeleteApiJob(accountsState()->account(), + "ocs/v2.php/apps/client_side_encryption/api/v1/encrypted/" + QString(fileId)); + connect(job, &OCC::DeleteApiJob::result, [this](int httpResponse) { + qCInfo(lcAccountSettings) << "Decrypt Http Response" << httpResponse; + }); + job->start(); }); } From 920047fa700e3ca7b2063520f322d665bdd4410c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 25 Aug 2017 16:56:34 +0200 Subject: [PATCH 007/262] [CSE] Remove uneeded includes --- src/libsync/account.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libsync/account.cpp b/src/libsync/account.cpp index dbe8b0019..bf3f67066 100644 --- a/src/libsync/account.cpp +++ b/src/libsync/account.cpp @@ -16,19 +16,17 @@ #include "cookiejar.h" #include "networkjobs.h" #include "configfile.h" -#include "accessmanager.h" #include "creds/abstractcredentials.h" #include "capabilities.h" #include "theme.h" #include "common/asserts.h" -#include #include -#include #include #include #include #include + #include #include #include From 1c85f94b007cfe4682f1017fc4c3c0f63f1f37c9 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 3 Sep 2017 13:33:50 +0200 Subject: [PATCH 008/262] [cse] Use server capabilities to query for cse --- src/gui/accountsettings.cpp | 5 +++-- src/libsync/account.cpp | 10 ---------- src/libsync/account.h | 4 ---- src/libsync/capabilities.cpp | 6 ++++++ src/libsync/capabilities.h | 3 +++ src/libsync/connectionvalidator.cpp | 24 ------------------------ src/libsync/connectionvalidator.h | 1 - 7 files changed, 12 insertions(+), 41 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index f64b584e7..091d7ce85 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -277,7 +277,7 @@ void AccountSettings::slotCustomContextMenuRequested(const QPoint &pos) } auto fileId = _model->data(index, FolderStatusModel::FileIdRole).toByteArray(); - if (accountsState()->account()->hasClientSideEncryption()) { + if (accountsState()->account()->capabilities().clientSideEncryptionAvaliable()) { ac = menu->addAction(tr("Encrypt")); connect(ac, &QAction::triggered, [this, &fileId](bool triggered) { Q_UNUSED(triggered); @@ -285,6 +285,7 @@ void AccountSettings::slotCustomContextMenuRequested(const QPoint &pos) "ocs/v2.php/apps/client_side_encryption/api/v1/encrypted/" + QString(fileId)); connect(job, &OCC::JsonApiJob::jsonReceived, [this](const QJsonDocument& json, int httpResponse) { + Q_UNUSED(json); qCInfo(lcAccountSettings) << "Encrypt Http Response" << httpResponse; }); job->start(); @@ -735,7 +736,7 @@ void AccountSettings::slotAccountStateChanged() * if it has, do not offer to create one. */ qCInfo(lcAccountSettings) << "Accout" << accountsState()->account()->displayName() - << "Client Side Encryption" << accountsState()->account()->hasClientSideEncryption(); + << "Client Side Encryption" << accountsState()->account()->capabilities().clientSideEncryptionAvaliable(); } } diff --git a/src/libsync/account.cpp b/src/libsync/account.cpp index bf3f67066..98190e653 100644 --- a/src/libsync/account.cpp +++ b/src/libsync/account.cpp @@ -482,14 +482,4 @@ void Account::setNonShib(bool nonShib) } } -bool Account::hasClientSideEncryption() const -{ - return _hasClientSideEncryption; -} - -void Account::setHasClientSideEncryption(bool cse) -{ - _hasClientSideEncryption = cse; -} - } // namespace OCC diff --git a/src/libsync/account.h b/src/libsync/account.h index e33d8baac..aa1009287 100644 --- a/src/libsync/account.h +++ b/src/libsync/account.h @@ -225,9 +225,6 @@ public: /// Called by network jobs on credential errors, emits invalidCredentials() void handleInvalidCredentials(); - bool hasClientSideEncryption() const; - void setHasClientSideEncryption(bool cse); - public slots: /// Used when forgetting credentials void clearQNAMCache(); @@ -278,7 +275,6 @@ private: QSharedPointer _am; QScopedPointer _credentials; bool _http2Supported = false; - bool _hasClientSideEncryption; /// Certificates that were explicitly rejected by the user QList _rejectedCertificates; diff --git a/src/libsync/capabilities.cpp b/src/libsync/capabilities.cpp index 4dfc33b89..09d9fe21f 100644 --- a/src/libsync/capabilities.cpp +++ b/src/libsync/capabilities.cpp @@ -81,6 +81,12 @@ bool Capabilities::shareResharing() const return _capabilities["files_sharing"].toMap()["resharing"].toBool(); } +bool Capabilities::clientSideEncryptionAvaliable() const +{ + return _capabilities.keys().indexOf("client-side-encryption") != 1 + ? _capabilities["client-side-encryption"].toMap()["enabled"].toBool() : false; +} + bool Capabilities::notificationsAvailable() const { // We require the OCS style API in 9.x, can't deal with the REST one only found in 8.2 diff --git a/src/libsync/capabilities.h b/src/libsync/capabilities.h index 9b15edb9c..8e7011913 100644 --- a/src/libsync/capabilities.h +++ b/src/libsync/capabilities.h @@ -53,6 +53,9 @@ public: /// returns true if the capabilities report notifications bool notificationsAvailable() const; + /// returns true if the server supports client side encryption + bool clientSideEncryptionAvaliable() const; + /// returns true if the capabilities are loaded already. bool isValid() const; diff --git a/src/libsync/connectionvalidator.cpp b/src/libsync/connectionvalidator.cpp index c0de1cb6c..920d247f5 100644 --- a/src/libsync/connectionvalidator.cpp +++ b/src/libsync/connectionvalidator.cpp @@ -246,36 +246,12 @@ void ConnectionValidator::slotAuthSuccess() checkServerCapabilities(); } -void ConnectionValidator::checkClientSideEncryption() -{ - JsonApiJob *job = new JsonApiJob(_account, QLatin1String("ocs/v2.php/cloud/apps"), this); - job->setTimeout(timeoutToUseMsec); - connect(job, &JsonApiJob::jsonReceived, [this](const QJsonDocument& json, int httpResponse) { - Q_UNUSED(httpResponse); - auto apps = json.object().value("ocs").toObject().value("data").toObject().value("apps").toArray().toVariantList(); - - bool hasClientSideEncryption = false; - for(const auto& app : qAsConst(apps)) { - if (app.toString() == "client_side_encryption") { - hasClientSideEncryption = true; - qCInfo(lcConnectionValidator()) << "Found Client Side Encryption"; - break; - } - } - - _account->setHasClientSideEncryption(hasClientSideEncryption); - }); - job->start(); -} - void ConnectionValidator::checkServerCapabilities() { JsonApiJob *job = new JsonApiJob(_account, QLatin1String("ocs/v1.php/cloud/capabilities"), this); job->setTimeout(timeoutToUseMsec); QObject::connect(job, &JsonApiJob::jsonReceived, this, &ConnectionValidator::slotCapabilitiesRecieved); job->start(); - - checkClientSideEncryption(); } void ConnectionValidator::slotCapabilitiesRecieved(const QJsonDocument &json) diff --git a/src/libsync/connectionvalidator.h b/src/libsync/connectionvalidator.h index 9513aef06..b8bca9af9 100644 --- a/src/libsync/connectionvalidator.h +++ b/src/libsync/connectionvalidator.h @@ -128,7 +128,6 @@ protected slots: private: void reportResult(Status status); void checkServerCapabilities(); - void checkClientSideEncryption(); void fetchUser(); /** Sets the account's server version From 7e2085375f5cbd9f42780ed058be31699d88c4f5 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 11 Sep 2017 11:31:25 +0200 Subject: [PATCH 009/262] [cse] Invalid read of 1 --- src/libsync/capabilities.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libsync/capabilities.cpp b/src/libsync/capabilities.cpp index 09d9fe21f..c7fc546d2 100644 --- a/src/libsync/capabilities.cpp +++ b/src/libsync/capabilities.cpp @@ -17,9 +17,13 @@ #include "configfile.h" #include +#include +#include namespace OCC { +Q_LOGGING_CATEGORY(lcServerCapabilities, "sync.server.capabilities", QtInfoMsg) + Capabilities::Capabilities(const QVariantMap &capabilities) : _capabilities(capabilities) @@ -83,7 +87,7 @@ bool Capabilities::shareResharing() const bool Capabilities::clientSideEncryptionAvaliable() const { - return _capabilities.keys().indexOf("client-side-encryption") != 1 + return _capabilities.keys().indexOf("client-side-encryption") != -1 ? _capabilities["client-side-encryption"].toMap()["enabled"].toBool() : false; } From 3f4d915a177ef8dc3bfdd0a3d59cc56760fac591 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 11 Sep 2017 15:00:01 +0200 Subject: [PATCH 010/262] [cse] Add files to handle client side encryption This will be the responsible for encryption, decryption and talking with the server. --- src/libsync/CMakeLists.txt | 1 + src/libsync/clientsideencryption.cpp | 4 ++++ src/libsync/clientsideencryption.h | 10 ++++++++++ 3 files changed, 15 insertions(+) create mode 100644 src/libsync/clientsideencryption.cpp create mode 100644 src/libsync/clientsideencryption.h diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index eeb8fa0a9..9a90a175d 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -57,6 +57,7 @@ set(libsync_SRCS syncresult.cpp theme.cpp excludedfiles.cpp + clientsideencryption.cpp creds/dummycredentials.cpp creds/abstractcredentials.cpp creds/credentialscommon.cpp diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp new file mode 100644 index 000000000..a912b0040 --- /dev/null +++ b/src/libsync/clientsideencryption.cpp @@ -0,0 +1,4 @@ +#include "clientsideencryption.h" + +namespace ClientSideEncryption { +} diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h new file mode 100644 index 000000000..233cf2b7b --- /dev/null +++ b/src/libsync/clientsideencryption.h @@ -0,0 +1,10 @@ +#ifndef CLIENTSIDEENCRYPTION_H +#define CLIENTSIDEENCRYPTION_H + +#include + +namespace ClientSideEncryption { + QString baseUrl = QStringLiteral("ocs/v2.php/apps/client_side_encryption/api/v1/"); +} + +#endif From 29b64640fac739e4262fb62965b1c2908e26711c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 11 Sep 2017 16:52:57 +0200 Subject: [PATCH 011/262] [cse] Start to fetch the basics to fetch the key from the server --- src/libsync/account.cpp | 8 +++++++ src/libsync/account.h | 3 +++ src/libsync/clientsideencryption.cpp | 32 +++++++++++++++++++++++++++- src/libsync/clientsideencryption.h | 22 +++++++++++++++++-- src/libsync/connectionvalidator.cpp | 5 +++++ src/libsync/connectionvalidator.h | 7 +++++- 6 files changed, 73 insertions(+), 4 deletions(-) diff --git a/src/libsync/account.cpp b/src/libsync/account.cpp index 98190e653..75cc9def1 100644 --- a/src/libsync/account.cpp +++ b/src/libsync/account.cpp @@ -19,7 +19,9 @@ #include "creds/abstractcredentials.h" #include "capabilities.h" #include "theme.h" + #include "common/asserts.h" +#include "clientsideencryption.h" #include #include @@ -39,6 +41,7 @@ Q_LOGGING_CATEGORY(lcAccount, "sync.account", QtInfoMsg) Account::Account(QObject *parent) : QObject(parent) , _capabilities(QVariantMap()) + , _encryption(new ClientSideEncryption(this)) , _davPath(Theme::instance()->webDavPath()) { qRegisterMetaType("AccountPtr"); @@ -482,4 +485,9 @@ void Account::setNonShib(bool nonShib) } } +ClientSideEncryption *Account::cse() const +{ + return _encryption; +} + } // namespace OCC diff --git a/src/libsync/account.h b/src/libsync/account.h index aa1009287..b8e1b39b2 100644 --- a/src/libsync/account.h +++ b/src/libsync/account.h @@ -31,6 +31,7 @@ #include "common/utility.h" #include #include "capabilities.h" +#include "clientsideencryption.h" class QSettings; class QNetworkReply; @@ -225,6 +226,7 @@ public: /// Called by network jobs on credential errors, emits invalidCredentials() void handleInvalidCredentials(); + ClientSideEncryption *cse() const; public slots: /// Used when forgetting credentials void clearQNAMCache(); @@ -274,6 +276,7 @@ private: QuotaInfo *_quotaInfo; QSharedPointer _am; QScopedPointer _credentials; + ClientSideEncryption *_encryption; bool _http2Supported = false; /// Certificates that were explicitly rejected by the user diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index a912b0040..8b8b17709 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1,4 +1,34 @@ #include "clientsideencryption.h" +#include "account.h" +#include "capabilities.h" + +#include +#include + +namespace OCC +{ + +Q_LOGGING_CATEGORY(lcCse, "sync.connectionvalidator", QtInfoMsg) + +QString baseUrl = QStringLiteral("ocs/v2.php/apps/client_side_encryption/api/v1/"); + +ClientSideEncryption::ClientSideEncryption(Account *parent) : _account(parent) +{ +} + +void OCC::ClientSideEncryption::initialize() +{ + if (!_account->capabilities().clientSideEncryptionAvaliable()) { + qCInfo(lcCse()) << "No client side encryption, do not initialize anything."; + emit initializationFinished(); + } + + fetchPrivateKey(); +} + +void ClientSideEncryption::fetchPrivateKey() +{ + qCInfo(lcCse()) << "Client side encryption enabled, trying to retrieve the key."; +} -namespace ClientSideEncryption { } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 233cf2b7b..4f5826997 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -2,9 +2,27 @@ #define CLIENTSIDEENCRYPTION_H #include +#include + +namespace OCC { + +class Account; + +class ClientSideEncryption : public QObject { + Q_OBJECT +public: + ClientSideEncryption(OCC::Account *parent); + void initialize(); + + void fetchPrivateKey(); +signals: + void initializationFinished(); + +private: + OCC::Account *_account; + bool isInitialized = false; +}; -namespace ClientSideEncryption { - QString baseUrl = QStringLiteral("ocs/v2.php/apps/client_side_encryption/api/v1/"); } #endif diff --git a/src/libsync/connectionvalidator.cpp b/src/libsync/connectionvalidator.cpp index 920d247f5..7d9d04e50 100644 --- a/src/libsync/connectionvalidator.cpp +++ b/src/libsync/connectionvalidator.cpp @@ -326,6 +326,11 @@ void ConnectionValidator::slotUserFetched(const QJsonDocument &json) void ConnectionValidator::slotAvatarImage(const QImage &img) { _account->setAvatar(img); + connect(_account->cse(), &ClientSideEncryption::initializationFinished, this, &ConnectionValidator::reportConnected); + _account->cse()->initialize(); +} + +void ConnectionValidator::reportConnected() { reportResult(Connected); } diff --git a/src/libsync/connectionvalidator.h b/src/libsync/connectionvalidator.h index b8bca9af9..24a43f926 100644 --- a/src/libsync/connectionvalidator.h +++ b/src/libsync/connectionvalidator.h @@ -65,7 +65,11 @@ namespace OCC { | +-----------------------------------+ | - +-> fetchUser + +-> Client Side Encryption Checks --+ + | + +---------------------------------+ + | + fetchUser PropfindJob | +-> slotUserFetched @@ -126,6 +130,7 @@ protected slots: void slotAvatarImage(const QImage &img); private: + void reportConnected(); void reportResult(Status status); void checkServerCapabilities(); void fetchUser(); From f4bbec1019e62bd7acb92fbde0c8476ea49de0b4 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Sep 2017 11:06:57 +0200 Subject: [PATCH 012/262] [bugfix] Return the error code, don't discard it. --- src/libsync/networkjobs.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index 3e968d0a0..ba9eba010 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -787,6 +787,7 @@ bool JsonApiJob::finished() if (reply()->error() != QNetworkReply::NoError) { qCWarning(lcJsonApiJob) << "Network error: " << path() << errorString() << reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute); + statusCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); emit jsonReceived(QJsonDocument(), statusCode); return true; } From 17693a75e59e6d0d92c5d4fa332b8a8a56c52dc8 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Sep 2017 11:21:53 +0200 Subject: [PATCH 013/262] [cse] Request public key from server This is the first step needed to properly communicate. Next, get private key. --- src/libsync/account.cpp | 6 -- src/libsync/account.h | 2 - src/libsync/clientsideencryption.cpp | 84 +++++++++++++++++++++++++--- src/libsync/clientsideencryption.h | 20 +++++-- src/libsync/connectionvalidator.cpp | 5 +- src/libsync/connectionvalidator.h | 13 +++-- 6 files changed, 103 insertions(+), 27 deletions(-) diff --git a/src/libsync/account.cpp b/src/libsync/account.cpp index 75cc9def1..d8b957e16 100644 --- a/src/libsync/account.cpp +++ b/src/libsync/account.cpp @@ -41,7 +41,6 @@ Q_LOGGING_CATEGORY(lcAccount, "sync.account", QtInfoMsg) Account::Account(QObject *parent) : QObject(parent) , _capabilities(QVariantMap()) - , _encryption(new ClientSideEncryption(this)) , _davPath(Theme::instance()->webDavPath()) { qRegisterMetaType("AccountPtr"); @@ -485,9 +484,4 @@ void Account::setNonShib(bool nonShib) } } -ClientSideEncryption *Account::cse() const -{ - return _encryption; -} - } // namespace OCC diff --git a/src/libsync/account.h b/src/libsync/account.h index b8e1b39b2..f172e5476 100644 --- a/src/libsync/account.h +++ b/src/libsync/account.h @@ -226,7 +226,6 @@ public: /// Called by network jobs on credential errors, emits invalidCredentials() void handleInvalidCredentials(); - ClientSideEncryption *cse() const; public slots: /// Used when forgetting credentials void clearQNAMCache(); @@ -276,7 +275,6 @@ private: QuotaInfo *_quotaInfo; QSharedPointer _am; QScopedPointer _credentials; - ClientSideEncryption *_encryption; bool _http2Supported = false; /// Certificates that were explicitly rejected by the user diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 8b8b17709..c92d53a2d 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1,34 +1,104 @@ #include "clientsideencryption.h" #include "account.h" #include "capabilities.h" +#include "networkjobs.h" #include #include +#include +#include namespace OCC { -Q_LOGGING_CATEGORY(lcCse, "sync.connectionvalidator", QtInfoMsg) +Q_LOGGING_CATEGORY(lcCse, "sync.clientsideencryption", QtInfoMsg) QString baseUrl = QStringLiteral("ocs/v2.php/apps/client_side_encryption/api/v1/"); +QString baseDirectory = QDir::homePath() + QStringLiteral("/.nextcloud-keys/"); -ClientSideEncryption::ClientSideEncryption(Account *parent) : _account(parent) +ClientSideEncryption::ClientSideEncryption() { } -void OCC::ClientSideEncryption::initialize() +void ClientSideEncryption::setAccount(AccountPtr account) { + _account = account; +} + +void ClientSideEncryption::initialize() +{ + qCInfo(lcCse()) << "Initializing"; if (!_account->capabilities().clientSideEncryptionAvaliable()) { - qCInfo(lcCse()) << "No client side encryption, do not initialize anything."; + qCInfo(lcCse()) << "No Client side encryption avaliable on server."; emit initializationFinished(); } - fetchPrivateKey(); + if (hasPrivateKey() && hasPublicKey()) { + qCInfo(lcCse()) << "Public and private keys already downloaded"; + emit initializationFinished(); + } + + getPublicKeyFromServer(); } -void ClientSideEncryption::fetchPrivateKey() +QString ClientSideEncryption::publicKeyPath() const { - qCInfo(lcCse()) << "Client side encryption enabled, trying to retrieve the key."; + return baseDirectory + _account->displayName() + ".pub"; +} + +QString ClientSideEncryption::privateKeyPath() const +{ + return baseDirectory + _account->displayName() + ".rsa"; +} + +bool ClientSideEncryption::hasPrivateKey() const +{ + return QFileInfo(privateKeyPath()).exists(); +} + +bool ClientSideEncryption::hasPublicKey() const +{ + return QFileInfo(publicKeyPath()).exists(); +} + +void ClientSideEncryption::generateKeyPair() +{ + +} + +QString ClientSideEncryption::generateSCR() +{ + return {}; +} + +void ClientSideEncryption::getPrivateKeyFromServer() +{ + +} + +void ClientSideEncryption::getPublicKeyFromServer() +{ + qCInfo(lcCse()) << "Retrieving public key from server"; + auto job = new JsonApiJob(_account, baseUrl + "public-key", this); + connect(job, &JsonApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { + switch(retCode) { + case 404: // no public key + qCInfo(lcCse()) << "No public key, generating a pair."; + break; + case 400: // internal error + qCInfo(lcCse()) << "Internal server error while requesting the public key, encryption aborted."; + break; + case 200: // ok + qCInfo(lcCse()) << "Found Public key, requesting Private Key."; + break; + } + }); + job->start(); +} + +void ClientSideEncryption::signPublicKey() +{ + } } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 4f5826997..efe67d97f 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -3,23 +3,35 @@ #include #include +#include + + +#include "accountfwd.h" namespace OCC { -class Account; class ClientSideEncryption : public QObject { Q_OBJECT public: - ClientSideEncryption(OCC::Account *parent); + ClientSideEncryption(); void initialize(); + void setAccount(AccountPtr account); + bool hasPrivateKey() const; + bool hasPublicKey() const; + void generateKeyPair(); + QString generateSCR(); + void getPrivateKeyFromServer(); + void getPublicKeyFromServer(); + void signPublicKey(); + QString privateKeyPath() const; + QString publicKeyPath() const; - void fetchPrivateKey(); signals: void initializationFinished(); private: - OCC::Account *_account; + OCC::AccountPtr _account; bool isInitialized = false; }; diff --git a/src/libsync/connectionvalidator.cpp b/src/libsync/connectionvalidator.cpp index 7d9d04e50..f4bd69360 100644 --- a/src/libsync/connectionvalidator.cpp +++ b/src/libsync/connectionvalidator.cpp @@ -326,8 +326,9 @@ void ConnectionValidator::slotUserFetched(const QJsonDocument &json) void ConnectionValidator::slotAvatarImage(const QImage &img) { _account->setAvatar(img); - connect(_account->cse(), &ClientSideEncryption::initializationFinished, this, &ConnectionValidator::reportConnected); - _account->cse()->initialize(); + cse.setAccount(_account); + connect(&cse, &ClientSideEncryption::initializationFinished, this, &ConnectionValidator::reportConnected); + cse.initialize(); } void ConnectionValidator::reportConnected() { diff --git a/src/libsync/connectionvalidator.h b/src/libsync/connectionvalidator.h index 24a43f926..b9dfda3ed 100644 --- a/src/libsync/connectionvalidator.h +++ b/src/libsync/connectionvalidator.h @@ -21,6 +21,7 @@ #include #include #include "accountfwd.h" +#include "clientsideencryption.h" namespace OCC { @@ -63,10 +64,6 @@ namespace OCC { | +-> slotCapabilitiesRecieved -+ | - +-----------------------------------+ - | - +-> Client Side Encryption Checks --+ - | +---------------------------------+ | fetchUser @@ -75,10 +72,13 @@ namespace OCC { +-> slotUserFetched AvatarJob | - +-> slotAvatarImage --> reportResult() - + +-> slotAvatarImage --> + +-----------------------------------+ + | + +-> Client Side Encryption Checks --+ --reportResult() \endcode */ + class OWNCLOUDSYNC_EXPORT ConnectionValidator : public QObject { Q_OBJECT @@ -144,6 +144,7 @@ private: QStringList _errors; AccountPtr _account; bool _isCheckingServerAndAuth; + ClientSideEncryption cse; }; } From 8c342cb1dd9055f47304712a4cd2deee9a048bb3 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Sep 2017 13:44:42 +0200 Subject: [PATCH 014/262] [cse] Generate the KeyPair Not stored anywhere yet, but it's correctly running. --- src/libsync/clientsideencryption.cpp | 50 +++++++++++++++++++++++++++- src/libsync/clientsideencryption.h | 2 ++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index c92d53a2d..ba72f57d2 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -3,6 +3,10 @@ #include "capabilities.h" #include "networkjobs.h" +#include +#include +#include + #include #include #include @@ -63,7 +67,50 @@ bool ClientSideEncryption::hasPublicKey() const void ClientSideEncryption::generateKeyPair() { + // AES/GCM/NoPadding, + // metadataKeys with RSA/ECB/OAEPWithSHA-256AndMGF1Padding + qCInfo(lcCse()) << "No public key, generating a pair."; + const int rsaKeyLen = 2048; + EVP_PKEY *localKeyPair = nullptr; + EVP_PKEY *serverPub = nullptr; + + EVP_CIPHER_CTX *rsaEncryptCtx = EVP_CIPHER_CTX_new(); + EVP_CIPHER_CTX *aesEncryptCtx = EVP_CIPHER_CTX_new(); + EVP_CIPHER_CTX *rsaDecryptCtx = EVP_CIPHER_CTX_new(); + EVP_CIPHER_CTX *aesDecryptCtx = EVP_CIPHER_CTX_new(); + + unsigned char *aesKey = nullptr; + unsigned char *aesIv = nullptr; + + for(auto ctx : {rsaEncryptCtx, aesEncryptCtx, rsaDecryptCtx, aesDecryptCtx}) { + if (!ctx) { + qCInfo(lcCse()) << "Failed to generate the cripto context"; + return; + } + } + + // Init RSA + EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL); + + if(EVP_PKEY_keygen_init(ctx) <= 0) { + qCInfo(lcCse()) << "Couldn't initialize the key generator"; + return; + } + + if(EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, rsaKeyLen) <= 0) { + qCInfo(lcCse()) << "Couldn't initialize the key generator bits"; + return; + } + + if(EVP_PKEY_keygen(ctx, &localKeyPair) <= 0) { + qCInfo(lcCse()) << "Could not generate the key"; + return; + } + EVP_PKEY_CTX_free(ctx); + qCInfo(lcCse()) << "Key correctly generated"; + + qCInfo(lcCse()) << "Keys generated correctly, sending to server."; } QString ClientSideEncryption::generateSCR() @@ -83,7 +130,8 @@ void ClientSideEncryption::getPublicKeyFromServer() connect(job, &JsonApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { switch(retCode) { case 404: // no public key - qCInfo(lcCse()) << "No public key, generating a pair."; + qCInfo(lcCse()) << "No public key on the server"; + generateKeyPair(); break; case 400: // internal error qCInfo(lcCse()) << "Internal server error while requesting the public key, encryption aborted."; diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index efe67d97f..7541e0ad0 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -5,6 +5,8 @@ #include #include +#include +#include #include "accountfwd.h" From 58e2e6b30b2fdaac72a51df1c143a6284810551e Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Sep 2017 14:35:05 +0200 Subject: [PATCH 015/262] [cse] Generate the public / private keys and store locally Now I need to understand what the hell I need to do to send this to the server. --- src/libsync/clientsideencryption.cpp | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index ba72f57d2..99b11f140 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -7,6 +7,8 @@ #include #include +#include + #include #include #include @@ -109,7 +111,37 @@ void ClientSideEncryption::generateKeyPair() } EVP_PKEY_CTX_free(ctx); qCInfo(lcCse()) << "Key correctly generated"; + qCInfo(lcCse()) << "Storing keys locally"; + QDir dir; + if (!dir.mkpath(baseDirectory)) { + qCInfo(lcCse()) << "Could not create the folder for the keys."; + return; + } + + auto privKeyPath = privateKeyPath().toLocal8Bit(); + auto pubKeyPath = publicKeyPath().toLocal8Bit(); + FILE *privKeyFile = fopen(privKeyPath.constData(), "w"); + FILE *pubKeyFile = fopen(pubKeyPath.constData(), "w"); + + qCInfo(lcCse()) << "Private key filename" << privKeyPath; + qCInfo(lcCse()) << "Public key filename" << pubKeyPath; + + //TODO: Verify if the key needs to be stored with a Cipher and Pass. + if (!PEM_write_PrivateKey(privKeyFile, localKeyPair, NULL, NULL, 0, 0, NULL)) { + qCInfo(lcCse()) << "Could not write the private key to a file."; + return; + } + + if (!PEM_write_PUBKEY(pubKeyFile, localKeyPair)) { + qCInfo(lcCse()) << "Could not write the public key to a file."; + return; + } + + fclose(privKeyFile); + fclose(pubKeyFile); + + //TODO: Send to server. qCInfo(lcCse()) << "Keys generated correctly, sending to server."; } From 42a3098595c38de8379da02de15d2053fff5a98d Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Sep 2017 17:08:04 +0200 Subject: [PATCH 016/262] [cse] s/scr/csr --- src/libsync/clientsideencryption.cpp | 2 +- src/libsync/clientsideencryption.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 99b11f140..e156f879e 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -145,7 +145,7 @@ void ClientSideEncryption::generateKeyPair() qCInfo(lcCse()) << "Keys generated correctly, sending to server."; } -QString ClientSideEncryption::generateSCR() +QString ClientSideEncryption::generateCSR() { return {}; } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 7541e0ad0..6dd88f642 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -22,7 +22,7 @@ public: bool hasPrivateKey() const; bool hasPublicKey() const; void generateKeyPair(); - QString generateSCR(); + QString generateCSR(); void getPrivateKeyFromServer(); void getPublicKeyFromServer(); void signPublicKey(); From c358980448c0204cd2aacdba3b3833db005c240c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Sep 2017 18:28:11 +0200 Subject: [PATCH 017/262] [cse] Link Against OpenSSL For some reason, this was working untill I added a call to X509_REQ_get_subject_name, then the linking suddenly stopped working (even tougth I'm using a ton of other OpenSSL calls) Force to link against 1.0 --- CMakeLists.txt | 4 +++- src/libsync/CMakeLists.txt | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c2540f689..f2fa757a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -189,7 +189,7 @@ if(BUILD_CLIENT) endif() find_package(Sphinx) find_package(PdfLatex) - + find_package(OpenSSL REQUIRED VERSION 1.0) find_package(SQLite3 3.8.0 REQUIRED) # On some OS, we want to use our own, not the system sqlite if (USE_OUR_OWN_SQLITE3) @@ -197,6 +197,8 @@ if(BUILD_CLIENT) if (WIN32) add_definitions(-DSQLITE_API=__declspec\(dllimport\)) endif() + + endif() find_package(ZLIB) diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index 9a90a175d..65209ffd4 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -99,6 +99,8 @@ ENDIF(NOT APPLE) list(APPEND libsync_LINK_TARGETS ${QT_LIBRARIES} ocsync + OpenSSL::Crypto + OpenSSL::SSL ${OS_SPECIFIC_LINK_LIBRARIES} ) From d2992d92bada9767beee18ff0cdc94fb1410227e Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Sep 2017 20:02:17 +0200 Subject: [PATCH 018/262] [cse] Generate the CSE I still need to send it to the server. It's been a long learning with the OpenSSL library. --- src/libsync/clientsideencryption.cpp | 77 +++++++++++++++++++++------- src/libsync/clientsideencryption.h | 2 +- 2 files changed, 60 insertions(+), 19 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index e156f879e..f6597741e 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -7,6 +7,9 @@ #include #include +#include +#include + #include #include @@ -75,22 +78,6 @@ void ClientSideEncryption::generateKeyPair() const int rsaKeyLen = 2048; EVP_PKEY *localKeyPair = nullptr; - EVP_PKEY *serverPub = nullptr; - - EVP_CIPHER_CTX *rsaEncryptCtx = EVP_CIPHER_CTX_new(); - EVP_CIPHER_CTX *aesEncryptCtx = EVP_CIPHER_CTX_new(); - EVP_CIPHER_CTX *rsaDecryptCtx = EVP_CIPHER_CTX_new(); - EVP_CIPHER_CTX *aesDecryptCtx = EVP_CIPHER_CTX_new(); - - unsigned char *aesKey = nullptr; - unsigned char *aesIv = nullptr; - - for(auto ctx : {rsaEncryptCtx, aesEncryptCtx, rsaDecryptCtx, aesDecryptCtx}) { - if (!ctx) { - qCInfo(lcCse()) << "Failed to generate the cripto context"; - return; - } - } // Init RSA EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL); @@ -141,13 +128,67 @@ void ClientSideEncryption::generateKeyPair() fclose(privKeyFile); fclose(pubKeyFile); + generateCSR(localKeyPair); + //TODO: Send to server. qCInfo(lcCse()) << "Keys generated correctly, sending to server."; } -QString ClientSideEncryption::generateCSR() +QString ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) { - return {}; + // OpenSSL expects const char. + auto certParams = std::map{ + {"C", "DE"}, + {"ST", "Baden-Wuerttemberg"}, + {"L", "Stuttgart"}, + {"O","Nextcloud"}, + {"CN", "www.nextcloud.com"} + }; + + int ret = 0; + int nVersion = 1; + + X509_REQ *x509_req = NULL; + BIO *out = NULL; + + // 2. set version of x509 req + x509_req = X509_REQ_new(); + ret = X509_REQ_set_version(x509_req, nVersion); + + // 3. set subject of x509 req + auto x509_name = X509_REQ_get_subject_name(x509_req); + + using ucharp = const unsigned char *; + for(const auto& v : certParams) { + ret = X509_NAME_add_entry_by_txt(x509_name, v.first, MBSTRING_ASC, (ucharp) v.second, -1, -1, 0); + if (ret != 1) { + qCInfo(lcCse()) << "Error Generating the Certificate while adding" << v.first << v.second; + goto free_all; + } + } + + ret = X509_REQ_set_pubkey(x509_req, keyPair); + if (ret != 1){ + qCInfo(lcCse()) << "Error setting the public key on the csr"; + goto free_all; + } + + ret = X509_REQ_sign(x509_req, keyPair, EVP_sha1()); // return x509_req->signature->length + if (ret <= 0){ + qCInfo(lcCse()) << "Error setting the public key on the csr"; + goto free_all; + } + + out = BIO_new_file("/home/tcanabrava/.nextcloud-keys/request.pem","w"); + ret = PEM_write_bio_X509_REQ(out, x509_req); + if (ret != 1) { + qCInfo(lcCse()) << "Error saving the csr file"; + } + +free_all: + X509_REQ_free(x509_req); + BIO_free_all(out); + return ""; } void ClientSideEncryption::getPrivateKeyFromServer() diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 6dd88f642..e21e3aabc 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -22,7 +22,7 @@ public: bool hasPrivateKey() const; bool hasPublicKey() const; void generateKeyPair(); - QString generateCSR(); + QString generateCSR(EVP_PKEY *keyPair); void getPrivateKeyFromServer(); void getPublicKeyFromServer(); void signPublicKey(); From ecb05020a9e27692f7bbd39ebcbf1ed104aa12b2 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Sep 2017 21:52:10 +0200 Subject: [PATCH 019/262] [cse] Do not save the certificate on disk Store it on memory, and discard it as soon as no longer needed. --- src/libsync/clientsideencryption.cpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index f6597741e..119e234aa 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -149,7 +149,12 @@ QString ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) int nVersion = 1; X509_REQ *x509_req = NULL; - BIO *out = NULL; + auto out = BIO_new(BIO_s_mem()); + QString output; + char data[80]; + + // auto out = BIO_new_file("/home/tcanabrava/.nextcloud-keys/request.pem","w"); + // 2. set version of x509 req x509_req = X509_REQ_new(); @@ -179,12 +184,22 @@ QString ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) goto free_all; } - out = BIO_new_file("/home/tcanabrava/.nextcloud-keys/request.pem","w"); ret = PEM_write_bio_X509_REQ(out, x509_req); + do { + ret = BIO_gets(out, data, 80); + output += data; + if (output.endsWith("-----END CERTIFICATE REQUEST-----")) + break; + } while (ret > 0 ); + if (ret != 1) { qCInfo(lcCse()) << "Error saving the csr file"; } + qCInfo(lcCse()) << "Returning the certificate"; + qCInfo(lcCse()) << output; + + free_all: X509_REQ_free(x509_req); BIO_free_all(out); From 304231811d9d2b829fb0f8d86a31244dd524880c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 13 Sep 2017 17:01:02 +0200 Subject: [PATCH 020/262] [cse] Start the job to sign the public key. --- src/libsync/networkjobs.cpp | 27 ++++++++++++++++++++++ src/libsync/networkjobs.h | 46 +++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index ba9eba010..73d8458ed 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -49,6 +49,7 @@ Q_LOGGING_CATEGORY(lcMkColJob, "sync.networkjob.mkcol", QtInfoMsg) Q_LOGGING_CATEGORY(lcProppatchJob, "sync.networkjob.proppatch", QtInfoMsg) Q_LOGGING_CATEGORY(lcJsonApiJob, "sync.networkjob.jsonapi", QtInfoMsg) Q_LOGGING_CATEGORY(lcDetermineAuthTypeJob, "sync.networkjob.determineauthtype", QtInfoMsg) +Q_LOGGING_CATEGORY(lcSignPublicKeyApiJob, "sync.networkjob.sendcsr", QtInfoMsg); RequestEtagJob::RequestEtagJob(AccountPtr account, const QString &path, QObject *parent) : AbstractNetworkJob(account, path, parent) @@ -941,4 +942,30 @@ bool DeleteApiJob::finished() const auto replyData = QString::fromUtf8(reply()->readAll()); qCInfo(lcJsonApiJob()) << "TMX Delete Job" << replyData; } + +SignPublicKeyApiJob::SignPublicKeyApiJob(const AccountPtr& account, const QString& path, QObject* parent) +: AbstractNetworkJob(account, path, parent) +{ +} + +void SignPublicKeyApiJob::setCsr(const QByteArray& csr) +{ + _csr.setData(csr); +} + +void SignPublicKeyApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + sendRequest("POST", url, req, &_csr); + AbstractNetworkJob::start(); +} + +bool SignPublicKeyApiJob::finished() +{ + qCInfo(lcSignPublicKeyApiJob()) << "Sending CSR ended with" << path() << errorString() << reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute); + qCInfo(lcSignPublicKeyApiJob()) << reply()->readAll(); +} + } // namespace OCC diff --git a/src/libsync/networkjobs.h b/src/libsync/networkjobs.h index eee55e144..1585b9156 100644 --- a/src/libsync/networkjobs.h +++ b/src/libsync/networkjobs.h @@ -18,6 +18,8 @@ #include "abstractnetworkjob.h" +#include + class QUrl; class QJsonObject; @@ -427,6 +429,50 @@ private slots: bool finished() Q_DECL_OVERRIDE; }; +/* + * @brief Job to sigh the CSR that return JSON + * + * To be used like this: + * \code + * _job = new SignPublicKeyApiJob(account, QLatin1String("ocs/v1.php/foo/bar"), this); + * _job->setCsr( csr ); + * connect(_job...); + * _job->start(); + * \encode + * + * @ingroup libsync + */ +class OWNCLOUDSYNC_EXPORT SignPublicKeyApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit SignPublicKeyApiJob(const AccountPtr &account, const QString &path, QObject *parent = 0); + + /** + * @brief setCsr - the CSR with the public key. + * This function needs to be called before start() obviously. + */ + void setCsr(const QByteArray& csr); + +public slots: + void start() override; + +protected: + bool finished() override; +signals: + + /** + * @brief jsonReceived - signal to report the json answer from ocs + * @param json - the parsed json document + * @param statusCode - the OCS status code: 100 (!) for success + */ + void jsonReceived(const QJsonDocument &json, int statusCode); + +private: + QBuffer _csr; + +}; + } // namespace OCC #endif // NETWORKJOBS_H From 307dfd195c23cbfcda43296a1fcc4e8182c937db Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 13 Sep 2017 19:57:39 +0200 Subject: [PATCH 021/262] [cse] Call the CSR job. There's something wrong on the CSR job that I need to discover. --- src/libsync/clientsideencryption.cpp | 20 +++++++++++--------- src/libsync/networkjobs.cpp | 10 +++++++++- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 119e234aa..ad53b59ba 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -148,13 +148,11 @@ QString ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) int ret = 0; int nVersion = 1; - X509_REQ *x509_req = NULL; + X509_REQ *x509_req = nullptr; auto out = BIO_new(BIO_s_mem()); - QString output; + QByteArray output("csr=\""); char data[80]; - - // auto out = BIO_new_file("/home/tcanabrava/.nextcloud-keys/request.pem","w"); - + SignPublicKeyApiJob *job = nullptr; // 2. set version of x509 req x509_req = X509_REQ_new(); @@ -191,14 +189,18 @@ QString ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) if (output.endsWith("-----END CERTIFICATE REQUEST-----")) break; } while (ret > 0 ); - - if (ret != 1) { - qCInfo(lcCse()) << "Error saving the csr file"; - } + output += "\""; qCInfo(lcCse()) << "Returning the certificate"; qCInfo(lcCse()) << output; + job = new SignPublicKeyApiJob(_account, baseUrl + "public-key", this); + job->setCsr(output); + connect(job, &SignPublicKeyApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { + qCInfo(lcCse()) << retCode; + qCInfo(lcCse()) << doc; + }); + job->start(); free_all: X509_REQ_free(x509_req); diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index 73d8458ed..48e49c2fa 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -958,6 +958,11 @@ void SignPublicKeyApiJob::start() QNetworkRequest req; req.setRawHeader("OCS-APIREQUEST", "true"); QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) + }; + url.setQueryItems(params); + sendRequest("POST", url, req, &_csr); AbstractNetworkJob::start(); } @@ -965,7 +970,10 @@ void SignPublicKeyApiJob::start() bool SignPublicKeyApiJob::finished() { qCInfo(lcSignPublicKeyApiJob()) << "Sending CSR ended with" << path() << errorString() << reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute); - qCInfo(lcSignPublicKeyApiJob()) << reply()->readAll(); + + QJsonParseError error; + auto json = QJsonDocument::fromJson(reply()->readAll(), &error); + emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); } } // namespace OCC From 090336c92830617a8535724e8b16894203d00c8f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 13 Sep 2017 20:52:55 +0200 Subject: [PATCH 022/262] [cse] Correctly send the CSR finally. Signed-off-by: Tomaz Canabrava --- src/libsync/clientsideencryption.cpp | 7 ++++--- src/libsync/networkjobs.cpp | 7 +++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index ad53b59ba..83e4c1540 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -150,7 +150,7 @@ QString ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) X509_REQ *x509_req = nullptr; auto out = BIO_new(BIO_s_mem()); - QByteArray output("csr=\""); + QByteArray output; char data[80]; SignPublicKeyApiJob *job = nullptr; @@ -186,10 +186,11 @@ QString ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) do { ret = BIO_gets(out, data, 80); output += data; - if (output.endsWith("-----END CERTIFICATE REQUEST-----")) + if (output.endsWith("-----END CERTIFICATE REQUEST-----")) { + output = output.trimmed(); break; + } } while (ret > 0 ); - output += "\""; qCInfo(lcCse()) << "Returning the certificate"; qCInfo(lcCse()) << output; diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index 48e49c2fa..52102acbd 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -30,7 +30,6 @@ #include #include #include - #include "networkjobs.h" #include "account.h" #include "owncloudpropagator.h" @@ -950,7 +949,9 @@ SignPublicKeyApiJob::SignPublicKeyApiJob(const AccountPtr& account, const QStrin void SignPublicKeyApiJob::setCsr(const QByteArray& csr) { - _csr.setData(csr); + QByteArray data = "csr="; + data += QUrl::toPercentEncoding(csr); + _csr.setData(data); } void SignPublicKeyApiJob::start() @@ -963,6 +964,7 @@ void SignPublicKeyApiJob::start() }; url.setQueryItems(params); + qCInfo(lcSignPublicKeyApiJob) << "Sending the CSR" << _csr.data(); sendRequest("POST", url, req, &_csr); AbstractNetworkJob::start(); } @@ -977,3 +979,4 @@ bool SignPublicKeyApiJob::finished() } } // namespace OCC + From 69c709714d9ee4d3c6545b930ff3b7740b84717f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 14 Sep 2017 15:57:41 +0200 Subject: [PATCH 023/262] [cse] Save signed key on disk --- src/libsync/clientsideencryption.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 83e4c1540..2704a100c 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -16,6 +16,7 @@ #include #include #include +#include namespace OCC { @@ -197,9 +198,20 @@ QString ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) job = new SignPublicKeyApiJob(_account, baseUrl + "public-key", this); job->setCsr(output); - connect(job, &SignPublicKeyApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { + // TODO: Extract this function, should not be a lambda. + connect(job, &SignPublicKeyApiJob::jsonReceived, [this](const QJsonDocument& json, int retCode) { + if (retCode == 200) { + auto caps = json.object().value("ocs").toObject().value("data").toObject().value("public-key").toString(); + qCInfo(lcCse()) << "Public Key Returned" << caps; + QFile file(publicKeyPath() + ".sign"); // TODO: Verify if I need to keep the old file. + if (file.open(QIODevice::WriteOnly)) { + QTextStream s(&file); + s << caps; + } + file.close(); + qCInfo(lcCse()) << "public key saved."; + } qCInfo(lcCse()) << retCode; - qCInfo(lcCse()) << doc; }); job->start(); From 2111aeaac176f27d2a0bbec60794f6a852a8aa4a Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 14 Sep 2017 16:50:25 +0200 Subject: [PATCH 024/262] [cse] Add wordlist & related methods. --- src/libsync/CMakeLists.txt | 1 + src/libsync/wordlist.cpp | 2073 ++++++++++++++++++++++++++++++++++++ src/libsync/wordlist.h | 13 + 3 files changed, 2087 insertions(+) create mode 100644 src/libsync/wordlist.cpp create mode 100644 src/libsync/wordlist.h diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index 65209ffd4..fabcef768 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -27,6 +27,7 @@ endif() set(libsync_SRCS account.cpp + wordlist.cpp bandwidthmanager.cpp capabilities.cpp clientproxy.cpp diff --git a/src/libsync/wordlist.cpp b/src/libsync/wordlist.cpp new file mode 100644 index 000000000..f33a3e1cb --- /dev/null +++ b/src/libsync/wordlist.cpp @@ -0,0 +1,2073 @@ +#include "wordlist.h" + +namespace OCC { +namespace WordList { + +#include + +QStringList getRandomWords() +{ + QStringList wordList = { + "abandon", + "ability", + "able", + "about", + "above", + "absent", + "absorb", + "abstract", + "absurd", + "abuse", + "access", + "accident", + "account", + "accuse", + "achieve", + "acid", + "acoustic", + "acquire", + "across", + "act", + "action", + "actor", + "actress", + "actual", + "adapt", + "add", + "addict", + "address", + "adjust", + "admit", + "adult", + "advance", + "advice", + "aerobic", + "affair", + "afford", + "afraid", + "again", + "age", + "agent", + "agree", + "ahead", + "aim", + "air", + "airport", + "aisle", + "alarm", + "album", + "alcohol", + "alert", + "alien", + "all", + "alley", + "allow", + "almost", + "alone", + "alpha", + "already", + "also", + "alter", + "always", + "amateur", + "amazing", + "among", + "amount", + "amused", + "analyst", + "anchor", + "ancient", + "anger", + "angle", + "angry", + "animal", + "ankle", + "announce", + "annual", + "another", + "answer", + "antenna", + "antique", + "anxiety", + "any", + "apart", + "apology", + "appear", + "apple", + "approve", + "april", + "arch", + "arctic", + "area", + "arena", + "argue", + "arm", + "armed", + "armor", + "army", + "around", + "arrange", + "arrest", + "arrive", + "arrow", + "art", + "artefact", + "artist", + "artwork", + "ask", + "aspect", + "assault", + "asset", + "assist", + "assume", + "asthma", + "athlete", + "atom", + "attack", + "attend", + "attitude", + "attract", + "auction", + "audit", + "august", + "aunt", + "author", + "auto", + "autumn", + "average", + "avocado", + "avoid", + "awake", + "aware", + "away", + "awesome", + "awful", + "awkward", + "axis", + "baby", + "bachelor", + "bacon", + "badge", + "bag", + "balance", + "balcony", + "ball", + "bamboo", + "banana", + "banner", + "bar", + "barely", + "bargain", + "barrel", + "base", + "basic", + "basket", + "battle", + "beach", + "bean", + "beauty", + "because", + "become", + "beef", + "before", + "begin", + "behave", + "behind", + "believe", + "below", + "belt", + "bench", + "benefit", + "best", + "betray", + "better", + "between", + "beyond", + "bicycle", + "bid", + "bike", + "bind", + "biology", + "bird", + "birth", + "bitter", + "black", + "blade", + "blame", + "blanket", + "blast", + "bleak", + "bless", + "blind", + "blood", + "blossom", + "blouse", + "blue", + "blur", + "blush", + "board", + "boat", + "body", + "boil", + "bomb", + "bone", + "bonus", + "book", + "boost", + "border", + "boring", + "borrow", + "boss", + "bottom", + "bounce", + "box", + "boy", + "bracket", + "brain", + "brand", + "brass", + "brave", + "bread", + "breeze", + "brick", + "bridge", + "brief", + "bright", + "bring", + "brisk", + "broccoli", + "broken", + "bronze", + "broom", + "brother", + "brown", + "brush", + "bubble", + "buddy", + "budget", + "buffalo", + "build", + "bulb", + "bulk", + "bullet", + "bundle", + "bunker", + "burden", + "burger", + "burst", + "bus", + "business", + "busy", + "butter", + "buyer", + "buzz", + "cabbage", + "cabin", + "cable", + "cactus", + "cage", + "cake", + "call", + "calm", + "camera", + "camp", + "can", + "canal", + "cancel", + "candy", + "cannon", + "canoe", + "canvas", + "canyon", + "capable", + "capital", + "captain", + "car", + "carbon", + "card", + "cargo", + "carpet", + "carry", + "cart", + "case", + "cash", + "casino", + "castle", + "casual", + "cat", + "catalog", + "catch", + "category", + "cattle", + "caught", + "cause", + "caution", + "cave", + "ceiling", + "celery", + "cement", + "census", + "century", + "cereal", + "certain", + "chair", + "chalk", + "champion", + "change", + "chaos", + "chapter", + "charge", + "chase", + "chat", + "cheap", + "check", + "cheese", + "chef", + "cherry", + "chest", + "chicken", + "chief", + "child", + "chimney", + "choice", + "choose", + "chronic", + "chuckle", + "chunk", + "churn", + "cigar", + "cinnamon", + "circle", + "citizen", + "city", + "civil", + "claim", + "clap", + "clarify", + "claw", + "clay", + "clean", + "clerk", + "clever", + "click", + "client", + "cliff", + "climb", + "clinic", + "clip", + "clock", + "clog", + "close", + "cloth", + "cloud", + "clown", + "club", + "clump", + "cluster", + "clutch", + "coach", + "coast", + "coconut", + "code", + "coffee", + "coil", + "coin", + "collect", + "color", + "column", + "combine", + "come", + "comfort", + "comic", + "common", + "company", + "concert", + "conduct", + "confirm", + "congress", + "connect", + "consider", + "control", + "convince", + "cook", + "cool", + "copper", + "copy", + "coral", + "core", + "corn", + "correct", + "cost", + "cotton", + "couch", + "country", + "couple", + "course", + "cousin", + "cover", + "coyote", + "crack", + "cradle", + "craft", + "cram", + "crane", + "crash", + "crater", + "crawl", + "crazy", + "cream", + "credit", + "creek", + "crew", + "cricket", + "crime", + "crisp", + "critic", + "crop", + "cross", + "crouch", + "crowd", + "crucial", + "cruel", + "cruise", + "crumble", + "crunch", + "crush", + "cry", + "crystal", + "cube", + "culture", + "cup", + "cupboard", + "curious", + "current", + "curtain", + "curve", + "cushion", + "custom", + "cute", + "cycle", + "dad", + "damage", + "damp", + "dance", + "danger", + "daring", + "dash", + "daughter", + "dawn", + "day", + "deal", + "debate", + "debris", + "decade", + "december", + "decide", + "decline", + "decorate", + "decrease", + "deer", + "defense", + "define", + "defy", + "degree", + "delay", + "deliver", + "demand", + "demise", + "denial", + "dentist", + "deny", + "depart", + "depend", + "deposit", + "depth", + "deputy", + "derive", + "describe", + "desert", + "design", + "desk", + "despair", + "destroy", + "detail", + "detect", + "develop", + "device", + "devote", + "diagram", + "dial", + "diamond", + "diary", + "dice", + "diesel", + "diet", + "differ", + "digital", + "dignity", + "dilemma", + "dinner", + "dinosaur", + "direct", + "dirt", + "disagree", + "discover", + "disease", + "dish", + "dismiss", + "disorder", + "display", + "distance", + "divert", + "divide", + "divorce", + "dizzy", + "doctor", + "document", + "dog", + "doll", + "dolphin", + "domain", + "donate", + "donkey", + "donor", + "door", + "dose", + "double", + "dove", + "draft", + "dragon", + "drama", + "drastic", + "draw", + "dream", + "dress", + "drift", + "drill", + "drink", + "drip", + "drive", + "drop", + "drum", + "dry", + "duck", + "dumb", + "dune", + "during", + "dust", + "dutch", + "duty", + "dwarf", + "dynamic", + "eager", + "eagle", + "early", + "earn", + "earth", + "easily", + "east", + "easy", + "echo", + "ecology", + "economy", + "edge", + "edit", + "educate", + "effort", + "egg", + "eight", + "either", + "elbow", + "elder", + "electric", + "elegant", + "element", + "elephant", + "elevator", + "elite", + "else", + "embark", + "embody", + "embrace", + "emerge", + "emotion", + "employ", + "empower", + "empty", + "enable", + "enact", + "end", + "endless", + "endorse", + "enemy", + "energy", + "enforce", + "engage", + "engine", + "enhance", + "enjoy", + "enlist", + "enough", + "enrich", + "enroll", + "ensure", + "enter", + "entire", + "entry", + "envelope", + "episode", + "equal", + "equip", + "era", + "erase", + "erode", + "erosion", + "error", + "erupt", + "escape", + "essay", + "essence", + "estate", + "eternal", + "ethics", + "evidence", + "evil", + "evoke", + "evolve", + "exact", + "example", + "excess", + "exchange", + "excite", + "exclude", + "excuse", + "execute", + "exercise", + "exhaust", + "exhibit", + "exile", + "exist", + "exit", + "exotic", + "expand", + "expect", + "expire", + "explain", + "expose", + "express", + "extend", + "extra", + "eye", + "eyebrow", + "fabric", + "face", + "faculty", + "fade", + "faint", + "faith", + "fall", + "false", + "fame", + "family", + "famous", + "fan", + "fancy", + "fantasy", + "farm", + "fashion", + "fat", + "fatal", + "father", + "fatigue", + "fault", + "favorite", + "feature", + "february", + "federal", + "fee", + "feed", + "feel", + "female", + "fence", + "festival", + "fetch", + "fever", + "few", + "fiber", + "fiction", + "field", + "figure", + "file", + "film", + "filter", + "final", + "find", + "fine", + "finger", + "finish", + "fire", + "firm", + "first", + "fiscal", + "fish", + "fit", + "fitness", + "fix", + "flag", + "flame", + "flash", + "flat", + "flavor", + "flee", + "flight", + "flip", + "float", + "flock", + "floor", + "flower", + "fluid", + "flush", + "fly", + "foam", + "focus", + "fog", + "foil", + "fold", + "follow", + "food", + "foot", + "force", + "forest", + "forget", + "fork", + "fortune", + "forum", + "forward", + "fossil", + "foster", + "found", + "fox", + "fragile", + "frame", + "frequent", + "fresh", + "friend", + "fringe", + "frog", + "front", + "frost", + "frown", + "frozen", + "fruit", + "fuel", + "fun", + "funny", + "furnace", + "fury", + "future", + "gadget", + "gain", + "galaxy", + "gallery", + "game", + "gap", + "garage", + "garbage", + "garden", + "garlic", + "garment", + "gas", + "gasp", + "gate", + "gather", + "gauge", + "gaze", + "general", + "genius", + "genre", + "gentle", + "genuine", + "gesture", + "ghost", + "giant", + "gift", + "giggle", + "ginger", + "giraffe", + "girl", + "give", + "glad", + "glance", + "glare", + "glass", + "glide", + "glimpse", + "globe", + "gloom", + "glory", + "glove", + "glow", + "glue", + "goat", + "goddess", + "gold", + "good", + "goose", + "gorilla", + "gospel", + "gossip", + "govern", + "gown", + "grab", + "grace", + "grain", + "grant", + "grape", + "grass", + "gravity", + "great", + "green", + "grid", + "grief", + "grit", + "grocery", + "group", + "grow", + "grunt", + "guard", + "guess", + "guide", + "guilt", + "guitar", + "gun", + "gym", + "habit", + "hair", + "half", + "hammer", + "hamster", + "hand", + "happy", + "harbor", + "hard", + "harsh", + "harvest", + "hat", + "have", + "hawk", + "hazard", + "head", + "health", + "heart", + "heavy", + "hedgehog", + "height", + "hello", + "helmet", + "help", + "hen", + "hero", + "hidden", + "high", + "hill", + "hint", + "hip", + "hire", + "history", + "hobby", + "hockey", + "hold", + "hole", + "holiday", + "hollow", + "home", + "honey", + "hood", + "hope", + "horn", + "horror", + "horse", + "hospital", + "host", + "hotel", + "hour", + "hover", + "hub", + "huge", + "human", + "humble", + "humor", + "hundred", + "hungry", + "hunt", + "hurdle", + "hurry", + "hurt", + "husband", + "hybrid", + "ice", + "icon", + "idea", + "identify", + "idle", + "ignore", + "ill", + "illegal", + "illness", + "image", + "imitate", + "immense", + "immune", + "impact", + "impose", + "improve", + "impulse", + "inch", + "include", + "income", + "increase", + "index", + "indicate", + "indoor", + "industry", + "infant", + "inflict", + "inform", + "inhale", + "inherit", + "initial", + "inject", + "injury", + "inmate", + "inner", + "innocent", + "input", + "inquiry", + "insane", + "insect", + "inside", + "inspire", + "install", + "intact", + "interest", + "into", + "invest", + "invite", + "involve", + "iron", + "island", + "isolate", + "issue", + "item", + "ivory", + "jacket", + "jaguar", + "jar", + "jazz", + "jealous", + "jeans", + "jelly", + "jewel", + "job", + "join", + "joke", + "journey", + "joy", + "judge", + "juice", + "jump", + "jungle", + "junior", + "junk", + "just", + "kangaroo", + "keen", + "keep", + "ketchup", + "key", + "kick", + "kid", + "kidney", + "kind", + "kingdom", + "kiss", + "kit", + "kitchen", + "kite", + "kitten", + "kiwi", + "knee", + "knife", + "knock", + "know", + "lab", + "label", + "labor", + "ladder", + "lady", + "lake", + "lamp", + "language", + "laptop", + "large", + "later", + "latin", + "laugh", + "laundry", + "lava", + "law", + "lawn", + "lawsuit", + "layer", + "lazy", + "leader", + "leaf", + "learn", + "leave", + "lecture", + "left", + "leg", + "legal", + "legend", + "leisure", + "lemon", + "lend", + "length", + "lens", + "leopard", + "lesson", + "letter", + "level", + "liar", + "liberty", + "library", + "license", + "life", + "lift", + "light", + "like", + "limb", + "limit", + "link", + "lion", + "liquid", + "list", + "little", + "live", + "lizard", + "load", + "loan", + "lobster", + "local", + "lock", + "logic", + "lonely", + "long", + "loop", + "lottery", + "loud", + "lounge", + "love", + "loyal", + "lucky", + "luggage", + "lumber", + "lunar", + "lunch", + "luxury", + "lyrics", + "machine", + "mad", + "magic", + "magnet", + "maid", + "mail", + "main", + "major", + "make", + "mammal", + "man", + "manage", + "mandate", + "mango", + "mansion", + "manual", + "maple", + "marble", + "march", + "margin", + "marine", + "market", + "marriage", + "mask", + "mass", + "master", + "match", + "material", + "math", + "matrix", + "matter", + "maximum", + "maze", + "meadow", + "mean", + "measure", + "meat", + "mechanic", + "medal", + "media", + "melody", + "melt", + "member", + "memory", + "mention", + "menu", + "mercy", + "merge", + "merit", + "merry", + "mesh", + "message", + "metal", + "method", + "middle", + "midnight", + "milk", + "million", + "mimic", + "mind", + "minimum", + "minor", + "minute", + "miracle", + "mirror", + "misery", + "miss", + "mistake", + "mix", + "mixed", + "mixture", + "mobile", + "model", + "modify", + "mom", + "moment", + "monitor", + "monkey", + "monster", + "month", + "moon", + "moral", + "more", + "morning", + "mosquito", + "mother", + "motion", + "motor", + "mountain", + "mouse", + "move", + "movie", + "much", + "muffin", + "mule", + "multiply", + "muscle", + "museum", + "mushroom", + "music", + "must", + "mutual", + "myself", + "mystery", + "myth", + "naive", + "name", + "napkin", + "narrow", + "nasty", + "nation", + "nature", + "near", + "neck", + "need", + "negative", + "neglect", + "neither", + "nephew", + "nerve", + "nest", + "net", + "network", + "neutral", + "never", + "news", + "next", + "nice", + "night", + "noble", + "noise", + "nominee", + "noodle", + "normal", + "north", + "nose", + "notable", + "note", + "nothing", + "notice", + "novel", + "now", + "nuclear", + "number", + "nurse", + "nut", + "oak", + "obey", + "object", + "oblige", + "obscure", + "observe", + "obtain", + "obvious", + "occur", + "ocean", + "october", + "odor", + "off", + "offer", + "office", + "often", + "oil", + "okay", + "old", + "olive", + "olympic", + "omit", + "once", + "one", + "onion", + "online", + "only", + "open", + "opera", + "opinion", + "oppose", + "option", + "orange", + "orbit", + "orchard", + "order", + "ordinary", + "organ", + "orient", + "original", + "orphan", + "ostrich", + "other", + "outdoor", + "outer", + "output", + "outside", + "oval", + "oven", + "over", + "own", + "owner", + "oxygen", + "oyster", + "ozone", + "pact", + "paddle", + "page", + "pair", + "palace", + "palm", + "panda", + "panel", + "panic", + "panther", + "paper", + "parade", + "parent", + "park", + "parrot", + "party", + "pass", + "patch", + "path", + "patient", + "patrol", + "pattern", + "pause", + "pave", + "payment", + "peace", + "peanut", + "pear", + "peasant", + "pelican", + "pen", + "penalty", + "pencil", + "people", + "pepper", + "perfect", + "permit", + "person", + "pet", + "phone", + "photo", + "phrase", + "physical", + "piano", + "picnic", + "picture", + "piece", + "pig", + "pigeon", + "pill", + "pilot", + "pink", + "pioneer", + "pipe", + "pistol", + "pitch", + "pizza", + "place", + "planet", + "plastic", + "plate", + "play", + "please", + "pledge", + "pluck", + "plug", + "plunge", + "poem", + "poet", + "point", + "polar", + "pole", + "police", + "pond", + "pony", + "pool", + "popular", + "portion", + "position", + "possible", + "post", + "potato", + "pottery", + "poverty", + "powder", + "power", + "practice", + "praise", + "predict", + "prefer", + "prepare", + "present", + "pretty", + "prevent", + "price", + "pride", + "primary", + "print", + "priority", + "prison", + "private", + "prize", + "problem", + "process", + "produce", + "profit", + "program", + "project", + "promote", + "proof", + "property", + "prosper", + "protect", + "proud", + "provide", + "public", + "pudding", + "pull", + "pulp", + "pulse", + "pumpkin", + "punch", + "pupil", + "puppy", + "purchase", + "purity", + "purpose", + "purse", + "push", + "put", + "puzzle", + "pyramid", + "quality", + "quantum", + "quarter", + "question", + "quick", + "quit", + "quiz", + "quote", + "rabbit", + "raccoon", + "race", + "rack", + "radar", + "radio", + "rail", + "rain", + "raise", + "rally", + "ramp", + "ranch", + "random", + "range", + "rapid", + "rare", + "rate", + "rather", + "raven", + "raw", + "razor", + "ready", + "real", + "reason", + "rebel", + "rebuild", + "recall", + "receive", + "recipe", + "record", + "recycle", + "reduce", + "reflect", + "reform", + "refuse", + "region", + "regret", + "regular", + "reject", + "relax", + "release", + "relief", + "rely", + "remain", + "remember", + "remind", + "remove", + "render", + "renew", + "rent", + "reopen", + "repair", + "repeat", + "replace", + "report", + "require", + "rescue", + "resemble", + "resist", + "resource", + "response", + "result", + "retire", + "retreat", + "return", + "reunion", + "reveal", + "review", + "reward", + "rhythm", + "rib", + "ribbon", + "rice", + "rich", + "ride", + "ridge", + "rifle", + "right", + "rigid", + "ring", + "riot", + "ripple", + "risk", + "ritual", + "rival", + "river", + "road", + "roast", + "robot", + "robust", + "rocket", + "romance", + "roof", + "rookie", + "room", + "rose", + "rotate", + "rough", + "round", + "route", + "royal", + "rubber", + "rude", + "rug", + "rule", + "run", + "runway", + "rural", + "sad", + "saddle", + "sadness", + "safe", + "sail", + "salad", + "salmon", + "salon", + "salt", + "salute", + "same", + "sample", + "sand", + "satisfy", + "satoshi", + "sauce", + "sausage", + "save", + "say", + "scale", + "scan", + "scare", + "scatter", + "scene", + "scheme", + "school", + "science", + "scissors", + "scorpion", + "scout", + "scrap", + "screen", + "script", + "scrub", + "sea", + "search", + "season", + "seat", + "second", + "secret", + "section", + "security", + "seed", + "seek", + "segment", + "select", + "sell", + "seminar", + "senior", + "sense", + "sentence", + "series", + "service", + "session", + "settle", + "setup", + "seven", + "shadow", + "shaft", + "shallow", + "share", + "shed", + "shell", + "sheriff", + "shield", + "shift", + "shine", + "ship", + "shiver", + "shock", + "shoe", + "shoot", + "shop", + "short", + "shoulder", + "shove", + "shrimp", + "shrug", + "shuffle", + "shy", + "sibling", + "sick", + "side", + "siege", + "sight", + "sign", + "silent", + "silk", + "silly", + "silver", + "similar", + "simple", + "since", + "sing", + "siren", + "sister", + "situate", + "six", + "size", + "skate", + "sketch", + "ski", + "skill", + "skin", + "skirt", + "skull", + "slab", + "slam", + "sleep", + "slender", + "slice", + "slide", + "slight", + "slim", + "slogan", + "slot", + "slow", + "slush", + "small", + "smart", + "smile", + "smoke", + "smooth", + "snack", + "snake", + "snap", + "sniff", + "snow", + "soap", + "soccer", + "social", + "sock", + "soda", + "soft", + "solar", + "soldier", + "solid", + "solution", + "solve", + "someone", + "song", + "soon", + "sorry", + "sort", + "soul", + "sound", + "soup", + "source", + "south", + "space", + "spare", + "spatial", + "spawn", + "speak", + "special", + "speed", + "spell", + "spend", + "sphere", + "spice", + "spider", + "spike", + "spin", + "spirit", + "split", + "spoil", + "sponsor", + "spoon", + "sport", + "spot", + "spray", + "spread", + "spring", + "spy", + "square", + "squeeze", + "squirrel", + "stable", + "stadium", + "staff", + "stage", + "stairs", + "stamp", + "stand", + "start", + "state", + "stay", + "steak", + "steel", + "stem", + "step", + "stereo", + "stick", + "still", + "sting", + "stock", + "stomach", + "stone", + "stool", + "story", + "stove", + "strategy", + "street", + "strike", + "strong", + "struggle", + "student", + "stuff", + "stumble", + "style", + "subject", + "submit", + "subway", + "success", + "such", + "sudden", + "suffer", + "sugar", + "suggest", + "suit", + "summer", + "sun", + "sunny", + "sunset", + "super", + "supply", + "supreme", + "sure", + "surface", + "surge", + "surprise", + "surround", + "survey", + "suspect", + "sustain", + "swallow", + "swamp", + "swap", + "swarm", + "swear", + "sweet", + "swift", + "swim", + "swing", + "switch", + "sword", + "symbol", + "symptom", + "syrup", + "system", + "table", + "tackle", + "tag", + "tail", + "talent", + "talk", + "tank", + "tape", + "target", + "task", + "taste", + "tattoo", + "taxi", + "teach", + "team", + "tell", + "ten", + "tenant", + "tennis", + "tent", + "term", + "test", + "text", + "thank", + "that", + "theme", + "then", + "theory", + "there", + "they", + "thing", + "this", + "thought", + "three", + "thrive", + "throw", + "thumb", + "thunder", + "ticket", + "tide", + "tiger", + "tilt", + "timber", + "time", + "tiny", + "tip", + "tired", + "tissue", + "title", + "toast", + "tobacco", + "today", + "toddler", + "toe", + "together", + "toilet", + "token", + "tomato", + "tomorrow", + "tone", + "tongue", + "tonight", + "tool", + "tooth", + "top", + "topic", + "topple", + "torch", + "tornado", + "tortoise", + "toss", + "total", + "tourist", + "toward", + "tower", + "town", + "toy", + "track", + "trade", + "traffic", + "tragic", + "train", + "transfer", + "trap", + "trash", + "travel", + "tray", + "treat", + "tree", + "trend", + "trial", + "tribe", + "trick", + "trigger", + "trim", + "trip", + "trophy", + "trouble", + "truck", + "true", + "truly", + "trumpet", + "trust", + "truth", + "try", + "tube", + "tuition", + "tumble", + "tuna", + "tunnel", + "turkey", + "turn", + "turtle", + "twelve", + "twenty", + "twice", + "twin", + "twist", + "two", + "type", + "typical", + "ugly", + "umbrella", + "unable", + "unaware", + "uncle", + "uncover", + "under", + "undo", + "unfair", + "unfold", + "unhappy", + "uniform", + "unique", + "unit", + "universe", + "unknown", + "unlock", + "until", + "unusual", + "unveil", + "update", + "upgrade", + "uphold", + "upon", + "upper", + "upset", + "urban", + "urge", + "usage", + "use", + "used", + "useful", + "useless", + "usual", + "utility", + "vacant", + "vacuum", + "vague", + "valid", + "valley", + "valve", + "van", + "vanish", + "vapor", + "various", + "vast", + "vault", + "vehicle", + "velvet", + "vendor", + "venture", + "venue", + "verb", + "verify", + "version", + "very", + "vessel", + "veteran", + "viable", + "vibrant", + "vicious", + "victory", + "video", + "view", + "village", + "vintage", + "violin", + "virtual", + "virus", + "visa", + "visit", + "visual", + "vital", + "vivid", + "vocal", + "voice", + "void", + "volcano", + "volume", + "vote", + "voyage", + "wage", + "wagon", + "wait", + "walk", + "wall", + "walnut", + "want", + "warfare", + "warm", + "warrior", + "wash", + "wasp", + "waste", + "water", + "wave", + "way", + "wealth", + "weapon", + "wear", + "weasel", + "weather", + "web", + "wedding", + "weekend", + "weird", + "welcome", + "west", + "wet", + "whale", + "what", + "wheat", + "wheel", + "when", + "where", + "whip", + "whisper", + "wide", + "width", + "wife", + "wild", + "will", + "win", + "window", + "wine", + "wing", + "wink", + "winner", + "winter", + "wire", + "wisdom", + "wise", + "wish", + "witness", + "wolf", + "woman", + "wonder", + "wood", + "wool", + "word", + "work", + "world", + "worry", + "worth", + "wrap", + "wreck", + "wrestle", + "wrist", + "write", + "wrong", + "yard", + "year", + "yellow", + "you", + "young", + "youth", + "zebra", + "zero", + "zone", + "zoo" + }; + + QStringList randomWords; + while(randomWords.size() != 16) { + QString currWord = wordList.at(rand()); + if (!randomWords.contains(currWord)) { + randomWords.append(currWord); + } + } + return randomWords; +} + +// Namespaces +} +} diff --git a/src/libsync/wordlist.h b/src/libsync/wordlist.h new file mode 100644 index 000000000..a27fb46ed --- /dev/null +++ b/src/libsync/wordlist.h @@ -0,0 +1,13 @@ +#ifndef WORDLIST_H +#define WORDLIST_H + +#include +#include + +namespace OCC { + namespace WordList { + QStringList getRandomWords(int nr); + } +} + +#endif From 418401a33c55ee1ff63cb63f975dc2230e063ebf Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 14 Sep 2017 17:23:40 +0200 Subject: [PATCH 025/262] [cse] new method, getUnifiedString Enter a string list, return a string. --- src/libsync/wordlist.cpp | 13 +++++++++++-- src/libsync/wordlist.h | 1 + 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/libsync/wordlist.cpp b/src/libsync/wordlist.cpp index f33a3e1cb..4d77286da 100644 --- a/src/libsync/wordlist.cpp +++ b/src/libsync/wordlist.cpp @@ -5,7 +5,7 @@ namespace WordList { #include -QStringList getRandomWords() +QStringList getRandomWords(int nr) { QStringList wordList = { "abandon", @@ -2059,7 +2059,7 @@ QStringList getRandomWords() }; QStringList randomWords; - while(randomWords.size() != 16) { + while(randomWords.size() != nr) { QString currWord = wordList.at(rand()); if (!randomWords.contains(currWord)) { randomWords.append(currWord); @@ -2068,6 +2068,15 @@ QStringList getRandomWords() return randomWords; } +QString getUnifiedString(const QStringList& wList) +{ + QString ret; + for(const auto& str : wList) { + ret += str; + } + return ret; +} + // Namespaces } } diff --git a/src/libsync/wordlist.h b/src/libsync/wordlist.h index a27fb46ed..b8891640b 100644 --- a/src/libsync/wordlist.h +++ b/src/libsync/wordlist.h @@ -7,6 +7,7 @@ namespace OCC { namespace WordList { QStringList getRandomWords(int nr); + QString getUnifiedString(const QStringList& l); } } From 60729f1fc44f314bb0b72a066450f327b95092bf Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 14 Sep 2017 17:37:43 +0200 Subject: [PATCH 026/262] [cse] Enable OpenSSL for encryption --- src/gui/main.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/gui/main.cpp b/src/gui/main.cpp index ba3947f14..6d3a2e335 100644 --- a/src/gui/main.cpp +++ b/src/gui/main.cpp @@ -21,6 +21,9 @@ #include #endif +#include +#include + #include "application.h" #include "theme.h" #include "common/utility.h" @@ -47,6 +50,11 @@ int main(int argc, char **argv) { Q_INIT_RESOURCE(client); + /* Initialise the library */ + ERR_load_crypto_strings(); + OpenSSL_add_all_algorithms(); + OPENSSL_config(NULL); + #ifdef Q_OS_WIN // If the font size ratio is set on Windows, we need to // enable the auto pixelRatio in Qt since we don't From fd00e180f54dd9dd54a2d8adb3643445a921b4ac Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 14 Sep 2017 18:30:00 +0200 Subject: [PATCH 027/262] [aes] fix out of bounds access. --- src/libsync/wordlist.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsync/wordlist.cpp b/src/libsync/wordlist.cpp index 4d77286da..245089bc7 100644 --- a/src/libsync/wordlist.cpp +++ b/src/libsync/wordlist.cpp @@ -2060,7 +2060,7 @@ QStringList getRandomWords(int nr) QStringList randomWords; while(randomWords.size() != nr) { - QString currWord = wordList.at(rand()); + QString currWord = wordList.at(rand() % wordList.size()); if (!randomWords.contains(currWord)) { randomWords.append(currWord); } From 5395fc56b17d6d0b486a98202b7098039fc79233 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 14 Sep 2017 18:39:18 +0200 Subject: [PATCH 028/262] [cse] Start the encryption algorithm for the Private Key --- src/libsync/clientsideencryption.cpp | 46 +++++++++++++++++++++++++--- src/libsync/clientsideencryption.h | 1 + 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 2704a100c..220f6a1b4 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -18,6 +18,8 @@ #include #include +#include "wordlist.h" + namespace OCC { @@ -198,18 +200,19 @@ QString ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) job = new SignPublicKeyApiJob(_account, baseUrl + "public-key", this); job->setCsr(output); - // TODO: Extract this function, should not be a lambda. - connect(job, &SignPublicKeyApiJob::jsonReceived, [this](const QJsonDocument& json, int retCode) { + + connect(job, &SignPublicKeyApiJob::jsonReceived, [this, keyPair](const QJsonDocument& json, int retCode) { if (retCode == 200) { auto caps = json.object().value("ocs").toObject().value("data").toObject().value("public-key").toString(); qCInfo(lcCse()) << "Public Key Returned" << caps; - QFile file(publicKeyPath() + ".sign"); // TODO: Verify if I need to keep the old file. + QFile file(publicKeyPath() + ".sign"); if (file.open(QIODevice::WriteOnly)) { QTextStream s(&file); s << caps; } file.close(); - qCInfo(lcCse()) << "public key saved."; + qCInfo(lcCse()) << "public key saved, Encrypting Private Key."; + encryptPrivateKey(keyPair); } qCInfo(lcCse()) << retCode; }); @@ -221,6 +224,41 @@ free_all: return ""; } +void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) +{ + // Write the Private File to a BIO + // Retrieve the BIO contents, and encrypt it. + // Send the encrypted key to the server. + // I have no idea what I'm doing. + + static const char* salt = "$4$YmBjm3hk$Qb74D5IUYwghUmzsMqeNFx5z0/8$"; + static const int iterationCount = 1024; + static const int keyStrength = 256; + BIO* bio = BIO_new(BIO_s_mem()); + + QString passPhrase = WordList::getUnifiedString(WordList::getRandomWords(12)); + const char* passPhrasePtr = qPrintable(passPhrase); + qCInfo(lcCse()) << "Passphrase Generated:"; + qCInfo(lcCse()) << passPhrase; + + // Extract the Private key from the key pair. + PEM_write_bio_PrivateKey(bio, keyPair, NULL, NULL, 0, 0, NULL); + char data[80]; + QString output; + int ret = 0; + do { + ret = BIO_gets(bio, data, 80); + output += data; + if (output.endsWith("-----END PRIVATE KEY-----")) { + output = output.trimmed(); + break; + } + } while (ret > 0 ); + + qCInfo(lcCse()) << "Private Key Extracted"; + qCInfo(lcCse()) << output; +} + void ClientSideEncryption::getPrivateKeyFromServer() { diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index e21e3aabc..5d83bdeff 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -26,6 +26,7 @@ public: void getPrivateKeyFromServer(); void getPublicKeyFromServer(); void signPublicKey(); + void encryptPrivateKey(EVP_PKEY *keyPair); QString privateKeyPath() const; QString publicKeyPath() const; From 77c0309e02159ddb09c28e25c270a98843b9592d Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 14 Sep 2017 21:41:31 +0200 Subject: [PATCH 029/262] [cse] use PKCS5_PBKDF2_HMAC_SHA1 on the random-word passphrase --- src/libsync/clientsideencryption.cpp | 29 +++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 220f6a1b4..9e63a45ff 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -231,9 +231,11 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) // Send the encrypted key to the server. // I have no idea what I'm doing. - static const char* salt = "$4$YmBjm3hk$Qb74D5IUYwghUmzsMqeNFx5z0/8$"; - static const int iterationCount = 1024; - static const int keyStrength = 256; + using ucharp = unsigned char *; + const char *salt = "$4$YmBjm3hk$Qb74D5IUYwghUmzsMqeNFx5z0/8$"; + const int saltLen = 40; + const int iterationCount = 1024; + const int keyStrength = 256; BIO* bio = BIO_new(BIO_s_mem()); QString passPhrase = WordList::getUnifiedString(WordList::getRandomWords(12)); @@ -257,6 +259,27 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) qCInfo(lcCse()) << "Private Key Extracted"; qCInfo(lcCse()) << output; + + /* Jesus. the OpenSSL docs do not help at all. + * This PKCS5_PBKDF2_HMAC_SHA1 call will generate + * a new password from the password that was submited. + */ + unsigned char secretKey[keyStrength]; + + ret = PKCS5_PBKDF2_HMAC_SHA1( + passPhrasePtr, // const char *password, + passPhrase.size(), // int password length, + (ucharp) salt, // const unsigned char *salt, + saltLen, // int saltlen, + iterationCount, // int iterations, + keyStrength, // int keylen, + secretKey // unsigned char *out + ); + qCInfo(lcCse()) << "Return of the PKCS5" << ret; + qCInfo(lcCse()) << "Result String" << secretKey; + + const EVP_CIPHER *cipher = EVP_get_cipherbyname("aes-256-cbc"); + // Now, Try to encrypt it. } void ClientSideEncryption::getPrivateKeyFromServer() From 6b53b4f25735f5dedbdc231072e8a6e04dd3db1c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 14 Sep 2017 22:45:12 +0200 Subject: [PATCH 030/262] [cse] Add the crypt method, based on OpenSSL Wiki --- src/libsync/clientsideencryption.cpp | 53 +++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 9e63a45ff..e9dff4468 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -28,6 +29,52 @@ Q_LOGGING_CATEGORY(lcCse, "sync.clientsideencryption", QtInfoMsg) QString baseUrl = QStringLiteral("ocs/v2.php/apps/client_side_encryption/api/v1/"); QString baseDirectory = QDir::homePath() + QStringLiteral("/.nextcloud-keys/"); +namespace { + void handleErrors(void) + { + ERR_print_errors_fp(stderr); + abort(); + } + + int encrypt(unsigned char *plaintext, int plaintext_len, unsigned char *key, unsigned char *iv, unsigned char *ciphertext) + { + EVP_CIPHER_CTX *ctx; + int len; + int ciphertext_len; + + /* Create and initialise the context */ + if(!(ctx = EVP_CIPHER_CTX_new())) { + handleErrors(); + } + + /* Initialise the encryption operation. IMPORTANT - ensure you use a key + * and IV size appropriate for your cipher + * In this example we are using 256 bit AES (i.e. a 256 bit key). The + * IV size for *most* modes is the same as the block size. For AES this + * is 128 bits */ + if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, key, iv)) { + handleErrors(); + } + + // Figure out if I need to call this multiple times. + if(1 != EVP_EncryptUpdate(ctx, ciphertext, &len, plaintext, plaintext_len)) { + handleErrors(); + } + ciphertext_len = len; + + // Finalize Encryption + if(1 != EVP_EncryptFinal_ex(ctx, ciphertext + len, &len)) { + handleErrors(); + } + ciphertext_len += len; + + /* Clean up */ + EVP_CIPHER_CTX_free(ctx); + + return ciphertext_len; + } +} + ClientSideEncryption::ClientSideEncryption() { } @@ -278,7 +325,11 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) qCInfo(lcCse()) << "Return of the PKCS5" << ret; qCInfo(lcCse()) << "Result String" << secretKey; - const EVP_CIPHER *cipher = EVP_get_cipherbyname("aes-256-cbc"); + //: FIRST TRY A SILLY PHRASE. + //: Hardcoed IV, really bad. + unsigned char *iv = (unsigned char *)"0123456789012345"; + const EVP_CIPHER *cipher = EVP_aes_256_gcm (); + // Now, Try to encrypt it. } From 4f7265c04f258d4e8e8acb82f7068d2a43d0a5f0 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 14 Sep 2017 22:50:56 +0200 Subject: [PATCH 031/262] [aes] Create the Decrypt function. --- src/libsync/clientsideencryption.cpp | 56 ++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index e9dff4468..fb9b07cbf 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -36,11 +36,15 @@ namespace { abort(); } - int encrypt(unsigned char *plaintext, int plaintext_len, unsigned char *key, unsigned char *iv, unsigned char *ciphertext) + int encrypt(unsigned char *plaintext, + int plaintext_len, + unsigned char *key, + unsigned char *iv, + unsigned char *ciphertext) { - EVP_CIPHER_CTX *ctx; - int len; - int ciphertext_len; + EVP_CIPHER_CTX *ctx = nullptr; + int len = 0; + int ciphertext_len = 0; /* Create and initialise the context */ if(!(ctx = EVP_CIPHER_CTX_new())) { @@ -73,6 +77,50 @@ namespace { return ciphertext_len; } + + int decrypt(unsigned char *ciphertext, + int ciphertext_len, + unsigned char *key, + unsigned char *iv, + unsigned char *plaintext) + { + EVP_CIPHER_CTX *ctx = nullptr; + + int len = 0; + + int plaintext_len = 0; + + /* Create and initialise the context */ + if(!(ctx = EVP_CIPHER_CTX_new())) { + handleErrors(); + } + + /* Initialise the decryption operation. IMPORTANT - ensure you use a key + * and IV size appropriate for your cipher + * In this example we are using 256 bit AES (i.e. a 256 bit key). The + * IV size for *most* modes is the same as the block size. For AES this + * is 128 bits */ + if(1 != EVP_DecryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, key, iv)) { + handleErrors(); + } + + // Verify if we need to call this more than one time + if(1 != EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len)) { + handleErrors(); + } + plaintext_len = len; + + // Finalize Decrypt + if(1 != EVP_DecryptFinal_ex(ctx, plaintext + len, &len)) { + handleErrors(); + } + plaintext_len += len; + + /* Clean up */ + EVP_CIPHER_CTX_free(ctx); + + return plaintext_len; + } } ClientSideEncryption::ClientSideEncryption() From e0d368cbb3c8d792599c2be311d8f8932641be45 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 15 Sep 2017 16:17:37 +0200 Subject: [PATCH 032/262] [cse] Update the encrypt and decrypt algorithm to GCM --- src/libsync/clientsideencryption.cpp | 114 ++++++++++++++++++++------- 1 file changed, 85 insertions(+), 29 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index fb9b07cbf..617d2bb46 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -38,40 +38,65 @@ namespace { int encrypt(unsigned char *plaintext, int plaintext_len, + unsigned char *aad, + int aad_len, unsigned char *key, unsigned char *iv, - unsigned char *ciphertext) + unsigned char *ciphertext, + unsigned char *tag) { - EVP_CIPHER_CTX *ctx = nullptr; - int len = 0; - int ciphertext_len = 0; + EVP_CIPHER_CTX *ctx; + int len; + int ciphertext_len; /* Create and initialise the context */ if(!(ctx = EVP_CIPHER_CTX_new())) { handleErrors(); } - /* Initialise the encryption operation. IMPORTANT - ensure you use a key - * and IV size appropriate for your cipher - * In this example we are using 256 bit AES (i.e. a 256 bit key). The - * IV size for *most* modes is the same as the block size. For AES this - * is 128 bits */ - if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, key, iv)) { + /* Initialise the encryption operation. */ + if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, NULL, NULL)) { handleErrors(); } - // Figure out if I need to call this multiple times. + /* Set IV length if default 12 bytes (96 bits) is not appropriate */ + if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL)) { + handleErrors(); + } + + /* Initialise key and IV */ + if(1 != EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv)) { + handleErrors(); + } + + /* Provide any AAD data. This can be called zero or more times as + * required + */ + if(1 != EVP_EncryptUpdate(ctx, NULL, &len, aad, aad_len)) { + handleErrors(); + } + + /* Provide the message to be encrypted, and obtain the encrypted output. + * EVP_EncryptUpdate can be called multiple times if necessary + */ if(1 != EVP_EncryptUpdate(ctx, ciphertext, &len, plaintext, plaintext_len)) { handleErrors(); } ciphertext_len = len; - // Finalize Encryption + /* Finalise the encryption. Normally ciphertext bytes may be written at + * this stage, but this does not occur in GCM mode + */ if(1 != EVP_EncryptFinal_ex(ctx, ciphertext + len, &len)) { handleErrors(); } ciphertext_len += len; + /* Get the tag */ + if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag)) { + handleErrors(); + } + /* Clean up */ EVP_CIPHER_CTX_free(ctx); @@ -80,46 +105,77 @@ namespace { int decrypt(unsigned char *ciphertext, int ciphertext_len, + unsigned char *aad, + int aad_len, + unsigned char *tag, unsigned char *key, unsigned char *iv, unsigned char *plaintext) { - EVP_CIPHER_CTX *ctx = nullptr; - - int len = 0; - - int plaintext_len = 0; + EVP_CIPHER_CTX *ctx; + int len; + int plaintext_len; + int ret; /* Create and initialise the context */ if(!(ctx = EVP_CIPHER_CTX_new())) { handleErrors(); } - /* Initialise the decryption operation. IMPORTANT - ensure you use a key - * and IV size appropriate for your cipher - * In this example we are using 256 bit AES (i.e. a 256 bit key). The - * IV size for *most* modes is the same as the block size. For AES this - * is 128 bits */ - if(1 != EVP_DecryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, key, iv)) { + /* Initialise the decryption operation. */ + if(!EVP_DecryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, NULL, NULL)) { handleErrors(); } - // Verify if we need to call this more than one time - if(1 != EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len)) { + /* Set IV length. Not necessary if this is 12 bytes (96 bits) */ + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL)) { + handleErrors(); + } + + /* Initialise key and IV */ + if(!EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv)) { + handleErrors(); + } + + /* Provide any AAD data. This can be called zero or more times as + * required + */ + if(!EVP_DecryptUpdate(ctx, NULL, &len, aad, aad_len)) { + handleErrors(); + } + + /* Provide the message to be decrypted, and obtain the plaintext output. + * EVP_DecryptUpdate can be called multiple times if necessary + */ + if(!EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len)) { handleErrors(); } plaintext_len = len; - // Finalize Decrypt - if(1 != EVP_DecryptFinal_ex(ctx, plaintext + len, &len)) { + /* Set expected tag value. Works in OpenSSL 1.0.1d and later */ + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag)) { handleErrors(); } - plaintext_len += len; + + /* Finalise the decryption. A positive return value indicates success, + * anything else is a failure - the plaintext is not trustworthy. + */ + ret = EVP_DecryptFinal_ex(ctx, plaintext + len, &len); /* Clean up */ EVP_CIPHER_CTX_free(ctx); - return plaintext_len; + if(ret > 0) + { + /* Success */ + plaintext_len += len; + return plaintext_len; + } + else + { + /* Verify failed */ + return -1; + } } } From ba3d2a61d52a6130cef2fe03591fc45e7b2797d9 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 15 Sep 2017 17:38:20 +0200 Subject: [PATCH 033/262] [cse] Call the encrypt and decrypt functions on fake data --- src/libsync/clientsideencryption.cpp | 31 +++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 617d2bb46..3efdd01f0 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -432,8 +432,37 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) //: FIRST TRY A SILLY PHRASE. //: Hardcoed IV, really bad. unsigned char *iv = (unsigned char *)"0123456789012345"; - const EVP_CIPHER *cipher = EVP_aes_256_gcm (); + unsigned char encryptTag[16]; + + const char *encryptTest = "a quick brown fox jumps over the lazy dog"; + // TODO: Find a way to + unsigned char cryptedText[128]; + unsigned char decryptedText[128]; + unsigned char tag[16]; + int cryptedText_len = encrypt( + (unsigned char*) encryptTest, //unsigned char *plaintext, + strlen(encryptTest), // int plaintext_len, + nullptr, // unsigned char *aad, + 0, // int aad_len, + secretKey, // unsigned char *key, + iv, // unsigned char *iv, + cryptedText, // unsigned char *ciphertext, + tag // unsigned char *tag + ); + + qCInfo(lcCse()) << "Encrypted Text" << QByteArray( (const char*) cryptedText, cryptedText_len); + int decryptedText_len = decrypt( + cryptedText, //unsigned char *ciphertext, + cryptedText_len, //int ciphertext_len, + NULL, //unsigned char *aad, + 0, //int aad_len, + tag, //unsigned char *tag, + secretKey, //unsigned char *key, + iv, //unsigned char *iv, + decryptedText //unsigned char *plaintext + ); + qCInfo(lcCse()) << "Decrypted Text" << QByteArray( (const char*) decryptedText, decryptedText_len); // Now, Try to encrypt it. } From 0f1480728eaf6eb848f9b5aa90ca8b4d8648038e Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 15 Sep 2017 17:59:14 +0200 Subject: [PATCH 034/262] [cse] Use fake data to verify the encryption beware, this is broken. --- src/gui/main.cpp | 4 +++- src/libsync/clientsideencryption.cpp | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/gui/main.cpp b/src/gui/main.cpp index 6d3a2e335..1c5815687 100644 --- a/src/gui/main.cpp +++ b/src/gui/main.cpp @@ -33,6 +33,7 @@ #include #include +#include using namespace OCC; @@ -50,7 +51,8 @@ int main(int argc, char **argv) { Q_INIT_RESOURCE(client); - /* Initialise the library */ + /* Initialise the library */ + qDebug() << "LOADING OPENSSL STUFF"; ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); OPENSSL_config(NULL); diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 3efdd01f0..740517951 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -32,8 +32,9 @@ QString baseDirectory = QDir::homePath() + QStringLiteral("/.nextcloud-keys/"); namespace { void handleErrors(void) { - ERR_print_errors_fp(stderr); - abort(); + qCInfo(lcCse()) << "Error handling encryption or decryption"; + ERR_print_errors_fp(stdout); // This line is not printing anything. + fflush(stdout); } int encrypt(unsigned char *plaintext, @@ -431,6 +432,7 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) //: FIRST TRY A SILLY PHRASE. //: Hardcoed IV, really bad. + unsigned char *fakepass = (unsigned char*) "qwertyuiasdfghjkzxcvbnm,qwertyui"; unsigned char *iv = (unsigned char *)"0123456789012345"; unsigned char encryptTag[16]; @@ -445,7 +447,7 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) strlen(encryptTest), // int plaintext_len, nullptr, // unsigned char *aad, 0, // int aad_len, - secretKey, // unsigned char *key, + fakepass, // unsigned char *key, iv, // unsigned char *iv, cryptedText, // unsigned char *ciphertext, tag // unsigned char *tag @@ -458,7 +460,7 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) NULL, //unsigned char *aad, 0, //int aad_len, tag, //unsigned char *tag, - secretKey, //unsigned char *key, + fakepass, //unsigned char *key, iv, //unsigned char *iv, decryptedText //unsigned char *plaintext ); From 08dfe86f37e36e617083527b70bce0570c771643 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 16 Oct 2017 19:50:00 +0200 Subject: [PATCH 035/262] [CSE] Adjust the calls to ent-to-end encryption We changed the name. --- src/gui/accountsettings.cpp | 4 ++-- src/libsync/capabilities.cpp | 4 ++-- src/libsync/clientsideencryption.cpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 091d7ce85..d664591e1 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -282,7 +282,7 @@ void AccountSettings::slotCustomContextMenuRequested(const QPoint &pos) connect(ac, &QAction::triggered, [this, &fileId](bool triggered) { Q_UNUSED(triggered); auto job = new OCC::JsonApiJob(accountsState()->account(), - "ocs/v2.php/apps/client_side_encryption/api/v1/encrypted/" + QString(fileId)); + "ocs/v2.php/apps/end_to_end_encryption/api/v1/encrypted/" + QString(fileId)); connect(job, &OCC::JsonApiJob::jsonReceived, [this](const QJsonDocument& json, int httpResponse) { Q_UNUSED(json); @@ -295,7 +295,7 @@ void AccountSettings::slotCustomContextMenuRequested(const QPoint &pos) connect(ac, &QAction::triggered, [this, &fileId](bool triggered) { Q_UNUSED(triggered); auto job = new OCC::DeleteApiJob(accountsState()->account(), - "ocs/v2.php/apps/client_side_encryption/api/v1/encrypted/" + QString(fileId)); + "ocs/v2.php/apps/end_to_end_encryption/api/v1/encrypted/" + QString(fileId)); connect(job, &OCC::DeleteApiJob::result, [this](int httpResponse) { qCInfo(lcAccountSettings) << "Decrypt Http Response" << httpResponse; }); diff --git a/src/libsync/capabilities.cpp b/src/libsync/capabilities.cpp index c7fc546d2..8a7cee951 100644 --- a/src/libsync/capabilities.cpp +++ b/src/libsync/capabilities.cpp @@ -87,8 +87,8 @@ bool Capabilities::shareResharing() const bool Capabilities::clientSideEncryptionAvaliable() const { - return _capabilities.keys().indexOf("client-side-encryption") != -1 - ? _capabilities["client-side-encryption"].toMap()["enabled"].toBool() : false; + return _capabilities.keys().indexOf("end-to-end-encryption") != -1 + ? _capabilities["end-to-end-encryption"].toMap()["enabled"].toBool() : false; } bool Capabilities::notificationsAvailable() const diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 740517951..f63ac3728 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -26,7 +26,7 @@ namespace OCC Q_LOGGING_CATEGORY(lcCse, "sync.clientsideencryption", QtInfoMsg) -QString baseUrl = QStringLiteral("ocs/v2.php/apps/client_side_encryption/api/v1/"); +QString baseUrl = QStringLiteral("ocs/v2.php/apps/end_to_end_encryption/api/v1/"); QString baseDirectory = QDir::homePath() + QStringLiteral("/.nextcloud-keys/"); namespace { From 78136a10b072a469f660fa7213aa3836cb9035da Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 16 Oct 2017 20:24:27 +0200 Subject: [PATCH 036/262] [CSE] Debug statements to help find the encrypt error --- src/libsync/clientsideencryption.cpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index f63ac3728..0fcf4ca1d 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -32,7 +32,6 @@ QString baseDirectory = QDir::homePath() + QStringLiteral("/.nextcloud-keys/"); namespace { void handleErrors(void) { - qCInfo(lcCse()) << "Error handling encryption or decryption"; ERR_print_errors_fp(stdout); // This line is not printing anything. fflush(stdout); } @@ -52,21 +51,25 @@ namespace { /* Create and initialise the context */ if(!(ctx = EVP_CIPHER_CTX_new())) { + qCInfo(lcCse()) << "Error creating the Cipher."; handleErrors(); } /* Initialise the encryption operation. */ if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, NULL, NULL)) { + qCInfo(lcCse()) << "Error initializing the context with aes_256"; handleErrors(); } /* Set IV length if default 12 bytes (96 bits) is not appropriate */ if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL)) { + qCInfo(lcCse()) << "Error setting the iv length to 16 bits. "; handleErrors(); } /* Initialise key and IV */ if(1 != EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv)) { + qCInfo(lcCse()) << "Error initializing encryption"; handleErrors(); } @@ -74,6 +77,7 @@ namespace { * required */ if(1 != EVP_EncryptUpdate(ctx, NULL, &len, aad, aad_len)) { + qCInfo(lcCse()) << "Error calling the Encrypt Update"; handleErrors(); } @@ -81,6 +85,7 @@ namespace { * EVP_EncryptUpdate can be called multiple times if necessary */ if(1 != EVP_EncryptUpdate(ctx, ciphertext, &len, plaintext, plaintext_len)) { + qCInfo(lcCse()) << "Error encrypting the cipher ext"; // Current error is here. handleErrors(); } ciphertext_len = len; @@ -89,12 +94,14 @@ namespace { * this stage, but this does not occur in GCM mode */ if(1 != EVP_EncryptFinal_ex(ctx, ciphertext + len, &len)) { + qCInfo(lcCse()) << "Error finalizing the encryption"; handleErrors(); } ciphertext_len += len; /* Get the tag */ if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag)) { + qCInfo(lcCse()) << "Error Retrieving the tag"; handleErrors(); } @@ -120,21 +127,25 @@ namespace { /* Create and initialise the context */ if(!(ctx = EVP_CIPHER_CTX_new())) { + qCInfo(lcCse()) << "Error Initializing the decrypt context"; handleErrors(); } /* Initialise the decryption operation. */ if(!EVP_DecryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, NULL, NULL)) { + qCInfo(lcCse()) << "Error initializing the decryption context"; handleErrors(); } /* Set IV length. Not necessary if this is 12 bytes (96 bits) */ if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL)) { + qCInfo(lcCse()) << "Error seting th iv length for the decrypt context"; handleErrors(); } /* Initialise key and IV */ if(!EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv)) { + qCInfo(lcCse()) << "Error setting the key and iv for decryption"; handleErrors(); } @@ -142,6 +153,7 @@ namespace { * required */ if(!EVP_DecryptUpdate(ctx, NULL, &len, aad, aad_len)) { + qCInfo(lcCse()) << "Error updating the decrypt context with aad information"; handleErrors(); } @@ -149,12 +161,14 @@ namespace { * EVP_DecryptUpdate can be called multiple times if necessary */ if(!EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len)) { + qCInfo(lcCse()) << "Error decrypting the text"; handleErrors(); } plaintext_len = len; /* Set expected tag value. Works in OpenSSL 1.0.1d and later */ if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag)) { + qCInfo(lcCse()) << "Error setting the tag on the decrupt context"; handleErrors(); } @@ -174,6 +188,7 @@ namespace { } else { + qCInfo(lcCse()) << "Error finalizing the decrypt"; /* Verify failed */ return -1; } @@ -465,7 +480,7 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) decryptedText //unsigned char *plaintext ); qCInfo(lcCse()) << "Decrypted Text" << QByteArray( (const char*) decryptedText, decryptedText_len); - // Now, Try to encrypt it. + } void ClientSideEncryption::getPrivateKeyFromServer() From cfb6e3be8cc471134da8e06ccbd3eb6ffa391cd7 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 16 Oct 2017 21:06:58 +0200 Subject: [PATCH 037/262] [CSE] Send the Private Key to the server --- src/libsync/clientsideencryption.cpp | 14 +++++++-- src/libsync/networkjobs.cpp | 40 +++++++++++++++++++++++- src/libsync/networkjobs.h | 46 +++++++++++++++++++++++++++- 3 files changed, 95 insertions(+), 5 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 0fcf4ca1d..fd72f0106 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -467,7 +467,7 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) cryptedText, // unsigned char *ciphertext, tag // unsigned char *tag ); - +/* qCInfo(lcCse()) << "Encrypted Text" << QByteArray( (const char*) cryptedText, cryptedText_len); int decryptedText_len = decrypt( cryptedText, //unsigned char *ciphertext, @@ -480,12 +480,20 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) decryptedText //unsigned char *plaintext ); qCInfo(lcCse()) << "Decrypted Text" << QByteArray( (const char*) decryptedText, decryptedText_len); - +*/ +// Pretend that the private key is actually encrypted and send it to the server. + auto job = new StorePrivateKeyApiJob(_account, baseUrl + "private-key", this); + job->setPrivateKey(QByteArray((const char*) cryptedText, 128)); + connect(job, &StorePrivateKeyApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { + qCInfo(lcCse()) << doc; + qCInfo(lcCse()) << "Store Private Key returned with" << retCode; + }); + job->start(); } void ClientSideEncryption::getPrivateKeyFromServer() { - + qCInfo(lcCse()) << "Trying to store the private key on the server."; } void ClientSideEncryption::getPublicKeyFromServer() diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index 52102acbd..a0be7ea9e 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -49,6 +49,7 @@ Q_LOGGING_CATEGORY(lcProppatchJob, "sync.networkjob.proppatch", QtInfoMsg) Q_LOGGING_CATEGORY(lcJsonApiJob, "sync.networkjob.jsonapi", QtInfoMsg) Q_LOGGING_CATEGORY(lcDetermineAuthTypeJob, "sync.networkjob.determineauthtype", QtInfoMsg) Q_LOGGING_CATEGORY(lcSignPublicKeyApiJob, "sync.networkjob.sendcsr", QtInfoMsg); +Q_LOGGING_CATEGORY(lcStorePrivateKeyApiJob, "sync.networkjob.storeprivatekey", QtInfoMsg); RequestEtagJob::RequestEtagJob(AccountPtr account, const QString &path, QObject *parent) : AbstractNetworkJob(account, path, parent) @@ -971,7 +972,44 @@ void SignPublicKeyApiJob::start() bool SignPublicKeyApiJob::finished() { - qCInfo(lcSignPublicKeyApiJob()) << "Sending CSR ended with" << path() << errorString() << reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute); + qCInfo(lcStorePrivateKeyApiJob()) << "Sending CSR ended with" << path() << errorString() << reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute); + + QJsonParseError error; + auto json = QJsonDocument::fromJson(reply()->readAll(), &error); + emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); +} + + +StorePrivateKeyApiJob::StorePrivateKeyApiJob(const AccountPtr& account, const QString& path, QObject* parent) +: AbstractNetworkJob(account, path, parent) +{ +} + +void StorePrivateKeyApiJob::setPrivateKey(const QByteArray& privKey) +{ + QByteArray data = "privateKey="; + data += QUrl::toPercentEncoding(privKey); + _privKey.setData(data); +} + +void StorePrivateKeyApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) + }; + url.setQueryItems(params); + + qCInfo(lcStorePrivateKeyApiJob) << "Sending the private key" << _privKey.data(); + sendRequest("POST", url, req, &_privKey); + AbstractNetworkJob::start(); +} + +bool StorePrivateKeyApiJob::finished() +{ + qCInfo(lcStorePrivateKeyApiJob()) << "Sending private key ended with" << path() << errorString() << reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute); QJsonParseError error; auto json = QJsonDocument::fromJson(reply()->readAll(), &error); diff --git a/src/libsync/networkjobs.h b/src/libsync/networkjobs.h index 1585b9156..685e89db2 100644 --- a/src/libsync/networkjobs.h +++ b/src/libsync/networkjobs.h @@ -470,9 +470,53 @@ signals: private: QBuffer _csr; - }; + +/* + * @brief Job to upload the PrivateKey that return JSON + * + * To be used like this: + * \code + * _job = new StorePrivateKeyApiJob(account, QLatin1String("ocs/v1.php/foo/bar"), this); + * _job->setPrivateKey( privKey ); + * connect(_job...); + * _job->start(); + * \encode + * + * @ingroup libsync + */ +class OWNCLOUDSYNC_EXPORT StorePrivateKeyApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit StorePrivateKeyApiJob(const AccountPtr &account, const QString &path, QObject *parent = 0); + + /** + * @brief setCsr - the CSR with the public key. + * This function needs to be called before start() obviously. + */ + void setPrivateKey(const QByteArray& privateKey); + +public slots: + void start() override; + +protected: + bool finished() override; +signals: + + /** + * @brief jsonReceived - signal to report the json answer from ocs + * @param json - the parsed json document + * @param statusCode - the OCS status code: 100 (!) for success + */ + void jsonReceived(const QJsonDocument &json, int statusCode); + +private: + QBuffer _privKey; +}; + + } // namespace OCC #endif // NETWORKJOBS_H From 0b50afe9153d13b1e19f7d6522447b378159ec44 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 18 Oct 2017 19:22:59 +0200 Subject: [PATCH 038/262] [CSE] Correctly Set's the client as connected If we manage to encrypt and upload the private key to the server, the client is in it's connected state. --- src/libsync/clientsideencryption.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index fd72f0106..520cd33de 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -485,8 +485,14 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) auto job = new StorePrivateKeyApiJob(_account, baseUrl + "private-key", this); job->setPrivateKey(QByteArray((const char*) cryptedText, 128)); connect(job, &StorePrivateKeyApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { - qCInfo(lcCse()) << doc; - qCInfo(lcCse()) << "Store Private Key returned with" << retCode; + switch(retCode) { + case 200: + qCInfo(lcCse()) << "Store private key working as expected."; + emit initializationFinished(); + break; + default: + qCInfo(lcCse()) << "Store private key failed, return code:" << retCode; + } }); job->start(); } From 1436d5bac15859fdf39a885f6581850ef9b35412 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 18 Oct 2017 19:55:03 +0200 Subject: [PATCH 039/262] [CSE] Better debug output --- src/libsync/networkjobs.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index a0be7ea9e..e2095a3a9 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -1009,7 +1009,9 @@ void StorePrivateKeyApiJob::start() bool StorePrivateKeyApiJob::finished() { - qCInfo(lcStorePrivateKeyApiJob()) << "Sending private key ended with" << path() << errorString() << reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute); + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) + qCInfo(lcStorePrivateKeyApiJob()) << "Sending private key ended with" << path() << errorString() << retCode; QJsonParseError error; auto json = QJsonDocument::fromJson(reply()->readAll(), &error); From 623eb29845fdf8581978eff88ba9078ba80c3867 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 18 Oct 2017 20:21:35 +0200 Subject: [PATCH 040/262] [CSE] Helper functions for the URL's --- src/libsync/clientsideencryption.cpp | 21 +++++++++++++-------- src/libsync/clientsideencryption.h | 3 ++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 520cd33de..f4d098bec 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -26,8 +26,13 @@ namespace OCC Q_LOGGING_CATEGORY(lcCse, "sync.clientsideencryption", QtInfoMsg) -QString baseUrl = QStringLiteral("ocs/v2.php/apps/end_to_end_encryption/api/v1/"); -QString baseDirectory = QDir::homePath() + QStringLiteral("/.nextcloud-keys/"); +QString baseUrl(){ + return QStringLiteral("ocs/v2.php/apps/end_to_end_encryption/api/v1/"); +} + +QString baseDirectory() { + return QDir::homePath() + QStringLiteral("/.nextcloud-keys/"); +} namespace { void handleErrors(void) @@ -222,12 +227,12 @@ void ClientSideEncryption::initialize() QString ClientSideEncryption::publicKeyPath() const { - return baseDirectory + _account->displayName() + ".pub"; + return baseDirectory() + _account->displayName() + ".pub"; } QString ClientSideEncryption::privateKeyPath() const { - return baseDirectory + _account->displayName() + ".rsa"; + return baseDirectory() + _account->displayName() + ".rsa"; } bool ClientSideEncryption::hasPrivateKey() const @@ -271,7 +276,7 @@ void ClientSideEncryption::generateKeyPair() qCInfo(lcCse()) << "Storing keys locally"; QDir dir; - if (!dir.mkpath(baseDirectory)) { + if (!dir.mkpath(baseDirectory())) { qCInfo(lcCse()) << "Could not create the folder for the keys."; return; } @@ -365,7 +370,7 @@ QString ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) qCInfo(lcCse()) << "Returning the certificate"; qCInfo(lcCse()) << output; - job = new SignPublicKeyApiJob(_account, baseUrl + "public-key", this); + job = new SignPublicKeyApiJob(_account, baseUrl() + "public-key", this); job->setCsr(output); connect(job, &SignPublicKeyApiJob::jsonReceived, [this, keyPair](const QJsonDocument& json, int retCode) { @@ -482,7 +487,7 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) qCInfo(lcCse()) << "Decrypted Text" << QByteArray( (const char*) decryptedText, decryptedText_len); */ // Pretend that the private key is actually encrypted and send it to the server. - auto job = new StorePrivateKeyApiJob(_account, baseUrl + "private-key", this); + auto job = new StorePrivateKeyApiJob(_account, baseUrl() + "private-key", this); job->setPrivateKey(QByteArray((const char*) cryptedText, 128)); connect(job, &StorePrivateKeyApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { switch(retCode) { @@ -505,7 +510,7 @@ void ClientSideEncryption::getPrivateKeyFromServer() void ClientSideEncryption::getPublicKeyFromServer() { qCInfo(lcCse()) << "Retrieving public key from server"; - auto job = new JsonApiJob(_account, baseUrl + "public-key", this); + auto job = new JsonApiJob(_account, baseUrl() + "public-key", this); connect(job, &JsonApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { switch(retCode) { case 404: // no public key diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 5d83bdeff..ea9b385b0 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -11,7 +11,8 @@ #include "accountfwd.h" namespace OCC { - +QString baseUrl(); +QString baseDirectory(); class ClientSideEncryption : public QObject { Q_OBJECT From d83e8819ce90fb3af1c3c7481f081da426fc87d0 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 18 Oct 2017 20:22:27 +0200 Subject: [PATCH 041/262] [CSE] SetEncryptionFlagApiJob This new job sets the Encryption Flag in a folder. --- src/libsync/networkjobs.cpp | 33 +++++++++++++++++++++++++++++++++ src/libsync/networkjobs.h | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index e2095a3a9..40b19f404 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -33,6 +33,7 @@ #include "networkjobs.h" #include "account.h" #include "owncloudpropagator.h" +#include "clientsideencryption.h" #include "creds/abstractcredentials.h" #include "creds/httpcredentials.h" @@ -50,6 +51,7 @@ Q_LOGGING_CATEGORY(lcJsonApiJob, "sync.networkjob.jsonapi", QtInfoMsg) Q_LOGGING_CATEGORY(lcDetermineAuthTypeJob, "sync.networkjob.determineauthtype", QtInfoMsg) Q_LOGGING_CATEGORY(lcSignPublicKeyApiJob, "sync.networkjob.sendcsr", QtInfoMsg); Q_LOGGING_CATEGORY(lcStorePrivateKeyApiJob, "sync.networkjob.storeprivatekey", QtInfoMsg); +Q_LOGGING_CATEGORY(lcCse, "sync.networkjob.clientsideencrypt", QtInfoMsg); RequestEtagJob::RequestEtagJob(AccountPtr account, const QString &path, QObject *parent) : AbstractNetworkJob(account, path, parent) @@ -1018,5 +1020,36 @@ bool StorePrivateKeyApiJob::finished() emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); } +SetEncryptionFlagApiJob::SetEncryptionFlagApiJob(const AccountPtr& account, int fileId, QObject* parent) +: AbstractNetworkJob(account, baseUrl() + "/encrypt/", parent), _fileId(fileId) +{ +} + +void SetEncryptionFlagApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) + }; + url.setQueryItems(params); + + qCInfo(lcCse()) << "marking the file with id" << _fileId << "as encrypted"; + sendRequest("PUT", url, req); + AbstractNetworkJob::start(); +} + +bool SetEncryptionFlagApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) + qCInfo(lcCse()) << "Setting the encrypted flag failed with" << path() << errorString() << retCode; + + QJsonParseError error; + auto json = QJsonDocument::fromJson(reply()->readAll(), &error); + emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); +} + } // namespace OCC diff --git a/src/libsync/networkjobs.h b/src/libsync/networkjobs.h index 685e89db2..5a7ca0637 100644 --- a/src/libsync/networkjobs.h +++ b/src/libsync/networkjobs.h @@ -516,6 +516,41 @@ private: QBuffer _privKey; }; +/* + * @brief Job to mark a folder as encrypted JSON + * + * To be used like this: + * \code + * _job = new SetEncryptionFlagApiJob(account, 2, this); + * connect(_job...); + * _job->start(); + * \encode + * + * @ingroup libsync + */ +class OWNCLOUDSYNC_EXPORT SetEncryptionFlagApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit SetEncryptionFlagApiJob(const AccountPtr &account, int fileId, QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + + /** + * @brief jsonReceived - signal to report the json answer from ocs + * @param json - the parsed json document + * @param statusCode - the OCS status code: 200 for success + */ + void jsonReceived(const QJsonDocument &json, int statusCode); +private: + int _fileId; +}; } // namespace OCC From e0988f482c1f86ceb8696574fb900032d5c25ee6 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 18 Oct 2017 20:29:13 +0200 Subject: [PATCH 042/262] [CSE] Build fix - change the name of one Qt Message Handler - changed parameter from int to QString --- src/gui/accountsettings.cpp | 5 ++--- src/libsync/networkjobs.cpp | 10 +++++----- src/libsync/networkjobs.h | 4 ++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index d664591e1..6fd4c616f 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -281,10 +281,9 @@ void AccountSettings::slotCustomContextMenuRequested(const QPoint &pos) ac = menu->addAction(tr("Encrypt")); connect(ac, &QAction::triggered, [this, &fileId](bool triggered) { Q_UNUSED(triggered); - auto job = new OCC::JsonApiJob(accountsState()->account(), - "ocs/v2.php/apps/end_to_end_encryption/api/v1/encrypted/" + QString(fileId)); + auto job = new OCC::SetEncryptionFlagApiJob(accountsState()->account(), QString(fileId)); - connect(job, &OCC::JsonApiJob::jsonReceived, [this](const QJsonDocument& json, int httpResponse) { + connect(job, &OCC::SetEncryptionFlagApiJob::jsonReceived, [this](const QJsonDocument& json, int httpResponse) { Q_UNUSED(json); qCInfo(lcAccountSettings) << "Encrypt Http Response" << httpResponse; }); diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index 40b19f404..bc17a54af 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -51,7 +51,7 @@ Q_LOGGING_CATEGORY(lcJsonApiJob, "sync.networkjob.jsonapi", QtInfoMsg) Q_LOGGING_CATEGORY(lcDetermineAuthTypeJob, "sync.networkjob.determineauthtype", QtInfoMsg) Q_LOGGING_CATEGORY(lcSignPublicKeyApiJob, "sync.networkjob.sendcsr", QtInfoMsg); Q_LOGGING_CATEGORY(lcStorePrivateKeyApiJob, "sync.networkjob.storeprivatekey", QtInfoMsg); -Q_LOGGING_CATEGORY(lcCse, "sync.networkjob.clientsideencrypt", QtInfoMsg); +Q_LOGGING_CATEGORY(lcCseJob, "sync.networkjob.clientsideencrypt", QtInfoMsg); RequestEtagJob::RequestEtagJob(AccountPtr account, const QString &path, QObject *parent) : AbstractNetworkJob(account, path, parent) @@ -1020,8 +1020,8 @@ bool StorePrivateKeyApiJob::finished() emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); } -SetEncryptionFlagApiJob::SetEncryptionFlagApiJob(const AccountPtr& account, int fileId, QObject* parent) -: AbstractNetworkJob(account, baseUrl() + "/encrypt/", parent), _fileId(fileId) +SetEncryptionFlagApiJob::SetEncryptionFlagApiJob(const AccountPtr& account, const QString& fileId, QObject* parent) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("encrypted/") + fileId, parent), _fileId(fileId) { } @@ -1035,7 +1035,7 @@ void SetEncryptionFlagApiJob::start() }; url.setQueryItems(params); - qCInfo(lcCse()) << "marking the file with id" << _fileId << "as encrypted"; + qCInfo(lcCseJob()) << "marking the file with id" << _fileId << "as encrypted"; sendRequest("PUT", url, req); AbstractNetworkJob::start(); } @@ -1044,7 +1044,7 @@ bool SetEncryptionFlagApiJob::finished() { int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); if (retCode != 200) - qCInfo(lcCse()) << "Setting the encrypted flag failed with" << path() << errorString() << retCode; + qCInfo(lcCseJob()) << "Setting the encrypted flag failed with" << path() << errorString() << retCode; QJsonParseError error; auto json = QJsonDocument::fromJson(reply()->readAll(), &error); diff --git a/src/libsync/networkjobs.h b/src/libsync/networkjobs.h index 5a7ca0637..b27af8b8a 100644 --- a/src/libsync/networkjobs.h +++ b/src/libsync/networkjobs.h @@ -532,7 +532,7 @@ class OWNCLOUDSYNC_EXPORT SetEncryptionFlagApiJob : public AbstractNetworkJob { Q_OBJECT public: - explicit SetEncryptionFlagApiJob(const AccountPtr &account, int fileId, QObject *parent = 0); + explicit SetEncryptionFlagApiJob(const AccountPtr &account, const QString& fileId, QObject *parent = 0); public slots: void start() override; @@ -549,7 +549,7 @@ signals: */ void jsonReceived(const QJsonDocument &json, int statusCode); private: - int _fileId; + QString _fileId; }; } // namespace OCC From 1a47052aa3fb1a1ee797eca231edfcdabe5100b5 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 18 Oct 2017 20:43:12 +0200 Subject: [PATCH 043/262] [CSE] Remove unused function This is already being called from somwhere else. --- src/libsync/clientsideencryption.cpp | 6 ------ src/libsync/clientsideencryption.h | 1 - 2 files changed, 7 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index f4d098bec..ed513f858 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -527,10 +527,4 @@ void ClientSideEncryption::getPublicKeyFromServer() }); job->start(); } - -void ClientSideEncryption::signPublicKey() -{ - -} - } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index ea9b385b0..7c2a8d223 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -26,7 +26,6 @@ public: QString generateCSR(EVP_PKEY *keyPair); void getPrivateKeyFromServer(); void getPublicKeyFromServer(); - void signPublicKey(); void encryptPrivateKey(EVP_PKEY *keyPair); QString privateKeyPath() const; QString publicKeyPath() const; From 23f5bb7ed9926dafe06a966ae88366887b544d82 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 23 Oct 2017 18:15:39 +0200 Subject: [PATCH 044/262] [CSE] Move network jobs out of networkjobs.h Since those networkjobs are all about client side encryption, mvoe them to clientsideencryption.h This will help with fewer conflicts with the origin. --- src/libsync/clientsideencryption.cpp | 111 +++++++++++++++++++++++ src/libsync/clientsideencryption.h | 127 ++++++++++++++++++++++++++- src/libsync/networkjobs.cpp | 109 ----------------------- src/libsync/networkjobs.h | 126 +------------------------- 4 files changed, 238 insertions(+), 235 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index ed513f858..020617eef 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -25,6 +25,9 @@ namespace OCC { Q_LOGGING_CATEGORY(lcCse, "sync.clientsideencryption", QtInfoMsg) +Q_LOGGING_CATEGORY(lcSignPublicKeyApiJob, "sync.networkjob.sendcsr", QtInfoMsg); +Q_LOGGING_CATEGORY(lcStorePrivateKeyApiJob, "sync.networkjob.storeprivatekey", QtInfoMsg); +Q_LOGGING_CATEGORY(lcCseJob, "sync.networkjob.clientsideencrypt", QtInfoMsg); QString baseUrl(){ return QStringLiteral("ocs/v2.php/apps/end_to_end_encryption/api/v1/"); @@ -527,4 +530,112 @@ void ClientSideEncryption::getPublicKeyFromServer() }); job->start(); } + + +SignPublicKeyApiJob::SignPublicKeyApiJob(const AccountPtr& account, const QString& path, QObject* parent) +: AbstractNetworkJob(account, path, parent) +{ +} + +void SignPublicKeyApiJob::setCsr(const QByteArray& csr) +{ + QByteArray data = "csr="; + data += QUrl::toPercentEncoding(csr); + _csr.setData(data); +} + +void SignPublicKeyApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) + }; + url.setQueryItems(params); + + qCInfo(lcSignPublicKeyApiJob) << "Sending the CSR" << _csr.data(); + sendRequest("POST", url, req, &_csr); + AbstractNetworkJob::start(); +} + +bool SignPublicKeyApiJob::finished() +{ + qCInfo(lcStorePrivateKeyApiJob()) << "Sending CSR ended with" << path() << errorString() << reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute); + + QJsonParseError error; + auto json = QJsonDocument::fromJson(reply()->readAll(), &error); + emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); +} + + +StorePrivateKeyApiJob::StorePrivateKeyApiJob(const AccountPtr& account, const QString& path, QObject* parent) +: AbstractNetworkJob(account, path, parent) +{ +} + +void StorePrivateKeyApiJob::setPrivateKey(const QByteArray& privKey) +{ + QByteArray data = "privateKey="; + data += QUrl::toPercentEncoding(privKey); + _privKey.setData(data); +} + +void StorePrivateKeyApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) + }; + url.setQueryItems(params); + + qCInfo(lcStorePrivateKeyApiJob) << "Sending the private key" << _privKey.data(); + sendRequest("POST", url, req, &_privKey); + AbstractNetworkJob::start(); +} + +bool StorePrivateKeyApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) + qCInfo(lcStorePrivateKeyApiJob()) << "Sending private key ended with" << path() << errorString() << retCode; + + QJsonParseError error; + auto json = QJsonDocument::fromJson(reply()->readAll(), &error); + emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); +} + +SetEncryptionFlagApiJob::SetEncryptionFlagApiJob(const AccountPtr& account, const QString& fileId, QObject* parent) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("encrypted/") + fileId, parent), _fileId(fileId) +{ +} + +void SetEncryptionFlagApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) + }; + url.setQueryItems(params); + + qCInfo(lcCseJob()) << "marking the file with id" << _fileId << "as encrypted"; + sendRequest("PUT", url, req); + AbstractNetworkJob::start(); +} + +bool SetEncryptionFlagApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) + qCInfo(lcCseJob()) << "Setting the encrypted flag failed with" << path() << errorString() << retCode; + + QJsonParseError error; + auto json = QJsonDocument::fromJson(reply()->readAll(), &error); + emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); +} + } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 7c2a8d223..e143d1aae 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -9,8 +9,10 @@ #include #include "accountfwd.h" +#include "networkjobs.h" namespace OCC { + QString baseUrl(); QString baseDirectory(); @@ -38,6 +40,129 @@ private: bool isInitialized = false; }; -} + +/* + * @brief Job to sigh the CSR that return JSON + * + * To be used like this: + * \code + * _job = new SignPublicKeyApiJob(account, QLatin1String("ocs/v1.php/foo/bar"), this); + * _job->setCsr( csr ); + * connect(_job...); + * _job->start(); + * \encode + * + * @ingroup libsync + */ +class OWNCLOUDSYNC_EXPORT SignPublicKeyApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit SignPublicKeyApiJob(const AccountPtr &account, const QString &path, QObject *parent = 0); + + /** + * @brief setCsr - the CSR with the public key. + * This function needs to be called before start() obviously. + */ + void setCsr(const QByteArray& csr); + +public slots: + void start() override; + +protected: + bool finished() override; +signals: + + /** + * @brief jsonReceived - signal to report the json answer from ocs + * @param json - the parsed json document + * @param statusCode - the OCS status code: 100 (!) for success + */ + void jsonReceived(const QJsonDocument &json, int statusCode); + +private: + QBuffer _csr; +}; + +/* + * @brief Job to upload the PrivateKey that return JSON + * + * To be used like this: + * \code + * _job = new StorePrivateKeyApiJob(account, QLatin1String("ocs/v1.php/foo/bar"), this); + * _job->setPrivateKey( privKey ); + * connect(_job...); + * _job->start(); + * \encode + * + * @ingroup libsync + */ +class OWNCLOUDSYNC_EXPORT StorePrivateKeyApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit StorePrivateKeyApiJob(const AccountPtr &account, const QString &path, QObject *parent = 0); + + /** + * @brief setCsr - the CSR with the public key. + * This function needs to be called before start() obviously. + */ + void setPrivateKey(const QByteArray& privateKey); + +public slots: + void start() override; + +protected: + bool finished() override; +signals: + + /** + * @brief jsonReceived - signal to report the json answer from ocs + * @param json - the parsed json document + * @param statusCode - the OCS status code: 100 (!) for success + */ + void jsonReceived(const QJsonDocument &json, int statusCode); + +private: + QBuffer _privKey; +}; + +/* + * @brief Job to mark a folder as encrypted JSON + * + * To be used like this: + * \code + * _job = new SetEncryptionFlagApiJob(account, 2, this); + * connect(_job...); + * _job->start(); + * \encode + * + * @ingroup libsync + */ +class OWNCLOUDSYNC_EXPORT SetEncryptionFlagApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit SetEncryptionFlagApiJob(const AccountPtr &account, const QString& fileId, QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + + /** + * @brief jsonReceived - signal to report the json answer from ocs + * @param json - the parsed json document + * @param statusCode - the OCS status code: 200 for success + */ + void jsonReceived(const QJsonDocument &json, int statusCode); +private: + QString _fileId; +}; + +} // namespace OCC #endif diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index bc17a54af..957cf4e99 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -49,9 +49,6 @@ Q_LOGGING_CATEGORY(lcMkColJob, "sync.networkjob.mkcol", QtInfoMsg) Q_LOGGING_CATEGORY(lcProppatchJob, "sync.networkjob.proppatch", QtInfoMsg) Q_LOGGING_CATEGORY(lcJsonApiJob, "sync.networkjob.jsonapi", QtInfoMsg) Q_LOGGING_CATEGORY(lcDetermineAuthTypeJob, "sync.networkjob.determineauthtype", QtInfoMsg) -Q_LOGGING_CATEGORY(lcSignPublicKeyApiJob, "sync.networkjob.sendcsr", QtInfoMsg); -Q_LOGGING_CATEGORY(lcStorePrivateKeyApiJob, "sync.networkjob.storeprivatekey", QtInfoMsg); -Q_LOGGING_CATEGORY(lcCseJob, "sync.networkjob.clientsideencrypt", QtInfoMsg); RequestEtagJob::RequestEtagJob(AccountPtr account, const QString &path, QObject *parent) : AbstractNetworkJob(account, path, parent) @@ -945,111 +942,5 @@ bool DeleteApiJob::finished() qCInfo(lcJsonApiJob()) << "TMX Delete Job" << replyData; } -SignPublicKeyApiJob::SignPublicKeyApiJob(const AccountPtr& account, const QString& path, QObject* parent) -: AbstractNetworkJob(account, path, parent) -{ -} - -void SignPublicKeyApiJob::setCsr(const QByteArray& csr) -{ - QByteArray data = "csr="; - data += QUrl::toPercentEncoding(csr); - _csr.setData(data); -} - -void SignPublicKeyApiJob::start() -{ - QNetworkRequest req; - req.setRawHeader("OCS-APIREQUEST", "true"); - QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) - }; - url.setQueryItems(params); - - qCInfo(lcSignPublicKeyApiJob) << "Sending the CSR" << _csr.data(); - sendRequest("POST", url, req, &_csr); - AbstractNetworkJob::start(); -} - -bool SignPublicKeyApiJob::finished() -{ - qCInfo(lcStorePrivateKeyApiJob()) << "Sending CSR ended with" << path() << errorString() << reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute); - - QJsonParseError error; - auto json = QJsonDocument::fromJson(reply()->readAll(), &error); - emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); -} - - -StorePrivateKeyApiJob::StorePrivateKeyApiJob(const AccountPtr& account, const QString& path, QObject* parent) -: AbstractNetworkJob(account, path, parent) -{ -} - -void StorePrivateKeyApiJob::setPrivateKey(const QByteArray& privKey) -{ - QByteArray data = "privateKey="; - data += QUrl::toPercentEncoding(privKey); - _privKey.setData(data); -} - -void StorePrivateKeyApiJob::start() -{ - QNetworkRequest req; - req.setRawHeader("OCS-APIREQUEST", "true"); - QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) - }; - url.setQueryItems(params); - - qCInfo(lcStorePrivateKeyApiJob) << "Sending the private key" << _privKey.data(); - sendRequest("POST", url, req, &_privKey); - AbstractNetworkJob::start(); -} - -bool StorePrivateKeyApiJob::finished() -{ - int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (retCode != 200) - qCInfo(lcStorePrivateKeyApiJob()) << "Sending private key ended with" << path() << errorString() << retCode; - - QJsonParseError error; - auto json = QJsonDocument::fromJson(reply()->readAll(), &error); - emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); -} - -SetEncryptionFlagApiJob::SetEncryptionFlagApiJob(const AccountPtr& account, const QString& fileId, QObject* parent) -: AbstractNetworkJob(account, baseUrl() + QStringLiteral("encrypted/") + fileId, parent), _fileId(fileId) -{ -} - -void SetEncryptionFlagApiJob::start() -{ - QNetworkRequest req; - req.setRawHeader("OCS-APIREQUEST", "true"); - QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) - }; - url.setQueryItems(params); - - qCInfo(lcCseJob()) << "marking the file with id" << _fileId << "as encrypted"; - sendRequest("PUT", url, req); - AbstractNetworkJob::start(); -} - -bool SetEncryptionFlagApiJob::finished() -{ - int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (retCode != 200) - qCInfo(lcCseJob()) << "Setting the encrypted flag failed with" << path() << errorString() << retCode; - - QJsonParseError error; - auto json = QJsonDocument::fromJson(reply()->readAll(), &error); - emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); -} - } // namespace OCC diff --git a/src/libsync/networkjobs.h b/src/libsync/networkjobs.h index b27af8b8a..958b93287 100644 --- a/src/libsync/networkjobs.h +++ b/src/libsync/networkjobs.h @@ -429,129 +429,5 @@ private slots: bool finished() Q_DECL_OVERRIDE; }; -/* - * @brief Job to sigh the CSR that return JSON - * - * To be used like this: - * \code - * _job = new SignPublicKeyApiJob(account, QLatin1String("ocs/v1.php/foo/bar"), this); - * _job->setCsr( csr ); - * connect(_job...); - * _job->start(); - * \encode - * - * @ingroup libsync - */ -class OWNCLOUDSYNC_EXPORT SignPublicKeyApiJob : public AbstractNetworkJob -{ - Q_OBJECT -public: - explicit SignPublicKeyApiJob(const AccountPtr &account, const QString &path, QObject *parent = 0); - - /** - * @brief setCsr - the CSR with the public key. - * This function needs to be called before start() obviously. - */ - void setCsr(const QByteArray& csr); - -public slots: - void start() override; - -protected: - bool finished() override; -signals: - - /** - * @brief jsonReceived - signal to report the json answer from ocs - * @param json - the parsed json document - * @param statusCode - the OCS status code: 100 (!) for success - */ - void jsonReceived(const QJsonDocument &json, int statusCode); - -private: - QBuffer _csr; -}; - - -/* - * @brief Job to upload the PrivateKey that return JSON - * - * To be used like this: - * \code - * _job = new StorePrivateKeyApiJob(account, QLatin1String("ocs/v1.php/foo/bar"), this); - * _job->setPrivateKey( privKey ); - * connect(_job...); - * _job->start(); - * \encode - * - * @ingroup libsync - */ -class OWNCLOUDSYNC_EXPORT StorePrivateKeyApiJob : public AbstractNetworkJob -{ - Q_OBJECT -public: - explicit StorePrivateKeyApiJob(const AccountPtr &account, const QString &path, QObject *parent = 0); - - /** - * @brief setCsr - the CSR with the public key. - * This function needs to be called before start() obviously. - */ - void setPrivateKey(const QByteArray& privateKey); - -public slots: - void start() override; - -protected: - bool finished() override; -signals: - - /** - * @brief jsonReceived - signal to report the json answer from ocs - * @param json - the parsed json document - * @param statusCode - the OCS status code: 100 (!) for success - */ - void jsonReceived(const QJsonDocument &json, int statusCode); - -private: - QBuffer _privKey; -}; - -/* - * @brief Job to mark a folder as encrypted JSON - * - * To be used like this: - * \code - * _job = new SetEncryptionFlagApiJob(account, 2, this); - * connect(_job...); - * _job->start(); - * \encode - * - * @ingroup libsync - */ -class OWNCLOUDSYNC_EXPORT SetEncryptionFlagApiJob : public AbstractNetworkJob -{ - Q_OBJECT -public: - explicit SetEncryptionFlagApiJob(const AccountPtr &account, const QString& fileId, QObject *parent = 0); - -public slots: - void start() override; - -protected: - bool finished() override; - -signals: - - /** - * @brief jsonReceived - signal to report the json answer from ocs - * @param json - the parsed json document - * @param statusCode - the OCS status code: 200 for success - */ - void jsonReceived(const QJsonDocument &json, int statusCode); -private: - QString _fileId; -}; - -} // namespace OCC - +} #endif // NETWORKJOBS_H From bb53c2586f3e7b354a4440e296971d775a604318 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 23 Oct 2017 19:14:46 +0200 Subject: [PATCH 045/262] [CSE] Add Nlohmann's json library Because nobody deservers to deal with Qt's JSON API --- src/3rdparty/nlohmann/json.hpp | 13003 +++++++++++++++++++++++++++++++ 1 file changed, 13003 insertions(+) create mode 100644 src/3rdparty/nlohmann/json.hpp diff --git a/src/3rdparty/nlohmann/json.hpp b/src/3rdparty/nlohmann/json.hpp new file mode 100644 index 000000000..6dfc1831f --- /dev/null +++ b/src/3rdparty/nlohmann/json.hpp @@ -0,0 +1,13003 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 2.1.1 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +Copyright (c) 2013-2017 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#ifndef NLOHMANN_JSON_HPP +#define NLOHMANN_JSON_HPP + +#include // all_of, copy, fill, find, for_each, none_of, remove, reverse, transform +#include // array +#include // assert +#include // isdigit +#include // and, not, or +#include // isfinite, labs, ldexp, signbit +#include // nullptr_t, ptrdiff_t, size_t +#include // int64_t, uint64_t +#include // abort, strtod, strtof, strtold, strtoul, strtoll, strtoull +#include // strlen +#include // forward_list +#include // function, hash, less +#include // initializer_list +#include // setw +#include // istream, ostream +#include // advance, begin, back_inserter, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator +#include // numeric_limits +#include // locale +#include // map +#include // addressof, allocator, allocator_traits, unique_ptr +#include // accumulate +#include // stringstream +#include // domain_error, invalid_argument, out_of_range +#include // getline, stoi, string, to_string +#include // add_pointer, conditional, decay, enable_if, false_type, integral_constant, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_default_constructible, is_enum, is_floating_point, is_integral, is_nothrow_move_assignable, is_nothrow_move_constructible, is_pointer, is_reference, is_same, is_scalar, is_signed, remove_const, remove_cv, remove_pointer, remove_reference, true_type, underlying_type +#include // declval, forward, make_pair, move, pair, swap +#include // vector + +// exclude unsupported compilers +#if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif +#elif defined(__GNUC__) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif +#endif + +// disable float-equal warnings on GCC/clang +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + +// disable documentation warnings on clang +#if defined(__clang__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdocumentation" +#endif + +// allow for portable deprecation warnings +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #define JSON_DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) + #define JSON_DEPRECATED __declspec(deprecated) +#else + #define JSON_DEPRECATED +#endif + +// allow to disable exceptions +#if not defined(JSON_NOEXCEPTION) || defined(__EXCEPTIONS) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) +#else + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) +#endif + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ + +/*! +@brief unnamed namespace with internal helper functions + +This namespace collects some functions that could not be defined inside the +@ref basic_json class. + +@since version 2.1.0 +*/ +namespace detail +{ +/////////////////////////// +// JSON type enumeration // +/////////////////////////// + +/*! +@brief the JSON type enumeration + +This enumeration collects the different JSON types. It is internally used to +distinguish the stored values, and the functions @ref basic_json::is_null(), +@ref basic_json::is_object(), @ref basic_json::is_array(), +@ref basic_json::is_string(), @ref basic_json::is_boolean(), +@ref basic_json::is_number() (with @ref basic_json::is_number_integer(), +@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()), +@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and +@ref basic_json::is_structured() rely on it. + +@note There are three enumeration entries (number_integer, number_unsigned, and +number_float), because the library distinguishes these three types for numbers: +@ref basic_json::number_unsigned_t is used for unsigned integers, +@ref basic_json::number_integer_t is used for signed integers, and +@ref basic_json::number_float_t is used for floating-point numbers or to +approximate integers which do not fit in the limits of their respective type. + +@sa @ref basic_json::basic_json(const value_t value_type) -- create a JSON +value with the default value for a given type + +@since version 1.0.0 +*/ +enum class value_t : uint8_t +{ + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + discarded ///< discarded by the the parser callback function +}; + +/*! +@brief comparison operator for JSON types + +Returns an ordering that is similar to Python: +- order: null < boolean < number < object < array < string +- furthermore, each type is not smaller than itself + +@since version 1.0.0 +*/ +inline bool operator<(const value_t lhs, const value_t rhs) noexcept +{ + static constexpr std::array order = {{ + 0, // null + 3, // object + 4, // array + 5, // string + 1, // boolean + 2, // integer + 2, // unsigned + 2, // float + } + }; + + // discarded values are not comparable + if (lhs == value_t::discarded or rhs == value_t::discarded) + { + return false; + } + + return order[static_cast(lhs)] < + order[static_cast(rhs)]; +} + + +///////////// +// helpers // +///////////// + +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; + +template +using uncvref_t = typename std::remove_cv::type>::type; + +// taken from http://stackoverflow.com/a/26936864/266378 +template +using is_unscoped_enum = + std::integral_constant::value and + std::is_enum::value>; + +/* +Implementation of two C++17 constructs: conjunction, negation. This is needed +to avoid evaluating all the traits in a condition + +For example: not std::is_same::value and has_value_type::value +will not compile when T = void (on MSVC at least). Whereas +conjunction>, has_value_type>::value will +stop evaluating if negation<...>::value == false + +Please note that those constructs must be used with caution, since symbols can +become very long quickly (which can slow down compilation and cause MSVC +internal compiler errors). Only use it when you have to (see example ahead). +*/ +template struct conjunction : std::true_type {}; +template struct conjunction : B1 {}; +template +struct conjunction : std::conditional, B1>::type {}; + +template struct negation : std::integral_constant < bool, !B::value > {}; + +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + + +////////////////// +// constructors // +////////////////// + +template struct external_constructor; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept + { + j.m_type = value_t::boolean; + j.m_value = b; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s) + { + j.m_type = value_t::string; + j.m_value = s; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept + { + // replace infinity and NAN by null + if (not std::isfinite(val)) + { + j = BasicJsonType{}; + } + else + { + j.m_type = value_t::number_float; + j.m_value = val; + } + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept + { + j.m_type = value_t::number_unsigned; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept + { + j.m_type = value_t::number_integer; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr) + { + j.m_type = value_t::array; + j.m_value = arr; + j.assert_invariant(); + } + + template::value, + int> = 0> + static void construct(BasicJsonType& j, const CompatibleArrayType& arr) + { + using std::begin; + using std::end; + j.m_type = value_t::array; + j.m_value.array = j.template create(begin(arr), end(arr)); + j.assert_invariant(); + } +}; + +template<> +struct external_constructor +{ + template + static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj) + { + j.m_type = value_t::object; + j.m_value = obj; + j.assert_invariant(); + } + + template::value, + int> = 0> + static void construct(BasicJsonType& j, const CompatibleObjectType& obj) + { + using std::begin; + using std::end; + + j.m_type = value_t::object; + j.m_value.object = j.template create(begin(obj), end(obj)); + j.assert_invariant(); + } +}; + + +//////////////////////// +// has_/is_ functions // +//////////////////////// + +/*! +@brief Helper to determine whether there's a key_type for T. + +This helper is used to tell associative containers apart from other containers +such as sequence containers. For instance, `std::map` passes the test as it +contains a `mapped_type`, whereas `std::vector` fails the test. + +@sa http://stackoverflow.com/a/7728728/266378 +@since version 1.0.0, overworked in version 2.0.6 +*/ +#define NLOHMANN_JSON_HAS_HELPER(type) \ + template struct has_##type { \ + private: \ + template \ + static int detect(U &&); \ + static void detect(...); \ + public: \ + static constexpr bool value = \ + std::is_integral()))>::value; \ + } + +NLOHMANN_JSON_HAS_HELPER(mapped_type); +NLOHMANN_JSON_HAS_HELPER(key_type); +NLOHMANN_JSON_HAS_HELPER(value_type); +NLOHMANN_JSON_HAS_HELPER(iterator); + +#undef NLOHMANN_JSON_HAS_HELPER + + +template +struct is_compatible_object_type_impl : std::false_type {}; + +template +struct is_compatible_object_type_impl +{ + static constexpr auto value = + std::is_constructible::value and + std::is_constructible::value; +}; + +template +struct is_compatible_object_type +{ + static auto constexpr value = is_compatible_object_type_impl < + conjunction>, + has_mapped_type, + has_key_type>::value, + typename BasicJsonType::object_t, CompatibleObjectType >::value; +}; + +template +struct is_basic_json_nested_type +{ + static auto constexpr value = std::is_same::value or + std::is_same::value or + std::is_same::value or + std::is_same::value or + std::is_same::value; +}; + +template +struct is_compatible_array_type +{ + static auto constexpr value = + conjunction>, + negation>, + negation>, + negation>, + has_value_type, + has_iterator>::value; +}; + +template +struct is_compatible_integer_type_impl : std::false_type {}; + +template +struct is_compatible_integer_type_impl +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits; + using CompatibleLimits = std::numeric_limits; + + static constexpr auto value = + std::is_constructible::value and + CompatibleLimits::is_integer and + RealLimits::is_signed == CompatibleLimits::is_signed; +}; + +template +struct is_compatible_integer_type +{ + static constexpr auto value = + is_compatible_integer_type_impl < + std::is_integral::value and + not std::is_same::value, + RealIntegerType, CompatibleNumberIntegerType > ::value; +}; + + +// trait checking if JSONSerializer::from_json(json const&, udt&) exists +template +struct has_from_json +{ + private: + // also check the return type of from_json + template::from_json( + std::declval(), std::declval()))>::value>> + static int detect(U&&); + static void detect(...); + + public: + static constexpr bool value = std::is_integral>()))>::value; +}; + +// This trait checks if JSONSerializer::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template +struct has_non_default_from_json +{ + private: + template < + typename U, + typename = enable_if_t::from_json(std::declval()))>::value >> + static int detect(U&&); + static void detect(...); + + public: + static constexpr bool value = std::is_integral>()))>::value; +}; + +// This trait checks if BasicJsonType::json_serializer::to_json exists +template +struct has_to_json +{ + private: + template::to_json( + std::declval(), std::declval()))> + static int detect(U&&); + static void detect(...); + + public: + static constexpr bool value = std::is_integral>()))>::value; +}; + + +///////////// +// to_json // +///////////// + +template::value, int> = 0> +void to_json(BasicJsonType& j, T b) noexcept +{ + external_constructor::construct(j, b); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, const CompatibleString& s) +{ + external_constructor::construct(j, s); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, FloatType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template < + typename BasicJsonType, typename CompatibleNumberUnsignedType, + enable_if_t::value, int> = 0 > +void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template < + typename BasicJsonType, typename CompatibleNumberIntegerType, + enable_if_t::value, int> = 0 > +void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept +{ + external_constructor::construct(j, static_cast(val)); +} + +template::value, int> = 0> +void to_json(BasicJsonType& j, UnscopedEnumType e) noexcept +{ + external_constructor::construct(j, e); +} + +template < + typename BasicJsonType, typename CompatibleArrayType, + enable_if_t < + is_compatible_array_type::value or + std::is_same::value, + int > = 0 > +void to_json(BasicJsonType& j, const CompatibleArrayType& arr) +{ + external_constructor::construct(j, arr); +} + +template < + typename BasicJsonType, typename CompatibleObjectType, + enable_if_t::value, + int> = 0 > +void to_json(BasicJsonType& j, const CompatibleObjectType& arr) +{ + external_constructor::construct(j, arr); +} + + +/////////////// +// from_json // +/////////////// + +// overloads for basic_json template parameters +template::value and + not std::is_same::value, + int> = 0> +void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast(j)) + { + case value_t::number_unsigned: + { + val = static_cast( + *j.template get_ptr()); + break; + } + case value_t::number_integer: + { + val = static_cast( + *j.template get_ptr()); + break; + } + case value_t::number_float: + { + val = static_cast( + *j.template get_ptr()); + break; + } + default: + { + JSON_THROW( + std::domain_error("type must be number, but is " + j.type_name())); + } + } +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b) +{ + if (not j.is_boolean()) + { + JSON_THROW(std::domain_error("type must be boolean, but is " + j.type_name())); + } + b = *j.template get_ptr(); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) +{ + if (not j.is_string()) + { + JSON_THROW(std::domain_error("type must be string, but is " + j.type_name())); + } + s = *j.template get_ptr(); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val) +{ + get_arithmetic_value(j, val); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +{ + get_arithmetic_value(j, val); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val) +{ + get_arithmetic_value(j, val); +} + +template::value, int> = 0> +void from_json(const BasicJsonType& j, UnscopedEnumType& e) +{ + typename std::underlying_type::type val; + get_arithmetic_value(j, val); + e = static_cast(val); +} + +template +void from_json(const BasicJsonType& j, typename BasicJsonType::array_t& arr) +{ + if (not j.is_array()) + { + JSON_THROW(std::domain_error("type must be array, but is " + j.type_name())); + } + arr = *j.template get_ptr(); +} + +// forward_list doesn't have an insert method +template +void from_json(const BasicJsonType& j, std::forward_list& l) +{ + // do not perform the check when user wants to retrieve jsons + // (except when it's null.. ?) + if (j.is_null()) + { + JSON_THROW(std::domain_error("type must be array, but is " + j.type_name())); + } + if (not std::is_same::value) + { + if (not j.is_array()) + { + JSON_THROW(std::domain_error("type must be array, but is " + j.type_name())); + } + } + for (auto it = j.rbegin(), end = j.rend(); it != end; ++it) + { + l.push_front(it->template get()); + } +} + +template +void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<0>) +{ + using std::begin; + using std::end; + + std::transform(j.begin(), j.end(), + std::inserter(arr, end(arr)), [](const BasicJsonType & i) + { + // get() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get(); + }); +} + +template +auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1>) +-> decltype( + arr.reserve(std::declval()), + void()) +{ + using std::begin; + using std::end; + + arr.reserve(j.size()); + std::transform( + j.begin(), j.end(), std::inserter(arr, end(arr)), [](const BasicJsonType & i) + { + // get() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get(); + }); +} + +template::value and + not std::is_same::value, int> = 0> +void from_json(const BasicJsonType& j, CompatibleArrayType& arr) +{ + if (j.is_null()) + { + JSON_THROW(std::domain_error("type must be array, but is " + j.type_name())); + } + + // when T == BasicJsonType, do not check if value_t is correct + if (not std::is_same::value) + { + if (not j.is_array()) + { + JSON_THROW(std::domain_error("type must be array, but is " + j.type_name())); + } + } + from_json_array_impl(j, arr, priority_tag<1> {}); +} + +template::value, int> = 0> +void from_json(const BasicJsonType& j, CompatibleObjectType& obj) +{ + if (not j.is_object()) + { + JSON_THROW(std::domain_error("type must be object, but is " + j.type_name())); + } + + auto inner_object = j.template get_ptr(); + using std::begin; + using std::end; + // we could avoid the assignment, but this might require a for loop, which + // might be less efficient than the container constructor for some + // containers (would it?) + obj = CompatibleObjectType(begin(*inner_object), end(*inner_object)); +} + +// overload for arithmetic types, not chosen for basic_json template arguments +// (BooleanType, etc..); note: Is it really necessary to provide explicit +// overloads for boolean_t etc. in case of a custom BooleanType which is not +// an arithmetic type? +template::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value and + not std::is_same::value, + int> = 0> +void from_json(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast(j)) + { + case value_t::number_unsigned: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_integer: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::number_float: + { + val = static_cast(*j.template get_ptr()); + break; + } + case value_t::boolean: + { + val = static_cast(*j.template get_ptr()); + break; + } + default: + { + JSON_THROW(std::domain_error("type must be number, but is " + j.type_name())); + } + } +} + +struct to_json_fn +{ + private: + template + auto call(BasicJsonType& j, T&& val, priority_tag<1>) const noexcept(noexcept(to_json(j, std::forward(val)))) + -> decltype(to_json(j, std::forward(val)), void()) + { + return to_json(j, std::forward(val)); + } + + template + void call(BasicJsonType&, T&&, priority_tag<0>) const noexcept + { + static_assert(sizeof(BasicJsonType) == 0, + "could not find to_json() method in T's namespace"); + } + + public: + template + void operator()(BasicJsonType& j, T&& val) const + noexcept(noexcept(std::declval().call(j, std::forward(val), priority_tag<1> {}))) + { + return call(j, std::forward(val), priority_tag<1> {}); + } +}; + +struct from_json_fn +{ + private: + template + auto call(const BasicJsonType& j, T& val, priority_tag<1>) const + noexcept(noexcept(from_json(j, val))) + -> decltype(from_json(j, val), void()) + { + return from_json(j, val); + } + + template + void call(const BasicJsonType&, T&, priority_tag<0>) const noexcept + { + static_assert(sizeof(BasicJsonType) == 0, + "could not find from_json() method in T's namespace"); + } + + public: + template + void operator()(const BasicJsonType& j, T& val) const + noexcept(noexcept(std::declval().call(j, val, priority_tag<1> {}))) + { + return call(j, val, priority_tag<1> {}); + } +}; + +// taken from ranges-v3 +template +struct static_const +{ + static constexpr T value{}; +}; + +template +constexpr T static_const::value; +} // namespace detail + + +/// namespace to hold default `to_json` / `from_json` functions +namespace +{ +constexpr const auto& to_json = detail::static_const::value; +constexpr const auto& from_json = detail::static_const::value; +} + + +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](http://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer +{ + /*! + @brief convert a JSON value to any value type + + This function is usually called by the `get()` function of the + @ref basic_json class (either explicit or via conversion operators). + + @param[in] j JSON value to read from + @param[in,out] val value to write to + */ + template + static void from_json(BasicJsonType&& j, ValueType& val) noexcept( + noexcept(::nlohmann::from_json(std::forward(j), val))) + { + ::nlohmann::from_json(std::forward(j), val); + } + + /*! + @brief convert any value type to a JSON value + + This function is usually called by the constructors of the @ref basic_json + class. + + @param[in,out] j JSON value to write to + @param[in] val value to read from + */ + template + static void to_json(BasicJsonType& j, ValueType&& val) noexcept( + noexcept(::nlohmann::to_json(j, std::forward(val)))) + { + ::nlohmann::to_json(j, std::forward(val)); + } +}; + + +/*! +@brief a class to store JSON values + +@tparam ObjectType type for JSON objects (`std::map` by default; will be used +in @ref object_t) +@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used +in @ref array_t) +@tparam StringType type for JSON strings and object keys (`std::string` by +default; will be used in @ref string_t) +@tparam BooleanType type for JSON booleans (`bool` by default; will be used +in @ref boolean_t) +@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by +default; will be used in @ref number_integer_t) +@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c +`uint64_t` by default; will be used in @ref number_unsigned_t) +@tparam NumberFloatType type for JSON floating-point numbers (`double` by +default; will be used in @ref number_float_t) +@tparam AllocatorType type of the allocator to use (`std::allocator` by +default) +@tparam JSONSerializer the serializer to resolve internal calls to `to_json()` +and `from_json()` (@ref adl_serializer by default) + +@requirement The class satisfies the following concept requirements: +- Basic + - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): + JSON values can be default constructed. The result will be a JSON null + value. + - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): + A JSON value can be constructed from an rvalue argument. + - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): + A JSON value can be copy-constructed from an lvalue expression. + - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): + A JSON value van be assigned from an rvalue argument. + - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): + A JSON value can be copy-assigned from an lvalue expression. + - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): + JSON values can be destructed. +- Layout + - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): + JSON values have + [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): + All non-static data members are private and standard layout types, the + class has no virtual functions or (virtual) base classes. +- Library-wide + - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): + JSON values can be compared with `==`, see @ref + operator==(const_reference,const_reference). + - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): + JSON values can be compared with `<`, see @ref + operator<(const_reference,const_reference). + - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): + Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of + other compatible types, using unqualified function call @ref swap(). + - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): + JSON values can be compared against `std::nullptr_t` objects which are used + to model the `null` value. +- Container + - [Container](http://en.cppreference.com/w/cpp/concept/Container): + JSON values can be used like STL containers and provide iterator access. + - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); + JSON values can be used like STL containers and provide reverse iterator + access. + +@invariant The member variables @a m_value and @a m_type have the following +relationship: +- If `m_type == value_t::object`, then `m_value.object != nullptr`. +- If `m_type == value_t::array`, then `m_value.array != nullptr`. +- If `m_type == value_t::string`, then `m_value.string != nullptr`. +The invariants are checked by member function assert_invariant(). + +@internal +@note ObjectType trick from http://stackoverflow.com/a/9860911 +@endinternal + +@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange +Format](http://rfc7159.net/rfc7159) + +@since version 1.0.0 + +@nosubgrouping +*/ +template < + template class ObjectType = std::map, + template class ArrayType = std::vector, + class StringType = std::string, + class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = adl_serializer + > +class basic_json +{ + private: + template friend struct detail::external_constructor; + /// workaround type for MSVC + using basic_json_t = basic_json; + + public: + using value_t = detail::value_t; + // forward declarations + template class iter_impl; + template class json_reverse_iterator; + class json_pointer; + template + using json_serializer = JSONSerializer; + + ///////////////////// + // container types // + ///////////////////// + + /// @name container types + /// The canonic container types to use @ref basic_json like any other STL + /// container. + /// @{ + + /// the type of elements in a basic_json container + using value_type = basic_json; + + /// the type of an element reference + using reference = value_type&; + /// the type of an element const reference + using const_reference = const value_type&; + + /// a type to represent differences between iterators + using difference_type = std::ptrdiff_t; + /// a type to represent container sizes + using size_type = std::size_t; + + /// the allocator type + using allocator_type = AllocatorType; + + /// the type of an element pointer + using pointer = typename std::allocator_traits::pointer; + /// the type of an element const pointer + using const_pointer = typename std::allocator_traits::const_pointer; + + /// an iterator for a basic_json container + using iterator = iter_impl; + /// a const iterator for a basic_json container + using const_iterator = iter_impl; + /// a reverse iterator for a basic_json container + using reverse_iterator = json_reverse_iterator; + /// a const reverse iterator for a basic_json container + using const_reverse_iterator = json_reverse_iterator; + + /// @} + + + /*! + @brief returns the allocator associated with the container + */ + static allocator_type get_allocator() + { + return allocator_type(); + } + + /*! + @brief returns version information on the library + + This function returns a JSON object with information about the library, + including the version number and information on the platform and compiler. + + @return JSON object holding version information + key | description + ----------- | --------------- + `compiler` | Information on the used compiler. It is an object with the following keys: `c++` (the used C++ standard), `family` (the compiler family; possible values are `clang`, `icc`, `gcc`, `ilecpp`, `msvc`, `pgcpp`, `sunpro`, and `unknown`), and `version` (the compiler version). + `copyright` | The copyright line for the library as string. + `name` | The name of the library as string. + `platform` | The used platform as string. Possible values are `win32`, `linux`, `apple`, `unix`, and `unknown`. + `url` | The URL of the project as string. + `version` | The version of the library. It is an object with the following keys: `major`, `minor`, and `patch` as defined by [Semantic Versioning](http://semver.org), and `string` (the version string). + + @liveexample{The following code shows an example output of the `meta()` + function.,meta} + + @complexity Constant. + + @since 2.1.0 + */ + static basic_json meta() + { + basic_json result; + + result["copyright"] = "(C) 2013-2017 Niels Lohmann"; + result["name"] = "JSON for Modern C++"; + result["url"] = "https://github.com/nlohmann/json"; + result["version"] = + { + {"string", "2.1.1"}, + {"major", 2}, + {"minor", 1}, + {"patch", 1} + }; + +#ifdef _WIN32 + result["platform"] = "win32"; +#elif defined __linux__ + result["platform"] = "linux"; +#elif defined __APPLE__ + result["platform"] = "apple"; +#elif defined __unix__ + result["platform"] = "unix"; +#else + result["platform"] = "unknown"; +#endif + +#if defined(__clang__) + result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}}; +#elif defined(__ICC) || defined(__INTEL_COMPILER) + result["compiler"] = {{"family", "icc"}, {"version", __INTEL_COMPILER}}; +#elif defined(__GNUC__) || defined(__GNUG__) + result["compiler"] = {{"family", "gcc"}, {"version", std::to_string(__GNUC__) + "." + std::to_string(__GNUC_MINOR__) + "." + std::to_string(__GNUC_PATCHLEVEL__)}}; +#elif defined(__HP_cc) || defined(__HP_aCC) + result["compiler"] = "hp" +#elif defined(__IBMCPP__) + result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}}; +#elif defined(_MSC_VER) + result["compiler"] = {{"family", "msvc"}, {"version", _MSC_VER}}; +#elif defined(__PGI) + result["compiler"] = {{"family", "pgcpp"}, {"version", __PGI}}; +#elif defined(__SUNPRO_CC) + result["compiler"] = {{"family", "sunpro"}, {"version", __SUNPRO_CC}}; +#else + result["compiler"] = {{"family", "unknown"}, {"version", "unknown"}}; +#endif + +#ifdef __cplusplus + result["compiler"]["c++"] = std::to_string(__cplusplus); +#else + result["compiler"]["c++"] = "unknown"; +#endif + return result; + } + + + /////////////////////////// + // JSON value data types // + /////////////////////////// + + /// @name JSON value data types + /// The data types to store a JSON value. These types are derived from + /// the template arguments passed to class @ref basic_json. + /// @{ + + /*! + @brief a type for an object + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows: + > An object is an unordered collection of zero or more name/value pairs, + > where a name is a string and a value is a string, number, boolean, null, + > object, or array. + + To store objects in C++, a type is defined by the template parameters + described below. + + @tparam ObjectType the container to store objects (e.g., `std::map` or + `std::unordered_map`) + @tparam StringType the type of the keys or names (e.g., `std::string`). + The comparison function `std::less` is used to order elements + inside the container. + @tparam AllocatorType the allocator to use for objects (e.g., + `std::allocator`) + + #### Default type + + With the default values for @a ObjectType (`std::map`), @a StringType + (`std::string`), and @a AllocatorType (`std::allocator`), the default + value for @a object_t is: + + @code {.cpp} + std::map< + std::string, // key_type + basic_json, // value_type + std::less, // key_compare + std::allocator> // allocator_type + > + @endcode + + #### Behavior + + The choice of @a object_t influences the behavior of the JSON class. With + the default type, objects have the following behavior: + + - When all names are unique, objects will be interoperable in the sense + that all software implementations receiving that object will agree on + the name-value mappings. + - When the names within an object are not unique, later stored name/value + pairs overwrite previously stored name/value pairs, leaving the used + names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will + be treated as equal and both stored as `{"key": 1}`. + - Internally, name/value pairs are stored in lexicographical order of the + names. Objects will also be serialized (see @ref dump) in this order. + For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored + and serialized as `{"a": 2, "b": 1}`. + - When comparing objects, the order of the name/value pairs is irrelevant. + This makes objects interoperable in the sense that they will not be + affected by these differences. For instance, `{"b": 1, "a": 2}` and + `{"a": 2, "b": 1}` will be treated as equal. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. + + In this class, the object's limit of nesting is not constraint explicitly. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON object. + + #### Storage + + Objects are stored as pointers in a @ref basic_json type. That is, for any + access to object values, a pointer of type `object_t*` must be + dereferenced. + + @sa @ref array_t -- type for an array value + + @since version 1.0.0 + + @note The order name/value pairs are added to the object is *not* + preserved by the library. Therefore, iterating an object may return + name/value pairs in a different order than they were originally stored. In + fact, keys will be traversed in alphabetical order as `std::map` with + `std::less` is used by default. Please note this behavior conforms to [RFC + 7159](http://rfc7159.net/rfc7159), because any order implements the + specified "unordered" nature of JSON objects. + */ + using object_t = ObjectType, + AllocatorType>>; + + /*! + @brief a type for an array + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows: + > An array is an ordered sequence of zero or more values. + + To store objects in C++, a type is defined by the template parameters + explained below. + + @tparam ArrayType container type to store arrays (e.g., `std::vector` or + `std::list`) + @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`) + + #### Default type + + With the default values for @a ArrayType (`std::vector`) and @a + AllocatorType (`std::allocator`), the default value for @a array_t is: + + @code {.cpp} + std::vector< + basic_json, // value_type + std::allocator // allocator_type + > + @endcode + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. + + In this class, the array's limit of nesting is not constraint explicitly. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON array. + + #### Storage + + Arrays are stored as pointers in a @ref basic_json type. That is, for any + access to array values, a pointer of type `array_t*` must be dereferenced. + + @sa @ref object_t -- type for an object value + + @since version 1.0.0 + */ + using array_t = ArrayType>; + + /*! + @brief a type for a string + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows: + > A string is a sequence of zero or more Unicode characters. + + To store objects in C++, a type is defined by the template parameter + described below. Unicode values are split by the JSON class into + byte-sized characters during deserialization. + + @tparam StringType the container to store strings (e.g., `std::string`). + Note this container is used for keys/names in objects, see @ref object_t. + + #### Default type + + With the default values for @a StringType (`std::string`), the default + value for @a string_t is: + + @code {.cpp} + std::string + @endcode + + #### Encoding + + Strings are stored in UTF-8 encoding. Therefore, functions like + `std::string::size()` or `std::string::length()` return the number of + bytes in the string rather than the number of characters or glyphs. + + #### String comparison + + [RFC 7159](http://rfc7159.net/rfc7159) states: + > Software implementations are typically required to test names of object + > members for equality. Implementations that transform the textual + > representation into sequences of Unicode code units and then perform the + > comparison numerically, code unit by code unit, are interoperable in the + > sense that implementations will agree in all cases on equality or + > inequality of two strings. For example, implementations that compare + > strings with escaped characters unconverted may incorrectly find that + > `"a\\b"` and `"a\u005Cb"` are not equal. + + This implementation is interoperable as it does compare strings code unit + by code unit. + + #### Storage + + String values are stored as pointers in a @ref basic_json type. That is, + for any access to string values, a pointer of type `string_t*` must be + dereferenced. + + @since version 1.0.0 + */ + using string_t = StringType; + + /*! + @brief a type for a boolean + + [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a + type which differentiates the two literals `true` and `false`. + + To store objects in C++, a type is defined by the template parameter @a + BooleanType which chooses the type to use. + + #### Default type + + With the default values for @a BooleanType (`bool`), the default value for + @a boolean_t is: + + @code {.cpp} + bool + @endcode + + #### Storage + + Boolean values are stored directly inside a @ref basic_json type. + + @since version 1.0.0 + */ + using boolean_t = BooleanType; + + /*! + @brief a type for a number (integer) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store integer numbers in C++, a type is defined by the template + parameter @a NumberIntegerType which chooses the type to use. + + #### Default type + + With the default values for @a NumberIntegerType (`int64_t`), the default + value for @a number_integer_t is: + + @code {.cpp} + int64_t + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. + + When the default type is used, the maximal integer number that can be + stored is `9223372036854775807` (INT64_MAX) and the minimal integer number + that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers + that are out of range will yield over/underflow when used in a + constructor. During deserialization, too large or small integer numbers + will be automatically be stored as @ref number_unsigned_t or @ref + number_float_t. + + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. + + As this range is a subrange of the exactly supported range [INT64_MIN, + INT64_MAX], this class's integer type is interoperable. + + #### Storage + + Integer number values are stored directly inside a @ref basic_json type. + + @sa @ref number_float_t -- type for number values (floating-point) + + @sa @ref number_unsigned_t -- type for number values (unsigned integer) + + @since version 1.0.0 + */ + using number_integer_t = NumberIntegerType; + + /*! + @brief a type for a number (unsigned) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store unsigned integer numbers in C++, a type is defined by the + template parameter @a NumberUnsignedType which chooses the type to use. + + #### Default type + + With the default values for @a NumberUnsignedType (`uint64_t`), the + default value for @a number_unsigned_t is: + + @code {.cpp} + uint64_t + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. + + When the default type is used, the maximal integer number that can be + stored is `18446744073709551615` (UINT64_MAX) and the minimal integer + number that can be stored is `0`. Integer numbers that are out of range + will yield over/underflow when used in a constructor. During + deserialization, too large or small integer numbers will be automatically + be stored as @ref number_integer_t or @ref number_float_t. + + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. + + As this range is a subrange (when considered in conjunction with the + number_integer_t type) of the exactly supported range [0, UINT64_MAX], + this class's integer type is interoperable. + + #### Storage + + Integer number values are stored directly inside a @ref basic_json type. + + @sa @ref number_float_t -- type for number values (floating-point) + @sa @ref number_integer_t -- type for number values (integer) + + @since version 2.0.0 + */ + using number_unsigned_t = NumberUnsignedType; + + /*! + @brief a type for a number (floating-point) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store floating-point numbers in C++, a type is defined by the template + parameter @a NumberFloatType which chooses the type to use. + + #### Default type + + With the default values for @a NumberFloatType (`double`), the default + value for @a number_float_t is: + + @code {.cpp} + double + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in floating-point literals will be ignored. Internally, + the value will be stored as decimal number. For instance, the C++ + floating-point literal `01.2` will be serialized to `1.2`. During + deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) states: + > This specification allows implementations to set limits on the range and + > precision of numbers accepted. Since software that implements IEEE + > 754-2008 binary64 (double precision) numbers is generally available and + > widely used, good interoperability can be achieved by implementations + > that expect no more precision or range than these provide, in the sense + > that implementations will approximate JSON numbers within the expected + > precision. + + This implementation does exactly follow this approach, as it uses double + precision floating-point numbers. Note values smaller than + `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` + will be stored as NaN internally and be serialized to `null`. + + #### Storage + + Floating-point number values are stored directly inside a @ref basic_json + type. + + @sa @ref number_integer_t -- type for number values (integer) + + @sa @ref number_unsigned_t -- type for number values (unsigned integer) + + @since version 1.0.0 + */ + using number_float_t = NumberFloatType; + + /// @} + + private: + + /// helper for exception-safe object creation + template + static T* create(Args&& ... args) + { + AllocatorType alloc; + auto deleter = [&](T * object) + { + alloc.deallocate(object, 1); + }; + std::unique_ptr object(alloc.allocate(1), deleter); + alloc.construct(object.get(), std::forward(args)...); + assert(object != nullptr); + return object.release(); + } + + //////////////////////// + // JSON value storage // + //////////////////////// + + /*! + @brief a JSON value + + The actual storage for a JSON value of the @ref basic_json class. This + union combines the different storage types for the JSON value types + defined in @ref value_t. + + JSON type | value_t type | used type + --------- | --------------- | ------------------------ + object | object | pointer to @ref object_t + array | array | pointer to @ref array_t + string | string | pointer to @ref string_t + boolean | boolean | @ref boolean_t + number | number_integer | @ref number_integer_t + number | number_unsigned | @ref number_unsigned_t + number | number_float | @ref number_float_t + null | null | *no value is stored* + + @note Variable-length types (objects, arrays, and strings) are stored as + pointers. The size of the union should not exceed 64 bits if the default + value types are used. + + @since version 1.0.0 + */ + union json_value + { + /// object (stored with pointer to save storage) + object_t* object; + /// array (stored with pointer to save storage) + array_t* array; + /// string (stored with pointer to save storage) + string_t* string; + /// boolean + boolean_t boolean; + /// number (integer) + number_integer_t number_integer; + /// number (unsigned integer) + number_unsigned_t number_unsigned; + /// number (floating-point) + number_float_t number_float; + + /// default constructor (for null values) + json_value() = default; + /// constructor for booleans + json_value(boolean_t v) noexcept : boolean(v) {} + /// constructor for numbers (integer) + json_value(number_integer_t v) noexcept : number_integer(v) {} + /// constructor for numbers (unsigned) + json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (floating-point) + json_value(number_float_t v) noexcept : number_float(v) {} + /// constructor for empty values of a given type + json_value(value_t t) + { + switch (t) + { + case value_t::object: + { + object = create(); + break; + } + + case value_t::array: + { + array = create(); + break; + } + + case value_t::string: + { + string = create(""); + break; + } + + case value_t::boolean: + { + boolean = boolean_t(false); + break; + } + + case value_t::number_integer: + { + number_integer = number_integer_t(0); + break; + } + + case value_t::number_unsigned: + { + number_unsigned = number_unsigned_t(0); + break; + } + + case value_t::number_float: + { + number_float = number_float_t(0.0); + break; + } + + case value_t::null: + { + break; + } + + default: + { + if (t == value_t::null) + { + JSON_THROW(std::domain_error("961c151d2e87f2686a955a9be24d316f1362bf21 2.1.1")); // LCOV_EXCL_LINE + } + break; + } + } + } + + /// constructor for strings + json_value(const string_t& value) + { + string = create(value); + } + + /// constructor for objects + json_value(const object_t& value) + { + object = create(value); + } + + /// constructor for arrays + json_value(const array_t& value) + { + array = create(value); + } + }; + + /*! + @brief checks the class invariants + + This function asserts the class invariants. It needs to be called at the + end of every constructor to make sure that created objects respect the + invariant. Furthermore, it has to be called each time the type of a JSON + value is changed, because the invariant expresses a relationship between + @a m_type and @a m_value. + */ + void assert_invariant() const + { + assert(m_type != value_t::object or m_value.object != nullptr); + assert(m_type != value_t::array or m_value.array != nullptr); + assert(m_type != value_t::string or m_value.string != nullptr); + } + + public: + ////////////////////////// + // JSON parser callback // + ////////////////////////// + + /*! + @brief JSON callback events + + This enumeration lists the parser events that can trigger calling a + callback function of type @ref parser_callback_t during parsing. + + @image html callback_events.png "Example when certain parse events are triggered" + + @since version 1.0.0 + */ + enum class parse_event_t : uint8_t + { + /// the parser read `{` and started to process a JSON object + object_start, + /// the parser read `}` and finished processing a JSON object + object_end, + /// the parser read `[` and started to process a JSON array + array_start, + /// the parser read `]` and finished processing a JSON array + array_end, + /// the parser read a key of a value in an object + key, + /// the parser finished reading a JSON value + value + }; + + /*! + @brief per-element parser callback type + + With a parser callback function, the result of parsing a JSON text can be + influenced. When passed to @ref parse(std::istream&, const + parser_callback_t) or @ref parse(const CharT, const parser_callback_t), + it is called on certain events (passed as @ref parse_event_t via parameter + @a event) with a set recursion depth @a depth and context JSON value + @a parsed. The return value of the callback function is a boolean + indicating whether the element that emitted the callback shall be kept or + not. + + We distinguish six scenarios (determined by the event type) in which the + callback function can be called. The following table describes the values + of the parameters @a depth, @a event, and @a parsed. + + parameter @a event | description | parameter @a depth | parameter @a parsed + ------------------ | ----------- | ------------------ | ------------------- + parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded + parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key + parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object + parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded + parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array + parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value + + @image html callback_events.png "Example when certain parse events are triggered" + + Discarding a value (i.e., returning `false`) has different effects + depending on the context in which function was called: + + - Discarded values in structured types are skipped. That is, the parser + will behave as if the discarded value was never read. + - In case a value outside a structured type is skipped, it is replaced + with `null`. This case happens if the top-level element is skipped. + + @param[in] depth the depth of the recursion during parsing + + @param[in] event an event of type parse_event_t indicating the context in + the callback function has been called + + @param[in,out] parsed the current intermediate parse result; note that + writing to this value has no effect for parse_event_t::key events + + @return Whether the JSON value which called the function during parsing + should be kept (`true`) or not (`false`). In the latter case, it is either + skipped completely or replaced by an empty discarded object. + + @sa @ref parse(std::istream&, parser_callback_t) or + @ref parse(const CharT, const parser_callback_t) for examples + + @since version 1.0.0 + */ + using parser_callback_t = std::function; + + + ////////////////// + // constructors // + ////////////////// + + /// @name constructors and destructors + /// Constructors of class @ref basic_json, copy/move constructor, copy + /// assignment, static functions creating objects, and the destructor. + /// @{ + + /*! + @brief create an empty value with a given type + + Create an empty JSON value with a given type. The value will be default + initialized with an empty value which depends on the type: + + Value type | initial value + ----------- | ------------- + null | `null` + boolean | `false` + string | `""` + number | `0` + object | `{}` + array | `[]` + + @param[in] value_type the type of the value to create + + @complexity Constant. + + @throw std::bad_alloc if allocation for object, array, or string value + fails + + @liveexample{The following code shows the constructor for different @ref + value_t values,basic_json__value_t} + + @since version 1.0.0 + */ + basic_json(const value_t value_type) + : m_type(value_type), m_value(value_type) + { + assert_invariant(); + } + + /*! + @brief create a null object + + Create a `null` JSON value. It either takes a null pointer as parameter + (explicitly creating `null`) or no parameter (implicitly creating `null`). + The passed null pointer itself is not read -- it is only used to choose + the right constructor. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this constructor never throws + exceptions. + + @liveexample{The following code shows the constructor with and without a + null pointer parameter.,basic_json__nullptr_t} + + @since version 1.0.0 + */ + basic_json(std::nullptr_t = nullptr) noexcept + : basic_json(value_t::null) + { + assert_invariant(); + } + + /*! + @brief create a JSON value + + This is a "catch all" constructor for all compatible JSON types; that is, + types for which a `to_json()` method exsits. The constructor forwards the + parameter @a val to that method (to `json_serializer::to_json` method + with `U = uncvref_t`, to be exact). + + Template type @a CompatibleType includes, but is not limited to, the + following types: + - **arrays**: @ref array_t and all kinds of compatible containers such as + `std::vector`, `std::deque`, `std::list`, `std::forward_list`, + `std::array`, `std::set`, `std::unordered_set`, `std::multiset`, and + `unordered_multiset` with a `value_type` from which a @ref basic_json + value can be constructed. + - **objects**: @ref object_t and all kinds of compatible associative + containers such as `std::map`, `std::unordered_map`, `std::multimap`, + and `std::unordered_multimap` with a `key_type` compatible to + @ref string_t and a `value_type` from which a @ref basic_json value can + be constructed. + - **strings**: @ref string_t, string literals, and all compatible string + containers can be used. + - **numbers**: @ref number_integer_t, @ref number_unsigned_t, + @ref number_float_t, and all convertible number types such as `int`, + `size_t`, `int64_t`, `float` or `double` can be used. + - **boolean**: @ref boolean_t / `bool` can be used. + + See the examples below. + + @tparam CompatibleType a type such that: + - @a CompatibleType is not derived from `std::istream`, + - @a CompatibleType is not @ref basic_json (to avoid hijacking copy/move + constructors), + - @a CompatibleType is not a @ref basic_json nested type (e.g., + @ref json_pointer, @ref iterator, etc ...) + - @ref @ref json_serializer has a + `to_json(basic_json_t&, CompatibleType&&)` method + + @tparam U = `uncvref_t` + + @param[in] val the value to be forwarded + + @complexity Usually linear in the size of the passed @a val, also + depending on the implementation of the called `to_json()` + method. + + @throw what `json_serializer::to_json()` throws + + @liveexample{The following code shows the constructor with several + compatible types.,basic_json__CompatibleType} + + @since version 2.1.0 + */ + template, + detail::enable_if_t::value and + not std::is_same::value and + not detail::is_basic_json_nested_type< + basic_json_t, U>::value and + detail::has_to_json::value, + int> = 0> + basic_json(CompatibleType && val) noexcept(noexcept(JSONSerializer::to_json( + std::declval(), std::forward(val)))) + { + JSONSerializer::to_json(*this, std::forward(val)); + assert_invariant(); + } + + /*! + @brief create a container (array or object) from an initializer list + + Creates a JSON value of type array or object from the passed initializer + list @a init. In case @a type_deduction is `true` (default), the type of + the JSON value to be created is deducted from the initializer list @a init + according to the following rules: + + 1. If the list is empty, an empty JSON object value `{}` is created. + 2. If the list consists of pairs whose first element is a string, a JSON + object value is created where the first elements of the pairs are + treated as keys and the second elements are as values. + 3. In all other cases, an array is created. + + The rules aim to create the best fit between a C++ initializer list and + JSON values. The rationale is as follows: + + 1. The empty initializer list is written as `{}` which is exactly an empty + JSON object. + 2. C++ has now way of describing mapped types other than to list a list of + pairs. As JSON requires that keys must be of type string, rule 2 is the + weakest constraint one can pose on initializer lists to interpret them + as an object. + 3. In all other cases, the initializer list could not be interpreted as + JSON object type, so interpreting it as JSON array type is safe. + + With the rules described above, the following JSON values cannot be + expressed by an initializer list: + + - the empty array (`[]`): use @ref array(std::initializer_list) + with an empty initializer list in this case + - arrays whose elements satisfy rule 2: use @ref + array(std::initializer_list) with the same initializer list + in this case + + @note When used without parentheses around an empty initializer list, @ref + basic_json() is called instead of this function, yielding the JSON null + value. + + @param[in] init initializer list with JSON values + + @param[in] type_deduction internal parameter; when set to `true`, the type + of the JSON value is deducted from the initializer list @a init; when set + to `false`, the type provided via @a manual_type is forced. This mode is + used by the functions @ref array(std::initializer_list) and + @ref object(std::initializer_list). + + @param[in] manual_type internal parameter; when @a type_deduction is set + to `false`, the created JSON value will use the provided type (only @ref + value_t::array and @ref value_t::object are valid); when @a type_deduction + is set to `true`, this parameter has no effect + + @throw std::domain_error if @a type_deduction is `false`, @a manual_type + is `value_t::object`, but @a init contains an element which is not a pair + whose first element is a string; example: `"cannot create object from + initializer list"` + + @complexity Linear in the size of the initializer list @a init. + + @liveexample{The example below shows how JSON values are created from + initializer lists.,basic_json__list_init_t} + + @sa @ref array(std::initializer_list) -- create a JSON array + value from an initializer list + @sa @ref object(std::initializer_list) -- create a JSON object + value from an initializer list + + @since version 1.0.0 + */ + basic_json(std::initializer_list init, + bool type_deduction = true, + value_t manual_type = value_t::array) + { + // check if each element is an array with two elements whose first + // element is a string + bool is_an_object = std::all_of(init.begin(), init.end(), + [](const basic_json & element) + { + return element.is_array() and element.size() == 2 and element[0].is_string(); + }); + + // adjust type if type deduction is not wanted + if (not type_deduction) + { + // if array is wanted, do not create an object though possible + if (manual_type == value_t::array) + { + is_an_object = false; + } + + // if object is wanted but impossible, throw an exception + if (manual_type == value_t::object and not is_an_object) + { + JSON_THROW(std::domain_error("cannot create object from initializer list")); + } + } + + if (is_an_object) + { + // the initializer list is a list of pairs -> create object + m_type = value_t::object; + m_value = value_t::object; + + std::for_each(init.begin(), init.end(), [this](const basic_json & element) + { + m_value.object->emplace(*(element[0].m_value.string), element[1]); + }); + } + else + { + // the initializer list describes an array -> create array + m_type = value_t::array; + m_value.array = create(init); + } + + assert_invariant(); + } + + /*! + @brief explicitly create an array from an initializer list + + Creates a JSON array value from a given initializer list. That is, given a + list of values `a, b, c`, creates the JSON value `[a, b, c]`. If the + initializer list is empty, the empty array `[]` is created. + + @note This function is only needed to express two edge cases that cannot + be realized with the initializer list constructor (@ref + basic_json(std::initializer_list, bool, value_t)). These cases + are: + 1. creating an array whose elements are all pairs whose first element is a + string -- in this case, the initializer list constructor would create an + object, taking the first elements as keys + 2. creating an empty array -- passing the empty initializer list to the + initializer list constructor yields an empty object + + @param[in] init initializer list with JSON values to create an array from + (optional) + + @return JSON array value + + @complexity Linear in the size of @a init. + + @liveexample{The following code shows an example for the `array` + function.,array} + + @sa @ref basic_json(std::initializer_list, bool, value_t) -- + create a JSON value from an initializer list + @sa @ref object(std::initializer_list) -- create a JSON object + value from an initializer list + + @since version 1.0.0 + */ + static basic_json array(std::initializer_list init = + std::initializer_list()) + { + return basic_json(init, false, value_t::array); + } + + /*! + @brief explicitly create an object from an initializer list + + Creates a JSON object value from a given initializer list. The initializer + lists elements must be pairs, and their first elements must be strings. If + the initializer list is empty, the empty object `{}` is created. + + @note This function is only added for symmetry reasons. In contrast to the + related function @ref array(std::initializer_list), there are + no cases which can only be expressed by this function. That is, any + initializer list @a init can also be passed to the initializer list + constructor @ref basic_json(std::initializer_list, bool, + value_t). + + @param[in] init initializer list to create an object from (optional) + + @return JSON object value + + @throw std::domain_error if @a init is not a pair whose first elements are + strings; thrown by + @ref basic_json(std::initializer_list, bool, value_t) + + @complexity Linear in the size of @a init. + + @liveexample{The following code shows an example for the `object` + function.,object} + + @sa @ref basic_json(std::initializer_list, bool, value_t) -- + create a JSON value from an initializer list + @sa @ref array(std::initializer_list) -- create a JSON array + value from an initializer list + + @since version 1.0.0 + */ + static basic_json object(std::initializer_list init = + std::initializer_list()) + { + return basic_json(init, false, value_t::object); + } + + /*! + @brief construct an array with count copies of given value + + Constructs a JSON array value by creating @a cnt copies of a passed value. + In case @a cnt is `0`, an empty array is created. As postcondition, + `std::distance(begin(),end()) == cnt` holds. + + @param[in] cnt the number of JSON copies of @a val to create + @param[in] val the JSON value to copy + + @complexity Linear in @a cnt. + + @liveexample{The following code shows examples for the @ref + basic_json(size_type\, const basic_json&) + constructor.,basic_json__size_type_basic_json} + + @since version 1.0.0 + */ + basic_json(size_type cnt, const basic_json& val) + : m_type(value_t::array) + { + m_value.array = create(cnt, val); + assert_invariant(); + } + + /*! + @brief construct a JSON container given an iterator range + + Constructs the JSON value with the contents of the range `[first, last)`. + The semantics depends on the different types a JSON value can have: + - In case of primitive types (number, boolean, or string), @a first must + be `begin()` and @a last must be `end()`. In this case, the value is + copied. Otherwise, std::out_of_range is thrown. + - In case of structured types (array, object), the constructor behaves as + similar versions for `std::vector`. + - In case of a null type, std::domain_error is thrown. + + @tparam InputIT an input iterator type (@ref iterator or @ref + const_iterator) + + @param[in] first begin of the range to copy from (included) + @param[in] last end of the range to copy from (excluded) + + @pre Iterators @a first and @a last must be initialized. **This + precondition is enforced with an assertion.** + + @throw std::domain_error if iterators are not compatible; that is, do not + belong to the same JSON value; example: `"iterators are not compatible"` + @throw std::out_of_range if iterators are for a primitive type (number, + boolean, or string) where an out of range error can be detected easily; + example: `"iterators out of range"` + @throw std::bad_alloc if allocation for object, array, or string fails + @throw std::domain_error if called with a null value; example: `"cannot + use construct with iterators from null"` + + @complexity Linear in distance between @a first and @a last. + + @liveexample{The example below shows several ways to create JSON values by + specifying a subrange with iterators.,basic_json__InputIt_InputIt} + + @since version 1.0.0 + */ + template::value or + std::is_same::value, int>::type = 0> + basic_json(InputIT first, InputIT last) + { + assert(first.m_object != nullptr); + assert(last.m_object != nullptr); + + // make sure iterator fits the current value + if (first.m_object != last.m_object) + { + JSON_THROW(std::domain_error("iterators are not compatible")); + } + + // copy type from first iterator + m_type = first.m_object->m_type; + + // check if iterator range is complete for primitive values + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) + { + JSON_THROW(std::out_of_range("iterators out of range")); + } + break; + } + + default: + { + break; + } + } + + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = first.m_object->m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = first.m_object->m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value.number_float = first.m_object->m_value.number_float; + break; + } + + case value_t::boolean: + { + m_value.boolean = first.m_object->m_value.boolean; + break; + } + + case value_t::string: + { + m_value = *first.m_object->m_value.string; + break; + } + + case value_t::object: + { + m_value.object = create(first.m_it.object_iterator, + last.m_it.object_iterator); + break; + } + + case value_t::array: + { + m_value.array = create(first.m_it.array_iterator, + last.m_it.array_iterator); + break; + } + + default: + { + JSON_THROW(std::domain_error("cannot use construct with iterators from " + first.m_object->type_name())); + } + } + + assert_invariant(); + } + + /*! + @brief construct a JSON value given an input stream + + @param[in,out] i stream to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @deprecated This constructor is deprecated and will be removed in version + 3.0.0 to unify the interface of the library. Deserialization will be + done by stream operators or by calling one of the `parse` functions, + e.g. @ref parse(std::istream&, const parser_callback_t). That is, calls + like `json j(i);` for an input stream @a i need to be replaced by + `json j = json::parse(i);`. See the example below. + + @liveexample{The example below demonstrates constructing a JSON value from + a `std::stringstream` with and without callback + function.,basic_json__istream} + + @since version 2.0.0, deprecated in version 2.0.3, to be removed in + version 3.0.0 + */ + JSON_DEPRECATED + explicit basic_json(std::istream& i, const parser_callback_t cb = nullptr) + { + *this = parser(i, cb).parse(); + assert_invariant(); + } + + /////////////////////////////////////// + // other constructors and destructor // + /////////////////////////////////////// + + /*! + @brief copy constructor + + Creates a copy of a given JSON value. + + @param[in] other the JSON value to copy + + @complexity Linear in the size of @a other. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + - As postcondition, it holds: `other == basic_json(other)`. + + @throw std::bad_alloc if allocation for object, array, or string fails. + + @liveexample{The following code shows an example for the copy + constructor.,basic_json__basic_json} + + @since version 1.0.0 + */ + basic_json(const basic_json& other) + : m_type(other.m_type) + { + // check of passed value is valid + other.assert_invariant(); + + switch (m_type) + { + case value_t::object: + { + m_value = *other.m_value.object; + break; + } + + case value_t::array: + { + m_value = *other.m_value.array; + break; + } + + case value_t::string: + { + m_value = *other.m_value.string; + break; + } + + case value_t::boolean: + { + m_value = other.m_value.boolean; + break; + } + + case value_t::number_integer: + { + m_value = other.m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value = other.m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value = other.m_value.number_float; + break; + } + + default: + { + break; + } + } + + assert_invariant(); + } + + /*! + @brief move constructor + + Move constructor. Constructs a JSON value with the contents of the given + value @a other using move semantics. It "steals" the resources from @a + other and leaves it as JSON null value. + + @param[in,out] other value to move to this object + + @post @a other is a JSON null value + + @complexity Constant. + + @liveexample{The code below shows the move constructor explicitly called + via std::move.,basic_json__moveconstructor} + + @since version 1.0.0 + */ + basic_json(basic_json&& other) noexcept + : m_type(std::move(other.m_type)), + m_value(std::move(other.m_value)) + { + // check that passed value is valid + other.assert_invariant(); + + // invalidate payload + other.m_type = value_t::null; + other.m_value = {}; + + assert_invariant(); + } + + /*! + @brief copy assignment + + Copy assignment operator. Copies a JSON value via the "copy and swap" + strategy: It is expressed in terms of the copy constructor, destructor, + and the swap() member function. + + @param[in] other value to copy from + + @complexity Linear. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + + @liveexample{The code below shows and example for the copy assignment. It + creates a copy of value `a` which is then swapped with `b`. Finally\, the + copy of `a` (which is the null value after the swap) is + destroyed.,basic_json__copyassignment} + + @since version 1.0.0 + */ + reference& operator=(basic_json other) noexcept ( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + // check that passed value is valid + other.assert_invariant(); + + using std::swap; + swap(m_type, other.m_type); + swap(m_value, other.m_value); + + assert_invariant(); + return *this; + } + + /*! + @brief destructor + + Destroys the JSON value and frees all allocated memory. + + @complexity Linear. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is linear. + - All stored elements are destroyed and all memory is freed. + + @since version 1.0.0 + */ + ~basic_json() + { + assert_invariant(); + + switch (m_type) + { + case value_t::object: + { + AllocatorType alloc; + alloc.destroy(m_value.object); + alloc.deallocate(m_value.object, 1); + break; + } + + case value_t::array: + { + AllocatorType alloc; + alloc.destroy(m_value.array); + alloc.deallocate(m_value.array, 1); + break; + } + + case value_t::string: + { + AllocatorType alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + break; + } + + default: + { + // all other types need no specific destructor + break; + } + } + } + + /// @} + + public: + /////////////////////// + // object inspection // + /////////////////////// + + /// @name object inspection + /// Functions to inspect the type of a JSON value. + /// @{ + + /*! + @brief serialization + + Serialization function for JSON values. The function tries to mimic + Python's `json.dumps()` function, and currently supports its @a indent + parameter. + + @param[in] indent If indent is nonnegative, then array elements and object + members will be pretty-printed with that indent level. An indent level of + `0` will only insert newlines. `-1` (the default) selects the most compact + representation. + + @return string containing the serialization of the JSON value + + @complexity Linear. + + @liveexample{The following example shows the effect of different @a indent + parameters to the result of the serialization.,dump} + + @see https://docs.python.org/2/library/json.html#json.dump + + @since version 1.0.0 + */ + string_t dump(const int indent = -1) const + { + std::stringstream ss; + + if (indent >= 0) + { + dump(ss, true, static_cast(indent)); + } + else + { + dump(ss, false, 0); + } + + return ss.str(); + } + + /*! + @brief return the type of the JSON value (explicit) + + Return the type of the JSON value as a value from the @ref value_t + enumeration. + + @return the type of the JSON value + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `type()` for all JSON + types.,type} + + @since version 1.0.0 + */ + constexpr value_t type() const noexcept + { + return m_type; + } + + /*! + @brief return whether type is primitive + + This function returns true iff the JSON type is primitive (string, number, + boolean, or null). + + @return `true` if type is primitive (string, number, boolean, or null), + `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_primitive()` for all JSON + types.,is_primitive} + + @sa @ref is_structured() -- returns whether JSON value is structured + @sa @ref is_null() -- returns whether JSON value is `null` + @sa @ref is_string() -- returns whether JSON value is a string + @sa @ref is_boolean() -- returns whether JSON value is a boolean + @sa @ref is_number() -- returns whether JSON value is a number + + @since version 1.0.0 + */ + constexpr bool is_primitive() const noexcept + { + return is_null() or is_string() or is_boolean() or is_number(); + } + + /*! + @brief return whether type is structured + + This function returns true iff the JSON type is structured (array or + object). + + @return `true` if type is structured (array or object), `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_structured()` for all JSON + types.,is_structured} + + @sa @ref is_primitive() -- returns whether value is primitive + @sa @ref is_array() -- returns whether value is an array + @sa @ref is_object() -- returns whether value is an object + + @since version 1.0.0 + */ + constexpr bool is_structured() const noexcept + { + return is_array() or is_object(); + } + + /*! + @brief return whether value is null + + This function returns true iff the JSON value is null. + + @return `true` if type is null, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_null()` for all JSON + types.,is_null} + + @since version 1.0.0 + */ + constexpr bool is_null() const noexcept + { + return m_type == value_t::null; + } + + /*! + @brief return whether value is a boolean + + This function returns true iff the JSON value is a boolean. + + @return `true` if type is boolean, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_boolean()` for all JSON + types.,is_boolean} + + @since version 1.0.0 + */ + constexpr bool is_boolean() const noexcept + { + return m_type == value_t::boolean; + } + + /*! + @brief return whether value is a number + + This function returns true iff the JSON value is a number. This includes + both integer and floating-point values. + + @return `true` if type is number (regardless whether integer, unsigned + integer or floating-type), `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number()` for all JSON + types.,is_number} + + @sa @ref is_number_integer() -- check if value is an integer or unsigned + integer number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 1.0.0 + */ + constexpr bool is_number() const noexcept + { + return is_number_integer() or is_number_float(); + } + + /*! + @brief return whether value is an integer number + + This function returns true iff the JSON value is an integer or unsigned + integer number. This excludes floating-point values. + + @return `true` if type is an integer or unsigned integer number, `false` + otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_integer()` for all + JSON types.,is_number_integer} + + @sa @ref is_number() -- check if value is a number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 1.0.0 + */ + constexpr bool is_number_integer() const noexcept + { + return m_type == value_t::number_integer or m_type == value_t::number_unsigned; + } + + /*! + @brief return whether value is an unsigned integer number + + This function returns true iff the JSON value is an unsigned integer + number. This excludes floating-point and (signed) integer values. + + @return `true` if type is an unsigned integer number, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_unsigned()` for all + JSON types.,is_number_unsigned} + + @sa @ref is_number() -- check if value is a number + @sa @ref is_number_integer() -- check if value is an integer or unsigned + integer number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 2.0.0 + */ + constexpr bool is_number_unsigned() const noexcept + { + return m_type == value_t::number_unsigned; + } + + /*! + @brief return whether value is a floating-point number + + This function returns true iff the JSON value is a floating-point number. + This excludes integer and unsigned integer values. + + @return `true` if type is a floating-point number, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_float()` for all + JSON types.,is_number_float} + + @sa @ref is_number() -- check if value is number + @sa @ref is_number_integer() -- check if value is an integer number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + + @since version 1.0.0 + */ + constexpr bool is_number_float() const noexcept + { + return m_type == value_t::number_float; + } + + /*! + @brief return whether value is an object + + This function returns true iff the JSON value is an object. + + @return `true` if type is object, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_object()` for all JSON + types.,is_object} + + @since version 1.0.0 + */ + constexpr bool is_object() const noexcept + { + return m_type == value_t::object; + } + + /*! + @brief return whether value is an array + + This function returns true iff the JSON value is an array. + + @return `true` if type is array, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_array()` for all JSON + types.,is_array} + + @since version 1.0.0 + */ + constexpr bool is_array() const noexcept + { + return m_type == value_t::array; + } + + /*! + @brief return whether value is a string + + This function returns true iff the JSON value is a string. + + @return `true` if type is string, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_string()` for all JSON + types.,is_string} + + @since version 1.0.0 + */ + constexpr bool is_string() const noexcept + { + return m_type == value_t::string; + } + + /*! + @brief return whether value is discarded + + This function returns true iff the JSON value was discarded during parsing + with a callback function (see @ref parser_callback_t). + + @note This function will always be `false` for JSON values after parsing. + That is, discarded values can only occur during parsing, but will be + removed when inside a structured value or replaced by null in other cases. + + @return `true` if type is discarded, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_discarded()` for all JSON + types.,is_discarded} + + @since version 1.0.0 + */ + constexpr bool is_discarded() const noexcept + { + return m_type == value_t::discarded; + } + + /*! + @brief return the type of the JSON value (implicit) + + Implicitly return the type of the JSON value as a value from the @ref + value_t enumeration. + + @return the type of the JSON value + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies the @ref value_t operator for + all JSON types.,operator__value_t} + + @since version 1.0.0 + */ + constexpr operator value_t() const noexcept + { + return m_type; + } + + /// @} + + private: + ////////////////// + // value access // + ////////////////// + + /// get a boolean (explicit) + boolean_t get_impl(boolean_t* /*unused*/) const + { + if (is_boolean()) + { + return m_value.boolean; + } + + JSON_THROW(std::domain_error("type must be boolean, but is " + type_name())); + } + + /// get a pointer to the value (object) + object_t* get_impl_ptr(object_t* /*unused*/) noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (object) + constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (array) + array_t* get_impl_ptr(array_t* /*unused*/) noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (array) + constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (string) + string_t* get_impl_ptr(string_t* /*unused*/) noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (string) + constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (boolean) + boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (boolean) + constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (integer number) + number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (integer number) + constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (unsigned number) + number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (unsigned number) + constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (floating-point number) + number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /// get a pointer to the value (floating-point number) + constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /*! + @brief helper function to implement get_ref() + + This funcion helps to implement get_ref() without code duplication for + const and non-const overloads + + @tparam ThisType will be deduced as `basic_json` or `const basic_json` + + @throw std::domain_error if ReferenceType does not match underlying value + type of the current JSON + */ + template + static ReferenceType get_ref_impl(ThisType& obj) + { + // helper type + using PointerType = typename std::add_pointer::type; + + // delegate the call to get_ptr<>() + auto ptr = obj.template get_ptr(); + + if (ptr != nullptr) + { + return *ptr; + } + + JSON_THROW(std::domain_error("incompatible ReferenceType for get_ref, actual type is " + + obj.type_name())); + } + + public: + /// @name value access + /// Direct access to the stored value of a JSON value. + /// @{ + + /*! + @brief get special-case overload + + This overloads avoids a lot of template boilerplate, it can be seen as the + identity method + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this + + @complexity Constant. + + @since version 2.1.0 + */ + template < + typename BasicJsonType, + detail::enable_if_t::type, + basic_json_t>::value, + int> = 0 > + basic_json get() const + { + return *this; + } + + /*! + @brief get a value (explicit) + + Explicit type conversion between the JSON value and a compatible value + which is [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible) + and [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). + The value is converted by calling the @ref json_serializer + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + ValueType ret; + JSONSerializer::from_json(*this, ret); + return ret; + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json, + - @ref json_serializer has a `from_json()` method of the form + `void from_json(const @ref basic_json&, ValueType&)`, and + - @ref json_serializer does not have a `from_json()` method of + the form `ValueType from_json(const @ref basic_json&)` + + @tparam ValueTypeCV the provided value type + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer `from_json()` method throws + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map`.,get__ValueType_const} + + @since version 2.1.0 + */ + template < + typename ValueTypeCV, + typename ValueType = detail::uncvref_t, + detail::enable_if_t < + not std::is_same::value and + detail::has_from_json::value and + not detail::has_non_default_from_json::value, + int > = 0 > + ValueType get() const noexcept(noexcept( + JSONSerializer::from_json(std::declval(), std::declval()))) + { + // we cannot static_assert on ValueTypeCV being non-const, because + // there is support for get(), which is why we + // still need the uncvref + static_assert(not std::is_reference::value, + "get() cannot be used with reference types, you might want to use get_ref()"); + static_assert(std::is_default_constructible::value, + "types must be DefaultConstructible when used with get()"); + + ValueType ret; + JSONSerializer::from_json(*this, ret); + return ret; + } + + /*! + @brief get a value (explicit); special case + + Explicit type conversion between the JSON value and a compatible value + which is **not** [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible) + and **not** [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). + The value is converted by calling the @ref json_serializer + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + return JSONSerializer::from_json(*this); + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json and + - @ref json_serializer has a `from_json()` method of the form + `ValueType from_json(const @ref basic_json&)` + + @note If @ref json_serializer has both overloads of + `from_json()`, this one is chosen. + + @tparam ValueTypeCV the provided value type + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer `from_json()` method throws + + @since version 2.1.0 + */ + template < + typename ValueTypeCV, + typename ValueType = detail::uncvref_t, + detail::enable_if_t::value and + detail::has_non_default_from_json::value, int> = 0 > + ValueType get() const noexcept(noexcept( + JSONSerializer::from_json(std::declval()))) + { + static_assert(not std::is_reference::value, + "get() cannot be used with reference types, you might want to use get_ref()"); + return JSONSerializer::from_json(*this); + } + + /*! + @brief get a pointer value (explicit) + + Explicit pointer access to the internally stored JSON value. No copies are + made. + + @warning The pointer becomes invalid if the underlying JSON object + changes. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get__PointerType} + + @sa @ref get_ptr() for explicit pointer-member access + + @since version 1.0.0 + */ + template::value, int>::type = 0> + PointerType get() noexcept + { + // delegate the call to get_ptr + return get_ptr(); + } + + /*! + @brief get a pointer value (explicit) + @copydoc get() + */ + template::value, int>::type = 0> + constexpr const PointerType get() const noexcept + { + // delegate the call to get_ptr + return get_ptr(); + } + + /*! + @brief get a pointer value (implicit) + + Implicit pointer access to the internally stored JSON value. No copies are + made. + + @warning Writing data to the pointee of the result yields an undefined + state. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. Enforced by a static + assertion. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get_ptr} + + @since version 1.0.0 + */ + template::value, int>::type = 0> + PointerType get_ptr() noexcept + { + // get the type of the PointerType (remove pointer and const) + using pointee_t = typename std::remove_const::type>::type>::type; + // make sure the type matches the allowed types + static_assert( + std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + , "incompatible pointer type"); + + // delegate the call to get_impl_ptr<>() + return get_impl_ptr(static_cast(nullptr)); + } + + /*! + @brief get a pointer value (implicit) + @copydoc get_ptr() + */ + template::value and + std::is_const::type>::value, int>::type = 0> + constexpr const PointerType get_ptr() const noexcept + { + // get the type of the PointerType (remove pointer and const) + using pointee_t = typename std::remove_const::type>::type>::type; + // make sure the type matches the allowed types + static_assert( + std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + or std::is_same::value + , "incompatible pointer type"); + + // delegate the call to get_impl_ptr<>() const + return get_impl_ptr(static_cast(nullptr)); + } + + /*! + @brief get a reference value (implicit) + + Implicit reference access to the internally stored JSON value. No copies + are made. + + @warning Writing data to the referee of the result yields an undefined + state. + + @tparam ReferenceType reference type; must be a reference to @ref array_t, + @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, or + @ref number_float_t. Enforced by static assertion. + + @return reference to the internally stored JSON value if the requested + reference type @a ReferenceType fits to the JSON value; throws + std::domain_error otherwise + + @throw std::domain_error in case passed type @a ReferenceType is + incompatible with the stored JSON value + + @complexity Constant. + + @liveexample{The example shows several calls to `get_ref()`.,get_ref} + + @since version 1.1.0 + */ + template::value, int>::type = 0> + ReferenceType get_ref() + { + // delegate call to get_ref_impl + return get_ref_impl(*this); + } + + /*! + @brief get a reference value (implicit) + @copydoc get_ref() + */ + template::value and + std::is_const::type>::value, int>::type = 0> + ReferenceType get_ref() const + { + // delegate call to get_ref_impl + return get_ref_impl(*this); + } + + /*! + @brief get a value (implicit) + + Implicit type conversion between the JSON value and a compatible value. + The call is realized by calling @ref get() const. + + @tparam ValueType non-pointer type compatible to the JSON value, for + instance `int` for JSON integer numbers, `bool` for JSON booleans, or + `std::vector` types for JSON arrays. The character type of @ref string_t + as well as an initializer list of this type is excluded to avoid + ambiguities as these types implicitly convert to `std::string`. + + @return copy of the JSON value, converted to type @a ValueType + + @throw std::domain_error in case passed type @a ValueType is incompatible + to JSON, thrown by @ref get() const + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map`.,operator__ValueType} + + @since version 1.0.0 + */ + template < typename ValueType, typename std::enable_if < + not std::is_pointer::value and + not std::is_same::value +#ifndef _MSC_VER // fix for issue #167 operator<< ambiguity under VS2015 + and not std::is_same>::value +#endif + , int >::type = 0 > + operator ValueType() const + { + // delegate the call to get<>() const + return get(); + } + + /// @} + + + //////////////////// + // element access // + //////////////////// + + /// @name element access + /// Access to the JSON value. + /// @{ + + /*! + @brief access specified array element with bounds checking + + Returns a reference to the element at specified location @a idx, with + bounds checking. + + @param[in] idx index of the element to access + + @return reference to the element at index @a idx + + @throw std::domain_error if the JSON value is not an array; example: + `"cannot use at() with string"` + @throw std::out_of_range if the index @a idx is out of range of the array; + that is, `idx >= size()`; example: `"array index 7 is out of range"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read and + written using `at()`.,at__size_type} + + @since version 1.0.0 + */ + reference at(size_type idx) + { + // at only works for arrays + if (is_array()) + { + JSON_TRY + { + return m_value.array->at(idx); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(std::out_of_range("array index " + std::to_string(idx) + " is out of range")); + } + } + else + { + JSON_THROW(std::domain_error("cannot use at() with " + type_name())); + } + } + + /*! + @brief access specified array element with bounds checking + + Returns a const reference to the element at specified location @a idx, + with bounds checking. + + @param[in] idx index of the element to access + + @return const reference to the element at index @a idx + + @throw std::domain_error if the JSON value is not an array; example: + `"cannot use at() with string"` + @throw std::out_of_range if the index @a idx is out of range of the array; + that is, `idx >= size()`; example: `"array index 7 is out of range"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read using + `at()`.,at__size_type_const} + + @since version 1.0.0 + */ + const_reference at(size_type idx) const + { + // at only works for arrays + if (is_array()) + { + JSON_TRY + { + return m_value.array->at(idx); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(std::out_of_range("array index " + std::to_string(idx) + " is out of range")); + } + } + else + { + JSON_THROW(std::domain_error("cannot use at() with " + type_name())); + } + } + + /*! + @brief access specified object element with bounds checking + + Returns a reference to the element at with specified key @a key, with + bounds checking. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if the JSON value is not an object; example: + `"cannot use at() with boolean"` + @throw std::out_of_range if the key @a key is is not stored in the object; + that is, `find(key) == end()`; example: `"key "the fast" not found"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using `at()`.,at__object_t_key_type} + + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + reference at(const typename object_t::key_type& key) + { + // at only works for objects + if (is_object()) + { + JSON_TRY + { + return m_value.object->at(key); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(std::out_of_range("key '" + key + "' not found")); + } + } + else + { + JSON_THROW(std::domain_error("cannot use at() with " + type_name())); + } + } + + /*! + @brief access specified object element with bounds checking + + Returns a const reference to the element at with specified key @a key, + with bounds checking. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if the JSON value is not an object; example: + `"cannot use at() with boolean"` + @throw std::out_of_range if the key @a key is is not stored in the object; + that is, `find(key) == end()`; example: `"key "the fast" not found"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + `at()`.,at__object_t_key_type_const} + + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + const_reference at(const typename object_t::key_type& key) const + { + // at only works for objects + if (is_object()) + { + JSON_TRY + { + return m_value.object->at(key); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(std::out_of_range("key '" + key + "' not found")); + } + } + else + { + JSON_THROW(std::domain_error("cannot use at() with " + type_name())); + } + } + + /*! + @brief access specified array element + + Returns a reference to the element at specified location @a idx. + + @note If @a idx is beyond the range of the array (i.e., `idx >= size()`), + then the array is silently filled up with `null` values to make `idx` a + valid reference to the last stored element. + + @param[in] idx index of the element to access + + @return reference to the element at index @a idx + + @throw std::domain_error if JSON is not an array or null; example: + `"cannot use operator[] with string"` + + @complexity Constant if @a idx is in the range of the array. Otherwise + linear in `idx - size()`. + + @liveexample{The example below shows how array elements can be read and + written using `[]` operator. Note the addition of `null` + values.,operatorarray__size_type} + + @since version 1.0.0 + */ + reference operator[](size_type idx) + { + // implicitly convert null value to an empty array + if (is_null()) + { + m_type = value_t::array; + m_value.array = create(); + assert_invariant(); + } + + // operator[] only works for arrays + if (is_array()) + { + // fill up array with null values if given idx is outside range + if (idx >= m_value.array->size()) + { + m_value.array->insert(m_value.array->end(), + idx - m_value.array->size() + 1, + basic_json()); + } + + return m_value.array->operator[](idx); + } + + JSON_THROW(std::domain_error("cannot use operator[] with " + type_name())); + } + + /*! + @brief access specified array element + + Returns a const reference to the element at specified location @a idx. + + @param[in] idx index of the element to access + + @return const reference to the element at index @a idx + + @throw std::domain_error if JSON is not an array; example: `"cannot use + operator[] with null"` + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read using + the `[]` operator.,operatorarray__size_type_const} + + @since version 1.0.0 + */ + const_reference operator[](size_type idx) const + { + // const operator[] only works for arrays + if (is_array()) + { + return m_value.array->operator[](idx); + } + + JSON_THROW(std::domain_error("cannot use operator[] with " + type_name())); + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + reference operator[](const typename object_t::key_type& key) + { + // implicitly convert null value to an empty object + if (is_null()) + { + m_type = value_t::object; + m_value.object = create(); + assert_invariant(); + } + + // operator[] only works for objects + if (is_object()) + { + return m_value.object->operator[](key); + } + + JSON_THROW(std::domain_error("cannot use operator[] with " + type_name())); + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @pre The element with key @a key must exist. **This precondition is + enforced with an assertion.** + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + const_reference operator[](const typename object_t::key_type& key) const + { + // const operator[] only works for objects + if (is_object()) + { + assert(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + + JSON_THROW(std::domain_error("cannot use operator[] with " + type_name())); + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + template + reference operator[](T * (&key)[n]) + { + return operator[](static_cast(key)); + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @note This function is required for compatibility reasons with Clang. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + template + const_reference operator[](T * (&key)[n]) const + { + return operator[](static_cast(key)); + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw std::domain_error if JSON is not an object or null; example: + `"cannot use operator[] with string"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.1.0 + */ + template + reference operator[](T* key) + { + // implicitly convert null to object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // at only works for objects + if (is_object()) + { + return m_value.object->operator[](key); + } + + JSON_THROW(std::domain_error("cannot use operator[] with " + type_name())); + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @pre The element with key @a key must exist. **This precondition is + enforced with an assertion.** + + @throw std::domain_error if JSON is not an object; example: `"cannot use + operator[] with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.1.0 + */ + template + const_reference operator[](T* key) const + { + // at only works for objects + if (is_object()) + { + assert(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + + JSON_THROW(std::domain_error("cannot use operator[] with " + type_name())); + } + + /*! + @brief access specified object element with default value + + Returns either a copy of an object's element at the specified key @a key + or a given default value if no element with key @a key exists. + + The function is basically equivalent to executing + @code {.cpp} + try { + return at(key); + } catch(std::out_of_range) { + return default_value; + } + @endcode + + @note Unlike @ref at(const typename object_t::key_type&), this function + does not throw if the given key @a key was not found. + + @note Unlike @ref operator[](const typename object_t::key_type& key), this + function does not implicitly add an element to the position defined by @a + key. This function is furthermore also applicable to const objects. + + @param[in] key key of the element to access + @param[in] default_value the value to return if @a key is not found + + @tparam ValueType type compatible to JSON values, for instance `int` for + JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for + JSON arrays. Note the type of the expected value at @a key and the default + value @a default_value must be compatible. + + @return copy of the element at key @a key or @a default_value if @a key + is not found + + @throw std::domain_error if JSON is not an object; example: `"cannot use + value() with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be queried + with a default value.,basic_json__value} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + + @since version 1.0.0 + */ + template::value, int>::type = 0> + ValueType value(const typename object_t::key_type& key, ValueType default_value) const + { + // at only works for objects + if (is_object()) + { + // if key is found, return value and given default value otherwise + const auto it = find(key); + if (it != end()) + { + return *it; + } + + return default_value; + } + else + { + JSON_THROW(std::domain_error("cannot use value() with " + type_name())); + } + } + + /*! + @brief overload for a default value of type const char* + @copydoc basic_json::value(const typename object_t::key_type&, ValueType) const + */ + string_t value(const typename object_t::key_type& key, const char* default_value) const + { + return value(key, string_t(default_value)); + } + + /*! + @brief access specified object element via JSON Pointer with default value + + Returns either a copy of an object's element at the specified key @a key + or a given default value if no element with key @a key exists. + + The function is basically equivalent to executing + @code {.cpp} + try { + return at(ptr); + } catch(std::out_of_range) { + return default_value; + } + @endcode + + @note Unlike @ref at(const json_pointer&), this function does not throw + if the given key @a key was not found. + + @param[in] ptr a JSON pointer to the element to access + @param[in] default_value the value to return if @a ptr found no value + + @tparam ValueType type compatible to JSON values, for instance `int` for + JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for + JSON arrays. Note the type of the expected value at @a key and the default + value @a default_value must be compatible. + + @return copy of the element at key @a key or @a default_value if @a key + is not found + + @throw std::domain_error if JSON is not an object; example: `"cannot use + value() with null"` + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be queried + with a default value.,basic_json__value_ptr} + + @sa @ref operator[](const json_pointer&) for unchecked access by reference + + @since version 2.0.2 + */ + template::value, int>::type = 0> + ValueType value(const json_pointer& ptr, ValueType default_value) const + { + // at only works for objects + if (is_object()) + { + // if pointer resolves a value, return it or use default value + JSON_TRY + { + return ptr.get_checked(this); + } + JSON_CATCH (std::out_of_range&) + { + return default_value; + } + } + + JSON_THROW(std::domain_error("cannot use value() with " + type_name())); + } + + /*! + @brief overload for a default value of type const char* + @copydoc basic_json::value(const json_pointer&, ValueType) const + */ + string_t value(const json_pointer& ptr, const char* default_value) const + { + return value(ptr, string_t(default_value)); + } + + /*! + @brief access the first element + + Returns a reference to the first element in the container. For a JSON + container `c`, the expression `c.front()` is equivalent to `*c.begin()`. + + @return In case of a structured type (array or object), a reference to the + first element is returned. In case of number, string, or boolean values, a + reference to the value is returned. + + @complexity Constant. + + @pre The JSON value must not be `null` (would throw `std::out_of_range`) + or an empty array or object (undefined behavior, **guarded by + assertions**). + @post The JSON value remains unchanged. + + @throw std::out_of_range when called on `null` value + + @liveexample{The following code shows an example for `front()`.,front} + + @sa @ref back() -- access the last element + + @since version 1.0.0 + */ + reference front() + { + return *begin(); + } + + /*! + @copydoc basic_json::front() + */ + const_reference front() const + { + return *cbegin(); + } + + /*! + @brief access the last element + + Returns a reference to the last element in the container. For a JSON + container `c`, the expression `c.back()` is equivalent to + @code {.cpp} + auto tmp = c.end(); + --tmp; + return *tmp; + @endcode + + @return In case of a structured type (array or object), a reference to the + last element is returned. In case of number, string, or boolean values, a + reference to the value is returned. + + @complexity Constant. + + @pre The JSON value must not be `null` (would throw `std::out_of_range`) + or an empty array or object (undefined behavior, **guarded by + assertions**). + @post The JSON value remains unchanged. + + @throw std::out_of_range when called on `null` value. + + @liveexample{The following code shows an example for `back()`.,back} + + @sa @ref front() -- access the first element + + @since version 1.0.0 + */ + reference back() + { + auto tmp = end(); + --tmp; + return *tmp; + } + + /*! + @copydoc basic_json::back() + */ + const_reference back() const + { + auto tmp = cend(); + --tmp; + return *tmp; + } + + /*! + @brief remove element given an iterator + + Removes the element specified by iterator @a pos. The iterator @a pos must + be valid and dereferenceable. Thus the `end()` iterator (which is valid, + but is not dereferenceable) cannot be used as a value for @a pos. + + If called on a primitive type other than `null`, the resulting JSON value + will be `null`. + + @param[in] pos iterator to the element to remove + @return Iterator following the last removed element. If the iterator @a + pos refers to the last element, the `end()` iterator is returned. + + @tparam IteratorType an @ref iterator or @ref const_iterator + + @post Invalidates iterators and references at or after the point of the + erase, including the `end()` iterator. + + @throw std::domain_error if called on a `null` value; example: `"cannot + use erase() with null"` + @throw std::domain_error if called on an iterator which does not belong to + the current JSON value; example: `"iterator does not fit current value"` + @throw std::out_of_range if called on a primitive type with invalid + iterator (i.e., any iterator which is not `begin()`); example: `"iterator + out of range"` + + @complexity The complexity depends on the type: + - objects: amortized constant + - arrays: linear in distance between @a pos and the end of the container + - strings: linear in the length of the string + - other types: constant + + @liveexample{The example shows the result of `erase()` for different JSON + types.,erase__IteratorType} + + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in + the given range + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + template::value or + std::is_same::value, int>::type + = 0> + IteratorType erase(IteratorType pos) + { + // make sure iterator fits the current value + if (this != pos.m_object) + { + JSON_THROW(std::domain_error("iterator does not fit current value")); + } + + IteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not pos.m_it.primitive_iterator.is_begin()) + { + JSON_THROW(std::out_of_range("iterator out of range")); + } + + if (is_string()) + { + AllocatorType alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + m_value.string = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator); + break; + } + + default: + { + JSON_THROW(std::domain_error("cannot use erase() with " + type_name())); + } + } + + return result; + } + + /*! + @brief remove elements given an iterator range + + Removes the element specified by the range `[first; last)`. The iterator + @a first does not need to be dereferenceable if `first == last`: erasing + an empty range is a no-op. + + If called on a primitive type other than `null`, the resulting JSON value + will be `null`. + + @param[in] first iterator to the beginning of the range to remove + @param[in] last iterator past the end of the range to remove + @return Iterator following the last removed element. If the iterator @a + second refers to the last element, the `end()` iterator is returned. + + @tparam IteratorType an @ref iterator or @ref const_iterator + + @post Invalidates iterators and references at or after the point of the + erase, including the `end()` iterator. + + @throw std::domain_error if called on a `null` value; example: `"cannot + use erase() with null"` + @throw std::domain_error if called on iterators which does not belong to + the current JSON value; example: `"iterators do not fit current value"` + @throw std::out_of_range if called on a primitive type with invalid + iterators (i.e., if `first != begin()` and `last != end()`); example: + `"iterators out of range"` + + @complexity The complexity depends on the type: + - objects: `log(size()) + std::distance(first, last)` + - arrays: linear in the distance between @a first and @a last, plus linear + in the distance between @a last and end of the container + - strings: linear in the length of the string + - other types: constant + + @liveexample{The example shows the result of `erase()` for different JSON + types.,erase__IteratorType_IteratorType} + + @sa @ref erase(IteratorType) -- removes the element at a given position + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + template::value or + std::is_same::value, int>::type + = 0> + IteratorType erase(IteratorType first, IteratorType last) + { + // make sure iterator fits the current value + if (this != first.m_object or this != last.m_object) + { + JSON_THROW(std::domain_error("iterators do not fit current value")); + } + + IteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end()) + { + JSON_THROW(std::out_of_range("iterators out of range")); + } + + if (is_string()) + { + AllocatorType alloc; + alloc.destroy(m_value.string); + alloc.deallocate(m_value.string, 1); + m_value.string = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator, + last.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator, + last.m_it.array_iterator); + break; + } + + default: + { + JSON_THROW(std::domain_error("cannot use erase() with " + type_name())); + } + } + + return result; + } + + /*! + @brief remove element from a JSON object given a key + + Removes elements from a JSON object with the key value @a key. + + @param[in] key value of the elements to remove + + @return Number of elements removed. If @a ObjectType is the default + `std::map` type, the return value will always be `0` (@a key was not + found) or `1` (@a key was found). + + @post References and iterators to the erased elements are invalidated. + Other references and iterators are not affected. + + @throw std::domain_error when called on a type other than JSON object; + example: `"cannot use erase() with null"` + + @complexity `log(size()) + count(key)` + + @liveexample{The example shows the effect of `erase()`.,erase__key_type} + + @sa @ref erase(IteratorType) -- removes the element at a given position + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in + the given range + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + size_type erase(const typename object_t::key_type& key) + { + // this erase only works for objects + if (is_object()) + { + return m_value.object->erase(key); + } + + JSON_THROW(std::domain_error("cannot use erase() with " + type_name())); + } + + /*! + @brief remove element from a JSON array given an index + + Removes element from a JSON array at the index @a idx. + + @param[in] idx index of the element to remove + + @throw std::domain_error when called on a type other than JSON array; + example: `"cannot use erase() with null"` + @throw std::out_of_range when `idx >= size()`; example: `"array index 17 + is out of range"` + + @complexity Linear in distance between @a idx and the end of the container. + + @liveexample{The example shows the effect of `erase()`.,erase__size_type} + + @sa @ref erase(IteratorType) -- removes the element at a given position + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in + the given range + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + + @since version 1.0.0 + */ + void erase(const size_type idx) + { + // this erase only works for arrays + if (is_array()) + { + if (idx >= size()) + { + JSON_THROW(std::out_of_range("array index " + std::to_string(idx) + " is out of range")); + } + + m_value.array->erase(m_value.array->begin() + static_cast(idx)); + } + else + { + JSON_THROW(std::domain_error("cannot use erase() with " + type_name())); + } + } + + /// @} + + + //////////// + // lookup // + //////////// + + /// @name lookup + /// @{ + + /*! + @brief find an element in a JSON object + + Finds an element in a JSON object with key equivalent to @a key. If the + element is not found or the JSON value is not an object, end() is + returned. + + @note This method always returns @ref end() when executed on a JSON type + that is not an object. + + @param[in] key key value of the element to search for + + @return Iterator to an element with key equivalent to @a key. If no such + element is found or the JSON value is not an object, past-the-end (see + @ref end()) iterator is returned. + + @complexity Logarithmic in the size of the JSON object. + + @liveexample{The example shows how `find()` is used.,find__key_type} + + @since version 1.0.0 + */ + iterator find(typename object_t::key_type key) + { + auto result = end(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(key); + } + + return result; + } + + /*! + @brief find an element in a JSON object + @copydoc find(typename object_t::key_type) + */ + const_iterator find(typename object_t::key_type key) const + { + auto result = cend(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(key); + } + + return result; + } + + /*! + @brief returns the number of occurrences of a key in a JSON object + + Returns the number of elements with key @a key. If ObjectType is the + default `std::map` type, the return value will always be `0` (@a key was + not found) or `1` (@a key was found). + + @note This method always returns `0` when executed on a JSON type that is + not an object. + + @param[in] key key value of the element to count + + @return Number of elements with key @a key. If the JSON value is not an + object, the return value will be `0`. + + @complexity Logarithmic in the size of the JSON object. + + @liveexample{The example shows how `count()` is used.,count} + + @since version 1.0.0 + */ + size_type count(typename object_t::key_type key) const + { + // return 0 for all nonobject types + return is_object() ? m_value.object->count(key) : 0; + } + + /// @} + + + /////////////// + // iterators // + /////////////// + + /// @name iterators + /// @{ + + /*! + @brief returns an iterator to the first element + + Returns an iterator to the first element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return iterator to the first element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + + @liveexample{The following code shows an example for `begin()`.,begin} + + @sa @ref cbegin() -- returns a const iterator to the beginning + @sa @ref end() -- returns an iterator to the end + @sa @ref cend() -- returns a const iterator to the end + + @since version 1.0.0 + */ + iterator begin() noexcept + { + iterator result(this); + result.set_begin(); + return result; + } + + /*! + @copydoc basic_json::cbegin() + */ + const_iterator begin() const noexcept + { + return cbegin(); + } + + /*! + @brief returns a const iterator to the first element + + Returns a const iterator to the first element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return const iterator to the first element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).begin()`. + + @liveexample{The following code shows an example for `cbegin()`.,cbegin} + + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref end() -- returns an iterator to the end + @sa @ref cend() -- returns a const iterator to the end + + @since version 1.0.0 + */ + const_iterator cbegin() const noexcept + { + const_iterator result(this); + result.set_begin(); + return result; + } + + /*! + @brief returns an iterator to one past the last element + + Returns an iterator to one past the last element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return iterator one past the last element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + + @liveexample{The following code shows an example for `end()`.,end} + + @sa @ref cend() -- returns a const iterator to the end + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref cbegin() -- returns a const iterator to the beginning + + @since version 1.0.0 + */ + iterator end() noexcept + { + iterator result(this); + result.set_end(); + return result; + } + + /*! + @copydoc basic_json::cend() + */ + const_iterator end() const noexcept + { + return cend(); + } + + /*! + @brief returns a const iterator to one past the last element + + Returns a const iterator to one past the last element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return const iterator one past the last element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).end()`. + + @liveexample{The following code shows an example for `cend()`.,cend} + + @sa @ref end() -- returns an iterator to the end + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref cbegin() -- returns a const iterator to the beginning + + @since version 1.0.0 + */ + const_iterator cend() const noexcept + { + const_iterator result(this); + result.set_end(); + return result; + } + + /*! + @brief returns an iterator to the reverse-beginning + + Returns an iterator to the reverse-beginning; that is, the last element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `reverse_iterator(end())`. + + @liveexample{The following code shows an example for `rbegin()`.,rbegin} + + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref crend() -- returns a const reverse iterator to the end + + @since version 1.0.0 + */ + reverse_iterator rbegin() noexcept + { + return reverse_iterator(end()); + } + + /*! + @copydoc basic_json::crbegin() + */ + const_reverse_iterator rbegin() const noexcept + { + return crbegin(); + } + + /*! + @brief returns an iterator to the reverse-end + + Returns an iterator to the reverse-end; that is, one before the first + element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `reverse_iterator(begin())`. + + @liveexample{The following code shows an example for `rend()`.,rend} + + @sa @ref crend() -- returns a const reverse iterator to the end + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + + @since version 1.0.0 + */ + reverse_iterator rend() noexcept + { + return reverse_iterator(begin()); + } + + /*! + @copydoc basic_json::crend() + */ + const_reverse_iterator rend() const noexcept + { + return crend(); + } + + /*! + @brief returns a const reverse iterator to the last element + + Returns a const iterator to the reverse-beginning; that is, the last + element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).rbegin()`. + + @liveexample{The following code shows an example for `crbegin()`.,crbegin} + + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref crend() -- returns a const reverse iterator to the end + + @since version 1.0.0 + */ + const_reverse_iterator crbegin() const noexcept + { + return const_reverse_iterator(cend()); + } + + /*! + @brief returns a const reverse iterator to one before the first + + Returns a const reverse iterator to the reverse-end; that is, one before + the first element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast(*this).rend()`. + + @liveexample{The following code shows an example for `crend()`.,crend} + + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + + @since version 1.0.0 + */ + const_reverse_iterator crend() const noexcept + { + return const_reverse_iterator(cbegin()); + } + + private: + // forward declaration + template class iteration_proxy; + + public: + /*! + @brief wrapper to access iterator member functions in range-based for + + This function allows to access @ref iterator::key() and @ref + iterator::value() during range-based for loops. In these loops, a + reference to the JSON values is returned, so there is no access to the + underlying iterator. + + @note The name of this function is not yet final and may change in the + future. + */ + static iteration_proxy iterator_wrapper(reference cont) + { + return iteration_proxy(cont); + } + + /*! + @copydoc iterator_wrapper(reference) + */ + static iteration_proxy iterator_wrapper(const_reference cont) + { + return iteration_proxy(cont); + } + + /// @} + + + ////////////// + // capacity // + ////////////// + + /// @name capacity + /// @{ + + /*! + @brief checks whether the container is empty + + Checks if a JSON value has no elements. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `true` + boolean | `false` + string | `false` + number | `false` + object | result of function `object_t::empty()` + array | result of function `array_t::empty()` + + @note This function does not return whether a string stored as JSON value + is empty - it returns whether the JSON container itself is empty which is + false in the case of a string. + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their `empty()` functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `begin() == end()`. + + @liveexample{The following code uses `empty()` to check if a JSON + object contains any elements.,empty} + + @sa @ref size() -- returns the number of elements + + @since version 1.0.0 + */ + bool empty() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return true; + } + + case value_t::array: + { + // delegate call to array_t::empty() + return m_value.array->empty(); + } + + case value_t::object: + { + // delegate call to object_t::empty() + return m_value.object->empty(); + } + + default: + { + // all other types are nonempty + return false; + } + } + } + + /*! + @brief returns the number of elements + + Returns the number of elements in a JSON value. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `0` + boolean | `1` + string | `1` + number | `1` + object | result of function object_t::size() + array | result of function array_t::size() + + @note This function does not return the length of a string stored as JSON + value - it returns the number of elements in the JSON value which is 1 in + the case of a string. + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their size() functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of `std::distance(begin(), end())`. + + @liveexample{The following code calls `size()` on the different value + types.,size} + + @sa @ref empty() -- checks whether the container is empty + @sa @ref max_size() -- returns the maximal number of elements + + @since version 1.0.0 + */ + size_type size() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return 0; + } + + case value_t::array: + { + // delegate call to array_t::size() + return m_value.array->size(); + } + + case value_t::object: + { + // delegate call to object_t::size() + return m_value.object->size(); + } + + default: + { + // all other types have size 1 + return 1; + } + } + } + + /*! + @brief returns the maximum possible number of elements + + Returns the maximum number of elements a JSON value is able to hold due to + system or library implementation limitations, i.e. `std::distance(begin(), + end())` for the JSON value. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `0` (same as `size()`) + boolean | `1` (same as `size()`) + string | `1` (same as `size()`) + number | `1` (same as `size()`) + object | result of function `object_t::max_size()` + array | result of function `array_t::max_size()` + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their `max_size()` functions have constant + complexity. + + @requirement This function helps `basic_json` satisfying the + [Container](http://en.cppreference.com/w/cpp/concept/Container) + requirements: + - The complexity is constant. + - Has the semantics of returning `b.size()` where `b` is the largest + possible JSON value. + + @liveexample{The following code calls `max_size()` on the different value + types. Note the output is implementation specific.,max_size} + + @sa @ref size() -- returns the number of elements + + @since version 1.0.0 + */ + size_type max_size() const noexcept + { + switch (m_type) + { + case value_t::array: + { + // delegate call to array_t::max_size() + return m_value.array->max_size(); + } + + case value_t::object: + { + // delegate call to object_t::max_size() + return m_value.object->max_size(); + } + + default: + { + // all other types have max_size() == size() + return size(); + } + } + } + + /// @} + + + /////////////// + // modifiers // + /////////////// + + /// @name modifiers + /// @{ + + /*! + @brief clears the contents + + Clears the content of a JSON value and resets it to the default value as + if @ref basic_json(value_t) would have been called: + + Value type | initial value + ----------- | ------------- + null | `null` + boolean | `false` + string | `""` + number | `0` + object | `{}` + array | `[]` + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows the effect of `clear()` to different + JSON types.,clear} + + @since version 1.0.0 + */ + void clear() noexcept + { + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = 0; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = 0; + break; + } + + case value_t::number_float: + { + m_value.number_float = 0.0; + break; + } + + case value_t::boolean: + { + m_value.boolean = false; + break; + } + + case value_t::string: + { + m_value.string->clear(); + break; + } + + case value_t::array: + { + m_value.array->clear(); + break; + } + + case value_t::object: + { + m_value.object->clear(); + break; + } + + default: + { + break; + } + } + } + + /*! + @brief add an object to an array + + Appends the given element @a val to the end of the JSON value. If the + function is called on a JSON null value, an empty array is created before + appending @a val. + + @param[in] val the value to add to the JSON array + + @throw std::domain_error when called on a type other than JSON array or + null; example: `"cannot use push_back() with number"` + + @complexity Amortized constant. + + @liveexample{The example shows how `push_back()` and `+=` can be used to + add elements to a JSON array. Note how the `null` value was silently + converted to a JSON array.,push_back} + + @since version 1.0.0 + */ + void push_back(basic_json&& val) + { + // push_back only works for null objects or arrays + if (not(is_null() or is_array())) + { + JSON_THROW(std::domain_error("cannot use push_back() with " + type_name())); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array (move semantics) + m_value.array->push_back(std::move(val)); + // invalidate object + val.m_type = value_t::null; + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + reference operator+=(basic_json&& val) + { + push_back(std::move(val)); + return *this; + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + void push_back(const basic_json& val) + { + // push_back only works for null objects or arrays + if (not(is_null() or is_array())) + { + JSON_THROW(std::domain_error("cannot use push_back() with " + type_name())); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array + m_value.array->push_back(val); + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + reference operator+=(const basic_json& val) + { + push_back(val); + return *this; + } + + /*! + @brief add an object to an object + + Inserts the given element @a val to the JSON object. If the function is + called on a JSON null value, an empty object is created before inserting + @a val. + + @param[in] val the value to add to the JSON object + + @throw std::domain_error when called on a type other than JSON object or + null; example: `"cannot use push_back() with number"` + + @complexity Logarithmic in the size of the container, O(log(`size()`)). + + @liveexample{The example shows how `push_back()` and `+=` can be used to + add elements to a JSON object. Note how the `null` value was silently + converted to a JSON object.,push_back__object_t__value} + + @since version 1.0.0 + */ + void push_back(const typename object_t::value_type& val) + { + // push_back only works for null objects or objects + if (not(is_null() or is_object())) + { + JSON_THROW(std::domain_error("cannot use push_back() with " + type_name())); + } + + // transform null object into an object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // add element to array + m_value.object->insert(val); + } + + /*! + @brief add an object to an object + @copydoc push_back(const typename object_t::value_type&) + */ + reference operator+=(const typename object_t::value_type& val) + { + push_back(val); + return *this; + } + + /*! + @brief add an object to an object + + This function allows to use `push_back` with an initializer list. In case + + 1. the current value is an object, + 2. the initializer list @a init contains only two elements, and + 3. the first element of @a init is a string, + + @a init is converted into an object element and added using + @ref push_back(const typename object_t::value_type&). Otherwise, @a init + is converted to a JSON value and added using @ref push_back(basic_json&&). + + @param init an initializer list + + @complexity Linear in the size of the initializer list @a init. + + @note This function is required to resolve an ambiguous overload error, + because pairs like `{"key", "value"}` can be both interpreted as + `object_t::value_type` or `std::initializer_list`, see + https://github.com/nlohmann/json/issues/235 for more information. + + @liveexample{The example shows how initializer lists are treated as + objects when possible.,push_back__initializer_list} + */ + void push_back(std::initializer_list init) + { + if (is_object() and init.size() == 2 and init.begin()->is_string()) + { + const string_t key = *init.begin(); + push_back(typename object_t::value_type(key, *(init.begin() + 1))); + } + else + { + push_back(basic_json(init)); + } + } + + /*! + @brief add an object to an object + @copydoc push_back(std::initializer_list) + */ + reference operator+=(std::initializer_list init) + { + push_back(init); + return *this; + } + + /*! + @brief add an object to an array + + Creates a JSON value from the passed parameters @a args to the end of the + JSON value. If the function is called on a JSON null value, an empty array + is created before appending the value created from @a args. + + @param[in] args arguments to forward to a constructor of @ref basic_json + @tparam Args compatible types to create a @ref basic_json object + + @throw std::domain_error when called on a type other than JSON array or + null; example: `"cannot use emplace_back() with number"` + + @complexity Amortized constant. + + @liveexample{The example shows how `push_back()` can be used to add + elements to a JSON array. Note how the `null` value was silently converted + to a JSON array.,emplace_back} + + @since version 2.0.8 + */ + template + void emplace_back(Args&& ... args) + { + // emplace_back only works for null objects or arrays + if (not(is_null() or is_array())) + { + JSON_THROW(std::domain_error("cannot use emplace_back() with " + type_name())); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array (perfect forwarding) + m_value.array->emplace_back(std::forward(args)...); + } + + /*! + @brief add an object to an object if key does not exist + + Inserts a new element into a JSON object constructed in-place with the + given @a args if there is no element with the key in the container. If the + function is called on a JSON null value, an empty object is created before + appending the value created from @a args. + + @param[in] args arguments to forward to a constructor of @ref basic_json + @tparam Args compatible types to create a @ref basic_json object + + @return a pair consisting of an iterator to the inserted element, or the + already-existing element if no insertion happened, and a bool + denoting whether the insertion took place. + + @throw std::domain_error when called on a type other than JSON object or + null; example: `"cannot use emplace() with number"` + + @complexity Logarithmic in the size of the container, O(log(`size()`)). + + @liveexample{The example shows how `emplace()` can be used to add elements + to a JSON object. Note how the `null` value was silently converted to a + JSON object. Further note how no value is added if there was already one + value stored with the same key.,emplace} + + @since version 2.0.8 + */ + template + std::pair emplace(Args&& ... args) + { + // emplace only works for null objects or arrays + if (not(is_null() or is_object())) + { + JSON_THROW(std::domain_error("cannot use emplace() with " + type_name())); + } + + // transform null object into an object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // add element to array (perfect forwarding) + auto res = m_value.object->emplace(std::forward(args)...); + // create result iterator and set iterator to the result of emplace + auto it = begin(); + it.m_it.object_iterator = res.first; + + // return pair of iterator and boolean + return {it, res.second}; + } + + /*! + @brief inserts element + + Inserts element @a val before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] val element to insert + @return iterator pointing to the inserted @a val. + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @complexity Constant plus linear in the distance between @a pos and end of + the container. + + @liveexample{The example shows how `insert()` is used.,insert} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, const basic_json& val) + { + // insert only works for arrays + if (is_array()) + { + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + JSON_THROW(std::domain_error("iterator does not fit current value")); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val); + return result; + } + + JSON_THROW(std::domain_error("cannot use insert() with " + type_name())); + } + + /*! + @brief inserts element + @copydoc insert(const_iterator, const basic_json&) + */ + iterator insert(const_iterator pos, basic_json&& val) + { + return insert(pos, val); + } + + /*! + @brief inserts elements + + Inserts @a cnt copies of @a val before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] cnt number of copies of @a val to insert + @param[in] val element to insert + @return iterator pointing to the first element inserted, or @a pos if + `cnt==0` + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @complexity Linear in @a cnt plus linear in the distance between @a pos + and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__count} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, size_type cnt, const basic_json& val) + { + // insert only works for arrays + if (is_array()) + { + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + JSON_THROW(std::domain_error("iterator does not fit current value")); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); + return result; + } + + JSON_THROW(std::domain_error("cannot use insert() with " + type_name())); + } + + /*! + @brief inserts elements + + Inserts elements from range `[first, last)` before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] first begin of the range of elements to insert + @param[in] last end of the range of elements to insert + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + @throw std::domain_error if @a first and @a last do not belong to the same + JSON value; example: `"iterators do not fit"` + @throw std::domain_error if @a first or @a last are iterators into + container for which insert is called; example: `"passed iterators may not + belong to container"` + + @return iterator pointing to the first element inserted, or @a pos if + `first==last` + + @complexity Linear in `std::distance(first, last)` plus linear in the + distance between @a pos and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__range} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, const_iterator first, const_iterator last) + { + // insert only works for arrays + if (not is_array()) + { + JSON_THROW(std::domain_error("cannot use insert() with " + type_name())); + } + + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + JSON_THROW(std::domain_error("iterator does not fit current value")); + } + + // check if range iterators belong to the same JSON object + if (first.m_object != last.m_object) + { + JSON_THROW(std::domain_error("iterators do not fit")); + } + + if (first.m_object == this or last.m_object == this) + { + JSON_THROW(std::domain_error("passed iterators may not belong to container")); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert( + pos.m_it.array_iterator, + first.m_it.array_iterator, + last.m_it.array_iterator); + return result; + } + + /*! + @brief inserts elements + + Inserts elements from initializer list @a ilist before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] ilist initializer list to insert the values from + + @throw std::domain_error if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw std::domain_error if @a pos is not an iterator of *this; example: + `"iterator does not fit current value"` + + @return iterator pointing to the first element inserted, or @a pos if + `ilist` is empty + + @complexity Linear in `ilist.size()` plus linear in the distance between + @a pos and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__ilist} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, std::initializer_list ilist) + { + // insert only works for arrays + if (not is_array()) + { + JSON_THROW(std::domain_error("cannot use insert() with " + type_name())); + } + + // check if iterator pos fits to this JSON value + if (pos.m_object != this) + { + JSON_THROW(std::domain_error("iterator does not fit current value")); + } + + // insert to array and return iterator + iterator result(this); + result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist); + return result; + } + + /*! + @brief exchanges the values + + Exchanges the contents of the JSON value with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other JSON value to exchange the contents with + + @complexity Constant. + + @liveexample{The example below shows how JSON values can be swapped with + `swap()`.,swap__reference} + + @since version 1.0.0 + */ + void swap(reference other) noexcept ( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + std::swap(m_type, other.m_type); + std::swap(m_value, other.m_value); + assert_invariant(); + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON array with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other array to exchange the contents with + + @throw std::domain_error when JSON value is not an array; example: + `"cannot use swap() with string"` + + @complexity Constant. + + @liveexample{The example below shows how arrays can be swapped with + `swap()`.,swap__array_t} + + @since version 1.0.0 + */ + void swap(array_t& other) + { + // swap only works for arrays + if (is_array()) + { + std::swap(*(m_value.array), other); + } + else + { + JSON_THROW(std::domain_error("cannot use swap() with " + type_name())); + } + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON object with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other object to exchange the contents with + + @throw std::domain_error when JSON value is not an object; example: + `"cannot use swap() with string"` + + @complexity Constant. + + @liveexample{The example below shows how objects can be swapped with + `swap()`.,swap__object_t} + + @since version 1.0.0 + */ + void swap(object_t& other) + { + // swap only works for objects + if (is_object()) + { + std::swap(*(m_value.object), other); + } + else + { + JSON_THROW(std::domain_error("cannot use swap() with " + type_name())); + } + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON string with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other string to exchange the contents with + + @throw std::domain_error when JSON value is not a string; example: `"cannot + use swap() with boolean"` + + @complexity Constant. + + @liveexample{The example below shows how strings can be swapped with + `swap()`.,swap__string_t} + + @since version 1.0.0 + */ + void swap(string_t& other) + { + // swap only works for strings + if (is_string()) + { + std::swap(*(m_value.string), other); + } + else + { + JSON_THROW(std::domain_error("cannot use swap() with " + type_name())); + } + } + + /// @} + + public: + ////////////////////////////////////////// + // lexicographical comparison operators // + ////////////////////////////////////////// + + /// @name lexicographical comparison operators + /// @{ + + /*! + @brief comparison: equal + + Compares two JSON values for equality according to the following rules: + - Two JSON values are equal if (1) they are from the same type and (2) + their stored values are the same. + - Integer and floating-point numbers are automatically converted before + comparison. Floating-point numbers are compared indirectly: two + floating-point numbers `f1` and `f2` are considered equal if neither + `f1 > f2` nor `f2 > f1` holds. + - Two JSON null values are equal. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are equal + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__equal} + + @since version 1.0.0 + */ + friend bool operator==(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + { + return *lhs.m_value.array == *rhs.m_value.array; + } + case value_t::object: + { + return *lhs.m_value.object == *rhs.m_value.object; + } + case value_t::null: + { + return true; + } + case value_t::string: + { + return *lhs.m_value.string == *rhs.m_value.string; + } + case value_t::boolean: + { + return lhs.m_value.boolean == rhs.m_value.boolean; + } + case value_t::number_integer: + { + return lhs.m_value.number_integer == rhs.m_value.number_integer; + } + case value_t::number_unsigned: + { + return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; + } + case value_t::number_float: + { + return lhs.m_value.number_float == rhs.m_value.number_float; + } + default: + { + return false; + } + } + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); + } + + return false; + } + + /*! + @brief comparison: equal + @copydoc operator==(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator==(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs == basic_json(rhs)); + } + + /*! + @brief comparison: equal + @copydoc operator==(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator==(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) == rhs); + } + + /*! + @brief comparison: not equal + + Compares two JSON values for inequality by calculating `not (lhs == rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are not equal + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__notequal} + + @since version 1.0.0 + */ + friend bool operator!=(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs == rhs); + } + + /*! + @brief comparison: not equal + @copydoc operator!=(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator!=(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs != basic_json(rhs)); + } + + /*! + @brief comparison: not equal + @copydoc operator!=(const_reference, const_reference) + */ + template::value, int>::type = 0> + friend bool operator!=(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) != rhs); + } + + /*! + @brief comparison: less than + + Compares whether one JSON value @a lhs is less than another JSON value @a + rhs according to the following rules: + - If @a lhs and @a rhs have the same type, the values are compared using + the default `<` operator. + - Integer and floating-point numbers are automatically converted before + comparison + - In case @a lhs and @a rhs have different types, the values are ignored + and the order of the types is considered, see + @ref operator<(const value_t, const value_t). + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is less than @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__less} + + @since version 1.0.0 + */ + friend bool operator<(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + { + return *lhs.m_value.array < *rhs.m_value.array; + } + case value_t::object: + { + return *lhs.m_value.object < *rhs.m_value.object; + } + case value_t::null: + { + return false; + } + case value_t::string: + { + return *lhs.m_value.string < *rhs.m_value.string; + } + case value_t::boolean: + { + return lhs.m_value.boolean < rhs.m_value.boolean; + } + case value_t::number_integer: + { + return lhs.m_value.number_integer < rhs.m_value.number_integer; + } + case value_t::number_unsigned: + { + return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned; + } + case value_t::number_float: + { + return lhs.m_value.number_float < rhs.m_value.number_float; + } + default: + { + return false; + } + } + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_integer) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float < static_cast(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) + { + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float < static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer < static_cast(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + { + return static_cast(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; + } + + // We only reach this line if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + return operator<(lhs_type, rhs_type); + } + + /*! + @brief comparison: less than or equal + + Compares whether one JSON value @a lhs is less than or equal to another + JSON value by calculating `not (rhs < lhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is less than or equal to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__greater} + + @since version 1.0.0 + */ + friend bool operator<=(const_reference lhs, const_reference rhs) noexcept + { + return not (rhs < lhs); + } + + /*! + @brief comparison: greater than + + Compares whether one JSON value @a lhs is greater than another + JSON value by calculating `not (lhs <= rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is greater than to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__lessequal} + + @since version 1.0.0 + */ + friend bool operator>(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs <= rhs); + } + + /*! + @brief comparison: greater than or equal + + Compares whether one JSON value @a lhs is greater than or equal to another + JSON value by calculating `not (lhs < rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is greater than or equal to @a rhs + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__greaterequal} + + @since version 1.0.0 + */ + friend bool operator>=(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs < rhs); + } + + /// @} + + + /////////////////// + // serialization // + /////////////////// + + /// @name serialization + /// @{ + + /*! + @brief serialize to stream + + Serialize the given JSON value @a j to the output stream @a o. The JSON + value will be serialized using the @ref dump member function. The + indentation of the output can be controlled with the member variable + `width` of the output stream @a o. For instance, using the manipulator + `std::setw(4)` on @a o sets the indentation level to `4` and the + serialization result is the same as calling `dump(4)`. + + @param[in,out] o stream to serialize to + @param[in] j JSON value to serialize + + @return the stream @a o + + @complexity Linear. + + @liveexample{The example below shows the serialization with different + parameters to `width` to adjust the indentation level.,operator_serialize} + + @since version 1.0.0 + */ + friend std::ostream& operator<<(std::ostream& o, const basic_json& j) + { + // read width member and use it as indentation parameter if nonzero + const bool pretty_print = (o.width() > 0); + const auto indentation = (pretty_print ? o.width() : 0); + + // reset width to 0 for subsequent calls to this stream + o.width(0); + + // do the actual serialization + j.dump(o, pretty_print, static_cast(indentation)); + + return o; + } + + /*! + @brief serialize to stream + @copydoc operator<<(std::ostream&, const basic_json&) + */ + friend std::ostream& operator>>(const basic_json& j, std::ostream& o) + { + return o << j; + } + + /// @} + + + ///////////////////// + // deserialization // + ///////////////////// + + /// @name deserialization + /// @{ + + /*! + @brief deserialize from an array + + This function reads from an array of 1-byte values. + + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @param[in] array array to read from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function reading + from an array.,parse__array__parser_callback_t} + + @since version 2.0.3 + */ + template + static basic_json parse(T (&array)[N], + const parser_callback_t cb = nullptr) + { + // delegate the call to the iterator-range parse overload + return parse(std::begin(array), std::end(array), cb); + } + + /*! + @brief deserialize from string literal + + @tparam CharT character/literal type with size of 1 byte + @param[in] s string literal to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + @note String containers like `std::string` or @ref string_t can be parsed + with @ref parse(const ContiguousContainer&, const parser_callback_t) + + @liveexample{The example below demonstrates the `parse()` function with + and without callback function.,parse__string__parser_callback_t} + + @sa @ref parse(std::istream&, const parser_callback_t) for a version that + reads from an input stream + + @since version 1.0.0 (originally for @ref string_t) + */ + template::value and + std::is_integral::type>::value and + sizeof(typename std::remove_pointer::type) == 1, int>::type = 0> + static basic_json parse(const CharT s, + const parser_callback_t cb = nullptr) + { + return parser(reinterpret_cast(s), cb).parse(); + } + + /*! + @brief deserialize from stream + + @param[in,out] i stream to read a serialized JSON value from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function with + and without callback function.,parse__istream__parser_callback_t} + + @sa @ref parse(const CharT, const parser_callback_t) for a version + that reads from a string + + @since version 1.0.0 + */ + static basic_json parse(std::istream& i, + const parser_callback_t cb = nullptr) + { + return parser(i, cb).parse(); + } + + /*! + @copydoc parse(std::istream&, const parser_callback_t) + */ + static basic_json parse(std::istream&& i, + const parser_callback_t cb = nullptr) + { + return parser(i, cb).parse(); + } + + /*! + @brief deserialize from an iterator range with contiguous storage + + This function reads from an iterator range of a container with contiguous + storage of 1-byte values. Compatible container types include + `std::vector`, `std::string`, `std::array`, `std::valarray`, and + `std::initializer_list`. Furthermore, C-style arrays can be used with + `std::begin()`/`std::end()`. User-defined containers can be used as long + as they implement random-access iterators and a contiguous storage. + + @pre The iterator range is contiguous. Violating this precondition yields + undefined behavior. **This precondition is enforced with an assertion.** + @pre Each element in the range has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @warning There is no way to enforce all preconditions at compile-time. If + the function is called with noncompliant iterators and with + assertions switched off, the behavior is undefined and will most + likely yield segmentation violation. + + @tparam IteratorType iterator of container with contiguous storage + @param[in] first begin of the range to parse (included) + @param[in] last end of the range to parse (excluded) + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function reading + from an iterator range.,parse__iteratortype__parser_callback_t} + + @since version 2.0.3 + */ + template::iterator_category>::value, int>::type = 0> + static basic_json parse(IteratorType first, IteratorType last, + const parser_callback_t cb = nullptr) + { + // assertion to check that the iterator range is indeed contiguous, + // see http://stackoverflow.com/a/35008842/266378 for more discussion + assert(std::accumulate(first, last, std::pair(true, 0), + [&first](std::pair res, decltype(*first) val) + { + res.first &= (val == *(std::next(std::addressof(*first), res.second++))); + return res; + }).first); + + // assertion to check that each element is 1 byte long + static_assert(sizeof(typename std::iterator_traits::value_type) == 1, + "each element in the iterator range must have the size of 1 byte"); + + // if iterator range is empty, create a parser with an empty string + // to generate "unexpected EOF" error message + if (std::distance(first, last) <= 0) + { + return parser("").parse(); + } + + return parser(first, last, cb).parse(); + } + + /*! + @brief deserialize from a container with contiguous storage + + This function reads from a container with contiguous storage of 1-byte + values. Compatible container types include `std::vector`, `std::string`, + `std::array`, and `std::initializer_list`. User-defined containers can be + used as long as they implement random-access iterators and a contiguous + storage. + + @pre The container storage is contiguous. Violating this precondition + yields undefined behavior. **This precondition is enforced with an + assertion.** + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @warning There is no way to enforce all preconditions at compile-time. If + the function is called with a noncompliant container and with + assertions switched off, the behavior is undefined and will most + likely yield segmentation violation. + + @tparam ContiguousContainer container type with contiguous storage + @param[in] c container to read from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + + @return result of the deserialization + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function reading + from a contiguous container.,parse__contiguouscontainer__parser_callback_t} + + @since version 2.0.3 + */ + template::value and + std::is_base_of< + std::random_access_iterator_tag, + typename std::iterator_traits()))>::iterator_category>::value + , int>::type = 0> + static basic_json parse(const ContiguousContainer& c, + const parser_callback_t cb = nullptr) + { + // delegate the call to the iterator-range parse overload + return parse(std::begin(c), std::end(c), cb); + } + + /*! + @brief deserialize from stream + + Deserializes an input stream to a JSON value. + + @param[in,out] i input stream to read a serialized JSON value from + @param[in,out] j JSON value to write the deserialized input to + + @throw std::invalid_argument in case of parse errors + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below shows how a JSON value is constructed by + reading a serialization from a stream.,operator_deserialize} + + @sa parse(std::istream&, const parser_callback_t) for a variant with a + parser callback function to filter values while parsing + + @since version 1.0.0 + */ + friend std::istream& operator<<(basic_json& j, std::istream& i) + { + j = parser(i).parse(); + return i; + } + + /*! + @brief deserialize from stream + @copydoc operator<<(basic_json&, std::istream&) + */ + friend std::istream& operator>>(std::istream& i, basic_json& j) + { + j = parser(i).parse(); + return i; + } + + /// @} + + ////////////////////////////////////////// + // binary serialization/deserialization // + ////////////////////////////////////////// + + /// @name binary serialization/deserialization support + /// @{ + + private: + /*! + @note Some code in the switch cases has been copied, because otherwise + copilers would complain about implicit fallthrough and there is no + portable attribute to mute such warnings. + */ + template + static void add_to_vector(std::vector& vec, size_t bytes, const T number) + { + assert(bytes == 1 or bytes == 2 or bytes == 4 or bytes == 8); + + switch (bytes) + { + case 8: + { + vec.push_back(static_cast((static_cast(number) >> 070) & 0xff)); + vec.push_back(static_cast((static_cast(number) >> 060) & 0xff)); + vec.push_back(static_cast((static_cast(number) >> 050) & 0xff)); + vec.push_back(static_cast((static_cast(number) >> 040) & 0xff)); + vec.push_back(static_cast((number >> 030) & 0xff)); + vec.push_back(static_cast((number >> 020) & 0xff)); + vec.push_back(static_cast((number >> 010) & 0xff)); + vec.push_back(static_cast(number & 0xff)); + break; + } + + case 4: + { + vec.push_back(static_cast((number >> 030) & 0xff)); + vec.push_back(static_cast((number >> 020) & 0xff)); + vec.push_back(static_cast((number >> 010) & 0xff)); + vec.push_back(static_cast(number & 0xff)); + break; + } + + case 2: + { + vec.push_back(static_cast((number >> 010) & 0xff)); + vec.push_back(static_cast(number & 0xff)); + break; + } + + case 1: + { + vec.push_back(static_cast(number & 0xff)); + break; + } + } + } + + /*! + @brief take sufficient bytes from a vector to fill an integer variable + + In the context of binary serialization formats, we need to read several + bytes from a byte vector and combine them to multi-byte integral data + types. + + @param[in] vec byte vector to read from + @param[in] current_index the position in the vector after which to read + + @return the next sizeof(T) bytes from @a vec, in reverse order as T + + @tparam T the integral return type + + @throw std::out_of_range if there are less than sizeof(T)+1 bytes in the + vector @a vec to read + + In the for loop, the bytes from the vector are copied in reverse order into + the return value. In the figures below, let sizeof(T)=4 and `i` be the loop + variable. + + Precondition: + + vec: | | | a | b | c | d | T: | | | | | + ^ ^ ^ ^ + current_index i ptr sizeof(T) + + Postcondition: + + vec: | | | a | b | c | d | T: | d | c | b | a | + ^ ^ ^ + | i ptr + current_index + + @sa Code adapted from . + */ + template + static T get_from_vector(const std::vector& vec, const size_t current_index) + { + if (current_index + sizeof(T) + 1 > vec.size()) + { + JSON_THROW(std::out_of_range("cannot read " + std::to_string(sizeof(T)) + " bytes from vector")); + } + + T result; + auto* ptr = reinterpret_cast(&result); + for (size_t i = 0; i < sizeof(T); ++i) + { + *ptr++ = vec[current_index + sizeof(T) - i]; + } + return result; + } + + /*! + @brief create a MessagePack serialization of a given JSON value + + This is a straightforward implementation of the MessagePack specification. + + @param[in] j JSON value to serialize + @param[in,out] v byte vector to write the serialization to + + @sa https://github.com/msgpack/msgpack/blob/master/spec.md + */ + static void to_msgpack_internal(const basic_json& j, std::vector& v) + { + switch (j.type()) + { + case value_t::null: + { + // nil + v.push_back(0xc0); + break; + } + + case value_t::boolean: + { + // true and false + v.push_back(j.m_value.boolean ? 0xc3 : 0xc2); + break; + } + + case value_t::number_integer: + { + if (j.m_value.number_integer >= 0) + { + // MessagePack does not differentiate between positive + // signed integers and unsigned integers. Therefore, we + // used the code from the value_t::number_unsigned case + // here. + if (j.m_value.number_unsigned < 128) + { + // positive fixnum + add_to_vector(v, 1, j.m_value.number_unsigned); + } + else if (j.m_value.number_unsigned <= std::numeric_limits::max()) + { + // uint 8 + v.push_back(0xcc); + add_to_vector(v, 1, j.m_value.number_unsigned); + } + else if (j.m_value.number_unsigned <= std::numeric_limits::max()) + { + // uint 16 + v.push_back(0xcd); + add_to_vector(v, 2, j.m_value.number_unsigned); + } + else if (j.m_value.number_unsigned <= std::numeric_limits::max()) + { + // uint 32 + v.push_back(0xce); + add_to_vector(v, 4, j.m_value.number_unsigned); + } + else if (j.m_value.number_unsigned <= std::numeric_limits::max()) + { + // uint 64 + v.push_back(0xcf); + add_to_vector(v, 8, j.m_value.number_unsigned); + } + } + else + { + if (j.m_value.number_integer >= -32) + { + // negative fixnum + add_to_vector(v, 1, j.m_value.number_integer); + } + else if (j.m_value.number_integer >= std::numeric_limits::min() and j.m_value.number_integer <= std::numeric_limits::max()) + { + // int 8 + v.push_back(0xd0); + add_to_vector(v, 1, j.m_value.number_integer); + } + else if (j.m_value.number_integer >= std::numeric_limits::min() and j.m_value.number_integer <= std::numeric_limits::max()) + { + // int 16 + v.push_back(0xd1); + add_to_vector(v, 2, j.m_value.number_integer); + } + else if (j.m_value.number_integer >= std::numeric_limits::min() and j.m_value.number_integer <= std::numeric_limits::max()) + { + // int 32 + v.push_back(0xd2); + add_to_vector(v, 4, j.m_value.number_integer); + } + else if (j.m_value.number_integer >= std::numeric_limits::min() and j.m_value.number_integer <= std::numeric_limits::max()) + { + // int 64 + v.push_back(0xd3); + add_to_vector(v, 8, j.m_value.number_integer); + } + } + break; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned < 128) + { + // positive fixnum + add_to_vector(v, 1, j.m_value.number_unsigned); + } + else if (j.m_value.number_unsigned <= std::numeric_limits::max()) + { + // uint 8 + v.push_back(0xcc); + add_to_vector(v, 1, j.m_value.number_unsigned); + } + else if (j.m_value.number_unsigned <= std::numeric_limits::max()) + { + // uint 16 + v.push_back(0xcd); + add_to_vector(v, 2, j.m_value.number_unsigned); + } + else if (j.m_value.number_unsigned <= std::numeric_limits::max()) + { + // uint 32 + v.push_back(0xce); + add_to_vector(v, 4, j.m_value.number_unsigned); + } + else if (j.m_value.number_unsigned <= std::numeric_limits::max()) + { + // uint 64 + v.push_back(0xcf); + add_to_vector(v, 8, j.m_value.number_unsigned); + } + break; + } + + case value_t::number_float: + { + // float 64 + v.push_back(0xcb); + const auto* helper = reinterpret_cast(&(j.m_value.number_float)); + for (size_t i = 0; i < 8; ++i) + { + v.push_back(helper[7 - i]); + } + break; + } + + case value_t::string: + { + const auto N = j.m_value.string->size(); + if (N <= 31) + { + // fixstr + v.push_back(static_cast(0xa0 | N)); + } + else if (N <= 255) + { + // str 8 + v.push_back(0xd9); + add_to_vector(v, 1, N); + } + else if (N <= 65535) + { + // str 16 + v.push_back(0xda); + add_to_vector(v, 2, N); + } + else if (N <= 4294967295) + { + // str 32 + v.push_back(0xdb); + add_to_vector(v, 4, N); + } + + // append string + std::copy(j.m_value.string->begin(), j.m_value.string->end(), + std::back_inserter(v)); + break; + } + + case value_t::array: + { + const auto N = j.m_value.array->size(); + if (N <= 15) + { + // fixarray + v.push_back(static_cast(0x90 | N)); + } + else if (N <= 0xffff) + { + // array 16 + v.push_back(0xdc); + add_to_vector(v, 2, N); + } + else if (N <= 0xffffffff) + { + // array 32 + v.push_back(0xdd); + add_to_vector(v, 4, N); + } + + // append each element + for (const auto& el : *j.m_value.array) + { + to_msgpack_internal(el, v); + } + break; + } + + case value_t::object: + { + const auto N = j.m_value.object->size(); + if (N <= 15) + { + // fixmap + v.push_back(static_cast(0x80 | (N & 0xf))); + } + else if (N <= 65535) + { + // map 16 + v.push_back(0xde); + add_to_vector(v, 2, N); + } + else if (N <= 4294967295) + { + // map 32 + v.push_back(0xdf); + add_to_vector(v, 4, N); + } + + // append each element + for (const auto& el : *j.m_value.object) + { + to_msgpack_internal(el.first, v); + to_msgpack_internal(el.second, v); + } + break; + } + + default: + { + break; + } + } + } + + /*! + @brief create a CBOR serialization of a given JSON value + + This is a straightforward implementation of the CBOR specification. + + @param[in] j JSON value to serialize + @param[in,out] v byte vector to write the serialization to + + @sa https://tools.ietf.org/html/rfc7049 + */ + static void to_cbor_internal(const basic_json& j, std::vector& v) + { + switch (j.type()) + { + case value_t::null: + { + v.push_back(0xf6); + break; + } + + case value_t::boolean: + { + v.push_back(j.m_value.boolean ? 0xf5 : 0xf4); + break; + } + + case value_t::number_integer: + { + if (j.m_value.number_integer >= 0) + { + // CBOR does not differentiate between positive signed + // integers and unsigned integers. Therefore, we used the + // code from the value_t::number_unsigned case here. + if (j.m_value.number_integer <= 0x17) + { + add_to_vector(v, 1, j.m_value.number_integer); + } + else if (j.m_value.number_integer <= std::numeric_limits::max()) + { + v.push_back(0x18); + // one-byte uint8_t + add_to_vector(v, 1, j.m_value.number_integer); + } + else if (j.m_value.number_integer <= std::numeric_limits::max()) + { + v.push_back(0x19); + // two-byte uint16_t + add_to_vector(v, 2, j.m_value.number_integer); + } + else if (j.m_value.number_integer <= std::numeric_limits::max()) + { + v.push_back(0x1a); + // four-byte uint32_t + add_to_vector(v, 4, j.m_value.number_integer); + } + else + { + v.push_back(0x1b); + // eight-byte uint64_t + add_to_vector(v, 8, j.m_value.number_integer); + } + } + else + { + // The conversions below encode the sign in the first + // byte, and the value is converted to a positive number. + const auto positive_number = -1 - j.m_value.number_integer; + if (j.m_value.number_integer >= -24) + { + v.push_back(static_cast(0x20 + positive_number)); + } + else if (positive_number <= std::numeric_limits::max()) + { + // int 8 + v.push_back(0x38); + add_to_vector(v, 1, positive_number); + } + else if (positive_number <= std::numeric_limits::max()) + { + // int 16 + v.push_back(0x39); + add_to_vector(v, 2, positive_number); + } + else if (positive_number <= std::numeric_limits::max()) + { + // int 32 + v.push_back(0x3a); + add_to_vector(v, 4, positive_number); + } + else + { + // int 64 + v.push_back(0x3b); + add_to_vector(v, 8, positive_number); + } + } + break; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= 0x17) + { + v.push_back(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= 0xff) + { + v.push_back(0x18); + // one-byte uint8_t + add_to_vector(v, 1, j.m_value.number_unsigned); + } + else if (j.m_value.number_unsigned <= 0xffff) + { + v.push_back(0x19); + // two-byte uint16_t + add_to_vector(v, 2, j.m_value.number_unsigned); + } + else if (j.m_value.number_unsigned <= 0xffffffff) + { + v.push_back(0x1a); + // four-byte uint32_t + add_to_vector(v, 4, j.m_value.number_unsigned); + } + else if (j.m_value.number_unsigned <= 0xffffffffffffffff) + { + v.push_back(0x1b); + // eight-byte uint64_t + add_to_vector(v, 8, j.m_value.number_unsigned); + } + break; + } + + case value_t::number_float: + { + // Double-Precision Float + v.push_back(0xfb); + const auto* helper = reinterpret_cast(&(j.m_value.number_float)); + for (size_t i = 0; i < 8; ++i) + { + v.push_back(helper[7 - i]); + } + break; + } + + case value_t::string: + { + const auto N = j.m_value.string->size(); + if (N <= 0x17) + { + v.push_back(0x60 + static_cast(N)); // 1 byte for string + size + } + else if (N <= 0xff) + { + v.push_back(0x78); // one-byte uint8_t for N + add_to_vector(v, 1, N); + } + else if (N <= 0xffff) + { + v.push_back(0x79); // two-byte uint16_t for N + add_to_vector(v, 2, N); + } + else if (N <= 0xffffffff) + { + v.push_back(0x7a); // four-byte uint32_t for N + add_to_vector(v, 4, N); + } + // LCOV_EXCL_START + else if (N <= 0xffffffffffffffff) + { + v.push_back(0x7b); // eight-byte uint64_t for N + add_to_vector(v, 8, N); + } + // LCOV_EXCL_STOP + + // append string + std::copy(j.m_value.string->begin(), j.m_value.string->end(), + std::back_inserter(v)); + break; + } + + case value_t::array: + { + const auto N = j.m_value.array->size(); + if (N <= 0x17) + { + v.push_back(0x80 + static_cast(N)); // 1 byte for array + size + } + else if (N <= 0xff) + { + v.push_back(0x98); // one-byte uint8_t for N + add_to_vector(v, 1, N); + } + else if (N <= 0xffff) + { + v.push_back(0x99); // two-byte uint16_t for N + add_to_vector(v, 2, N); + } + else if (N <= 0xffffffff) + { + v.push_back(0x9a); // four-byte uint32_t for N + add_to_vector(v, 4, N); + } + // LCOV_EXCL_START + else if (N <= 0xffffffffffffffff) + { + v.push_back(0x9b); // eight-byte uint64_t for N + add_to_vector(v, 8, N); + } + // LCOV_EXCL_STOP + + // append each element + for (const auto& el : *j.m_value.array) + { + to_cbor_internal(el, v); + } + break; + } + + case value_t::object: + { + const auto N = j.m_value.object->size(); + if (N <= 0x17) + { + v.push_back(0xa0 + static_cast(N)); // 1 byte for object + size + } + else if (N <= 0xff) + { + v.push_back(0xb8); + add_to_vector(v, 1, N); // one-byte uint8_t for N + } + else if (N <= 0xffff) + { + v.push_back(0xb9); + add_to_vector(v, 2, N); // two-byte uint16_t for N + } + else if (N <= 0xffffffff) + { + v.push_back(0xba); + add_to_vector(v, 4, N); // four-byte uint32_t for N + } + // LCOV_EXCL_START + else if (N <= 0xffffffffffffffff) + { + v.push_back(0xbb); + add_to_vector(v, 8, N); // eight-byte uint64_t for N + } + // LCOV_EXCL_STOP + + // append each element + for (const auto& el : *j.m_value.object) + { + to_cbor_internal(el.first, v); + to_cbor_internal(el.second, v); + } + break; + } + + default: + { + break; + } + } + } + + + /* + @brief checks if given lengths do not exceed the size of a given vector + + To secure the access to the byte vector during CBOR/MessagePack + deserialization, bytes are copied from the vector into buffers. This + function checks if the number of bytes to copy (@a len) does not exceed + the size @s size of the vector. Additionally, an @a offset is given from + where to start reading the bytes. + + This function checks whether reading the bytes is safe; that is, offset is + a valid index in the vector, offset+len + + @param[in] size size of the byte vector + @param[in] len number of bytes to read + @param[in] offset offset where to start reading + + vec: x x x x x X X X X X + ^ ^ ^ + 0 offset len + + @throws out_of_range if `len > v.size()` + */ + static void check_length(const size_t size, const size_t len, const size_t offset) + { + // simple case: requested length is greater than the vector's length + if (len > size or offset > size) + { + JSON_THROW(std::out_of_range("len out of range")); + } + + // second case: adding offset would result in overflow + if ((size > (std::numeric_limits::max() - offset))) + { + JSON_THROW(std::out_of_range("len+offset out of range")); + } + + // last case: reading past the end of the vector + if (len + offset > size) + { + JSON_THROW(std::out_of_range("len+offset out of range")); + } + } + + /*! + @brief create a JSON value from a given MessagePack vector + + @param[in] v MessagePack serialization + @param[in] idx byte index to start reading from @a v + + @return deserialized JSON value + + @throw std::invalid_argument if unsupported features from MessagePack were + used in the given vector @a v or if the input is not valid MessagePack + @throw std::out_of_range if the given vector ends prematurely + + @sa https://github.com/msgpack/msgpack/blob/master/spec.md + */ + static basic_json from_msgpack_internal(const std::vector& v, size_t& idx) + { + // make sure reading 1 byte is safe + check_length(v.size(), 1, idx); + + // store and increment index + const size_t current_idx = idx++; + + if (v[current_idx] <= 0xbf) + { + if (v[current_idx] <= 0x7f) // positive fixint + { + return v[current_idx]; + } + if (v[current_idx] <= 0x8f) // fixmap + { + basic_json result = value_t::object; + const size_t len = v[current_idx] & 0x0f; + for (size_t i = 0; i < len; ++i) + { + std::string key = from_msgpack_internal(v, idx); + result[key] = from_msgpack_internal(v, idx); + } + return result; + } + else if (v[current_idx] <= 0x9f) // fixarray + { + basic_json result = value_t::array; + const size_t len = v[current_idx] & 0x0f; + for (size_t i = 0; i < len; ++i) + { + result.push_back(from_msgpack_internal(v, idx)); + } + return result; + } + else // fixstr + { + const size_t len = v[current_idx] & 0x1f; + const size_t offset = current_idx + 1; + idx += len; // skip content bytes + check_length(v.size(), len, offset); + return std::string(reinterpret_cast(v.data()) + offset, len); + } + } + else if (v[current_idx] >= 0xe0) // negative fixint + { + return static_cast(v[current_idx]); + } + else + { + switch (v[current_idx]) + { + case 0xc0: // nil + { + return value_t::null; + } + + case 0xc2: // false + { + return false; + } + + case 0xc3: // true + { + return true; + } + + case 0xca: // float 32 + { + // copy bytes in reverse order into the double variable + float res; + for (size_t byte = 0; byte < sizeof(float); ++byte) + { + reinterpret_cast(&res)[sizeof(float) - byte - 1] = v.at(current_idx + 1 + byte); + } + idx += sizeof(float); // skip content bytes + return res; + } + + case 0xcb: // float 64 + { + // copy bytes in reverse order into the double variable + double res; + for (size_t byte = 0; byte < sizeof(double); ++byte) + { + reinterpret_cast(&res)[sizeof(double) - byte - 1] = v.at(current_idx + 1 + byte); + } + idx += sizeof(double); // skip content bytes + return res; + } + + case 0xcc: // uint 8 + { + idx += 1; // skip content byte + return get_from_vector(v, current_idx); + } + + case 0xcd: // uint 16 + { + idx += 2; // skip 2 content bytes + return get_from_vector(v, current_idx); + } + + case 0xce: // uint 32 + { + idx += 4; // skip 4 content bytes + return get_from_vector(v, current_idx); + } + + case 0xcf: // uint 64 + { + idx += 8; // skip 8 content bytes + return get_from_vector(v, current_idx); + } + + case 0xd0: // int 8 + { + idx += 1; // skip content byte + return get_from_vector(v, current_idx); + } + + case 0xd1: // int 16 + { + idx += 2; // skip 2 content bytes + return get_from_vector(v, current_idx); + } + + case 0xd2: // int 32 + { + idx += 4; // skip 4 content bytes + return get_from_vector(v, current_idx); + } + + case 0xd3: // int 64 + { + idx += 8; // skip 8 content bytes + return get_from_vector(v, current_idx); + } + + case 0xd9: // str 8 + { + const auto len = static_cast(get_from_vector(v, current_idx)); + const size_t offset = current_idx + 2; + idx += len + 1; // skip size byte + content bytes + check_length(v.size(), len, offset); + return std::string(reinterpret_cast(v.data()) + offset, len); + } + + case 0xda: // str 16 + { + const auto len = static_cast(get_from_vector(v, current_idx)); + const size_t offset = current_idx + 3; + idx += len + 2; // skip 2 size bytes + content bytes + check_length(v.size(), len, offset); + return std::string(reinterpret_cast(v.data()) + offset, len); + } + + case 0xdb: // str 32 + { + const auto len = static_cast(get_from_vector(v, current_idx)); + const size_t offset = current_idx + 5; + idx += len + 4; // skip 4 size bytes + content bytes + check_length(v.size(), len, offset); + return std::string(reinterpret_cast(v.data()) + offset, len); + } + + case 0xdc: // array 16 + { + basic_json result = value_t::array; + const auto len = static_cast(get_from_vector(v, current_idx)); + idx += 2; // skip 2 size bytes + for (size_t i = 0; i < len; ++i) + { + result.push_back(from_msgpack_internal(v, idx)); + } + return result; + } + + case 0xdd: // array 32 + { + basic_json result = value_t::array; + const auto len = static_cast(get_from_vector(v, current_idx)); + idx += 4; // skip 4 size bytes + for (size_t i = 0; i < len; ++i) + { + result.push_back(from_msgpack_internal(v, idx)); + } + return result; + } + + case 0xde: // map 16 + { + basic_json result = value_t::object; + const auto len = static_cast(get_from_vector(v, current_idx)); + idx += 2; // skip 2 size bytes + for (size_t i = 0; i < len; ++i) + { + std::string key = from_msgpack_internal(v, idx); + result[key] = from_msgpack_internal(v, idx); + } + return result; + } + + case 0xdf: // map 32 + { + basic_json result = value_t::object; + const auto len = static_cast(get_from_vector(v, current_idx)); + idx += 4; // skip 4 size bytes + for (size_t i = 0; i < len; ++i) + { + std::string key = from_msgpack_internal(v, idx); + result[key] = from_msgpack_internal(v, idx); + } + return result; + } + + default: + { + JSON_THROW(std::invalid_argument("error parsing a msgpack @ " + std::to_string(current_idx) + ": " + std::to_string(static_cast(v[current_idx])))); + } + } + } + } + + /*! + @brief create a JSON value from a given CBOR vector + + @param[in] v CBOR serialization + @param[in] idx byte index to start reading from @a v + + @return deserialized JSON value + + @throw std::invalid_argument if unsupported features from CBOR were used in + the given vector @a v or if the input is not valid CBOR + @throw std::out_of_range if the given vector ends prematurely + + @sa https://tools.ietf.org/html/rfc7049 + */ + static basic_json from_cbor_internal(const std::vector& v, size_t& idx) + { + // store and increment index + const size_t current_idx = idx++; + + switch (v.at(current_idx)) + { + // Integer 0x00..0x17 (0..23) + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0a: + case 0x0b: + case 0x0c: + case 0x0d: + case 0x0e: + case 0x0f: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + { + return v[current_idx]; + } + + case 0x18: // Unsigned integer (one-byte uint8_t follows) + { + idx += 1; // skip content byte + return get_from_vector(v, current_idx); + } + + case 0x19: // Unsigned integer (two-byte uint16_t follows) + { + idx += 2; // skip 2 content bytes + return get_from_vector(v, current_idx); + } + + case 0x1a: // Unsigned integer (four-byte uint32_t follows) + { + idx += 4; // skip 4 content bytes + return get_from_vector(v, current_idx); + } + + case 0x1b: // Unsigned integer (eight-byte uint64_t follows) + { + idx += 8; // skip 8 content bytes + return get_from_vector(v, current_idx); + } + + // Negative integer -1-0x00..-1-0x17 (-1..-24) + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2a: + case 0x2b: + case 0x2c: + case 0x2d: + case 0x2e: + case 0x2f: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + { + return static_cast(0x20 - 1 - v[current_idx]); + } + + case 0x38: // Negative integer (one-byte uint8_t follows) + { + idx += 1; // skip content byte + // must be uint8_t ! + return static_cast(-1) - get_from_vector(v, current_idx); + } + + case 0x39: // Negative integer -1-n (two-byte uint16_t follows) + { + idx += 2; // skip 2 content bytes + return static_cast(-1) - get_from_vector(v, current_idx); + } + + case 0x3a: // Negative integer -1-n (four-byte uint32_t follows) + { + idx += 4; // skip 4 content bytes + return static_cast(-1) - get_from_vector(v, current_idx); + } + + case 0x3b: // Negative integer -1-n (eight-byte uint64_t follows) + { + idx += 8; // skip 8 content bytes + return static_cast(-1) - static_cast(get_from_vector(v, current_idx)); + } + + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6a: + case 0x6b: + case 0x6c: + case 0x6d: + case 0x6e: + case 0x6f: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + { + const auto len = static_cast(v[current_idx] - 0x60); + const size_t offset = current_idx + 1; + idx += len; // skip content bytes + check_length(v.size(), len, offset); + return std::string(reinterpret_cast(v.data()) + offset, len); + } + + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + { + const auto len = static_cast(get_from_vector(v, current_idx)); + const size_t offset = current_idx + 2; + idx += len + 1; // skip size byte + content bytes + check_length(v.size(), len, offset); + return std::string(reinterpret_cast(v.data()) + offset, len); + } + + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + { + const auto len = static_cast(get_from_vector(v, current_idx)); + const size_t offset = current_idx + 3; + idx += len + 2; // skip 2 size bytes + content bytes + check_length(v.size(), len, offset); + return std::string(reinterpret_cast(v.data()) + offset, len); + } + + case 0x7a: // UTF-8 string (four-byte uint32_t for n follow) + { + const auto len = static_cast(get_from_vector(v, current_idx)); + const size_t offset = current_idx + 5; + idx += len + 4; // skip 4 size bytes + content bytes + check_length(v.size(), len, offset); + return std::string(reinterpret_cast(v.data()) + offset, len); + } + + case 0x7b: // UTF-8 string (eight-byte uint64_t for n follow) + { + const auto len = static_cast(get_from_vector(v, current_idx)); + const size_t offset = current_idx + 9; + idx += len + 8; // skip 8 size bytes + content bytes + check_length(v.size(), len, offset); + return std::string(reinterpret_cast(v.data()) + offset, len); + } + + case 0x7f: // UTF-8 string (indefinite length) + { + std::string result; + while (v.at(idx) != 0xff) + { + string_t s = from_cbor_internal(v, idx); + result += s; + } + // skip break byte (0xFF) + idx += 1; + return result; + } + + // array (0x00..0x17 data items follow) + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8a: + case 0x8b: + case 0x8c: + case 0x8d: + case 0x8e: + case 0x8f: + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + { + basic_json result = value_t::array; + const auto len = static_cast(v[current_idx] - 0x80); + for (size_t i = 0; i < len; ++i) + { + result.push_back(from_cbor_internal(v, idx)); + } + return result; + } + + case 0x98: // array (one-byte uint8_t for n follows) + { + basic_json result = value_t::array; + const auto len = static_cast(get_from_vector(v, current_idx)); + idx += 1; // skip 1 size byte + for (size_t i = 0; i < len; ++i) + { + result.push_back(from_cbor_internal(v, idx)); + } + return result; + } + + case 0x99: // array (two-byte uint16_t for n follow) + { + basic_json result = value_t::array; + const auto len = static_cast(get_from_vector(v, current_idx)); + idx += 2; // skip 4 size bytes + for (size_t i = 0; i < len; ++i) + { + result.push_back(from_cbor_internal(v, idx)); + } + return result; + } + + case 0x9a: // array (four-byte uint32_t for n follow) + { + basic_json result = value_t::array; + const auto len = static_cast(get_from_vector(v, current_idx)); + idx += 4; // skip 4 size bytes + for (size_t i = 0; i < len; ++i) + { + result.push_back(from_cbor_internal(v, idx)); + } + return result; + } + + case 0x9b: // array (eight-byte uint64_t for n follow) + { + basic_json result = value_t::array; + const auto len = static_cast(get_from_vector(v, current_idx)); + idx += 8; // skip 8 size bytes + for (size_t i = 0; i < len; ++i) + { + result.push_back(from_cbor_internal(v, idx)); + } + return result; + } + + case 0x9f: // array (indefinite length) + { + basic_json result = value_t::array; + while (v.at(idx) != 0xff) + { + result.push_back(from_cbor_internal(v, idx)); + } + // skip break byte (0xFF) + idx += 1; + return result; + } + + // map (0x00..0x17 pairs of data items follow) + case 0xa0: + case 0xa1: + case 0xa2: + case 0xa3: + case 0xa4: + case 0xa5: + case 0xa6: + case 0xa7: + case 0xa8: + case 0xa9: + case 0xaa: + case 0xab: + case 0xac: + case 0xad: + case 0xae: + case 0xaf: + case 0xb0: + case 0xb1: + case 0xb2: + case 0xb3: + case 0xb4: + case 0xb5: + case 0xb6: + case 0xb7: + { + basic_json result = value_t::object; + const auto len = static_cast(v[current_idx] - 0xa0); + for (size_t i = 0; i < len; ++i) + { + std::string key = from_cbor_internal(v, idx); + result[key] = from_cbor_internal(v, idx); + } + return result; + } + + case 0xb8: // map (one-byte uint8_t for n follows) + { + basic_json result = value_t::object; + const auto len = static_cast(get_from_vector(v, current_idx)); + idx += 1; // skip 1 size byte + for (size_t i = 0; i < len; ++i) + { + std::string key = from_cbor_internal(v, idx); + result[key] = from_cbor_internal(v, idx); + } + return result; + } + + case 0xb9: // map (two-byte uint16_t for n follow) + { + basic_json result = value_t::object; + const auto len = static_cast(get_from_vector(v, current_idx)); + idx += 2; // skip 2 size bytes + for (size_t i = 0; i < len; ++i) + { + std::string key = from_cbor_internal(v, idx); + result[key] = from_cbor_internal(v, idx); + } + return result; + } + + case 0xba: // map (four-byte uint32_t for n follow) + { + basic_json result = value_t::object; + const auto len = static_cast(get_from_vector(v, current_idx)); + idx += 4; // skip 4 size bytes + for (size_t i = 0; i < len; ++i) + { + std::string key = from_cbor_internal(v, idx); + result[key] = from_cbor_internal(v, idx); + } + return result; + } + + case 0xbb: // map (eight-byte uint64_t for n follow) + { + basic_json result = value_t::object; + const auto len = static_cast(get_from_vector(v, current_idx)); + idx += 8; // skip 8 size bytes + for (size_t i = 0; i < len; ++i) + { + std::string key = from_cbor_internal(v, idx); + result[key] = from_cbor_internal(v, idx); + } + return result; + } + + case 0xbf: // map (indefinite length) + { + basic_json result = value_t::object; + while (v.at(idx) != 0xff) + { + std::string key = from_cbor_internal(v, idx); + result[key] = from_cbor_internal(v, idx); + } + // skip break byte (0xFF) + idx += 1; + return result; + } + + case 0xf4: // false + { + return false; + } + + case 0xf5: // true + { + return true; + } + + case 0xf6: // null + { + return value_t::null; + } + + case 0xf9: // Half-Precision Float (two-byte IEEE 754) + { + idx += 2; // skip two content bytes + + // code from RFC 7049, Appendix D, Figure 3: + // As half-precision floating-point numbers were only added to + // IEEE 754 in 2008, today's programming platforms often still + // only have limited support for them. It is very easy to + // include at least decoding support for them even without such + // support. An example of a small decoder for half-precision + // floating-point numbers in the C language is shown in Fig. 3. + const int half = (v.at(current_idx + 1) << 8) + v.at(current_idx + 2); + const int exp = (half >> 10) & 0x1f; + const int mant = half & 0x3ff; + double val; + if (exp == 0) + { + val = std::ldexp(mant, -24); + } + else if (exp != 31) + { + val = std::ldexp(mant + 1024, exp - 25); + } + else + { + val = mant == 0 + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + } + return (half & 0x8000) != 0 ? -val : val; + } + + case 0xfa: // Single-Precision Float (four-byte IEEE 754) + { + // copy bytes in reverse order into the float variable + float res; + for (size_t byte = 0; byte < sizeof(float); ++byte) + { + reinterpret_cast(&res)[sizeof(float) - byte - 1] = v.at(current_idx + 1 + byte); + } + idx += sizeof(float); // skip content bytes + return res; + } + + case 0xfb: // Double-Precision Float (eight-byte IEEE 754) + { + // copy bytes in reverse order into the double variable + double res; + for (size_t byte = 0; byte < sizeof(double); ++byte) + { + reinterpret_cast(&res)[sizeof(double) - byte - 1] = v.at(current_idx + 1 + byte); + } + idx += sizeof(double); // skip content bytes + return res; + } + + default: // anything else (0xFF is handled inside the other types) + { + JSON_THROW(std::invalid_argument("error parsing a CBOR @ " + std::to_string(current_idx) + ": " + std::to_string(static_cast(v[current_idx])))); + } + } + } + + public: + /*! + @brief create a MessagePack serialization of a given JSON value + + Serializes a given JSON value @a j to a byte vector using the MessagePack + serialization format. MessagePack is a binary serialization format which + aims to be more compact than JSON itself, yet more efficient to parse. + + @param[in] j JSON value to serialize + @return MessagePack serialization as byte vector + + @complexity Linear in the size of the JSON value @a j. + + @liveexample{The example shows the serialization of a JSON value to a byte + vector in MessagePack format.,to_msgpack} + + @sa http://msgpack.org + @sa @ref from_msgpack(const std::vector&, const size_t) for the + analogous deserialization + @sa @ref to_cbor(const basic_json& for the related CBOR format + + @since version 2.0.9 + */ + static std::vector to_msgpack(const basic_json& j) + { + std::vector result; + to_msgpack_internal(j, result); + return result; + } + + /*! + @brief create a JSON value from a byte vector in MessagePack format + + Deserializes a given byte vector @a v to a JSON value using the MessagePack + serialization format. + + @param[in] v a byte vector in MessagePack format + @param[in] start_index the index to start reading from @a v (0 by default) + @return deserialized JSON value + + @throw std::invalid_argument if unsupported features from MessagePack were + used in the given vector @a v or if the input is not valid MessagePack + @throw std::out_of_range if the given vector ends prematurely + + @complexity Linear in the size of the byte vector @a v. + + @liveexample{The example shows the deserialization of a byte vector in + MessagePack format to a JSON value.,from_msgpack} + + @sa http://msgpack.org + @sa @ref to_msgpack(const basic_json&) for the analogous serialization + @sa @ref from_cbor(const std::vector&, const size_t) for the + related CBOR format + + @since version 2.0.9, parameter @a start_index since 2.1.1 + */ + static basic_json from_msgpack(const std::vector& v, + const size_t start_index = 0) + { + size_t i = start_index; + return from_msgpack_internal(v, i); + } + + /*! + @brief create a MessagePack serialization of a given JSON value + + Serializes a given JSON value @a j to a byte vector using the CBOR (Concise + Binary Object Representation) serialization format. CBOR is a binary + serialization format which aims to be more compact than JSON itself, yet + more efficient to parse. + + @param[in] j JSON value to serialize + @return MessagePack serialization as byte vector + + @complexity Linear in the size of the JSON value @a j. + + @liveexample{The example shows the serialization of a JSON value to a byte + vector in CBOR format.,to_cbor} + + @sa http://cbor.io + @sa @ref from_cbor(const std::vector&, const size_t) for the + analogous deserialization + @sa @ref to_msgpack(const basic_json& for the related MessagePack format + + @since version 2.0.9 + */ + static std::vector to_cbor(const basic_json& j) + { + std::vector result; + to_cbor_internal(j, result); + return result; + } + + /*! + @brief create a JSON value from a byte vector in CBOR format + + Deserializes a given byte vector @a v to a JSON value using the CBOR + (Concise Binary Object Representation) serialization format. + + @param[in] v a byte vector in CBOR format + @param[in] start_index the index to start reading from @a v (0 by default) + @return deserialized JSON value + + @throw std::invalid_argument if unsupported features from CBOR were used in + the given vector @a v or if the input is not valid MessagePack + @throw std::out_of_range if the given vector ends prematurely + + @complexity Linear in the size of the byte vector @a v. + + @liveexample{The example shows the deserialization of a byte vector in CBOR + format to a JSON value.,from_cbor} + + @sa http://cbor.io + @sa @ref to_cbor(const basic_json&) for the analogous serialization + @sa @ref from_msgpack(const std::vector&, const size_t) for the + related MessagePack format + + @since version 2.0.9, parameter @a start_index since 2.1.1 + */ + static basic_json from_cbor(const std::vector& v, + const size_t start_index = 0) + { + size_t i = start_index; + return from_cbor_internal(v, i); + } + + /// @} + + /////////////////////////// + // convenience functions // + /////////////////////////// + + /*! + @brief return the type as string + + Returns the type name as string to be used in error messages - usually to + indicate that a function was called on a wrong JSON type. + + @return basically a string representation of a the @a m_type member + + @complexity Constant. + + @liveexample{The following code exemplifies `type_name()` for all JSON + types.,type_name} + + @since version 1.0.0, public since 2.1.0 + */ + std::string type_name() const + { + { + switch (m_type) + { + case value_t::null: + return "null"; + case value_t::object: + return "object"; + case value_t::array: + return "array"; + case value_t::string: + return "string"; + case value_t::boolean: + return "boolean"; + case value_t::discarded: + return "discarded"; + default: + return "number"; + } + } + } + + private: + /*! + @brief calculates the extra space to escape a JSON string + + @param[in] s the string to escape + @return the number of characters required to escape string @a s + + @complexity Linear in the length of string @a s. + */ + static std::size_t extra_space(const string_t& s) noexcept + { + return std::accumulate(s.begin(), s.end(), size_t{}, + [](size_t res, typename string_t::value_type c) + { + switch (c) + { + case '"': + case '\\': + case '\b': + case '\f': + case '\n': + case '\r': + case '\t': + { + // from c (1 byte) to \x (2 bytes) + return res + 1; + } + + default: + { + if (c >= 0x00 and c <= 0x1f) + { + // from c (1 byte) to \uxxxx (6 bytes) + return res + 5; + } + + return res; + } + } + }); + } + + /*! + @brief escape a string + + Escape a string by replacing certain special characters by a sequence of + an escape character (backslash) and another character and other control + characters by a sequence of "\u" followed by a four-digit hex + representation. + + @param[in] s the string to escape + @return the escaped string + + @complexity Linear in the length of string @a s. + */ + static string_t escape_string(const string_t& s) + { + const auto space = extra_space(s); + if (space == 0) + { + return s; + } + + // create a result string of necessary size + string_t result(s.size() + space, '\\'); + std::size_t pos = 0; + + for (const auto& c : s) + { + switch (c) + { + // quotation mark (0x22) + case '"': + { + result[pos + 1] = '"'; + pos += 2; + break; + } + + // reverse solidus (0x5c) + case '\\': + { + // nothing to change + pos += 2; + break; + } + + // backspace (0x08) + case '\b': + { + result[pos + 1] = 'b'; + pos += 2; + break; + } + + // formfeed (0x0c) + case '\f': + { + result[pos + 1] = 'f'; + pos += 2; + break; + } + + // newline (0x0a) + case '\n': + { + result[pos + 1] = 'n'; + pos += 2; + break; + } + + // carriage return (0x0d) + case '\r': + { + result[pos + 1] = 'r'; + pos += 2; + break; + } + + // horizontal tab (0x09) + case '\t': + { + result[pos + 1] = 't'; + pos += 2; + break; + } + + default: + { + if (c >= 0x00 and c <= 0x1f) + { + // convert a number 0..15 to its hex representation + // (0..f) + static const char hexify[16] = + { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' + }; + + // print character c as \uxxxx + for (const char m : + { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f] + }) + { + result[++pos] = m; + } + + ++pos; + } + else + { + // all other characters are added as-is + result[pos++] = c; + } + break; + } + } + } + + return result; + } + + + /*! + @brief locale-independent serialization for built-in arithmetic types + */ + struct numtostr + { + public: + template + numtostr(NumberType value) + { + x_write(value, std::is_integral()); + } + + const char* c_str() const + { + return m_buf.data(); + } + + private: + /// a (hopefully) large enough character buffer + std::array < char, 64 > m_buf{{}}; + + template + void x_write(NumberType x, /*is_integral=*/std::true_type) + { + // special case for "0" + if (x == 0) + { + m_buf[0] = '0'; + return; + } + + const bool is_negative = x < 0; + size_t i = 0; + + // spare 1 byte for '\0' + while (x != 0 and i < m_buf.size() - 1) + { + const auto digit = std::labs(static_cast(x % 10)); + m_buf[i++] = static_cast('0' + digit); + x /= 10; + } + + // make sure the number has been processed completely + assert(x == 0); + + if (is_negative) + { + // make sure there is capacity for the '-' + assert(i < m_buf.size() - 2); + m_buf[i++] = '-'; + } + + std::reverse(m_buf.begin(), m_buf.begin() + i); + } + + template + void x_write(NumberType x, /*is_integral=*/std::false_type) + { + // special case for 0.0 and -0.0 + if (x == 0) + { + size_t i = 0; + if (std::signbit(x)) + { + m_buf[i++] = '-'; + } + m_buf[i++] = '0'; + m_buf[i++] = '.'; + m_buf[i] = '0'; + return; + } + + // get number of digits for a text -> float -> text round-trip + static constexpr auto d = std::numeric_limits::digits10; + + // the actual conversion + const auto written_bytes = snprintf(m_buf.data(), m_buf.size(), "%.*g", d, x); + + // negative value indicates an error + assert(written_bytes > 0); + // check if buffer was large enough + assert(static_cast(written_bytes) < m_buf.size()); + + // read information from locale + const auto loc = localeconv(); + assert(loc != nullptr); + const char thousands_sep = !loc->thousands_sep ? '\0' + : loc->thousands_sep[0]; + + const char decimal_point = !loc->decimal_point ? '\0' + : loc->decimal_point[0]; + + // erase thousands separator + if (thousands_sep != '\0') + { + const auto end = std::remove(m_buf.begin(), m_buf.begin() + written_bytes, thousands_sep); + std::fill(end, m_buf.end(), '\0'); + } + + // convert decimal point to '.' + if (decimal_point != '\0' and decimal_point != '.') + { + for (auto& c : m_buf) + { + if (c == decimal_point) + { + c = '.'; + break; + } + } + } + + // determine if need to append ".0" + size_t i = 0; + bool value_is_int_like = true; + for (i = 0; i < m_buf.size(); ++i) + { + // break when end of number is reached + if (m_buf[i] == '\0') + { + break; + } + + // check if we find non-int character + value_is_int_like = value_is_int_like and m_buf[i] != '.' and + m_buf[i] != 'e' and m_buf[i] != 'E'; + } + + if (value_is_int_like) + { + // there must be 2 bytes left for ".0" + assert((i + 2) < m_buf.size()); + // we write to the end of the number + assert(m_buf[i] == '\0'); + assert(m_buf[i - 1] != '\0'); + + // add ".0" + m_buf[i] = '.'; + m_buf[i + 1] = '0'; + + // the resulting string is properly terminated + assert(m_buf[i + 2] == '\0'); + } + } + }; + + + /*! + @brief internal implementation of the serialization function + + This function is called by the public member function dump and organizes + the serialization internally. The indentation level is propagated as + additional parameter. In case of arrays and objects, the function is + called recursively. Note that + + - strings and object keys are escaped using `escape_string()` + - integer numbers are converted implicitly via `operator<<` + - floating-point numbers are converted to a string using `"%g"` format + + @param[out] o stream to write to + @param[in] pretty_print whether the output shall be pretty-printed + @param[in] indent_step the indent level + @param[in] current_indent the current indent level (only used internally) + */ + void dump(std::ostream& o, + const bool pretty_print, + const unsigned int indent_step, + const unsigned int current_indent = 0) const + { + // variable to hold indentation for recursive calls + unsigned int new_indent = current_indent; + + switch (m_type) + { + case value_t::object: + { + if (m_value.object->empty()) + { + o << "{}"; + return; + } + + o << "{"; + + // increase indentation + if (pretty_print) + { + new_indent += indent_step; + o << "\n"; + } + + for (auto i = m_value.object->cbegin(); i != m_value.object->cend(); ++i) + { + if (i != m_value.object->cbegin()) + { + o << (pretty_print ? ",\n" : ","); + } + o << string_t(new_indent, ' ') << "\"" + << escape_string(i->first) << "\":" + << (pretty_print ? " " : ""); + i->second.dump(o, pretty_print, indent_step, new_indent); + } + + // decrease indentation + if (pretty_print) + { + new_indent -= indent_step; + o << "\n"; + } + + o << string_t(new_indent, ' ') + "}"; + return; + } + + case value_t::array: + { + if (m_value.array->empty()) + { + o << "[]"; + return; + } + + o << "["; + + // increase indentation + if (pretty_print) + { + new_indent += indent_step; + o << "\n"; + } + + for (auto i = m_value.array->cbegin(); i != m_value.array->cend(); ++i) + { + if (i != m_value.array->cbegin()) + { + o << (pretty_print ? ",\n" : ","); + } + o << string_t(new_indent, ' '); + i->dump(o, pretty_print, indent_step, new_indent); + } + + // decrease indentation + if (pretty_print) + { + new_indent -= indent_step; + o << "\n"; + } + + o << string_t(new_indent, ' ') << "]"; + return; + } + + case value_t::string: + { + o << string_t("\"") << escape_string(*m_value.string) << "\""; + return; + } + + case value_t::boolean: + { + o << (m_value.boolean ? "true" : "false"); + return; + } + + case value_t::number_integer: + { + o << numtostr(m_value.number_integer).c_str(); + return; + } + + case value_t::number_unsigned: + { + o << numtostr(m_value.number_unsigned).c_str(); + return; + } + + case value_t::number_float: + { + o << numtostr(m_value.number_float).c_str(); + return; + } + + case value_t::discarded: + { + o << ""; + return; + } + + case value_t::null: + { + o << "null"; + return; + } + } + } + + private: + ////////////////////// + // member variables // + ////////////////////// + + /// the type of the current element + value_t m_type = value_t::null; + + /// the value of the current element + json_value m_value = {}; + + + private: + /////////////// + // iterators // + /////////////// + + /*! + @brief an iterator for primitive JSON types + + This class models an iterator for primitive JSON types (boolean, number, + string). It's only purpose is to allow the iterator/const_iterator classes + to "iterate" over primitive values. Internally, the iterator is modeled by + a `difference_type` variable. Value begin_value (`0`) models the begin, + end_value (`1`) models past the end. + */ + class primitive_iterator_t + { + public: + + difference_type get_value() const noexcept + { + return m_it; + } + /// set iterator to a defined beginning + void set_begin() noexcept + { + m_it = begin_value; + } + + /// set iterator to a defined past the end + void set_end() noexcept + { + m_it = end_value; + } + + /// return whether the iterator can be dereferenced + constexpr bool is_begin() const noexcept + { + return (m_it == begin_value); + } + + /// return whether the iterator is at end + constexpr bool is_end() const noexcept + { + return (m_it == end_value); + } + + friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it == rhs.m_it; + } + + friend constexpr bool operator!=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return !(lhs == rhs); + } + + friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it < rhs.m_it; + } + + friend constexpr bool operator<=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it <= rhs.m_it; + } + + friend constexpr bool operator>(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it > rhs.m_it; + } + + friend constexpr bool operator>=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it >= rhs.m_it; + } + + primitive_iterator_t operator+(difference_type i) + { + auto result = *this; + result += i; + return result; + } + + friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it - rhs.m_it; + } + + friend std::ostream& operator<<(std::ostream& os, primitive_iterator_t it) + { + return os << it.m_it; + } + + primitive_iterator_t& operator++() + { + ++m_it; + return *this; + } + + primitive_iterator_t operator++(int) + { + auto result = *this; + m_it++; + return result; + } + + primitive_iterator_t& operator--() + { + --m_it; + return *this; + } + + primitive_iterator_t operator--(int) + { + auto result = *this; + m_it--; + return result; + } + + primitive_iterator_t& operator+=(difference_type n) + { + m_it += n; + return *this; + } + + primitive_iterator_t& operator-=(difference_type n) + { + m_it -= n; + return *this; + } + + private: + static constexpr difference_type begin_value = 0; + static constexpr difference_type end_value = begin_value + 1; + + /// iterator as signed integer type + difference_type m_it = std::numeric_limits::denorm_min(); + }; + + /*! + @brief an iterator value + + @note This structure could easily be a union, but MSVC currently does not + allow unions members with complex constructors, see + https://github.com/nlohmann/json/pull/105. + */ + struct internal_iterator + { + /// iterator for JSON objects + typename object_t::iterator object_iterator; + /// iterator for JSON arrays + typename array_t::iterator array_iterator; + /// generic iterator for all other types + primitive_iterator_t primitive_iterator; + + /// create an uninitialized internal_iterator + internal_iterator() noexcept + : object_iterator(), array_iterator(), primitive_iterator() + {} + }; + + /// proxy class for the iterator_wrapper functions + template + class iteration_proxy + { + private: + /// helper class for iteration + class iteration_proxy_internal + { + private: + /// the iterator + IteratorType anchor; + /// an index for arrays (used to create key names) + size_t array_index = 0; + + public: + explicit iteration_proxy_internal(IteratorType it) noexcept + : anchor(it) + {} + + /// dereference operator (needed for range-based for) + iteration_proxy_internal& operator*() + { + return *this; + } + + /// increment operator (needed for range-based for) + iteration_proxy_internal& operator++() + { + ++anchor; + ++array_index; + + return *this; + } + + /// inequality operator (needed for range-based for) + bool operator!= (const iteration_proxy_internal& o) const + { + return anchor != o.anchor; + } + + /// return key of the iterator + typename basic_json::string_t key() const + { + assert(anchor.m_object != nullptr); + + switch (anchor.m_object->type()) + { + // use integer array index as key + case value_t::array: + { + return std::to_string(array_index); + } + + // use key from the object + case value_t::object: + { + return anchor.key(); + } + + // use an empty key for all primitive types + default: + { + return ""; + } + } + } + + /// return value of the iterator + typename IteratorType::reference value() const + { + return anchor.value(); + } + }; + + /// the container to iterate + typename IteratorType::reference container; + + public: + /// construct iteration proxy from a container + explicit iteration_proxy(typename IteratorType::reference cont) + : container(cont) + {} + + /// return iterator begin (needed for range-based for) + iteration_proxy_internal begin() noexcept + { + return iteration_proxy_internal(container.begin()); + } + + /// return iterator end (needed for range-based for) + iteration_proxy_internal end() noexcept + { + return iteration_proxy_internal(container.end()); + } + }; + + public: + /*! + @brief a template for a random access iterator for the @ref basic_json class + + This class implements a both iterators (iterator and const_iterator) for the + @ref basic_json class. + + @note An iterator is called *initialized* when a pointer to a JSON value + has been set (e.g., by a constructor or a copy assignment). If the + iterator is default-constructed, it is *uninitialized* and most + methods are undefined. **The library uses assertions to detect calls + on uninitialized iterators.** + + @requirement The class satisfies the following concept requirements: + - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): + The iterator that can be moved to point (forward and backward) to any + element in constant time. + + @since version 1.0.0, simplified in version 2.0.9 + */ + template + class iter_impl : public std::iterator + { + /// allow basic_json to access private members + friend class basic_json; + + // make sure U is basic_json or const basic_json + static_assert(std::is_same::value + or std::is_same::value, + "iter_impl only accepts (const) basic_json"); + + public: + /// the type of the values when the iterator is dereferenced + using value_type = typename basic_json::value_type; + /// a type to represent differences between iterators + using difference_type = typename basic_json::difference_type; + /// defines a pointer to the type iterated over (value_type) + using pointer = typename std::conditional::value, + typename basic_json::const_pointer, + typename basic_json::pointer>::type; + /// defines a reference to the type iterated over (value_type) + using reference = typename std::conditional::value, + typename basic_json::const_reference, + typename basic_json::reference>::type; + /// the category of the iterator + using iterator_category = std::bidirectional_iterator_tag; + + /// default constructor + iter_impl() = default; + + /*! + @brief constructor for a given JSON instance + @param[in] object pointer to a JSON object for this iterator + @pre object != nullptr + @post The iterator is initialized; i.e. `m_object != nullptr`. + */ + explicit iter_impl(pointer object) noexcept + : m_object(object) + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = typename object_t::iterator(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = typename array_t::iterator(); + break; + } + + default: + { + m_it.primitive_iterator = primitive_iterator_t(); + break; + } + } + } + + /* + Use operator `const_iterator` instead of `const_iterator(const iterator& + other) noexcept` to avoid two class definitions for @ref iterator and + @ref const_iterator. + + This function is only called if this class is an @ref iterator. If this + class is a @ref const_iterator this function is not called. + */ + operator const_iterator() const + { + const_iterator ret; + + if (m_object) + { + ret.m_object = m_object; + ret.m_it = m_it; + } + + return ret; + } + + /*! + @brief copy constructor + @param[in] other iterator to copy from + @note It is not checked whether @a other is initialized. + */ + iter_impl(const iter_impl& other) noexcept + : m_object(other.m_object), m_it(other.m_it) + {} + + /*! + @brief copy assignment + @param[in,out] other iterator to copy from + @note It is not checked whether @a other is initialized. + */ + iter_impl& operator=(iter_impl other) noexcept( + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value and + std::is_nothrow_move_constructible::value and + std::is_nothrow_move_assignable::value + ) + { + std::swap(m_object, other.m_object); + std::swap(m_it, other.m_it); + return *this; + } + + private: + /*! + @brief set the iterator to the first value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_begin() noexcept + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = m_object->m_value.object->begin(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = m_object->m_value.array->begin(); + break; + } + + case basic_json::value_t::null: + { + // set to end so begin()==end() is true: null is empty + m_it.primitive_iterator.set_end(); + break; + } + + default: + { + m_it.primitive_iterator.set_begin(); + break; + } + } + } + + /*! + @brief set the iterator past the last value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_end() noexcept + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + m_it.object_iterator = m_object->m_value.object->end(); + break; + } + + case basic_json::value_t::array: + { + m_it.array_iterator = m_object->m_value.array->end(); + break; + } + + default: + { + m_it.primitive_iterator.set_end(); + break; + } + } + } + + public: + /*! + @brief return a reference to the value pointed to by the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator*() const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + assert(m_it.object_iterator != m_object->m_value.object->end()); + return m_it.object_iterator->second; + } + + case basic_json::value_t::array: + { + assert(m_it.array_iterator != m_object->m_value.array->end()); + return *m_it.array_iterator; + } + + case basic_json::value_t::null: + { + JSON_THROW(std::out_of_range("cannot get value")); + } + + default: + { + if (m_it.primitive_iterator.is_begin()) + { + return *m_object; + } + + JSON_THROW(std::out_of_range("cannot get value")); + } + } + } + + /*! + @brief dereference the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + pointer operator->() const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + assert(m_it.object_iterator != m_object->m_value.object->end()); + return &(m_it.object_iterator->second); + } + + case basic_json::value_t::array: + { + assert(m_it.array_iterator != m_object->m_value.array->end()); + return &*m_it.array_iterator; + } + + default: + { + if (m_it.primitive_iterator.is_begin()) + { + return m_object; + } + + JSON_THROW(std::out_of_range("cannot get value")); + } + } + } + + /*! + @brief post-increment (it++) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl operator++(int) + { + auto result = *this; + ++(*this); + return result; + } + + /*! + @brief pre-increment (++it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator++() + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + std::advance(m_it.object_iterator, 1); + break; + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, 1); + break; + } + + default: + { + ++m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief post-decrement (it--) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl operator--(int) + { + auto result = *this; + --(*this); + return result; + } + + /*! + @brief pre-decrement (--it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator--() + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + std::advance(m_it.object_iterator, -1); + break; + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, -1); + break; + } + + default: + { + --m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief comparison: equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator==(const iter_impl& other) const + { + // if objects are not the same, the comparison is undefined + if (m_object != other.m_object) + { + JSON_THROW(std::domain_error("cannot compare iterators of different containers")); + } + + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + return (m_it.object_iterator == other.m_it.object_iterator); + } + + case basic_json::value_t::array: + { + return (m_it.array_iterator == other.m_it.array_iterator); + } + + default: + { + return (m_it.primitive_iterator == other.m_it.primitive_iterator); + } + } + } + + /*! + @brief comparison: not equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator!=(const iter_impl& other) const + { + return not operator==(other); + } + + /*! + @brief comparison: smaller + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<(const iter_impl& other) const + { + // if objects are not the same, the comparison is undefined + if (m_object != other.m_object) + { + JSON_THROW(std::domain_error("cannot compare iterators of different containers")); + } + + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + JSON_THROW(std::domain_error("cannot compare order of object iterators")); + } + + case basic_json::value_t::array: + { + return (m_it.array_iterator < other.m_it.array_iterator); + } + + default: + { + return (m_it.primitive_iterator < other.m_it.primitive_iterator); + } + } + } + + /*! + @brief comparison: less than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<=(const iter_impl& other) const + { + return not other.operator < (*this); + } + + /*! + @brief comparison: greater than + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>(const iter_impl& other) const + { + return not operator<=(other); + } + + /*! + @brief comparison: greater than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>=(const iter_impl& other) const + { + return not operator<(other); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator+=(difference_type i) + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + JSON_THROW(std::domain_error("cannot use offsets with object iterators")); + } + + case basic_json::value_t::array: + { + std::advance(m_it.array_iterator, i); + break; + } + + default: + { + m_it.primitive_iterator += i; + break; + } + } + + return *this; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator-=(difference_type i) + { + return operator+=(-i); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl operator+(difference_type i) + { + auto result = *this; + result += i; + return result; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl operator-(difference_type i) + { + auto result = *this; + result -= i; + return result; + } + + /*! + @brief return difference + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + difference_type operator-(const iter_impl& other) const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + JSON_THROW(std::domain_error("cannot use offsets with object iterators")); + } + + case basic_json::value_t::array: + { + return m_it.array_iterator - other.m_it.array_iterator; + } + + default: + { + return m_it.primitive_iterator - other.m_it.primitive_iterator; + } + } + } + + /*! + @brief access to successor + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator[](difference_type n) const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case basic_json::value_t::object: + { + JSON_THROW(std::domain_error("cannot use operator[] for object iterators")); + } + + case basic_json::value_t::array: + { + return *std::next(m_it.array_iterator, n); + } + + case basic_json::value_t::null: + { + JSON_THROW(std::out_of_range("cannot get value")); + } + + default: + { + if (m_it.primitive_iterator.get_value() == -n) + { + return *m_object; + } + + JSON_THROW(std::out_of_range("cannot get value")); + } + } + } + + /*! + @brief return the key of an object iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + typename object_t::key_type key() const + { + assert(m_object != nullptr); + + if (m_object->is_object()) + { + return m_it.object_iterator->first; + } + + JSON_THROW(std::domain_error("cannot use key() for non-object iterators")); + } + + /*! + @brief return the value of an iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference value() const + { + return operator*(); + } + + private: + /// associated JSON instance + pointer m_object = nullptr; + /// the actual iterator of the associated instance + internal_iterator m_it = internal_iterator(); + }; + + /*! + @brief a template for a reverse iterator class + + @tparam Base the base iterator type to reverse. Valid types are @ref + iterator (to create @ref reverse_iterator) and @ref const_iterator (to + create @ref const_reverse_iterator). + + @requirement The class satisfies the following concept requirements: + - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator): + The iterator that can be moved to point (forward and backward) to any + element in constant time. + - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): + It is possible to write to the pointed-to element (only if @a Base is + @ref iterator). + + @since version 1.0.0 + */ + template + class json_reverse_iterator : public std::reverse_iterator + { + public: + /// shortcut to the reverse iterator adaptor + using base_iterator = std::reverse_iterator; + /// the reference type for the pointed-to element + using reference = typename Base::reference; + + /// create reverse iterator from iterator + json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept + : base_iterator(it) + {} + + /// create reverse iterator from base class + json_reverse_iterator(const base_iterator& it) noexcept + : base_iterator(it) + {} + + /// post-increment (it++) + json_reverse_iterator operator++(int) + { + return base_iterator::operator++(1); + } + + /// pre-increment (++it) + json_reverse_iterator& operator++() + { + base_iterator::operator++(); + return *this; + } + + /// post-decrement (it--) + json_reverse_iterator operator--(int) + { + return base_iterator::operator--(1); + } + + /// pre-decrement (--it) + json_reverse_iterator& operator--() + { + base_iterator::operator--(); + return *this; + } + + /// add to iterator + json_reverse_iterator& operator+=(difference_type i) + { + base_iterator::operator+=(i); + return *this; + } + + /// add to iterator + json_reverse_iterator operator+(difference_type i) const + { + auto result = *this; + result += i; + return result; + } + + /// subtract from iterator + json_reverse_iterator operator-(difference_type i) const + { + auto result = *this; + result -= i; + return result; + } + + /// return difference + difference_type operator-(const json_reverse_iterator& other) const + { + return this->base() - other.base(); + } + + /// access to successor + reference operator[](difference_type n) const + { + return *(this->operator+(n)); + } + + /// return the key of an object iterator + typename object_t::key_type key() const + { + auto it = --this->base(); + return it.key(); + } + + /// return the value of an iterator + reference value() const + { + auto it = --this->base(); + return it.operator * (); + } + }; + + + private: + ////////////////////// + // lexer and parser // + ////////////////////// + + /*! + @brief lexical analysis + + This class organizes the lexical analysis during JSON deserialization. The + core of it is a scanner generated by [re2c](http://re2c.org) that + processes a buffer and recognizes tokens according to RFC 7159. + */ + class lexer + { + public: + /// token types for the parser + enum class token_type + { + uninitialized, ///< indicating the scanner is uninitialized + literal_true, ///< the `true` literal + literal_false, ///< the `false` literal + literal_null, ///< the `null` literal + value_string, ///< a string -- use get_string() for actual value + value_unsigned, ///< an unsigned integer -- use get_number() for actual value + value_integer, ///< a signed integer -- use get_number() for actual value + value_float, ///< an floating point number -- use get_number() for actual value + begin_array, ///< the character for array begin `[` + begin_object, ///< the character for object begin `{` + end_array, ///< the character for array end `]` + end_object, ///< the character for object end `}` + name_separator, ///< the name separator `:` + value_separator, ///< the value separator `,` + parse_error, ///< indicating a parse error + end_of_input ///< indicating the end of the input buffer + }; + + /// the char type to use in the lexer + using lexer_char_t = unsigned char; + + /// a lexer from a buffer with given length + lexer(const lexer_char_t* buff, const size_t len) noexcept + : m_content(buff) + { + assert(m_content != nullptr); + m_start = m_cursor = m_content; + m_limit = m_content + len; + } + + /// a lexer from an input stream + explicit lexer(std::istream& s) + : m_stream(&s), m_line_buffer() + { + // immediately abort if stream is erroneous + if (s.fail()) + { + JSON_THROW(std::invalid_argument("stream error")); + } + + // fill buffer + fill_line_buffer(); + + // skip UTF-8 byte-order mark + if (m_line_buffer.size() >= 3 and m_line_buffer.substr(0, 3) == "\xEF\xBB\xBF") + { + m_line_buffer[0] = ' '; + m_line_buffer[1] = ' '; + m_line_buffer[2] = ' '; + } + } + + // switch off unwanted functions (due to pointer members) + lexer() = delete; + lexer(const lexer&) = delete; + lexer operator=(const lexer&) = delete; + + /*! + @brief create a string from one or two Unicode code points + + There are two cases: (1) @a codepoint1 is in the Basic Multilingual + Plane (U+0000 through U+FFFF) and @a codepoint2 is 0, or (2) + @a codepoint1 and @a codepoint2 are a UTF-16 surrogate pair to + represent a code point above U+FFFF. + + @param[in] codepoint1 the code point (can be high surrogate) + @param[in] codepoint2 the code point (can be low surrogate or 0) + + @return string representation of the code point; the length of the + result string is between 1 and 4 characters. + + @throw std::out_of_range if code point is > 0x10ffff; example: `"code + points above 0x10FFFF are invalid"` + @throw std::invalid_argument if the low surrogate is invalid; example: + `""missing or wrong low surrogate""` + + @complexity Constant. + + @see + */ + static string_t to_unicode(const std::size_t codepoint1, + const std::size_t codepoint2 = 0) + { + // calculate the code point from the given code points + std::size_t codepoint = codepoint1; + + // check if codepoint1 is a high surrogate + if (codepoint1 >= 0xD800 and codepoint1 <= 0xDBFF) + { + // check if codepoint2 is a low surrogate + if (codepoint2 >= 0xDC00 and codepoint2 <= 0xDFFF) + { + codepoint = + // high surrogate occupies the most significant 22 bits + (codepoint1 << 10) + // low surrogate occupies the least significant 15 bits + + codepoint2 + // there is still the 0xD800, 0xDC00 and 0x10000 noise + // in the result so we have to subtract with: + // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 + - 0x35FDC00; + } + else + { + JSON_THROW(std::invalid_argument("missing or wrong low surrogate")); + } + } + + string_t result; + + if (codepoint < 0x80) + { + // 1-byte characters: 0xxxxxxx (ASCII) + result.append(1, static_cast(codepoint)); + } + else if (codepoint <= 0x7ff) + { + // 2-byte characters: 110xxxxx 10xxxxxx + result.append(1, static_cast(0xC0 | ((codepoint >> 6) & 0x1F))); + result.append(1, static_cast(0x80 | (codepoint & 0x3F))); + } + else if (codepoint <= 0xffff) + { + // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx + result.append(1, static_cast(0xE0 | ((codepoint >> 12) & 0x0F))); + result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + result.append(1, static_cast(0x80 | (codepoint & 0x3F))); + } + else if (codepoint <= 0x10ffff) + { + // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + result.append(1, static_cast(0xF0 | ((codepoint >> 18) & 0x07))); + result.append(1, static_cast(0x80 | ((codepoint >> 12) & 0x3F))); + result.append(1, static_cast(0x80 | ((codepoint >> 6) & 0x3F))); + result.append(1, static_cast(0x80 | (codepoint & 0x3F))); + } + else + { + JSON_THROW(std::out_of_range("code points above 0x10FFFF are invalid")); + } + + return result; + } + + /// return name of values of type token_type (only used for errors) + static std::string token_type_name(const token_type t) + { + switch (t) + { + case token_type::uninitialized: + return ""; + case token_type::literal_true: + return "true literal"; + case token_type::literal_false: + return "false literal"; + case token_type::literal_null: + return "null literal"; + case token_type::value_string: + return "string literal"; + case lexer::token_type::value_unsigned: + case lexer::token_type::value_integer: + case lexer::token_type::value_float: + return "number literal"; + case token_type::begin_array: + return "'['"; + case token_type::begin_object: + return "'{'"; + case token_type::end_array: + return "']'"; + case token_type::end_object: + return "'}'"; + case token_type::name_separator: + return "':'"; + case token_type::value_separator: + return "','"; + case token_type::parse_error: + return ""; + case token_type::end_of_input: + return "end of input"; + default: + { + // catch non-enum values + return "unknown token"; // LCOV_EXCL_LINE + } + } + } + + /*! + This function implements a scanner for JSON. It is specified using + regular expressions that try to follow RFC 7159 as close as possible. + These regular expressions are then translated into a minimized + deterministic finite automaton (DFA) by the tool + [re2c](http://re2c.org). As a result, the translated code for this + function consists of a large block of code with `goto` jumps. + + @return the class of the next token read from the buffer + + @complexity Linear in the length of the input.\n + + Proposition: The loop below will always terminate for finite input.\n + + Proof (by contradiction): Assume a finite input. To loop forever, the + loop must never hit code with a `break` statement. The only code + snippets without a `break` statement are the continue statements for + whitespace and byte-order-marks. To loop forever, the input must be an + infinite sequence of whitespace or byte-order-marks. This contradicts + the assumption of finite input, q.e.d. + */ + token_type scan() + { + while (true) + { + // pointer for backtracking information + m_marker = nullptr; + + // remember the begin of the token + m_start = m_cursor; + assert(m_start != nullptr); + + + { + lexer_char_t yych; + unsigned int yyaccept = 0; + static const unsigned char yybm[] = + { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 32, 32, 0, 0, 32, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 128, 0, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 0, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + }; + if ((m_limit - m_cursor) < 5) + { + fill_line_buffer(5); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yybm[0 + yych] & 32) + { + goto basic_json_parser_6; + } + if (yych <= '[') + { + if (yych <= '-') + { + if (yych <= '"') + { + if (yych <= 0x00) + { + goto basic_json_parser_2; + } + if (yych <= '!') + { + goto basic_json_parser_4; + } + goto basic_json_parser_9; + } + else + { + if (yych <= '+') + { + goto basic_json_parser_4; + } + if (yych <= ',') + { + goto basic_json_parser_10; + } + goto basic_json_parser_12; + } + } + else + { + if (yych <= '9') + { + if (yych <= '/') + { + goto basic_json_parser_4; + } + if (yych <= '0') + { + goto basic_json_parser_13; + } + goto basic_json_parser_15; + } + else + { + if (yych <= ':') + { + goto basic_json_parser_17; + } + if (yych <= 'Z') + { + goto basic_json_parser_4; + } + goto basic_json_parser_19; + } + } + } + else + { + if (yych <= 'n') + { + if (yych <= 'e') + { + if (yych == ']') + { + goto basic_json_parser_21; + } + goto basic_json_parser_4; + } + else + { + if (yych <= 'f') + { + goto basic_json_parser_23; + } + if (yych <= 'm') + { + goto basic_json_parser_4; + } + goto basic_json_parser_24; + } + } + else + { + if (yych <= 'z') + { + if (yych == 't') + { + goto basic_json_parser_25; + } + goto basic_json_parser_4; + } + else + { + if (yych <= '{') + { + goto basic_json_parser_26; + } + if (yych == '}') + { + goto basic_json_parser_28; + } + goto basic_json_parser_4; + } + } + } +basic_json_parser_2: + ++m_cursor; + { + last_token_type = token_type::end_of_input; + break; + } +basic_json_parser_4: + ++m_cursor; +basic_json_parser_5: + { + last_token_type = token_type::parse_error; + break; + } +basic_json_parser_6: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yybm[0 + yych] & 32) + { + goto basic_json_parser_6; + } + { + continue; + } +basic_json_parser_9: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych <= 0x1F) + { + goto basic_json_parser_5; + } + if (yych <= 0x7F) + { + goto basic_json_parser_31; + } + if (yych <= 0xC1) + { + goto basic_json_parser_5; + } + if (yych <= 0xF4) + { + goto basic_json_parser_31; + } + goto basic_json_parser_5; +basic_json_parser_10: + ++m_cursor; + { + last_token_type = token_type::value_separator; + break; + } +basic_json_parser_12: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_5; + } + if (yych <= '0') + { + goto basic_json_parser_43; + } + if (yych <= '9') + { + goto basic_json_parser_45; + } + goto basic_json_parser_5; +basic_json_parser_13: + yyaccept = 1; + yych = *(m_marker = ++m_cursor); + if (yych <= '9') + { + if (yych == '.') + { + goto basic_json_parser_47; + } + if (yych >= '0') + { + goto basic_json_parser_48; + } + } + else + { + if (yych <= 'E') + { + if (yych >= 'E') + { + goto basic_json_parser_51; + } + } + else + { + if (yych == 'e') + { + goto basic_json_parser_51; + } + } + } +basic_json_parser_14: + { + last_token_type = token_type::value_unsigned; + break; + } +basic_json_parser_15: + yyaccept = 1; + m_marker = ++m_cursor; + if ((m_limit - m_cursor) < 3) + { + fill_line_buffer(3); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yybm[0 + yych] & 64) + { + goto basic_json_parser_15; + } + if (yych <= 'D') + { + if (yych == '.') + { + goto basic_json_parser_47; + } + goto basic_json_parser_14; + } + else + { + if (yych <= 'E') + { + goto basic_json_parser_51; + } + if (yych == 'e') + { + goto basic_json_parser_51; + } + goto basic_json_parser_14; + } +basic_json_parser_17: + ++m_cursor; + { + last_token_type = token_type::name_separator; + break; + } +basic_json_parser_19: + ++m_cursor; + { + last_token_type = token_type::begin_array; + break; + } +basic_json_parser_21: + ++m_cursor; + { + last_token_type = token_type::end_array; + break; + } +basic_json_parser_23: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'a') + { + goto basic_json_parser_52; + } + goto basic_json_parser_5; +basic_json_parser_24: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'u') + { + goto basic_json_parser_53; + } + goto basic_json_parser_5; +basic_json_parser_25: + yyaccept = 0; + yych = *(m_marker = ++m_cursor); + if (yych == 'r') + { + goto basic_json_parser_54; + } + goto basic_json_parser_5; +basic_json_parser_26: + ++m_cursor; + { + last_token_type = token_type::begin_object; + break; + } +basic_json_parser_28: + ++m_cursor; + { + last_token_type = token_type::end_object; + break; + } +basic_json_parser_30: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; +basic_json_parser_31: + if (yybm[0 + yych] & 128) + { + goto basic_json_parser_30; + } + if (yych <= 0xE0) + { + if (yych <= '\\') + { + if (yych <= 0x1F) + { + goto basic_json_parser_32; + } + if (yych <= '"') + { + goto basic_json_parser_33; + } + goto basic_json_parser_35; + } + else + { + if (yych <= 0xC1) + { + goto basic_json_parser_32; + } + if (yych <= 0xDF) + { + goto basic_json_parser_36; + } + goto basic_json_parser_37; + } + } + else + { + if (yych <= 0xEF) + { + if (yych == 0xED) + { + goto basic_json_parser_39; + } + goto basic_json_parser_38; + } + else + { + if (yych <= 0xF0) + { + goto basic_json_parser_40; + } + if (yych <= 0xF3) + { + goto basic_json_parser_41; + } + if (yych <= 0xF4) + { + goto basic_json_parser_42; + } + } + } +basic_json_parser_32: + m_cursor = m_marker; + if (yyaccept <= 1) + { + if (yyaccept == 0) + { + goto basic_json_parser_5; + } + else + { + goto basic_json_parser_14; + } + } + else + { + if (yyaccept == 2) + { + goto basic_json_parser_44; + } + else + { + goto basic_json_parser_58; + } + } +basic_json_parser_33: + ++m_cursor; + { + last_token_type = token_type::value_string; + break; + } +basic_json_parser_35: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 'e') + { + if (yych <= '/') + { + if (yych == '"') + { + goto basic_json_parser_30; + } + if (yych <= '.') + { + goto basic_json_parser_32; + } + goto basic_json_parser_30; + } + else + { + if (yych <= '\\') + { + if (yych <= '[') + { + goto basic_json_parser_32; + } + goto basic_json_parser_30; + } + else + { + if (yych == 'b') + { + goto basic_json_parser_30; + } + goto basic_json_parser_32; + } + } + } + else + { + if (yych <= 'q') + { + if (yych <= 'f') + { + goto basic_json_parser_30; + } + if (yych == 'n') + { + goto basic_json_parser_30; + } + goto basic_json_parser_32; + } + else + { + if (yych <= 's') + { + if (yych <= 'r') + { + goto basic_json_parser_30; + } + goto basic_json_parser_32; + } + else + { + if (yych <= 't') + { + goto basic_json_parser_30; + } + if (yych <= 'u') + { + goto basic_json_parser_55; + } + goto basic_json_parser_32; + } + } + } +basic_json_parser_36: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x7F) + { + goto basic_json_parser_32; + } + if (yych <= 0xBF) + { + goto basic_json_parser_30; + } + goto basic_json_parser_32; +basic_json_parser_37: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x9F) + { + goto basic_json_parser_32; + } + if (yych <= 0xBF) + { + goto basic_json_parser_36; + } + goto basic_json_parser_32; +basic_json_parser_38: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x7F) + { + goto basic_json_parser_32; + } + if (yych <= 0xBF) + { + goto basic_json_parser_36; + } + goto basic_json_parser_32; +basic_json_parser_39: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x7F) + { + goto basic_json_parser_32; + } + if (yych <= 0x9F) + { + goto basic_json_parser_36; + } + goto basic_json_parser_32; +basic_json_parser_40: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x8F) + { + goto basic_json_parser_32; + } + if (yych <= 0xBF) + { + goto basic_json_parser_38; + } + goto basic_json_parser_32; +basic_json_parser_41: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x7F) + { + goto basic_json_parser_32; + } + if (yych <= 0xBF) + { + goto basic_json_parser_38; + } + goto basic_json_parser_32; +basic_json_parser_42: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 0x7F) + { + goto basic_json_parser_32; + } + if (yych <= 0x8F) + { + goto basic_json_parser_38; + } + goto basic_json_parser_32; +basic_json_parser_43: + yyaccept = 2; + yych = *(m_marker = ++m_cursor); + if (yych <= '9') + { + if (yych == '.') + { + goto basic_json_parser_47; + } + if (yych >= '0') + { + goto basic_json_parser_48; + } + } + else + { + if (yych <= 'E') + { + if (yych >= 'E') + { + goto basic_json_parser_51; + } + } + else + { + if (yych == 'e') + { + goto basic_json_parser_51; + } + } + } +basic_json_parser_44: + { + last_token_type = token_type::value_integer; + break; + } +basic_json_parser_45: + yyaccept = 2; + m_marker = ++m_cursor; + if ((m_limit - m_cursor) < 3) + { + fill_line_buffer(3); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '9') + { + if (yych == '.') + { + goto basic_json_parser_47; + } + if (yych <= '/') + { + goto basic_json_parser_44; + } + goto basic_json_parser_45; + } + else + { + if (yych <= 'E') + { + if (yych <= 'D') + { + goto basic_json_parser_44; + } + goto basic_json_parser_51; + } + else + { + if (yych == 'e') + { + goto basic_json_parser_51; + } + goto basic_json_parser_44; + } + } +basic_json_parser_47: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_56; + } + goto basic_json_parser_32; +basic_json_parser_48: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '/') + { + goto basic_json_parser_50; + } + if (yych <= '9') + { + goto basic_json_parser_48; + } +basic_json_parser_50: + { + last_token_type = token_type::parse_error; + break; + } +basic_json_parser_51: + yych = *++m_cursor; + if (yych <= ',') + { + if (yych == '+') + { + goto basic_json_parser_59; + } + goto basic_json_parser_32; + } + else + { + if (yych <= '-') + { + goto basic_json_parser_59; + } + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_60; + } + goto basic_json_parser_32; + } +basic_json_parser_52: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_62; + } + goto basic_json_parser_32; +basic_json_parser_53: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_63; + } + goto basic_json_parser_32; +basic_json_parser_54: + yych = *++m_cursor; + if (yych == 'u') + { + goto basic_json_parser_64; + } + goto basic_json_parser_32; +basic_json_parser_55: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_65; + } + goto basic_json_parser_32; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_65; + } + if (yych <= '`') + { + goto basic_json_parser_32; + } + if (yych <= 'f') + { + goto basic_json_parser_65; + } + goto basic_json_parser_32; + } +basic_json_parser_56: + yyaccept = 3; + m_marker = ++m_cursor; + if ((m_limit - m_cursor) < 3) + { + fill_line_buffer(3); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= 'D') + { + if (yych <= '/') + { + goto basic_json_parser_58; + } + if (yych <= '9') + { + goto basic_json_parser_56; + } + } + else + { + if (yych <= 'E') + { + goto basic_json_parser_51; + } + if (yych == 'e') + { + goto basic_json_parser_51; + } + } +basic_json_parser_58: + { + last_token_type = token_type::value_float; + break; + } +basic_json_parser_59: + yych = *++m_cursor; + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych >= ':') + { + goto basic_json_parser_32; + } +basic_json_parser_60: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '/') + { + goto basic_json_parser_58; + } + if (yych <= '9') + { + goto basic_json_parser_60; + } + goto basic_json_parser_58; +basic_json_parser_62: + yych = *++m_cursor; + if (yych == 's') + { + goto basic_json_parser_66; + } + goto basic_json_parser_32; +basic_json_parser_63: + yych = *++m_cursor; + if (yych == 'l') + { + goto basic_json_parser_67; + } + goto basic_json_parser_32; +basic_json_parser_64: + yych = *++m_cursor; + if (yych == 'e') + { + goto basic_json_parser_69; + } + goto basic_json_parser_32; +basic_json_parser_65: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_71; + } + goto basic_json_parser_32; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_71; + } + if (yych <= '`') + { + goto basic_json_parser_32; + } + if (yych <= 'f') + { + goto basic_json_parser_71; + } + goto basic_json_parser_32; + } +basic_json_parser_66: + yych = *++m_cursor; + if (yych == 'e') + { + goto basic_json_parser_72; + } + goto basic_json_parser_32; +basic_json_parser_67: + ++m_cursor; + { + last_token_type = token_type::literal_null; + break; + } +basic_json_parser_69: + ++m_cursor; + { + last_token_type = token_type::literal_true; + break; + } +basic_json_parser_71: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_74; + } + goto basic_json_parser_32; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_74; + } + if (yych <= '`') + { + goto basic_json_parser_32; + } + if (yych <= 'f') + { + goto basic_json_parser_74; + } + goto basic_json_parser_32; + } +basic_json_parser_72: + ++m_cursor; + { + last_token_type = token_type::literal_false; + break; + } +basic_json_parser_74: + ++m_cursor; + if (m_limit <= m_cursor) + { + fill_line_buffer(1); // LCOV_EXCL_LINE + } + yych = *m_cursor; + if (yych <= '@') + { + if (yych <= '/') + { + goto basic_json_parser_32; + } + if (yych <= '9') + { + goto basic_json_parser_30; + } + goto basic_json_parser_32; + } + else + { + if (yych <= 'F') + { + goto basic_json_parser_30; + } + if (yych <= '`') + { + goto basic_json_parser_32; + } + if (yych <= 'f') + { + goto basic_json_parser_30; + } + goto basic_json_parser_32; + } + } + + } + + return last_token_type; + } + + /*! + @brief append data from the stream to the line buffer + + This function is called by the scan() function when the end of the + buffer (`m_limit`) is reached and the `m_cursor` pointer cannot be + incremented without leaving the limits of the line buffer. Note re2c + decides when to call this function. + + If the lexer reads from contiguous storage, there is no trailing null + byte. Therefore, this function must make sure to add these padding + null bytes. + + If the lexer reads from an input stream, this function reads the next + line of the input. + + @pre + p p p p p p u u u u u x . . . . . . + ^ ^ ^ ^ + m_content m_start | m_limit + m_cursor + + @post + u u u u u x x x x x x x . . . . . . + ^ ^ ^ + | m_cursor m_limit + m_start + m_content + */ + void fill_line_buffer(size_t n = 0) + { + // if line buffer is used, m_content points to its data + assert(m_line_buffer.empty() + or m_content == reinterpret_cast(m_line_buffer.data())); + + // if line buffer is used, m_limit is set past the end of its data + assert(m_line_buffer.empty() + or m_limit == m_content + m_line_buffer.size()); + + // pointer relationships + assert(m_content <= m_start); + assert(m_start <= m_cursor); + assert(m_cursor <= m_limit); + assert(m_marker == nullptr or m_marker <= m_limit); + + // number of processed characters (p) + const auto num_processed_chars = static_cast(m_start - m_content); + // offset for m_marker wrt. to m_start + const auto offset_marker = (m_marker == nullptr) ? 0 : m_marker - m_start; + // number of unprocessed characters (u) + const auto offset_cursor = m_cursor - m_start; + + // no stream is used or end of file is reached + if (m_stream == nullptr or m_stream->eof()) + { + // m_start may or may not be pointing into m_line_buffer at + // this point. We trust the standard library to do the right + // thing. See http://stackoverflow.com/q/28142011/266378 + m_line_buffer.assign(m_start, m_limit); + + // append n characters to make sure that there is sufficient + // space between m_cursor and m_limit + m_line_buffer.append(1, '\x00'); + if (n > 0) + { + m_line_buffer.append(n - 1, '\x01'); + } + } + else + { + // delete processed characters from line buffer + m_line_buffer.erase(0, num_processed_chars); + // read next line from input stream + m_line_buffer_tmp.clear(); + std::getline(*m_stream, m_line_buffer_tmp, '\n'); + + // add line with newline symbol to the line buffer + m_line_buffer += m_line_buffer_tmp; + m_line_buffer.push_back('\n'); + } + + // set pointers + m_content = reinterpret_cast(m_line_buffer.data()); + assert(m_content != nullptr); + m_start = m_content; + m_marker = m_start + offset_marker; + m_cursor = m_start + offset_cursor; + m_limit = m_start + m_line_buffer.size(); + } + + /// return string representation of last read token + string_t get_token_string() const + { + assert(m_start != nullptr); + return string_t(reinterpret_cast(m_start), + static_cast(m_cursor - m_start)); + } + + /*! + @brief return string value for string tokens + + The function iterates the characters between the opening and closing + quotes of the string value. The complete string is the range + [m_start,m_cursor). Consequently, we iterate from m_start+1 to + m_cursor-1. + + We differentiate two cases: + + 1. Escaped characters. In this case, a new character is constructed + according to the nature of the escape. Some escapes create new + characters (e.g., `"\\n"` is replaced by `"\n"`), some are copied + as is (e.g., `"\\\\"`). Furthermore, Unicode escapes of the shape + `"\\uxxxx"` need special care. In this case, to_unicode takes care + of the construction of the values. + 2. Unescaped characters are copied as is. + + @pre `m_cursor - m_start >= 2`, meaning the length of the last token + is at least 2 bytes which is trivially true for any string (which + consists of at least two quotes). + + " c1 c2 c3 ... " + ^ ^ + m_start m_cursor + + @complexity Linear in the length of the string.\n + + Lemma: The loop body will always terminate.\n + + Proof (by contradiction): Assume the loop body does not terminate. As + the loop body does not contain another loop, one of the called + functions must never return. The called functions are `std::strtoul` + and to_unicode. Neither function can loop forever, so the loop body + will never loop forever which contradicts the assumption that the loop + body does not terminate, q.e.d.\n + + Lemma: The loop condition for the for loop is eventually false.\n + + Proof (by contradiction): Assume the loop does not terminate. Due to + the above lemma, this can only be due to a tautological loop + condition; that is, the loop condition i < m_cursor - 1 must always be + true. Let x be the change of i for any loop iteration. Then + m_start + 1 + x < m_cursor - 1 must hold to loop indefinitely. This + can be rephrased to m_cursor - m_start - 2 > x. With the + precondition, we x <= 0, meaning that the loop condition holds + indefinitely if i is always decreased. However, observe that the value + of i is strictly increasing with each iteration, as it is incremented + by 1 in the iteration expression and never decremented inside the loop + body. Hence, the loop condition will eventually be false which + contradicts the assumption that the loop condition is a tautology, + q.e.d. + + @return string value of current token without opening and closing + quotes + @throw std::out_of_range if to_unicode fails + */ + string_t get_string() const + { + assert(m_cursor - m_start >= 2); + + string_t result; + result.reserve(static_cast(m_cursor - m_start - 2)); + + // iterate the result between the quotes + for (const lexer_char_t* i = m_start + 1; i < m_cursor - 1; ++i) + { + // find next escape character + auto e = std::find(i, m_cursor - 1, '\\'); + if (e != i) + { + // see https://github.com/nlohmann/json/issues/365#issuecomment-262874705 + for (auto k = i; k < e; k++) + { + result.push_back(static_cast(*k)); + } + i = e - 1; // -1 because of ++i + } + else + { + // processing escaped character + // read next character + ++i; + + switch (*i) + { + // the default escapes + case 't': + { + result += "\t"; + break; + } + case 'b': + { + result += "\b"; + break; + } + case 'f': + { + result += "\f"; + break; + } + case 'n': + { + result += "\n"; + break; + } + case 'r': + { + result += "\r"; + break; + } + case '\\': + { + result += "\\"; + break; + } + case '/': + { + result += "/"; + break; + } + case '"': + { + result += "\""; + break; + } + + // unicode + case 'u': + { + // get code xxxx from uxxxx + auto codepoint = std::strtoul(std::string(reinterpret_cast(i + 1), + 4).c_str(), nullptr, 16); + + // check if codepoint is a high surrogate + if (codepoint >= 0xD800 and codepoint <= 0xDBFF) + { + // make sure there is a subsequent unicode + if ((i + 6 >= m_limit) or * (i + 5) != '\\' or * (i + 6) != 'u') + { + JSON_THROW(std::invalid_argument("missing low surrogate")); + } + + // get code yyyy from uxxxx\uyyyy + auto codepoint2 = std::strtoul(std::string(reinterpret_cast + (i + 7), 4).c_str(), nullptr, 16); + result += to_unicode(codepoint, codepoint2); + // skip the next 10 characters (xxxx\uyyyy) + i += 10; + } + else if (codepoint >= 0xDC00 and codepoint <= 0xDFFF) + { + // we found a lone low surrogate + JSON_THROW(std::invalid_argument("missing high surrogate")); + } + else + { + // add unicode character(s) + result += to_unicode(codepoint); + // skip the next four characters (xxxx) + i += 4; + } + break; + } + } + } + } + + return result; + } + + + /*! + @brief parse string into a built-in arithmetic type as if the current + locale is POSIX. + + @note in floating-point case strtod may parse past the token's end - + this is not an error + + @note any leading blanks are not handled + */ + struct strtonum + { + public: + strtonum(const char* start, const char* end) + : m_start(start), m_end(end) + {} + + /*! + @return true iff parsed successfully as number of type T + + @param[in,out] val shall contain parsed value, or undefined value + if could not parse + */ + template::value>::type> + bool to(T& val) const + { + return parse(val, std::is_integral()); + } + + private: + const char* const m_start = nullptr; + const char* const m_end = nullptr; + + // floating-point conversion + + // overloaded wrappers for strtod/strtof/strtold + // that will be called from parse + static void strtof(float& f, const char* str, char** endptr) + { + f = std::strtof(str, endptr); + } + + static void strtof(double& f, const char* str, char** endptr) + { + f = std::strtod(str, endptr); + } + + static void strtof(long double& f, const char* str, char** endptr) + { + f = std::strtold(str, endptr); + } + + template + bool parse(T& value, /*is_integral=*/std::false_type) const + { + // replace decimal separator with locale-specific version, + // when necessary; data will point to either the original + // string, or buf, or tempstr containing the fixed string. + std::string tempstr; + std::array buf; + const size_t len = static_cast(m_end - m_start); + + // lexer will reject empty numbers + assert(len > 0); + + // since dealing with strtod family of functions, we're + // getting the decimal point char from the C locale facilities + // instead of C++'s numpunct facet of the current std::locale + const auto loc = localeconv(); + assert(loc != nullptr); + const char decimal_point_char = (loc->decimal_point == nullptr) ? '.' : loc->decimal_point[0]; + + const char* data = m_start; + + if (decimal_point_char != '.') + { + const size_t ds_pos = static_cast(std::find(m_start, m_end, '.') - m_start); + + if (ds_pos != len) + { + // copy the data into the local buffer or tempstr, if + // buffer is too small; replace decimal separator, and + // update data to point to the modified bytes + if ((len + 1) < buf.size()) + { + std::copy(m_start, m_end, buf.begin()); + buf[len] = 0; + buf[ds_pos] = decimal_point_char; + data = buf.data(); + } + else + { + tempstr.assign(m_start, m_end); + tempstr[ds_pos] = decimal_point_char; + data = tempstr.c_str(); + } + } + } + + char* endptr = nullptr; + value = 0; + // this calls appropriate overload depending on T + strtof(value, data, &endptr); + + // parsing was successful iff strtof parsed exactly the number + // of characters determined by the lexer (len) + const bool ok = (endptr == (data + len)); + + if (ok and (value == static_cast(0.0)) and (*data == '-')) + { + // some implementations forget to negate the zero + value = -0.0; + } + + return ok; + } + + // integral conversion + + signed long long parse_integral(char** endptr, /*is_signed*/std::true_type) const + { + return std::strtoll(m_start, endptr, 10); + } + + unsigned long long parse_integral(char** endptr, /*is_signed*/std::false_type) const + { + return std::strtoull(m_start, endptr, 10); + } + + template + bool parse(T& value, /*is_integral=*/std::true_type) const + { + char* endptr = nullptr; + errno = 0; // these are thread-local + const auto x = parse_integral(&endptr, std::is_signed()); + + // called right overload? + static_assert(std::is_signed() == std::is_signed(), ""); + + value = static_cast(x); + + return (x == static_cast(value)) // x fits into destination T + and (x < 0) == (value < 0) // preserved sign + //and ((x != 0) or is_integral()) // strto[u]ll did nto fail + and (errno == 0) // strto[u]ll did not overflow + and (m_start < m_end) // token was not empty + and (endptr == m_end); // parsed entire token exactly + } + }; + + /*! + @brief return number value for number tokens + + This function translates the last token into the most appropriate + number type (either integer, unsigned integer or floating point), + which is passed back to the caller via the result parameter. + + integral numbers that don't fit into the the range of the respective + type are parsed as number_float_t + + floating-point values do not satisfy std::isfinite predicate + are converted to value_t::null + + throws if the entire string [m_start .. m_cursor) cannot be + interpreted as a number + + @param[out] result @ref basic_json object to receive the number. + @param[in] token the type of the number token + */ + bool get_number(basic_json& result, const token_type token) const + { + assert(m_start != nullptr); + assert(m_start < m_cursor); + assert((token == token_type::value_unsigned) or + (token == token_type::value_integer) or + (token == token_type::value_float)); + + strtonum num_converter(reinterpret_cast(m_start), + reinterpret_cast(m_cursor)); + + switch (token) + { + case lexer::token_type::value_unsigned: + { + number_unsigned_t val; + if (num_converter.to(val)) + { + // parsing successful + result.m_type = value_t::number_unsigned; + result.m_value = val; + return true; + } + break; + } + + case lexer::token_type::value_integer: + { + number_integer_t val; + if (num_converter.to(val)) + { + // parsing successful + result.m_type = value_t::number_integer; + result.m_value = val; + return true; + } + break; + } + + default: + { + break; + } + } + + // parse float (either explicitly or because a previous conversion + // failed) + number_float_t val; + if (num_converter.to(val)) + { + // parsing successful + result.m_type = value_t::number_float; + result.m_value = val; + + // replace infinity and NAN by null + if (not std::isfinite(result.m_value.number_float)) + { + result.m_type = value_t::null; + result.m_value = basic_json::json_value(); + } + + return true; + } + + // couldn't parse number in any format + return false; + } + + private: + /// optional input stream + std::istream* m_stream = nullptr; + /// line buffer buffer for m_stream + string_t m_line_buffer {}; + /// used for filling m_line_buffer + string_t m_line_buffer_tmp {}; + /// the buffer pointer + const lexer_char_t* m_content = nullptr; + /// pointer to the beginning of the current symbol + const lexer_char_t* m_start = nullptr; + /// pointer for backtracking information + const lexer_char_t* m_marker = nullptr; + /// pointer to the current symbol + const lexer_char_t* m_cursor = nullptr; + /// pointer to the end of the buffer + const lexer_char_t* m_limit = nullptr; + /// the last token type + token_type last_token_type = token_type::end_of_input; + }; + + /*! + @brief syntax analysis + + This class implements a recursive decent parser. + */ + class parser + { + public: + /// a parser reading from a string literal + parser(const char* buff, const parser_callback_t cb = nullptr) + : callback(cb), + m_lexer(reinterpret_cast(buff), std::strlen(buff)) + {} + + /// a parser reading from an input stream + parser(std::istream& is, const parser_callback_t cb = nullptr) + : callback(cb), m_lexer(is) + {} + + /// a parser reading from an iterator range with contiguous storage + template::iterator_category, std::random_access_iterator_tag>::value + , int>::type + = 0> + parser(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr) + : callback(cb), + m_lexer(reinterpret_cast(&(*first)), + static_cast(std::distance(first, last))) + {} + + /// public parser interface + basic_json parse() + { + // read first token + get_token(); + + basic_json result = parse_internal(true); + result.assert_invariant(); + + expect(lexer::token_type::end_of_input); + + // return parser result and replace it with null in case the + // top-level value was discarded by the callback function + return result.is_discarded() ? basic_json() : std::move(result); + } + + private: + /// the actual parser + basic_json parse_internal(bool keep) + { + auto result = basic_json(value_t::discarded); + + switch (last_token) + { + case lexer::token_type::begin_object: + { + if (keep and (not callback + or ((keep = callback(depth++, parse_event_t::object_start, result)) != 0))) + { + // explicitly set result to object to cope with {} + result.m_type = value_t::object; + result.m_value = value_t::object; + } + + // read next token + get_token(); + + // closing } -> we are done + if (last_token == lexer::token_type::end_object) + { + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + // no comma is expected here + unexpect(lexer::token_type::value_separator); + + // otherwise: parse key-value pairs + do + { + // ugly, but could be fixed with loop reorganization + if (last_token == lexer::token_type::value_separator) + { + get_token(); + } + + // store key + expect(lexer::token_type::value_string); + const auto key = m_lexer.get_string(); + + bool keep_tag = false; + if (keep) + { + if (callback) + { + basic_json k(key); + keep_tag = callback(depth, parse_event_t::key, k); + } + else + { + keep_tag = true; + } + } + + // parse separator (:) + get_token(); + expect(lexer::token_type::name_separator); + + // parse and add value + get_token(); + auto value = parse_internal(keep); + if (keep and keep_tag and not value.is_discarded()) + { + result[key] = std::move(value); + } + } + while (last_token == lexer::token_type::value_separator); + + // closing } + expect(lexer::token_type::end_object); + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) + { + result = basic_json(value_t::discarded); + } + + return result; + } + + case lexer::token_type::begin_array: + { + if (keep and (not callback + or ((keep = callback(depth++, parse_event_t::array_start, result)) != 0))) + { + // explicitly set result to object to cope with [] + result.m_type = value_t::array; + result.m_value = value_t::array; + } + + // read next token + get_token(); + + // closing ] -> we are done + if (last_token == lexer::token_type::end_array) + { + get_token(); + if (callback and not callback(--depth, parse_event_t::array_end, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + // no comma is expected here + unexpect(lexer::token_type::value_separator); + + // otherwise: parse values + do + { + // ugly, but could be fixed with loop reorganization + if (last_token == lexer::token_type::value_separator) + { + get_token(); + } + + // parse value + auto value = parse_internal(keep); + if (keep and not value.is_discarded()) + { + result.push_back(std::move(value)); + } + } + while (last_token == lexer::token_type::value_separator); + + // closing ] + expect(lexer::token_type::end_array); + get_token(); + if (keep and callback and not callback(--depth, parse_event_t::array_end, result)) + { + result = basic_json(value_t::discarded); + } + + return result; + } + + case lexer::token_type::literal_null: + { + get_token(); + result.m_type = value_t::null; + break; + } + + case lexer::token_type::value_string: + { + const auto s = m_lexer.get_string(); + get_token(); + result = basic_json(s); + break; + } + + case lexer::token_type::literal_true: + { + get_token(); + result.m_type = value_t::boolean; + result.m_value = true; + break; + } + + case lexer::token_type::literal_false: + { + get_token(); + result.m_type = value_t::boolean; + result.m_value = false; + break; + } + + case lexer::token_type::value_unsigned: + case lexer::token_type::value_integer: + case lexer::token_type::value_float: + { + m_lexer.get_number(result, last_token); + get_token(); + break; + } + + default: + { + // the last token was unexpected + unexpect(last_token); + } + } + + if (keep and callback and not callback(depth, parse_event_t::value, result)) + { + result = basic_json(value_t::discarded); + } + return result; + } + + /// get next token from lexer + typename lexer::token_type get_token() + { + last_token = m_lexer.scan(); + return last_token; + } + + void expect(typename lexer::token_type t) const + { + if (t != last_token) + { + std::string error_msg = "parse error - unexpected "; + error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + + "'") : + lexer::token_type_name(last_token)); + error_msg += "; expected " + lexer::token_type_name(t); + JSON_THROW(std::invalid_argument(error_msg)); + } + } + + void unexpect(typename lexer::token_type t) const + { + if (t == last_token) + { + std::string error_msg = "parse error - unexpected "; + error_msg += (last_token == lexer::token_type::parse_error ? ("'" + m_lexer.get_token_string() + + "'") : + lexer::token_type_name(last_token)); + JSON_THROW(std::invalid_argument(error_msg)); + } + } + + private: + /// current level of recursion + int depth = 0; + /// callback function + const parser_callback_t callback = nullptr; + /// the type of the last read token + typename lexer::token_type last_token = lexer::token_type::uninitialized; + /// the lexer + lexer m_lexer; + }; + + public: + /*! + @brief JSON Pointer + + A JSON pointer defines a string syntax for identifying a specific value + within a JSON document. It can be used with functions `at` and + `operator[]`. Furthermore, JSON pointers are the base for JSON patches. + + @sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + + @since version 2.0.0 + */ + class json_pointer + { + /// allow basic_json to access private members + friend class basic_json; + + public: + /*! + @brief create JSON pointer + + Create a JSON pointer according to the syntax described in + [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3). + + @param[in] s string representing the JSON pointer; if omitted, the + empty string is assumed which references the whole JSON + value + + @throw std::domain_error if reference token is nonempty and does not + begin with a slash (`/`); example: `"JSON pointer must be empty or + begin with /"` + @throw std::domain_error if a tilde (`~`) is not followed by `0` + (representing `~`) or `1` (representing `/`); example: `"escape error: + ~ must be followed with 0 or 1"` + + @liveexample{The example shows the construction several valid JSON + pointers as well as the exceptional behavior.,json_pointer} + + @since version 2.0.0 + */ + explicit json_pointer(const std::string& s = "") + : reference_tokens(split(s)) + {} + + /*! + @brief return a string representation of the JSON pointer + + @invariant For each JSON pointer `ptr`, it holds: + @code {.cpp} + ptr == json_pointer(ptr.to_string()); + @endcode + + @return a string representation of the JSON pointer + + @liveexample{The example shows the result of `to_string`., + json_pointer__to_string} + + @since version 2.0.0 + */ + std::string to_string() const noexcept + { + return std::accumulate(reference_tokens.begin(), + reference_tokens.end(), std::string{}, + [](const std::string & a, const std::string & b) + { + return a + "/" + escape(b); + }); + } + + /// @copydoc to_string() + operator std::string() const + { + return to_string(); + } + + private: + /// remove and return last reference pointer + std::string pop_back() + { + if (is_root()) + { + JSON_THROW(std::domain_error("JSON pointer has no parent")); + } + + auto last = reference_tokens.back(); + reference_tokens.pop_back(); + return last; + } + + /// return whether pointer points to the root document + bool is_root() const + { + return reference_tokens.empty(); + } + + json_pointer top() const + { + if (is_root()) + { + JSON_THROW(std::domain_error("JSON pointer has no parent")); + } + + json_pointer result = *this; + result.reference_tokens = {reference_tokens[0]}; + return result; + } + + /*! + @brief create and return a reference to the pointed to value + + @complexity Linear in the number of reference tokens. + */ + reference get_and_create(reference j) const + { + pointer result = &j; + + // in case no reference tokens exist, return a reference to the + // JSON value j which will be overwritten by a primitive value + for (const auto& reference_token : reference_tokens) + { + switch (result->m_type) + { + case value_t::null: + { + if (reference_token == "0") + { + // start a new array if reference token is 0 + result = &result->operator[](0); + } + else + { + // start a new object otherwise + result = &result->operator[](reference_token); + } + break; + } + + case value_t::object: + { + // create an entry in the object + result = &result->operator[](reference_token); + break; + } + + case value_t::array: + { + // create an entry in the array + result = &result->operator[](static_cast(std::stoi(reference_token))); + break; + } + + /* + The following code is only reached if there exists a + reference token _and_ the current value is primitive. In + this case, we have an error situation, because primitive + values may only occur as single value; that is, with an + empty list of reference tokens. + */ + default: + { + JSON_THROW(std::domain_error("invalid value to unflatten")); + } + } + } + + return *result; + } + + /*! + @brief return a reference to the pointed to value + + @note This version does not throw if a value is not present, but tries + to create nested values instead. For instance, calling this function + with pointer `"/this/that"` on a null value is equivalent to calling + `operator[]("this").operator[]("that")` on that value, effectively + changing the null value to an object. + + @param[in] ptr a JSON value + + @return reference to the JSON value pointed to by the JSON pointer + + @complexity Linear in the length of the JSON pointer. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + */ + reference get_unchecked(pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + // convert null values to arrays or objects before continuing + if (ptr->m_type == value_t::null) + { + // check if reference token is a number + const bool nums = std::all_of(reference_token.begin(), + reference_token.end(), + [](const char x) + { + return std::isdigit(x); + }); + + // change value to array for numbers or "-" or to object + // otherwise + if (nums or reference_token == "-") + { + *ptr = value_t::array; + } + else + { + *ptr = value_t::object; + } + } + + switch (ptr->m_type) + { + case value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case value_t::array: + { + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + JSON_THROW(std::domain_error("array index must not begin with '0'")); + } + + if (reference_token == "-") + { + // explicitly treat "-" as index beyond the end + ptr = &ptr->operator[](ptr->m_value.array->size()); + } + else + { + // convert array index to number; unchecked access + ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + } + break; + } + + default: + { + JSON_THROW(std::out_of_range("unresolved reference token '" + reference_token + "'")); + } + } + } + + return *ptr; + } + + reference get_checked(pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" always fails the range check + JSON_THROW(std::out_of_range("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + JSON_THROW(std::domain_error("array index must not begin with '0'")); + } + + // note: at performs range check + ptr = &ptr->at(static_cast(std::stoi(reference_token))); + break; + } + + default: + { + JSON_THROW(std::out_of_range("unresolved reference token '" + reference_token + "'")); + } + } + } + + return *ptr; + } + + /*! + @brief return a const reference to the pointed to value + + @param[in] ptr a JSON value + + @return const reference to the JSON value pointed to by the JSON + pointer + */ + const_reference get_unchecked(const_pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" cannot be used for const access + JSON_THROW(std::out_of_range("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + JSON_THROW(std::domain_error("array index must not begin with '0'")); + } + + // use unchecked array access + ptr = &ptr->operator[](static_cast(std::stoi(reference_token))); + break; + } + + default: + { + JSON_THROW(std::out_of_range("unresolved reference token '" + reference_token + "'")); + } + } + } + + return *ptr; + } + + const_reference get_checked(const_pointer ptr) const + { + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case value_t::array: + { + if (reference_token == "-") + { + // "-" always fails the range check + JSON_THROW(std::out_of_range("array index '-' (" + + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (reference_token.size() > 1 and reference_token[0] == '0') + { + JSON_THROW(std::domain_error("array index must not begin with '0'")); + } + + // note: at performs range check + ptr = &ptr->at(static_cast(std::stoi(reference_token))); + break; + } + + default: + { + JSON_THROW(std::out_of_range("unresolved reference token '" + reference_token + "'")); + } + } + } + + return *ptr; + } + + /// split the string input to reference tokens + static std::vector split(const std::string& reference_string) + { + std::vector result; + + // special case: empty reference string -> no reference tokens + if (reference_string.empty()) + { + return result; + } + + // check if nonempty reference string begins with slash + if (reference_string[0] != '/') + { + JSON_THROW(std::domain_error("JSON pointer must be empty or begin with '/'")); + } + + // extract the reference tokens: + // - slash: position of the last read slash (or end of string) + // - start: position after the previous slash + for ( + // search for the first slash after the first character + size_t slash = reference_string.find_first_of('/', 1), + // set the beginning of the first reference token + start = 1; + // we can stop if start == string::npos+1 = 0 + start != 0; + // set the beginning of the next reference token + // (will eventually be 0 if slash == std::string::npos) + start = slash + 1, + // find next slash + slash = reference_string.find_first_of('/', start)) + { + // use the text between the beginning of the reference token + // (start) and the last slash (slash). + auto reference_token = reference_string.substr(start, slash - start); + + // check reference tokens are properly escaped + for (size_t pos = reference_token.find_first_of('~'); + pos != std::string::npos; + pos = reference_token.find_first_of('~', pos + 1)) + { + assert(reference_token[pos] == '~'); + + // ~ must be followed by 0 or 1 + if (pos == reference_token.size() - 1 or + (reference_token[pos + 1] != '0' and + reference_token[pos + 1] != '1')) + { + JSON_THROW(std::domain_error("escape error: '~' must be followed with '0' or '1'")); + } + } + + // finally, store the reference token + unescape(reference_token); + result.push_back(reference_token); + } + + return result; + } + + private: + /*! + @brief replace all occurrences of a substring by another string + + @param[in,out] s the string to manipulate; changed so that all + occurrences of @a f are replaced with @a t + @param[in] f the substring to replace with @a t + @param[in] t the string to replace @a f + + @pre The search string @a f must not be empty. + + @since version 2.0.0 + */ + static void replace_substring(std::string& s, + const std::string& f, + const std::string& t) + { + assert(not f.empty()); + + for ( + size_t pos = s.find(f); // find first occurrence of f + pos != std::string::npos; // make sure f was found + s.replace(pos, f.size(), t), // replace with t + pos = s.find(f, pos + t.size()) // find next occurrence of f + ); + } + + /// escape tilde and slash + static std::string escape(std::string s) + { + // escape "~"" to "~0" and "/" to "~1" + replace_substring(s, "~", "~0"); + replace_substring(s, "/", "~1"); + return s; + } + + /// unescape tilde and slash + static void unescape(std::string& s) + { + // first transform any occurrence of the sequence '~1' to '/' + replace_substring(s, "~1", "/"); + // then transform any occurrence of the sequence '~0' to '~' + replace_substring(s, "~0", "~"); + } + + /*! + @param[in] reference_string the reference string to the current value + @param[in] value the value to consider + @param[in,out] result the result object to insert values to + + @note Empty objects or arrays are flattened to `null`. + */ + static void flatten(const std::string& reference_string, + const basic_json& value, + basic_json& result) + { + switch (value.m_type) + { + case value_t::array: + { + if (value.m_value.array->empty()) + { + // flatten empty array as null + result[reference_string] = nullptr; + } + else + { + // iterate array and use index as reference string + for (size_t i = 0; i < value.m_value.array->size(); ++i) + { + flatten(reference_string + "/" + std::to_string(i), + value.m_value.array->operator[](i), result); + } + } + break; + } + + case value_t::object: + { + if (value.m_value.object->empty()) + { + // flatten empty object as null + result[reference_string] = nullptr; + } + else + { + // iterate object and use keys as reference string + for (const auto& element : *value.m_value.object) + { + flatten(reference_string + "/" + escape(element.first), + element.second, result); + } + } + break; + } + + default: + { + // add primitive value with its reference string + result[reference_string] = value; + break; + } + } + } + + /*! + @param[in] value flattened JSON + + @return unflattened JSON + */ + static basic_json unflatten(const basic_json& value) + { + if (not value.is_object()) + { + JSON_THROW(std::domain_error("only objects can be unflattened")); + } + + basic_json result; + + // iterate the JSON object values + for (const auto& element : *value.m_value.object) + { + if (not element.second.is_primitive()) + { + JSON_THROW(std::domain_error("values in object must be primitive")); + } + + // assign value to reference pointed to by JSON pointer; Note + // that if the JSON pointer is "" (i.e., points to the whole + // value), function get_and_create returns a reference to + // result itself. An assignment will then create a primitive + // value. + json_pointer(element.first).get_and_create(result) = element.second; + } + + return result; + } + + private: + friend bool operator==(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return lhs.reference_tokens == rhs.reference_tokens; + } + + friend bool operator!=(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return !(lhs == rhs); + } + + /// the reference tokens + std::vector reference_tokens {}; + }; + + ////////////////////////// + // JSON Pointer support // + ////////////////////////// + + /// @name JSON Pointer functions + /// @{ + + /*! + @brief access specified element via JSON Pointer + + Uses a JSON pointer to retrieve a reference to the respective JSON value. + No bound checking is performed. Similar to @ref operator[](const typename + object_t::key_type&), `null` values are created in arrays and objects if + necessary. + + In particular: + - If the JSON pointer points to an object key that does not exist, it + is created an filled with a `null` value before a reference to it + is returned. + - If the JSON pointer points to an array index that does not exist, it + is created an filled with a `null` value before a reference to it + is returned. All indices between the current maximum and the given + index are also filled with `null`. + - The special value `-` is treated as a synonym for the index past the + end. + + @param[in] ptr a JSON pointer + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,operatorjson_pointer} + + @since version 2.0.0 + */ + reference operator[](const json_pointer& ptr) + { + return ptr.get_unchecked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Uses a JSON pointer to retrieve a reference to the respective JSON value. + No bound checking is performed. The function does not change the JSON + value; no `null` values are created. In particular, the the special value + `-` yields an exception. + + @param[in] ptr JSON pointer to the desired element + + @return const reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,operatorjson_pointer_const} + + @since version 2.0.0 + */ + const_reference operator[](const json_pointer& ptr) const + { + return ptr.get_unchecked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Returns a reference to the element at with specified JSON pointer @a ptr, + with bounds checking. + + @param[in] ptr JSON pointer to the desired element + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,at_json_pointer} + + @since version 2.0.0 + */ + reference at(const json_pointer& ptr) + { + return ptr.get_checked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Returns a const reference to the element at with specified JSON pointer @a + ptr, with bounds checking. + + @param[in] ptr JSON pointer to the desired element + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw std::out_of_range if the JSON pointer can not be resolved + @throw std::domain_error if an array index begins with '0' + @throw std::invalid_argument if an array index was not a number + + @liveexample{The behavior is shown in the example.,at_json_pointer_const} + + @since version 2.0.0 + */ + const_reference at(const json_pointer& ptr) const + { + return ptr.get_checked(this); + } + + /*! + @brief return flattened JSON value + + The function creates a JSON object whose keys are JSON pointers (see [RFC + 6901](https://tools.ietf.org/html/rfc6901)) and whose values are all + primitive. The original JSON value can be restored using the @ref + unflatten() function. + + @return an object that maps JSON pointers to primitive values + + @note Empty objects and arrays are flattened to `null` and will not be + reconstructed correctly by the @ref unflatten() function. + + @complexity Linear in the size the JSON value. + + @liveexample{The following code shows how a JSON object is flattened to an + object whose keys consist of JSON pointers.,flatten} + + @sa @ref unflatten() for the reverse function + + @since version 2.0.0 + */ + basic_json flatten() const + { + basic_json result(value_t::object); + json_pointer::flatten("", *this, result); + return result; + } + + /*! + @brief unflatten a previously flattened JSON value + + The function restores the arbitrary nesting of a JSON value that has been + flattened before using the @ref flatten() function. The JSON value must + meet certain constraints: + 1. The value must be an object. + 2. The keys must be JSON pointers (see + [RFC 6901](https://tools.ietf.org/html/rfc6901)) + 3. The mapped values must be primitive JSON types. + + @return the original JSON from a flattened version + + @note Empty objects and arrays are flattened by @ref flatten() to `null` + values and can not unflattened to their original type. Apart from + this example, for a JSON value `j`, the following is always true: + `j == j.flatten().unflatten()`. + + @complexity Linear in the size the JSON value. + + @liveexample{The following code shows how a flattened JSON object is + unflattened into the original nested JSON object.,unflatten} + + @sa @ref flatten() for the reverse function + + @since version 2.0.0 + */ + basic_json unflatten() const + { + return json_pointer::unflatten(*this); + } + + /// @} + + ////////////////////////// + // JSON Patch functions // + ////////////////////////// + + /// @name JSON Patch functions + /// @{ + + /*! + @brief applies a JSON patch + + [JSON Patch](http://jsonpatch.com) defines a JSON document structure for + expressing a sequence of operations to apply to a JSON) document. With + this function, a JSON Patch is applied to the current JSON value by + executing all operations from the patch. + + @param[in] json_patch JSON patch document + @return patched document + + @note The application of a patch is atomic: Either all operations succeed + and the patched document is returned or an exception is thrown. In + any case, the original value is not changed: the patch is applied + to a copy of the value. + + @throw std::out_of_range if a JSON pointer inside the patch could not + be resolved successfully in the current JSON value; example: `"key baz + not found"` + @throw invalid_argument if the JSON patch is malformed (e.g., mandatory + attributes are missing); example: `"operation add must have member path"` + + @complexity Linear in the size of the JSON value and the length of the + JSON patch. As usually only a fraction of the JSON value is affected by + the patch, the complexity can usually be neglected. + + @liveexample{The following code shows how a JSON patch is applied to a + value.,patch} + + @sa @ref diff -- create a JSON patch by comparing two JSON values + + @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) + @sa [RFC 6901 (JSON Pointer)](https://tools.ietf.org/html/rfc6901) + + @since version 2.0.0 + */ + basic_json patch(const basic_json& json_patch) const + { + // make a working copy to apply the patch to + basic_json result = *this; + + // the valid JSON Patch operations + enum class patch_operations {add, remove, replace, move, copy, test, invalid}; + + const auto get_op = [](const std::string op) + { + if (op == "add") + { + return patch_operations::add; + } + if (op == "remove") + { + return patch_operations::remove; + } + if (op == "replace") + { + return patch_operations::replace; + } + if (op == "move") + { + return patch_operations::move; + } + if (op == "copy") + { + return patch_operations::copy; + } + if (op == "test") + { + return patch_operations::test; + } + + return patch_operations::invalid; + }; + + // wrapper for "add" operation; add value at ptr + const auto operation_add = [&result](json_pointer & ptr, basic_json val) + { + // adding to the root of the target document means replacing it + if (ptr.is_root()) + { + result = val; + } + else + { + // make sure the top element of the pointer exists + json_pointer top_pointer = ptr.top(); + if (top_pointer != ptr) + { + result.at(top_pointer); + } + + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.pop_back(); + basic_json& parent = result[ptr]; + + switch (parent.m_type) + { + case value_t::null: + case value_t::object: + { + // use operator[] to add value + parent[last_path] = val; + break; + } + + case value_t::array: + { + if (last_path == "-") + { + // special case: append to back + parent.push_back(val); + } + else + { + const auto idx = std::stoi(last_path); + if (static_cast(idx) > parent.size()) + { + // avoid undefined behavior + JSON_THROW(std::out_of_range("array index " + std::to_string(idx) + " is out of range")); + } + else + { + // default case: insert add offset + parent.insert(parent.begin() + static_cast(idx), val); + } + } + break; + } + + default: + { + // if there exists a parent it cannot be primitive + assert(false); // LCOV_EXCL_LINE + } + } + } + }; + + // wrapper for "remove" operation; remove value at ptr + const auto operation_remove = [&result](json_pointer & ptr) + { + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.pop_back(); + basic_json& parent = result.at(ptr); + + // remove child + if (parent.is_object()) + { + // perform range check + auto it = parent.find(last_path); + if (it != parent.end()) + { + parent.erase(it); + } + else + { + JSON_THROW(std::out_of_range("key '" + last_path + "' not found")); + } + } + else if (parent.is_array()) + { + // note erase performs range check + parent.erase(static_cast(std::stoi(last_path))); + } + }; + + // type check + if (not json_patch.is_array()) + { + // a JSON patch must be an array of objects + JSON_THROW(std::invalid_argument("JSON patch must be an array of objects")); + } + + // iterate and apply the operations + for (const auto& val : json_patch) + { + // wrapper to get a value for an operation + const auto get_value = [&val](const std::string & op, + const std::string & member, + bool string_type) -> basic_json& + { + // find value + auto it = val.m_value.object->find(member); + + // context-sensitive error message + const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'"; + + // check if desired value is present + if (it == val.m_value.object->end()) + { + JSON_THROW(std::invalid_argument(error_msg + " must have member '" + member + "'")); + } + + // check if result is of type string + if (string_type and not it->second.is_string()) + { + JSON_THROW(std::invalid_argument(error_msg + " must have string member '" + member + "'")); + } + + // no error: return value + return it->second; + }; + + // type check + if (not val.is_object()) + { + JSON_THROW(std::invalid_argument("JSON patch must be an array of objects")); + } + + // collect mandatory members + const std::string op = get_value("op", "op", true); + const std::string path = get_value(op, "path", true); + json_pointer ptr(path); + + switch (get_op(op)) + { + case patch_operations::add: + { + operation_add(ptr, get_value("add", "value", false)); + break; + } + + case patch_operations::remove: + { + operation_remove(ptr); + break; + } + + case patch_operations::replace: + { + // the "path" location must exist - use at() + result.at(ptr) = get_value("replace", "value", false); + break; + } + + case patch_operations::move: + { + const std::string from_path = get_value("move", "from", true); + json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + basic_json v = result.at(from_ptr); + + // The move operation is functionally identical to a + // "remove" operation on the "from" location, followed + // immediately by an "add" operation at the target + // location with the value that was just removed. + operation_remove(from_ptr); + operation_add(ptr, v); + break; + } + + case patch_operations::copy: + { + const std::string from_path = get_value("copy", "from", true);; + const json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + result[ptr] = result.at(from_ptr); + break; + } + + case patch_operations::test: + { + bool success = false; + JSON_TRY + { + // check if "value" matches the one at "path" + // the "path" location must exist - use at() + success = (result.at(ptr) == get_value("test", "value", false)); + } + JSON_CATCH (std::out_of_range&) + { + // ignore out of range errors: success remains false + } + + // throw an exception if test fails + if (not success) + { + JSON_THROW(std::domain_error("unsuccessful: " + val.dump())); + } + + break; + } + + case patch_operations::invalid: + { + // op must be "add", "remove", "replace", "move", "copy", or + // "test" + JSON_THROW(std::invalid_argument("operation value '" + op + "' is invalid")); + } + } + } + + return result; + } + + /*! + @brief creates a diff as a JSON patch + + Creates a [JSON Patch](http://jsonpatch.com) so that value @a source can + be changed into the value @a target by calling @ref patch function. + + @invariant For two JSON values @a source and @a target, the following code + yields always `true`: + @code {.cpp} + source.patch(diff(source, target)) == target; + @endcode + + @note Currently, only `remove`, `add`, and `replace` operations are + generated. + + @param[in] source JSON value to compare from + @param[in] target JSON value to compare against + @param[in] path helper value to create JSON pointers + + @return a JSON patch to convert the @a source to @a target + + @complexity Linear in the lengths of @a source and @a target. + + @liveexample{The following code shows how a JSON patch is created as a + diff for two JSON values.,diff} + + @sa @ref patch -- apply a JSON patch + + @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) + + @since version 2.0.0 + */ + static basic_json diff(const basic_json& source, + const basic_json& target, + const std::string& path = "") + { + // the patch + basic_json result(value_t::array); + + // if the values are the same, return empty patch + if (source == target) + { + return result; + } + + if (source.type() != target.type()) + { + // different types: replace value + result.push_back( + { + {"op", "replace"}, + {"path", path}, + {"value", target} + }); + } + else + { + switch (source.type()) + { + case value_t::array: + { + // first pass: traverse common elements + size_t i = 0; + while (i < source.size() and i < target.size()) + { + // recursive call to compare array values at index i + auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + ++i; + } + + // i now reached the end of at least one array + // in a second pass, traverse the remaining elements + + // remove my remaining elements + const auto end_index = static_cast(result.size()); + while (i < source.size()) + { + // add operations in reverse order to avoid invalid + // indices + result.insert(result.begin() + end_index, object( + { + {"op", "remove"}, + {"path", path + "/" + std::to_string(i)} + })); + ++i; + } + + // add other remaining elements + while (i < target.size()) + { + result.push_back( + { + {"op", "add"}, + {"path", path + "/" + std::to_string(i)}, + {"value", target[i]} + }); + ++i; + } + + break; + } + + case value_t::object: + { + // first pass: traverse this object's elements + for (auto it = source.begin(); it != source.end(); ++it) + { + // escape the key name to be used in a JSON patch + const auto key = json_pointer::escape(it.key()); + + if (target.find(it.key()) != target.end()) + { + // recursive call to compare object values at key it + auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + } + else + { + // found a key that is not in o -> remove it + result.push_back(object( + { + {"op", "remove"}, + {"path", path + "/" + key} + })); + } + } + + // second pass: traverse other object's elements + for (auto it = target.begin(); it != target.end(); ++it) + { + if (source.find(it.key()) == source.end()) + { + // found a key that is not in this -> add it + const auto key = json_pointer::escape(it.key()); + result.push_back( + { + {"op", "add"}, + {"path", path + "/" + key}, + {"value", it.value()} + }); + } + } + + break; + } + + default: + { + // both primitive type: replace value + result.push_back( + { + {"op", "replace"}, + {"path", path}, + {"value", target} + }); + break; + } + } + } + + return result; + } + + /// @} +}; + +///////////// +// presets // +///////////// + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} // namespace nlohmann + + +/////////////////////// +// nonmember support // +/////////////////////// + +// specialization of std::swap, and std::hash +namespace std +{ +/*! +@brief exchanges the values of two JSON objects + +@since version 1.0.0 +*/ +template<> +inline void swap(nlohmann::json& j1, + nlohmann::json& j2) noexcept( + is_nothrow_move_constructible::value and + is_nothrow_move_assignable::value + ) +{ + j1.swap(j2); +} + +/// hash value for JSON objects +template<> +struct hash +{ + /*! + @brief return a hash value for a JSON object + + @since version 1.0.0 + */ + std::size_t operator()(const nlohmann::json& j) const + { + // a naive hashing via the string representation + const auto& h = hash(); + return h(j.dump()); + } +}; +} // namespace std + +/*! +@brief user-defined string literal for JSON values + +This operator implements a user-defined string literal for JSON objects. It +can be used by adding `"_json"` to a string literal and returns a JSON object +if no parse error occurred. + +@param[in] s a string representation of a JSON object +@param[in] n the length of string @a s +@return a JSON object + +@since version 1.0.0 +*/ +inline nlohmann::json operator "" _json(const char* s, std::size_t n) +{ + return nlohmann::json::parse(s, s + n); +} + +/*! +@brief user-defined string literal for JSON pointer + +This operator implements a user-defined string literal for JSON Pointers. It +can be used by adding `"_json_pointer"` to a string literal and returns a JSON pointer +object if no parse error occurred. + +@param[in] s a string representation of a JSON Pointer +@param[in] n the length of string @a s +@return a JSON pointer object + +@since version 2.0.0 +*/ +inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n) +{ + return nlohmann::json::json_pointer(std::string(s, n)); +} + +// restore GCC/clang diagnostic settings +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic pop +#endif + +// clean up +#undef JSON_CATCH +#undef JSON_DEPRECATED +#undef JSON_THROW +#undef JSON_TRY + +#endif From 476fe66043d746bbd61717ec4a7280a52a26b4ef Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 23 Oct 2017 19:27:34 +0200 Subject: [PATCH 046/262] [CSE] Add beginnign of the code to deal with the metadata --- src/libsync/CMakeLists.txt | 8 +++-- src/libsync/clientsideencryption.cpp | 52 ++++++++++++++++++++++++++++ src/libsync/clientsideencryption.h | 28 +++++++++++++-- 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index fabcef768..0a392acd0 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -3,9 +3,11 @@ set(CMAKE_AUTOMOC TRUE) include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) # csync is required. -include_directories(${CMAKE_SOURCE_DIR}/src/csync - ${CMAKE_BINARY_DIR}/src/csync - ) +include_directories( + ${CMAKE_SOURCE_DIR}/src/csync + ${CMAKE_BINARY_DIR}/src/csync + ${CMAKE_SOURCE_DIR}/src/3rdparty +) if ( APPLE ) list(APPEND OS_SPECIFIC_LINK_LIBRARIES diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 020617eef..93a102050 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -19,6 +19,7 @@ #include #include +#include #include "wordlist.h" namespace OCC @@ -638,4 +639,55 @@ bool SetEncryptionFlagApiJob::finished() emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); } +/* Test metdata: +{ + // Metadata about the share + "metadata": { + // Encryption algorithm: RSA/ECB/OAEPWithSHA-256AndMGF1Padding, encrypted via private/public key (asymmetric) + "metadataKeys": { + "0": "OLDESTMETADATAKEY", + "2": "…", + "3": "NEWESTMETADATAKEY" + }, + // Encryption algorithm: AES/GCM/NoPadding (128 bit key size) with metadata key from above (symmetric) + "sharing": { + // Name of recipients as well as public keys of the recipients + "recipient": { + "recipient1@example.com": "PUBLIC KEY", + "recipient2@example.com": "PUBLIC KEY" + }, + }, + "version": 1 + }, + // A JSON blob referencing all files + "files": { + "ia7OEEEyXMoRa1QWQk8r": { + // Encryption algorithm: AES/GCM/NoPadding (128 bit key size) with metadata key from above (symmetric) + "encrypted": { + "key": "jtboLmgGR1OQf2uneqCVHpklQLlIwWL5TXAQ0keK", + "filename": "/foo/test.txt", + "mimetype": "plain/text", + "version": 1 + }, + "initializationVector": "+mHu52HyZq+pAAIN", + "authenticationTag": "GCM authentication tag", + "metadataKey": 1 + } + } +} + +*/ +FolderMetadata::FolderMetadata(const QByteArray& metadata) +{ + // This is a new folder + /* + if (metadata.isEmpty()) { + + } + QJsonParseError err; + _doc = QJsonDocument::fromJson(metadata, err); + */ + +} + } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index e143d1aae..a6501c150 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -40,7 +40,6 @@ private: bool isInitialized = false; }; - /* * @brief Job to sigh the CSR that return JSON * @@ -163,6 +162,31 @@ private: QString _fileId; }; -} // namespace OCC +/* Generates the Metadata for the folder */ +struct EncryptedFile { + QByteArray encryptionKey; + QByteArray mimetype; + QByteArray initializationVector; + QByteArray authenticationTag; + QString encryptedFilename; + QString originalFilename; + int fileVersion; + int metadataKey; +}; +class FolderMetadata { + FolderMetadata(const QByteArray& metadata = QByteArray()); + QByteArray encryptedMetadata(); + + void addEncryptedFile(const EncryptedFile& f); + QVector files() const; + +private: + QVector _files; + QVector _metadataKeys; + + QJsonDocument _jsonMetadata; +}; + +} // namespace OCC #endif From bacbf337d26ddcaac27a9ba1c1f57c9e05c90c02 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Tue, 24 Oct 2017 15:44:34 +0200 Subject: [PATCH 047/262] Update encryption function * Do not use padding * Do not use the ADD data * Append the tag to the ciphertext to be compatible with Android Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 93a102050..af7e49426 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -47,8 +47,6 @@ namespace { int encrypt(unsigned char *plaintext, int plaintext_len, - unsigned char *aad, - int aad_len, unsigned char *key, unsigned char *iv, unsigned char *ciphertext, @@ -70,9 +68,12 @@ namespace { handleErrors(); } - /* Set IV length if default 12 bytes (96 bits) is not appropriate */ + // We don't do padding + EVP_CIPHER_CTX_set_padding(ctx, 0); + + /* Set IV length to 16 bytes */ if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL)) { - qCInfo(lcCse()) << "Error setting the iv length to 16 bits. "; + qCInfo(lcCse()) << "Error setting the iv length to 16 bytes. "; handleErrors(); } @@ -82,14 +83,6 @@ namespace { handleErrors(); } - /* Provide any AAD data. This can be called zero or more times as - * required - */ - if(1 != EVP_EncryptUpdate(ctx, NULL, &len, aad, aad_len)) { - qCInfo(lcCse()) << "Error calling the Encrypt Update"; - handleErrors(); - } - /* Provide the message to be encrypted, and obtain the encrypted output. * EVP_EncryptUpdate can be called multiple times if necessary */ @@ -114,6 +107,10 @@ namespace { handleErrors(); } + /* Add tag to cypher text to be compatible with the Android implementation */ + memcpy(ciphertext + ciphertext_len, tag, 16); + ciphertext_len += 16; + /* Clean up */ EVP_CIPHER_CTX_free(ctx); @@ -469,8 +466,6 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) int cryptedText_len = encrypt( (unsigned char*) encryptTest, //unsigned char *plaintext, strlen(encryptTest), // int plaintext_len, - nullptr, // unsigned char *aad, - 0, // int aad_len, fakepass, // unsigned char *key, iv, // unsigned char *iv, cryptedText, // unsigned char *ciphertext, From ca6fa7b3410d4b9bcc8fcc5146242faf17ea3789 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Tue, 24 Oct 2017 15:48:44 +0200 Subject: [PATCH 048/262] Update decryp function * Do not use AAD * Do not try to decrypt the last 16 bytes as Android adds the tag there by default Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index af7e49426..c94aa7681 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -119,8 +119,6 @@ namespace { int decrypt(unsigned char *ciphertext, int ciphertext_len, - unsigned char *aad, - int aad_len, unsigned char *tag, unsigned char *key, unsigned char *iv, @@ -143,7 +141,7 @@ namespace { handleErrors(); } - /* Set IV length. Not necessary if this is 12 bytes (96 bits) */ + /* Set IV length to 16 bytes */ if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL)) { qCInfo(lcCse()) << "Error seting th iv length for the decrypt context"; handleErrors(); @@ -155,18 +153,13 @@ namespace { handleErrors(); } - /* Provide any AAD data. This can be called zero or more times as - * required - */ - if(!EVP_DecryptUpdate(ctx, NULL, &len, aad, aad_len)) { - qCInfo(lcCse()) << "Error updating the decrypt context with aad information"; - handleErrors(); - } - - /* Provide the message to be decrypted, and obtain the plaintext output. + /* Provide the message to be decrypted, and obtain the plaintext output. * EVP_DecryptUpdate can be called multiple times if necessary + * + * Do not try to decrypt the last 16 bytes. The tag is appended by Android. + * So we ignore the last 16 bytes. */ - if(!EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len)) { + if(!EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len - 16)) { qCInfo(lcCse()) << "Error decrypting the text"; handleErrors(); } From 817baf292d0a60eb8a8d8d057d68e6186b938b61 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Tue, 24 Oct 2017 15:49:45 +0200 Subject: [PATCH 049/262] Use EVP_aes_128_gcm Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index c94aa7681..c5ea8b7d5 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -63,7 +63,7 @@ namespace { } /* Initialise the encryption operation. */ - if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, NULL, NULL)) { + if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { qCInfo(lcCse()) << "Error initializing the context with aes_256"; handleErrors(); } @@ -136,7 +136,7 @@ namespace { } /* Initialise the decryption operation. */ - if(!EVP_DecryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, NULL, NULL)) { + if(!EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { qCInfo(lcCse()) << "Error initializing the decryption context"; handleErrors(); } From 238f0b3610abd1a8d25baef0bbd5eb98b3d4f646 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Tue, 24 Oct 2017 15:53:17 +0200 Subject: [PATCH 050/262] Add note Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index c5ea8b7d5..7ff58efba 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -444,6 +444,11 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) qCInfo(lcCse()) << "Return of the PKCS5" << ret; qCInfo(lcCse()) << "Result String" << secretKey; + + /** + * NOTES: the key + iv have to be base64 encoded in the metadata. + */ + //: FIRST TRY A SILLY PHRASE. //: Hardcoed IV, really bad. unsigned char *fakepass = (unsigned char*) "qwertyuiasdfghjkzxcvbnm,qwertyui"; From c695c50c33dce84113d985fe73a0d6b76c8d0d3c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 30 Oct 2017 15:39:40 +0100 Subject: [PATCH 051/262] [CSE] Implement the empty metadata Signed-off-by: Tomaz Canabrava --- src/CMakeLists.txt | 4 +++ src/libsync/clientsideencryption.cpp | 52 +++++++++++++++++++++++----- src/libsync/clientsideencryption.h | 15 ++++++-- 3 files changed, 60 insertions(+), 11 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5319fbdc2..072e43963 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -26,6 +26,10 @@ elseif(UNIX AND NOT APPLE) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-z,relro -Wl,-z,now") endif() +include_directories( + ${CMAKE_SOURCE_DIR}/src/3rdparty +) + add_subdirectory(csync) add_subdirectory(libsync) if (NOT BUILD_LIBRARIES_ONLY) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 7ff58efba..fecd719ee 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -9,7 +9,6 @@ #include #include -#include #include @@ -19,7 +18,6 @@ #include #include -#include #include "wordlist.h" namespace OCC @@ -668,19 +666,55 @@ bool SetEncryptionFlagApiJob::finished() } } } - */ + +//TODO: Create an actuall encryption here. +auto metadataKeyEnc(const QByteArray& data) -> QByteArray +{ + return data; +} + +auto metadataKeyDec(const QByteArray& data) -> QByteArray +{ + return data; +} + FolderMetadata::FolderMetadata(const QByteArray& metadata) { - // This is a new folder - /* if (metadata.isEmpty()) { - + setupEmptyMetadata(); } - QJsonParseError err; - _doc = QJsonDocument::fromJson(metadata, err); - */ +} +// RSA/ECB/OAEPWithSHA-256AndMGF1Padding using private / public key. +std::string FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKeys) const { + // pretend to encrypt for now. + return metadataKeys.dump(); +} + +std::string FolderMetadata::genMetadataPass() const { + return "4randomdiceroll"; +} + +// AES/GCM/NoPadding (128 bit key size) +std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const std::string& pass) const { + return obj.dump(); +} + +void FolderMetadata::setupEmptyMetadata() { + using namespace nlohmann; + std::string newMetadataPass = genMetadataPass(); + json metadataKeyObj = {"0", newMetadataPass}; + json recepient = {"recipient", {}}; + json m = { + {"metadata", { + {"metadataKeys", encryptMetadataKeys(metadataKeyObj)}, + {"sharing", encryptJsonObject(recepient, newMetadataPass)}, + {"version",1} + }}, + {"files", { + }} + }; } } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index a6501c150..61ad3bfac 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -7,10 +7,13 @@ #include #include +#include #include "accountfwd.h" #include "networkjobs.h" +#include + namespace OCC { QString baseUrl(); @@ -174,6 +177,8 @@ struct EncryptedFile { int metadataKey; }; + + class FolderMetadata { FolderMetadata(const QByteArray& metadata = QByteArray()); QByteArray encryptedMetadata(); @@ -182,10 +187,16 @@ class FolderMetadata { QVector files() const; private: + /* Use std::string and std::vector internally on this class + * to ease the port to Nlohmann Json API + */ + void setupEmptyMetadata(); + std::string encryptMetadataKeys(const nlohmann::json& metadataKeys) const; + std::string genMetadataPass() const; + std::string encryptJsonObject(const nlohmann::json& obj, const std::string& pass) const; + QVector _files; QVector _metadataKeys; - - QJsonDocument _jsonMetadata; }; } // namespace OCC From edb04c137c5cff101d56374433509f1d83834468 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 30 Oct 2017 19:05:32 +0100 Subject: [PATCH 052/262] [CSE] Call the FolderMetadata --- src/gui/accountsettings.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 6ef950eb4..f537d73fb 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -286,6 +286,11 @@ void AccountSettings::slotCustomContextMenuRequested(const QPoint &pos) connect(job, &OCC::SetEncryptionFlagApiJob::jsonReceived, [this](const QJsonDocument& json, int httpResponse) { Q_UNUSED(json); qCInfo(lcAccountSettings) << "Encrypt Http Response" << httpResponse; + + // prepare and send the metadata to the folder + if (httpResponse == 200) { + FolderMetadata emptyMetadata; + } }); job->start(); }); From cf56d58241554eb0d856c9c37340536d73ec97ac Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 30 Oct 2017 19:05:55 +0100 Subject: [PATCH 053/262] [CSE] Start the encryption of the metadataKey --- src/libsync/clientsideencryption.cpp | 53 ++++++++++++++++++++++++++-- src/libsync/clientsideencryption.h | 1 + 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index fecd719ee..9610beda6 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -688,8 +689,54 @@ FolderMetadata::FolderMetadata(const QByteArray& metadata) // RSA/ECB/OAEPWithSHA-256AndMGF1Padding using private / public key. std::string FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKeys) const { - // pretend to encrypt for now. - return metadataKeys.dump(); + std::string metadata = metadataKeys.dump(); + const char *metadataPtr = metadata.c_str(); + + unsigned char *out; + unsigned char *in; + size_t outLen; + size_t inLen; + int err = -1; + const int rsaOeapPadding = 1; + EVP_PKEY *key = nullptr; + ENGINE *eng = nullptr; + + auto ctx = EVP_PKEY_CTX_new(key, eng); + if (!ctx) { + qCInfo(lcCse()) << "Could not initialize the pkey context."; + exit(1); + } + + err = EVP_PKEY_encrypt_init(ctx); + if (err <= 0) { + qCInfo(lcCse()) << "Error initilaizing the encryption."; + exit(1); + } + + err = EVP_PKEY_CTX_set_rsa_padding(ctx, rsaOeapPadding); + if (err <= 0) { + qCInfo(lcCse()) << "Error setting the encryption padding."; + exit(1); + } + + err = EVP_PKEY_encrypt(ctx, NULL, &outLen, in, inLen); + if (err <= 0) { + qCInfo(lcCse()) << "Error retrieving the size of the encrypted data"; + exit(1); + } + + out = (uchar*) OPENSSL_malloc(outLen); + if (!out) { + qCInfo(lcCse()) << "Error requesting memory for the encrypted contents"; + exit(1); + } + + err = EVP_PKEY_encrypt(ctx, out, &outLen, in, inLen); + if (err <= 0) { + qCInfo(lcCse()) << "Could not encrypt key."; + exit(1); + } + qCInfo(lcCse()) << "Data encrypted successfully"; } std::string FolderMetadata::genMetadataPass() const { @@ -715,6 +762,8 @@ void FolderMetadata::setupEmptyMetadata() { {"files", { }} }; + + qCInfo(lcCse()) << QString::fromStdString(m.dump()); } } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 61ad3bfac..0faaa5f85 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -180,6 +180,7 @@ struct EncryptedFile { class FolderMetadata { +public: FolderMetadata(const QByteArray& metadata = QByteArray()); QByteArray encryptedMetadata(); From a9865324427bb50eaa7f7ea7075c6dc052cafa95 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 30 Oct 2017 19:08:03 +0100 Subject: [PATCH 054/262] [CSE] Generate a random password for the metadata. --- src/libsync/clientsideencryption.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 9610beda6..1d14342c0 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -740,7 +740,15 @@ std::string FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKe } std::string FolderMetadata::genMetadataPass() const { - return "4randomdiceroll"; + const char* charmap = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + const size_t charmapLength = strlen(charmap); + const int bytes = 16; + std::string result; + result.reserve(bytes); + generate_n(back_inserter(result), bytes, [&](){ + return charmap[rand() % charmapLength]; + }); + return result; } // AES/GCM/NoPadding (128 bit key size) From c7d9abbea32e655672aa4d5cb0000ba4b7fdf14b Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 30 Oct 2017 20:02:55 +0100 Subject: [PATCH 055/262] [CSE] Rework pubkeyPath to be able to fetch pubKey --- src/gui/accountsettings.cpp | 2 +- src/libsync/clientsideencryption.cpp | 31 +++++++++++++++++++--------- src/libsync/clientsideencryption.h | 7 ++++--- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index f537d73fb..84023425a 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -289,7 +289,7 @@ void AccountSettings::slotCustomContextMenuRequested(const QPoint &pos) // prepare and send the metadata to the folder if (httpResponse == 200) { - FolderMetadata emptyMetadata; + FolderMetadata emptyMetadata(accountsState()->account()); } }); job->start(); diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 1d14342c0..ca983a2ce 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -218,24 +218,24 @@ void ClientSideEncryption::initialize() getPublicKeyFromServer(); } -QString ClientSideEncryption::publicKeyPath() const +QString publicKeyPath(AccountPtr account) { - return baseDirectory() + _account->displayName() + ".pub"; + return baseDirectory() + account->displayName() + ".pub"; } -QString ClientSideEncryption::privateKeyPath() const +QString privateKeyPath(AccountPtr account) { - return baseDirectory() + _account->displayName() + ".rsa"; + return baseDirectory() + account->displayName() + ".rsa"; } bool ClientSideEncryption::hasPrivateKey() const { - return QFileInfo(privateKeyPath()).exists(); + return QFileInfo(privateKeyPath(_account)).exists(); } bool ClientSideEncryption::hasPublicKey() const { - return QFileInfo(publicKeyPath()).exists(); + return QFileInfo(publicKeyPath(_account)).exists(); } void ClientSideEncryption::generateKeyPair() @@ -274,8 +274,8 @@ void ClientSideEncryption::generateKeyPair() return; } - auto privKeyPath = privateKeyPath().toLocal8Bit(); - auto pubKeyPath = publicKeyPath().toLocal8Bit(); + auto privKeyPath = privateKeyPath(_account).toLocal8Bit(); + auto pubKeyPath = publicKeyPath(_account).toLocal8Bit(); FILE *privKeyFile = fopen(privKeyPath.constData(), "w"); FILE *pubKeyFile = fopen(pubKeyPath.constData(), "w"); @@ -370,7 +370,7 @@ QString ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) if (retCode == 200) { auto caps = json.object().value("ocs").toObject().value("data").toObject().value("public-key").toString(); qCInfo(lcCse()) << "Public Key Returned" << caps; - QFile file(publicKeyPath() + ".sign"); + QFile file(publicKeyPath(_account) + ".sign"); if (file.open(QIODevice::WriteOnly)) { QTextStream s(&file); s << caps; @@ -680,7 +680,7 @@ auto metadataKeyDec(const QByteArray& data) -> QByteArray return data; } -FolderMetadata::FolderMetadata(const QByteArray& metadata) +FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata) : _account(account) { if (metadata.isEmpty()) { setupEmptyMetadata(); @@ -701,6 +701,17 @@ std::string FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKe EVP_PKEY *key = nullptr; ENGINE *eng = nullptr; + auto path = publicKeyPath(_account); + const char *pathC = qPrintable(path); + + FILE* pkeyFile = fopen(pathC, "r"); + if (!pkeyFile) { + qCInfo(lcCse()) << "Could not open the public key"; + exit(1); + } + + key = PEM_read_PUBKEY(pkeyFile, NULL, NULL, NULL); + auto ctx = EVP_PKEY_CTX_new(key, eng); if (!ctx) { qCInfo(lcCse()) << "Could not initialize the pkey context."; diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 0faaa5f85..f7eb62f5a 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -18,6 +18,8 @@ namespace OCC { QString baseUrl(); QString baseDirectory(); +QString privateKeyPath(AccountPtr account); +QString publicKeyPath(AccountPtr account); class ClientSideEncryption : public QObject { Q_OBJECT @@ -32,8 +34,6 @@ public: void getPrivateKeyFromServer(); void getPublicKeyFromServer(); void encryptPrivateKey(EVP_PKEY *keyPair); - QString privateKeyPath() const; - QString publicKeyPath() const; signals: void initializationFinished(); @@ -181,7 +181,7 @@ struct EncryptedFile { class FolderMetadata { public: - FolderMetadata(const QByteArray& metadata = QByteArray()); + FolderMetadata(AccountPtr account, const QByteArray& metadata = QByteArray()); QByteArray encryptedMetadata(); void addEncryptedFile(const EncryptedFile& f); @@ -198,6 +198,7 @@ private: QVector _files; QVector _metadataKeys; + AccountPtr _account; }; } // namespace OCC From 69dc099b1751dc7662fcc31aa52208ccf389ccac Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 31 Oct 2017 11:03:24 +0100 Subject: [PATCH 056/262] [CSE] Dedicated function for the subfolder menu --- src/gui/accountsettings.cpp | 91 +++++++++++++++++++------------------ src/gui/accountsettings.h | 1 + 2 files changed, 47 insertions(+), 45 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 84023425a..e0bf1afa1 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -255,6 +255,51 @@ void AccountSettings::doExpand() ui->_folderList->expandToDepth(0); } +void AccountSettings::slotSubfolderContextMenuRequested(const QModelIndex& index, const QPoint& pos) +{ + QMenu menu; + auto ac = menu.addAction(tr("Open folder")); + connect(ac, &QAction::triggered, this, &AccountSettings::slotOpenCurrentLocalSubFolder); + + auto fileName = _model->data(index, FolderStatusDelegate::FolderPathRole).toString(); + if (!QFile::exists(fileName)) { + ac->setEnabled(false); + } + auto fileId = _model->data(index, FolderStatusModel::FileIdRole).toByteArray(); + + if (accountsState()->account()->capabilities().clientSideEncryptionAvaliable()) { + ac = menu.addAction(tr("Encrypt")); + connect(ac, &QAction::triggered, [this, &fileId](bool triggered) { + Q_UNUSED(triggered); + auto job = new OCC::SetEncryptionFlagApiJob(accountsState()->account(), QString(fileId)); + + connect(job, &OCC::SetEncryptionFlagApiJob::jsonReceived, [this](const QJsonDocument& json, int httpResponse) { + Q_UNUSED(json); + qCInfo(lcAccountSettings) << "Encrypt Http Response" << httpResponse; + + // prepare and send the metadata to the folder + if (httpResponse == 200) { + FolderMetadata emptyMetadata(accountsState()->account()); + } + }); + job->start(); + }); + + ac = menu.addAction(tr("Decrypt")); + connect(ac, &QAction::triggered, [this, &fileId](bool triggered) { + Q_UNUSED(triggered); + auto job = new OCC::DeleteApiJob(accountsState()->account(), + "ocs/v2.php/apps/end_to_end_encryption/api/v1/encrypted/" + QString(fileId)); + connect(job, &OCC::DeleteApiJob::result, [this](int httpResponse) { + qCInfo(lcAccountSettings) << "Decrypt Http Response" << httpResponse; + }); + job->start(); + }); + + } + menu.exec(QCursor::pos()); +} + void AccountSettings::slotCustomContextMenuRequested(const QPoint &pos) { QTreeView *tv = ui->_folderList; @@ -264,51 +309,7 @@ void AccountSettings::slotCustomContextMenuRequested(const QPoint &pos) } if (_model->classify(index) == FolderStatusModel::SubFolder) { - QTreeView *tv = ui->_folderList; - QMenu *menu = new QMenu(tv); - menu->setAttribute(Qt::WA_DeleteOnClose); - - QAction *ac = menu->addAction(tr("Open folder")); - connect(ac, &QAction::triggered, this, &AccountSettings::slotOpenCurrentLocalSubFolder); - - QString fileName = _model->data(index, FolderStatusDelegate::FolderPathRole).toString(); - if (!QFile::exists(fileName)) { - ac->setEnabled(false); - } - auto fileId = _model->data(index, FolderStatusModel::FileIdRole).toByteArray(); - - if (accountsState()->account()->capabilities().clientSideEncryptionAvaliable()) { - ac = menu->addAction(tr("Encrypt")); - connect(ac, &QAction::triggered, [this, &fileId](bool triggered) { - Q_UNUSED(triggered); - auto job = new OCC::SetEncryptionFlagApiJob(accountsState()->account(), QString(fileId)); - - connect(job, &OCC::SetEncryptionFlagApiJob::jsonReceived, [this](const QJsonDocument& json, int httpResponse) { - Q_UNUSED(json); - qCInfo(lcAccountSettings) << "Encrypt Http Response" << httpResponse; - - // prepare and send the metadata to the folder - if (httpResponse == 200) { - FolderMetadata emptyMetadata(accountsState()->account()); - } - }); - job->start(); - }); - - ac = menu->addAction(tr("Decrypt")); - connect(ac, &QAction::triggered, [this, &fileId](bool triggered) { - Q_UNUSED(triggered); - auto job = new OCC::DeleteApiJob(accountsState()->account(), - "ocs/v2.php/apps/end_to_end_encryption/api/v1/encrypted/" + QString(fileId)); - connect(job, &OCC::DeleteApiJob::result, [this](int httpResponse) { - qCInfo(lcAccountSettings) << "Decrypt Http Response" << httpResponse; - }); - job->start(); - }); - - } - menu->exec(QCursor::pos()); - + slotSubfolderContextMenuRequested(index, pos); return; } diff --git a/src/gui/accountsettings.h b/src/gui/accountsettings.h index eaef07eb4..96e8a3904 100644 --- a/src/gui/accountsettings.h +++ b/src/gui/accountsettings.h @@ -85,6 +85,7 @@ protected slots: void slotOpenAccountWizard(); void slotAccountAdded(AccountState *); void refreshSelectiveSyncStatus(); + void slotSubfolderContextMenuRequested(const QModelIndex& idx, const QPoint& point); void slotCustomContextMenuRequested(const QPoint &); void slotFolderListClicked(const QModelIndex &indx); void doExpand(); From 69d73162aa394943d5032abba1a8b2d7114acbb1 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 31 Oct 2017 11:09:34 +0100 Subject: [PATCH 057/262] [CSE] functions for encrypt / decrypt folder The code was scattered around the slot for the context menu request and had already too many indentation levels. --- src/gui/accountsettings.cpp | 52 +++++++++++++++++++++---------------- src/gui/accountsettings.h | 2 ++ 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index e0bf1afa1..6d2c2ab5b 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -255,6 +255,32 @@ void AccountSettings::doExpand() ui->_folderList->expandToDepth(0); } +void AccountSettings::slotMarkSubfolderEncrpted(const QByteArray& fileId) +{ + auto job = new OCC::SetEncryptionFlagApiJob(accountsState()->account(), QString(fileId)); + connect(job, &OCC::SetEncryptionFlagApiJob::jsonReceived, + [this](const QJsonDocument& json, int httpResponse) { + Q_UNUSED(json); + qCInfo(lcAccountSettings) << "Encrypt Http Response" << httpResponse; + + // prepare and send the metadata to the folder + if (httpResponse == 200) { + FolderMetadata emptyMetadata(accountsState()->account()); + } + }); + job->start(); +} + +void AccountSettings::slotMarkSubfolderDecrypted(const QByteArray& fileId) +{ + auto job = new OCC::DeleteApiJob(accountsState()->account(), + "ocs/v2.php/apps/end_to_end_encryption/api/v1/encrypted/" + QString(fileId)); + connect(job, &OCC::DeleteApiJob::result, [this](int httpResponse) { + qCInfo(lcAccountSettings) << "Decrypt Http Response" << httpResponse; + }); + job->start(); +} + void AccountSettings::slotSubfolderContextMenuRequested(const QModelIndex& index, const QPoint& pos) { QMenu menu; @@ -269,31 +295,13 @@ void AccountSettings::slotSubfolderContextMenuRequested(const QModelIndex& index if (accountsState()->account()->capabilities().clientSideEncryptionAvaliable()) { ac = menu.addAction(tr("Encrypt")); - connect(ac, &QAction::triggered, [this, &fileId](bool triggered) { - Q_UNUSED(triggered); - auto job = new OCC::SetEncryptionFlagApiJob(accountsState()->account(), QString(fileId)); - - connect(job, &OCC::SetEncryptionFlagApiJob::jsonReceived, [this](const QJsonDocument& json, int httpResponse) { - Q_UNUSED(json); - qCInfo(lcAccountSettings) << "Encrypt Http Response" << httpResponse; - - // prepare and send the metadata to the folder - if (httpResponse == 200) { - FolderMetadata emptyMetadata(accountsState()->account()); - } - }); - job->start(); + connect(ac, &QAction::triggered, [this, &fileId] { + slotMarkSubfolderEncrpted(fileId); }); ac = menu.addAction(tr("Decrypt")); - connect(ac, &QAction::triggered, [this, &fileId](bool triggered) { - Q_UNUSED(triggered); - auto job = new OCC::DeleteApiJob(accountsState()->account(), - "ocs/v2.php/apps/end_to_end_encryption/api/v1/encrypted/" + QString(fileId)); - connect(job, &OCC::DeleteApiJob::result, [this](int httpResponse) { - qCInfo(lcAccountSettings) << "Decrypt Http Response" << httpResponse; - }); - job->start(); + connect(ac, &QAction::triggered, [this, &fileId] { + slotMarkSubfolderDecrypted(fileId); }); } diff --git a/src/gui/accountsettings.h b/src/gui/accountsettings.h index 96e8a3904..074b93bb2 100644 --- a/src/gui/accountsettings.h +++ b/src/gui/accountsettings.h @@ -85,6 +85,8 @@ protected slots: void slotOpenAccountWizard(); void slotAccountAdded(AccountState *); void refreshSelectiveSyncStatus(); + void slotMarkSubfolderEncrpted(const QByteArray& fileId); + void slotMarkSubfolderDecrypted(const QByteArray& fileId); void slotSubfolderContextMenuRequested(const QModelIndex& idx, const QPoint& point); void slotCustomContextMenuRequested(const QPoint &); void slotFolderListClicked(const QModelIndex &indx); From 560ce958def79e240bd3844f4fb681d697b08cb3 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 31 Oct 2017 11:24:10 +0100 Subject: [PATCH 058/262] [CSE] Remove warning. --- src/gui/accountsettings.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 6d2c2ab5b..a2a4edf0f 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -275,7 +275,7 @@ void AccountSettings::slotMarkSubfolderDecrypted(const QByteArray& fileId) { auto job = new OCC::DeleteApiJob(accountsState()->account(), "ocs/v2.php/apps/end_to_end_encryption/api/v1/encrypted/" + QString(fileId)); - connect(job, &OCC::DeleteApiJob::result, [this](int httpResponse) { + connect(job, &OCC::DeleteApiJob::result, [](int httpResponse) { qCInfo(lcAccountSettings) << "Decrypt Http Response" << httpResponse; }); job->start(); From b28b4705ded15baa017d7291e6c9e987ec3c7d27 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 31 Oct 2017 12:07:47 +0100 Subject: [PATCH 059/262] [CSE] Correctly store the encrypted metadata Missing the conversion to base64. --- src/libsync/clientsideencryption.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index ca983a2ce..73c8b7cc1 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -683,7 +683,10 @@ auto metadataKeyDec(const QByteArray& data) -> QByteArray FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata) : _account(account) { if (metadata.isEmpty()) { + qCInfo(lcCse()) << "Setupping Empty Metadata"; setupEmptyMetadata(); + } else { + qCInfo(lcCse()) << "Metadata already exists, deal with it later."; } } @@ -747,7 +750,14 @@ std::string FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKe qCInfo(lcCse()) << "Could not encrypt key."; exit(1); } - qCInfo(lcCse()) << "Data encrypted successfully"; + + // Transform the encrypted data into base64. + const auto raw = QByteArray((const char*) out, outLen); + const auto b64 = raw.toBase64(); + const auto ret = std::string(b64.constData(), b64.length()); + + qCInfo(lcCse()) << raw.toBase64(); + return ret; } std::string FolderMetadata::genMetadataPass() const { @@ -782,7 +792,10 @@ void FolderMetadata::setupEmptyMetadata() { }} }; - qCInfo(lcCse()) << QString::fromStdString(m.dump()); + std::string result = m.dump(); + QString output = QString::fromStdString(result); + + qCInfo(lcCse()) << "Current Output" << output; } } From 3628f3739d1eb88be4d2b7f1a32aa89fc2c58a59 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 31 Oct 2017 13:06:20 +0100 Subject: [PATCH 060/262] [CSE] Start the decryption of the metadata --- src/libsync/clientsideencryption.cpp | 57 ++++++++++++++++++++++++++++ src/libsync/clientsideencryption.h | 2 + 2 files changed, 59 insertions(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 73c8b7cc1..4444dbbfe 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -772,6 +772,63 @@ std::string FolderMetadata::genMetadataPass() const { return result; } +std::string FolderMetadata::decryptMetadataKeys(const std::string& encryptedMetadata) const +{ + qCInfo(lcCse()) << "Starting to decrypt the metadata key"; + unsigned char *out; + unsigned char *in; + size_t outlen; + size_t inlen; + int err = -1; + + auto path = privateKeyPath(_account); + auto pathC = qPrintable(path); + auto pkeyFile = fopen(pathC, "r"); + auto key = PEM_read_PrivateKey(pkeyFile, NULL, NULL, NULL); + + /* NB: assumes key in, inlen are already set up + * and that key is an RSA private key + */ + auto ctx = EVP_PKEY_CTX_new(key, nullptr); + if (!ctx) { + qCInfo(lcCse()) << "Could not create the PKEY context."; + exit(1); + } + + err = EVP_PKEY_decrypt_init(ctx); + if (err <= 0) { + qCInfo(lcCse()) << "Could not init the decryption of the metadata"; + exit(1); + } + + err = EVP_PKEY_CTX_set_rsa_padding(ctx, 1); //TODO: Make this a class variable. + if (err <= 0) { + qCInfo(lcCse()) << "Could not set the RSA padding"; + exit(1); + } + + err = EVP_PKEY_decrypt(ctx, NULL, &outlen, in, inlen); + if (err <= 0) { + qCInfo(lcCse()) << "Could not determine the buffer length"; + exit(1); + } + + out = (unsigned char *) OPENSSL_malloc(outlen); + if (!out) { + qCInfo(lcCse()) << "Could not alloc space for the decrypted metadata"; + exit(1); + } + + err = EVP_PKEY_decrypt(ctx, out, &outlen, in, inlen); + if (err <= 0) { + qCInfo(lcCse()) << "Could not decrypt the metadata"; + exit(1); + } + + qCInfo(lcCse()) << "Metadata decrypted successfully"; + return ""; +} + // AES/GCM/NoPadding (128 bit key size) std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const std::string& pass) const { return obj.dump(); diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index f7eb62f5a..05664de9f 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -193,6 +193,8 @@ private: */ void setupEmptyMetadata(); std::string encryptMetadataKeys(const nlohmann::json& metadataKeys) const; + std::string decryptMetadataKeys(const std::string& encryptedMetadataKeys) const; + std::string genMetadataPass() const; std::string encryptJsonObject(const nlohmann::json& obj, const std::string& pass) const; From 4bb7ebb6aaa6f79557cfed073bf15a3d3b32ccd9 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 31 Oct 2017 13:17:22 +0100 Subject: [PATCH 061/262] [CSE] Convert the base64 data to raw, for the decyrption --- src/libsync/clientsideencryption.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 4444dbbfe..96a4bbce8 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -776,9 +776,7 @@ std::string FolderMetadata::decryptMetadataKeys(const std::string& encryptedMeta { qCInfo(lcCse()) << "Starting to decrypt the metadata key"; unsigned char *out; - unsigned char *in; size_t outlen; - size_t inlen; int err = -1; auto path = privateKeyPath(_account); @@ -786,6 +784,12 @@ std::string FolderMetadata::decryptMetadataKeys(const std::string& encryptedMeta auto pkeyFile = fopen(pathC, "r"); auto key = PEM_read_PrivateKey(pkeyFile, NULL, NULL, NULL); + // Data is base64 encoded. + auto raw = QByteArray(encryptedMetadata.c_str(), encryptedMetadata.length()); + auto b64d = QByteArray::fromBase64(raw); + auto in = (unsigned char *) b64d.constData(); + size_t inlen = b64d.length(); + /* NB: assumes key in, inlen are already set up * and that key is an RSA private key */ From 8793fdbc69f07883a6bd2184d0f1132d1c77e504 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 31 Oct 2017 16:06:01 +0100 Subject: [PATCH 062/262] [CSE] Remember to actually use the correct variable --- src/libsync/clientsideencryption.cpp | 54 +++++++++++++++++++++------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 96a4bbce8..185d2e397 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -21,6 +21,12 @@ #include "wordlist.h" +QDebug operator<<(QDebug out, const std::string& str) +{ + out << QString::fromStdString(str); + return out; +} + namespace OCC { @@ -693,15 +699,15 @@ FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata) : // RSA/ECB/OAEPWithSHA-256AndMGF1Padding using private / public key. std::string FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKeys) const { std::string metadata = metadataKeys.dump(); - const char *metadataPtr = metadata.c_str(); + unsigned char *metadataPtr = (unsigned char *) metadata.c_str(); + size_t metadataPtrLen = metadata.size(); - unsigned char *out; - unsigned char *in; - size_t outLen; - size_t inLen; int err = -1; const int rsaOeapPadding = 1; EVP_PKEY *key = nullptr; + + /*TODO: Verify if we need to setup a RSA engine. + * by default it's RSA OEAP */ ENGINE *eng = nullptr; auto path = publicKeyPath(_account); @@ -733,21 +739,24 @@ std::string FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKe exit(1); } - err = EVP_PKEY_encrypt(ctx, NULL, &outLen, in, inLen); + size_t outLen = 0; + err = EVP_PKEY_encrypt(ctx, NULL, &outLen, metadataPtr, metadataPtrLen); if (err <= 0) { qCInfo(lcCse()) << "Error retrieving the size of the encrypted data"; exit(1); + } else { + qCInfo(lcCse()) << "Encrption Length:" << outLen; } - out = (uchar*) OPENSSL_malloc(outLen); + unsigned char *out = (uchar*) OPENSSL_malloc(outLen); if (!out) { qCInfo(lcCse()) << "Error requesting memory for the encrypted contents"; exit(1); } - err = EVP_PKEY_encrypt(ctx, out, &outLen, in, inLen); + err = EVP_PKEY_encrypt(ctx, out, &outLen, metadataPtr, metadataPtrLen); if (err <= 0) { - qCInfo(lcCse()) << "Could not encrypt key."; + qCInfo(lcCse()) << "Could not encrypt key." << err; exit(1); } @@ -775,21 +784,27 @@ std::string FolderMetadata::genMetadataPass() const { std::string FolderMetadata::decryptMetadataKeys(const std::string& encryptedMetadata) const { qCInfo(lcCse()) << "Starting to decrypt the metadata key"; - unsigned char *out; - size_t outlen; + unsigned char *out = nullptr; + size_t outlen = 0; int err = -1; auto path = privateKeyPath(_account); auto pathC = qPrintable(path); auto pkeyFile = fopen(pathC, "r"); auto key = PEM_read_PrivateKey(pkeyFile, NULL, NULL, NULL); + if (!key) { + qCInfo(lcCse()) << "Error reading private key"; + } // Data is base64 encoded. + qCInfo(lcCse()) << "encryptedMetadata" << encryptedMetadata; auto raw = QByteArray(encryptedMetadata.c_str(), encryptedMetadata.length()); auto b64d = QByteArray::fromBase64(raw); auto in = (unsigned char *) b64d.constData(); size_t inlen = b64d.length(); + qCInfo(lcCse()) << "Encrypted metadata length: " << inlen; + /* NB: assumes key in, inlen are already set up * and that key is an RSA private key */ @@ -815,6 +830,8 @@ std::string FolderMetadata::decryptMetadataKeys(const std::string& encryptedMeta if (err <= 0) { qCInfo(lcCse()) << "Could not determine the buffer length"; exit(1); + } else { + qCInfo(lcCse()) << "Size of output is: " << outlen; } out = (unsigned char *) OPENSSL_malloc(outlen); @@ -830,7 +847,9 @@ std::string FolderMetadata::decryptMetadataKeys(const std::string& encryptedMeta } qCInfo(lcCse()) << "Metadata decrypted successfully"; - return ""; + const auto ret = std::string((char*) out, outlen); + + return ret; } // AES/GCM/NoPadding (128 bit key size) @@ -841,8 +860,19 @@ std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const st void FolderMetadata::setupEmptyMetadata() { using namespace nlohmann; std::string newMetadataPass = genMetadataPass(); + qCInfo(lcCse()) << "Key Generated for the Metadata" << newMetadataPass; + json metadataKeyObj = {"0", newMetadataPass}; json recepient = {"recipient", {}}; + + auto b64String = encryptMetadataKeys(metadataKeyObj); + + qCInfo(lcCse()) << "========================================="; + qCInfo(lcCse()) << "Original:" << metadataKeyObj.dump(); + qCInfo(lcCse()) << "Encrypted:" << b64String; + qCInfo(lcCse()) << "Decrypted" << decryptMetadataKeys(b64String); + qCInfo(lcCse()) << "=========================================="; + json m = { {"metadata", { {"metadataKeys", encryptMetadataKeys(metadataKeyObj)}, From eb43fa1459253f0b1d557cc6fa32b91307b4e6f6 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 31 Oct 2017 17:52:01 +0100 Subject: [PATCH 063/262] [CSE] Internal metadata encryption working --- src/libsync/clientsideencryption.cpp | 43 ++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 185d2e397..07d5344ed 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -854,7 +854,40 @@ std::string FolderMetadata::decryptMetadataKeys(const std::string& encryptedMeta // AES/GCM/NoPadding (128 bit key size) std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const std::string& pass) const { - return obj.dump(); + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + if (!ctx) { + qCInfo(lcCse()) << "Coult not create encryption context, aborting."; + exit(1); + } + unsigned char *iv = (unsigned char *)"0123456789012345"; + auto key = (const unsigned char*) pass.c_str(); + int err = EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, key, iv); + if (err != 1) { + qCInfo(lcCse()) << "Error initializing the encryption, aborting."; + exit(1); + } + + std::string metadata = obj.dump(); + int metadataLen = metadata.size(); + int outLen = 0; + unsigned char *metadataPtr = (unsigned char *) metadata.c_str(); + + qCInfo(lcCse()) << "Metadata to be encrypted" << metadata; + qCInfo(lcCse()) << "Metadata length: " << metadataLen; + + err = EVP_EncryptUpdate(ctx, NULL, &outLen, metadataPtr, metadataLen); + if (err != 1) { + qCInfo(lcCse()) << "Error getting the size of the output text, aborting."; + exit(1); + } + auto out = (unsigned char *) OPENSSL_malloc(outLen); + + err = EVP_EncryptUpdate(ctx, out, &outLen, metadataPtr, metadataLen); + if (err != 1) { + qCInfo(lcCse()) << "Error encrypting the metadata, aborting."; + exit(1); + } + qCInfo(lcCse()) << "Successfully encrypted the internal json blob."; } void FolderMetadata::setupEmptyMetadata() { @@ -867,15 +900,9 @@ void FolderMetadata::setupEmptyMetadata() { auto b64String = encryptMetadataKeys(metadataKeyObj); - qCInfo(lcCse()) << "========================================="; - qCInfo(lcCse()) << "Original:" << metadataKeyObj.dump(); - qCInfo(lcCse()) << "Encrypted:" << b64String; - qCInfo(lcCse()) << "Decrypted" << decryptMetadataKeys(b64String); - qCInfo(lcCse()) << "=========================================="; - json m = { {"metadata", { - {"metadataKeys", encryptMetadataKeys(metadataKeyObj)}, + {"metadataKeys", b64String}, {"sharing", encryptJsonObject(recepient, newMetadataPass)}, {"version",1} }}, From 88d87bf0ca2d4a21e008b4930d98a69698eda5b3 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 1 Nov 2017 14:42:39 +0100 Subject: [PATCH 064/262] [CSE] Convert encrypted bitearray to Base64 also, start the decrypt. --- src/libsync/clientsideencryption.cpp | 12 ++++++++++++ src/libsync/clientsideencryption.h | 1 + 2 files changed, 13 insertions(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 07d5344ed..201b9f3ff 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -888,6 +888,18 @@ std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const st exit(1); } qCInfo(lcCse()) << "Successfully encrypted the internal json blob."; + + // Transform the encrypted data into base64. + const auto raw = QByteArray((const char*) out, outLen); + const auto b64 = raw.toBase64(); + const auto ret = std::string(b64.constData(), b64.length()); + + qCInfo(lcCse()) << raw.toBase64(); + return ret; +} + +std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetadata, const std::string& pass) const +{ } void FolderMetadata::setupEmptyMetadata() { diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 05664de9f..7d55adf42 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -197,6 +197,7 @@ private: std::string genMetadataPass() const; std::string encryptJsonObject(const nlohmann::json& obj, const std::string& pass) const; + std::string decryptJsonObject(const std::string& encryptedJsonBlob, const std::string& pass) const; QVector _files; QVector _metadataKeys; From 77ec3b086d1f4ab49b1ef534a6d9f9dfb9e8d2b9 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 1 Nov 2017 15:24:19 +0100 Subject: [PATCH 065/262] [CSE] Remember to finalize encryption --- src/libsync/clientsideencryption.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 201b9f3ff..17f3d1193 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -870,6 +870,7 @@ std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const st std::string metadata = obj.dump(); int metadataLen = metadata.size(); int outLen = 0; + unsigned char *metadataPtr = (unsigned char *) metadata.c_str(); qCInfo(lcCse()) << "Metadata to be encrypted" << metadata; @@ -888,9 +889,18 @@ std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const st exit(1); } qCInfo(lcCse()) << "Successfully encrypted the internal json blob."; + int totalOutputLen = outLen; + + qCInfo(lcCse()) << "Current output length: " << totalOutputLen; + err = EVP_EncryptFinal_ex(ctx, out + outLen, &outLen); + if (err != 1) { + qCInfo(lcCse()) << "Error finalyzing the encryption."; + } + totalOutputLen += outLen; + qCInfo(lcCse()) << "Final output length: " << totalOutputLen; // Transform the encrypted data into base64. - const auto raw = QByteArray((const char*) out, outLen); + const auto raw = QByteArray((const char*) out, totalOutputLen); const auto b64 = raw.toBase64(); const auto ret = std::string(b64.constData(), b64.length()); From 41ebcd0b7e51ecd3d738505e6e41973088ee5503 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 1 Nov 2017 15:24:34 +0100 Subject: [PATCH 066/262] [CSE] Decryption of the metadata blob --- src/libsync/clientsideencryption.cpp | 44 ++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 17f3d1193..f026140c6 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -910,6 +910,50 @@ std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const st std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetadata, const std::string& pass) const { + // Jesus, should I encrypt here in 128kb chunks? + // perhaps. + + // Data is base64 encoded. + // TODO: Transform this bit into a function to remove duplicated code. + qCInfo(lcCse()) << "encryptedMetadata" << encryptedMetadata; + auto raw = QByteArray(encryptedMetadata.c_str(), encryptedMetadata.length()); + auto b64d = QByteArray::fromBase64(raw); + auto in = (unsigned char *) b64d.constData(); + size_t inlen = b64d.length(); + + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + if (!ctx) { + qCInfo(lcCse()) << "Coult not create decryptioncontext, aborting."; + exit(1); + } + unsigned char *iv = (unsigned char *)"0123456789012345"; + auto key = (const unsigned char*) pass.c_str(); + int err = EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, key, iv); + if (err != 1) { + qCInfo(lcCse()) << "Error initializing the decryption, aborting."; + exit(1); + } + + int outlen = 0; + err = EVP_DecryptUpdate(ctx, NULL, &outlen, in, inlen); + if (err != 1) { + qCInfo(lcCse()) << "Error retrieving the length of the decryption text, aborting."; + exit(1); + } + + auto out = (unsigned char *) OPENSSL_malloc(outlen); + err = EVP_DecryptUpdate(ctx, out, &outlen, in, inlen); + if (err != 1) { + qCInfo(lcCse()) << "Error decrypting the json blob, aborting."; + exit(1); + } + + err = EVP_DecryptFinal(ctx, out + outlen, &outlen); + if (err != 1) { + qCInfo(lcCse()) << "Error finalyzing the decryption, aborting."; + exit(1); + } + qCInfo(lcCse()) << "Decryption finalized."; } void FolderMetadata::setupEmptyMetadata() { From 1a891423e5788f190a0df2fcbd9dc44407287545 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 1 Nov 2017 15:48:40 +0100 Subject: [PATCH 067/262] [CSE] Call the decryption function to test. --- src/libsync/clientsideencryption.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index f026140c6..7c8146b27 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -947,6 +947,7 @@ std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetada qCInfo(lcCse()) << "Error decrypting the json blob, aborting."; exit(1); } + qCInfo(lcCse()) << "currently decrypted" << std::string( (char*) out, outlen); err = EVP_DecryptFinal(ctx, out + outlen, &outlen); if (err != 1) { @@ -954,6 +955,8 @@ std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetada exit(1); } qCInfo(lcCse()) << "Decryption finalized."; + const auto ret = std::string((char*) out, outlen); + return ret; } void FolderMetadata::setupEmptyMetadata() { @@ -966,10 +969,19 @@ void FolderMetadata::setupEmptyMetadata() { auto b64String = encryptMetadataKeys(metadataKeyObj); + auto sharingEncrypted = encryptJsonObject(recepient, newMetadataPass); + auto sharingDecrypted = decryptJsonObject(sharingEncrypted, newMetadataPass); + + qCInfo(lcCse()) << "====================="; + qCInfo(lcCse()) << "Original Json blob:" << recepient.dump(); + qCInfo(lcCse()) << "encrypted json blob:" << sharingEncrypted; + qCInfo(lcCse()) << "decrypted json blob:" << sharingDecrypted; + qCInfo(lcCse()) << "===================="; + json m = { {"metadata", { {"metadataKeys", b64String}, - {"sharing", encryptJsonObject(recepient, newMetadataPass)}, + {"sharing", sharingEncrypted}, {"version",1} }}, {"files", { From ef2529ca440d64dc9638b7526fd7c80bc1daf87d Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 1 Nov 2017 16:13:17 +0100 Subject: [PATCH 068/262] [CSE] Set padding to 0 for the Rsa encryption Also, commented out the finalization of the decrypt operation because that was messing with the encryption. There's something wrong here but I need to get this working and I can fix stuff later. --- src/libsync/clientsideencryption.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 7c8146b27..94b4056eb 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -474,6 +474,7 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) cryptedText, // unsigned char *ciphertext, tag // unsigned char *tag ); + /* qCInfo(lcCse()) << "Encrypted Text" << QByteArray( (const char*) cryptedText, cryptedText_len); int decryptedText_len = decrypt( @@ -488,6 +489,7 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) ); qCInfo(lcCse()) << "Decrypted Text" << QByteArray( (const char*) decryptedText, decryptedText_len); */ + // Pretend that the private key is actually encrypted and send it to the server. auto job = new StorePrivateKeyApiJob(_account, baseUrl() + "private-key", this); job->setPrivateKey(QByteArray((const char*) cryptedText, 128)); @@ -859,6 +861,8 @@ std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const st qCInfo(lcCse()) << "Coult not create encryption context, aborting."; exit(1); } + EVP_CIPHER_CTX_set_padding(ctx, 0); + unsigned char *iv = (unsigned char *)"0123456789012345"; auto key = (const unsigned char*) pass.c_str(); int err = EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, key, iv); @@ -926,6 +930,8 @@ std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetada qCInfo(lcCse()) << "Coult not create decryptioncontext, aborting."; exit(1); } + EVP_CIPHER_CTX_set_padding(ctx, 0); + unsigned char *iv = (unsigned char *)"0123456789012345"; auto key = (const unsigned char*) pass.c_str(); int err = EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, key, iv); @@ -948,12 +954,13 @@ std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetada exit(1); } qCInfo(lcCse()) << "currently decrypted" << std::string( (char*) out, outlen); + qCInfo(lcCse()) << "Current decrypt length" << outlen; - err = EVP_DecryptFinal(ctx, out + outlen, &outlen); - if (err != 1) { - qCInfo(lcCse()) << "Error finalyzing the decryption, aborting."; - exit(1); - } +// err = EVP_DecryptFinal_ex(ctx, out + outlen, &outlen); +// if (err != 1) { +// qCInfo(lcCse()) << "Error finalyzing the decryption, aborting."; +// exit(1); +// } qCInfo(lcCse()) << "Decryption finalized."; const auto ret = std::string((char*) out, outlen); return ret; From e5fdcd2f38ad9950776e50077b368688edbc63e9 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 1 Nov 2017 16:48:19 +0100 Subject: [PATCH 069/262] [CSE] Add TODO: --- src/libsync/clientsideencryption.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 94b4056eb..d717bca22 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -956,6 +956,7 @@ std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetada qCInfo(lcCse()) << "currently decrypted" << std::string( (char*) out, outlen); qCInfo(lcCse()) << "Current decrypt length" << outlen; +// TODO: figure out why this is not working. // err = EVP_DecryptFinal_ex(ctx, out + outlen, &outlen); // if (err != 1) { // qCInfo(lcCse()) << "Error finalyzing the decryption, aborting."; From 1b1add5ead05441511c698f3e9744ca8d360513c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 1 Nov 2017 17:36:54 +0100 Subject: [PATCH 070/262] [CSE] Add api to lock file --- src/gui/accountsettings.cpp | 5 +++++ src/libsync/clientsideencryption.cpp | 32 ++++++++++++++++++++++++++++ src/libsync/clientsideencryption.h | 26 +++++++++++++++++++++- 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index a2a4edf0f..cd224c563 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -265,6 +265,11 @@ void AccountSettings::slotMarkSubfolderEncrpted(const QByteArray& fileId) // prepare and send the metadata to the folder if (httpResponse == 200) { + /* TODO: Last preparations for today: + * Lock the folder + * Send the metadata + * Unlock the folder. + */ FolderMetadata emptyMetadata(accountsState()->account()); } }); diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index d717bca22..e7dc1f1a1 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1002,4 +1002,36 @@ void FolderMetadata::setupEmptyMetadata() { qCInfo(lcCse()) << "Current Output" << output; } + +LockEncryptFolderApiJob::LockEncryptFolderApiJob(const AccountPtr& account, const QString& fileId, QObject* parent) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("lock/") + fileId, parent), _fileId(fileId) +{ +} + +void LockEncryptFolderApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) + }; + url.setQueryItems(params); + + qCInfo(lcCseJob()) << "locking the folder with id" << _fileId << "as encrypted"; + sendRequest("POST", url, req); + AbstractNetworkJob::start(); +} + +bool LockEncryptFolderApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) + qCInfo(lcCseJob()) << "error locking file" << path() << errorString() << retCode; + + QJsonParseError error; + auto json = QJsonDocument::fromJson(reply()->readAll(), &error); + emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); +} + } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 7d55adf42..61cc3282c 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -183,10 +183,10 @@ class FolderMetadata { public: FolderMetadata(AccountPtr account, const QByteArray& metadata = QByteArray()); QByteArray encryptedMetadata(); - void addEncryptedFile(const EncryptedFile& f); QVector files() const; + private: /* Use std::string and std::vector internally on this class * to ease the port to Nlohmann Json API @@ -204,5 +204,29 @@ private: AccountPtr _account; }; +class OWNCLOUDSYNC_EXPORT LockEncryptFolderApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit LockEncryptFolderApiJob(const AccountPtr &account, const QString& fileId, QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + + /** + * @brief jsonReceived - signal to report the json answer from ocs + * @param json - the parsed json document + * @param statusCode - the OCS status code: 200 for success + */ + void jsonReceived(const QJsonDocument &json, int statusCode); +private: + QString _fileId; +}; + } // namespace OCC #endif From 26987595252ddca1f235266bdc63fa5ae8c5da27 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 1 Nov 2017 17:54:17 +0100 Subject: [PATCH 071/262] [CSE] Implement the Folder Unlock api job --- src/libsync/clientsideencryption.cpp | 37 ++++++++++++++++++++++++++++ src/libsync/clientsideencryption.h | 29 ++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index e7dc1f1a1..ec4c0a113 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1032,6 +1032,43 @@ bool LockEncryptFolderApiJob::finished() QJsonParseError error; auto json = QJsonDocument::fromJson(reply()->readAll(), &error); emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); + //TODO: Parse the token and submit. +} + + +UnlockEncryptFolderApiJob::UnlockEncryptFolderApiJob(const AccountPtr& account, + const QString& fileId, + const QString& token, + QObject* parent) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("lock/") + fileId, parent), _fileId(fileId), _token(token) +{ +} + +void UnlockEncryptFolderApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")), + qMakePair(QString::fromLatin1("token"), _token) + }; + url.setQueryItems(params); + + qCInfo(lcCseJob()) << "unlocking the folder with id" << _fileId << "as encrypted"; + sendRequest("DELETE", url, req); + AbstractNetworkJob::start(); +} + +bool UnlockEncryptFolderApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) + qCInfo(lcCseJob()) << "error unlocking file" << path() << errorString() << retCode; + + QJsonParseError error; + auto json = QJsonDocument::fromJson(reply()->readAll(), &error); + emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); } } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 61cc3282c..b6f3e6143 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -228,5 +228,34 @@ private: QString _fileId; }; +class OWNCLOUDSYNC_EXPORT UnlockEncryptFolderApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit UnlockEncryptFolderApiJob ( + const AccountPtr &account, + const QString& fileId, + const QString& token, + QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + + /** + * @brief jsonReceived - signal to report the json answer from ocs + * @param json - the parsed json document + * @param statusCode - the OCS status code: 200 for success + */ + void jsonReceived(const QJsonDocument &json, int statusCode); +private: + QString _fileId; + QString _token; +}; + } // namespace OCC #endif From 4a66cf11d235712fcb296d3afc39008fd05de763 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 1 Nov 2017 18:21:30 +0100 Subject: [PATCH 072/262] [CSE] Send Metadata to the server --- src/libsync/clientsideencryption.cpp | 38 ++++++++++++++++++++++++++++ src/libsync/clientsideencryption.h | 25 ++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index ec4c0a113..5859e5a5b 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1071,4 +1071,42 @@ bool UnlockEncryptFolderApiJob::finished() emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); } +StoreMetaDataApiJob::StoreMetaDataApiJob(const AccountPtr& account, + const QString& fileId, + const QByteArray& b64Metadata, + QObject* parent) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("meta-data/") + fileId, parent), _fileId(fileId), _b64Metadata(b64Metadata) +{ +} + +void StoreMetaDataApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")), + }; + url.setQueryItems(params); + + QByteArray data = QByteArray("metaData=") + _b64Metadata; + QBuffer buffer; + buffer.setData(data); + + qCInfo(lcCseJob()) << "sending the metadata for the fileId" << _fileId << "as encrypted"; + sendRequest("POST", url, req, &buffer); + AbstractNetworkJob::start(); +} + +bool StoreMetaDataApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) + qCInfo(lcCseJob()) << "error sending the metadata" << path() << errorString() << retCode; + + QJsonParseError error; + auto json = QJsonDocument::fromJson(reply()->readAll(), &error); + emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); +} + } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index b6f3e6143..395fe4b0d 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -257,5 +257,30 @@ private: QString _token; }; +class OWNCLOUDSYNC_EXPORT StoreMetaDataApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit StoreMetaDataApiJob ( + const AccountPtr &account, + const QString& fileId, + const QByteArray& b64Metadata, + QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + void jsonReceived(const QJsonDocument &json, int statusCode); + +private: + QString _fileId; + QByteArray _b64Metadata; +}; + + } // namespace OCC #endif From 8d537fdd3c681fcd68ef0324cf5b7c55e9ed27ee Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 1 Nov 2017 18:32:33 +0100 Subject: [PATCH 073/262] [CSE] Get Metadata from the server --- src/libsync/clientsideencryption.cpp | 32 ++++++++++++++++++++++++++++ src/libsync/clientsideencryption.h | 21 ++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 5859e5a5b..2eaecaea7 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1109,4 +1109,36 @@ bool StoreMetaDataApiJob::finished() emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); } +GetMetadataApiJob::GetMetadataApiJob(const AccountPtr& account, + const QString& fileId, + QObject* parent) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("meta-data/") + fileId, parent), _fileId(fileId) +{ +} + +void GetMetadataApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")), + }; + url.setQueryItems(params); + + qCInfo(lcCseJob()) << "Requesting the metadata for the fileId" << _fileId << "as encrypted"; + sendRequest("GET", url, req); + AbstractNetworkJob::start(); +} + +bool GetMetadataApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) + qCInfo(lcCseJob()) << "error requesting the metadata the metadata" << path() << errorString() << retCode; + + QJsonParseError error; + auto json = QJsonDocument::fromJson(reply()->readAll(), &error); + emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); +} } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 395fe4b0d..6959110c1 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -281,6 +281,27 @@ private: QByteArray _b64Metadata; }; +class OWNCLOUDSYNC_EXPORT GetMetadataApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit GetMetadataApiJob ( + const AccountPtr &account, + const QString& fileId, + QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + void jsonReceived(const QJsonDocument &json, int statusCode); + +private: + QString _fileId; +}; } // namespace OCC #endif From 8e3e3a4575c8999fd7b39749dc27bed9c75c42c9 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Thu, 2 Nov 2017 12:39:42 +0100 Subject: [PATCH 074/262] Be java compatible and store the tag at the end of the cipher text Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 36 +++++++++++++++++++++------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 2eaecaea7..1ed114955 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -885,7 +885,8 @@ std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const st qCInfo(lcCse()) << "Error getting the size of the output text, aborting."; exit(1); } - auto out = (unsigned char *) OPENSSL_malloc(outLen); + //+16 since we append the tag + auto out = (unsigned char *) OPENSSL_malloc(outLen + 16); err = EVP_EncryptUpdate(ctx, out, &outLen, metadataPtr, metadataLen); if (err != 1) { @@ -903,6 +904,17 @@ std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const st totalOutputLen += outLen; qCInfo(lcCse()) << "Final output length: " << totalOutputLen; + /* Get the tag */ + unsigned char tag[16]; + if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag)) { + qCInfo(lcCse()) << "Error Retrieving the tag"; + handleErrors(); + } + + /* Add tag to cypher text to be compatible with the Android implementation */ + memcpy(out + totalOutputLen, tag, 16); + totalOutputLen += 16; + // Transform the encrypted data into base64. const auto raw = QByteArray((const char*) out, totalOutputLen); const auto b64 = raw.toBase64(); @@ -923,7 +935,9 @@ std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetada auto raw = QByteArray(encryptedMetadata.c_str(), encryptedMetadata.length()); auto b64d = QByteArray::fromBase64(raw); auto in = (unsigned char *) b64d.constData(); - size_t inlen = b64d.length(); + // The tag is appended but it is not part of the cipher text + size_t inlen = b64d.length() - 16; + auto tag = in + inlen; EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); if (!ctx) { @@ -956,12 +970,18 @@ std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetada qCInfo(lcCse()) << "currently decrypted" << std::string( (char*) out, outlen); qCInfo(lcCse()) << "Current decrypt length" << outlen; -// TODO: figure out why this is not working. -// err = EVP_DecryptFinal_ex(ctx, out + outlen, &outlen); -// if (err != 1) { -// qCInfo(lcCse()) << "Error finalyzing the decryption, aborting."; -// exit(1); -// } + /* Set expected tag value. Works in OpenSSL 1.0.1d and later */ + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag)) { + qCInfo(lcCse()) << "Error decrypting the json blob, aborting."; + exit(1); + } + + err = EVP_DecryptFinal_ex(ctx, out + outlen, &outlen); + if (err != 1) { + qCInfo(lcCse()) << "Error finalyzing the decryption, aborting."; + exit(1); + } + qCInfo(lcCse()) << "Decryption finalized."; const auto ret = std::string((char*) out, outlen); return ret; From d7e05c9b056146ebdbbcc6467411147b692d9d8a Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 3 Nov 2017 11:20:44 +0100 Subject: [PATCH 075/262] [CSE] Comment out broken code. But why it's broken? --- src/libsync/clientsideencryption.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 1ed114955..2e7ab7d7d 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -976,11 +976,11 @@ std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetada exit(1); } - err = EVP_DecryptFinal_ex(ctx, out + outlen, &outlen); - if (err != 1) { - qCInfo(lcCse()) << "Error finalyzing the decryption, aborting."; - exit(1); - } +// // err = EVP_DecryptFinal_ex(ctx, out + outlen, &outlen); +// // if (err != 1) { +// // qCInfo(lcCse()) << "Error finalyzing the decryption, aborting."; +// // exit(1); +// // } qCInfo(lcCse()) << "Decryption finalized."; const auto ret = std::string((char*) out, outlen); From 3b157caf09bbee12c3625757ea1ddfc792c15ced Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 3 Nov 2017 11:21:01 +0100 Subject: [PATCH 076/262] [CSE] Handle encrypt / lock / unlock --- src/gui/accountsettings.cpp | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index cd224c563..117c08b59 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -257,9 +257,12 @@ void AccountSettings::doExpand() void AccountSettings::slotMarkSubfolderEncrpted(const QByteArray& fileId) { + //TODO: No good, we are three levels down in signal/slot hell. + // as soon as this works, rework this function to be way less + // deep. auto job = new OCC::SetEncryptionFlagApiJob(accountsState()->account(), QString(fileId)); connect(job, &OCC::SetEncryptionFlagApiJob::jsonReceived, - [this](const QJsonDocument& json, int httpResponse) { + [this, &fileId](const QJsonDocument& json, int httpResponse) { Q_UNUSED(json); qCInfo(lcAccountSettings) << "Encrypt Http Response" << httpResponse; @@ -270,7 +273,32 @@ void AccountSettings::slotMarkSubfolderEncrpted(const QByteArray& fileId) * Send the metadata * Unlock the folder. */ - FolderMetadata emptyMetadata(accountsState()->account()); + auto lockJob = new LockEncryptFolderApiJob(accountsState()->account(), fileId); + //TODO: create the signals locked and error. + connect(lockJob, &LockEncryptFolderApiJob::jsonReceived, + [this, &fileId] (const QJsonDocument& json, int httpResponse) { + qCInfo(lcAccountSettings()) << "Client side encryption lock response"; + qCInfo(lcAccountSettings()) << json; + + if (httpResponse == 200) { + qCInfo(lcAccountSettings()) << "Locked Successfully"; + FolderMetadata emptyMetadata(accountsState()->account()); + + // Send the metadata. + // Currently only try to unlock the folder. + auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), fileId, QString("token here")); + connect(unlockJob, &UnlockEncryptFolderApiJob::jsonReceived, + [this, &fileId] (const QJsonDocument& json, int httpResponse) { + qCInfo(lcAccountSettings()) << "Unlock Folder response"; + qCInfo(lcAccountSettings()) << json; + qCInfo(lcAccountSettings()) << httpResponse; + }); + unlockJob->start(); + } else { + qCInfo(lcAccountSettings()) << "Problem locking."; + } + }); + lockJob->start(); } }); job->start(); From 6facd29663c03b2a35bd3a60d494e36b7f94fc80 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 3 Nov 2017 12:00:25 +0100 Subject: [PATCH 077/262] [CSE] Start to break the lambdas Lambda within a lambda is a terrible idea, Use default signal / slot connections with a method instead. --- src/gui/accountsettings.cpp | 94 ++++++++++++++++------------ src/gui/accountsettings.h | 4 ++ src/libsync/clientsideencryption.cpp | 17 +++-- src/libsync/clientsideencryption.h | 12 ++-- 4 files changed, 68 insertions(+), 59 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 117c08b59..fb0d94060 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -255,52 +255,64 @@ void AccountSettings::doExpand() ui->_folderList->expandToDepth(0); } +void AccountSettings::slotEncryptionFlagSuccess(const QByteArray& fileId) +{ + /* TODO: Last preparations for today: + * Lock the folder + * Send the metadata + * Unlock the folder. + */ + auto lockJob = new LockEncryptFolderApiJob(accountsState()->account(), fileId); + //TODO: create the signals locked and error. + connect(lockJob, &LockEncryptFolderApiJob::jsonReceived, + [this, &fileId] (const QJsonDocument& json, int httpResponse) { + qCInfo(lcAccountSettings()) << "Client side encryption lock response"; + qCInfo(lcAccountSettings()) << json; + + if (httpResponse == 200) { + qCInfo(lcAccountSettings()) << "Locked Successfully"; + FolderMetadata emptyMetadata(accountsState()->account()); + + // Send the metadata. + // Currently only try to unlock the folder. + auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), fileId, QString("token here")); + connect(unlockJob, &UnlockEncryptFolderApiJob::jsonReceived, + [this, &fileId] (const QJsonDocument& json, int httpResponse) { + qCInfo(lcAccountSettings()) << "Unlock Folder response"; + qCInfo(lcAccountSettings()) << json; + qCInfo(lcAccountSettings()) << httpResponse; + }); + unlockJob->start(); + } else { + qCInfo(lcAccountSettings()) << "Problem locking."; + } + }); + lockJob->start(); +} + +void AccountSettings::slotEncryptionFlagError(const QByteArray& fileId, int httpErrorCode) +{ + qDebug() << "Error on the encryption flag"; +} + +void AccountSettings::slotLockFolderSuccess(const QByteArray& fileId, const QByteArray &token) +{ + +} + +void AccountSettings::slotLockFolderError(const QByteArray& fileId) +{ + +} + void AccountSettings::slotMarkSubfolderEncrpted(const QByteArray& fileId) { //TODO: No good, we are three levels down in signal/slot hell. // as soon as this works, rework this function to be way less // deep. - auto job = new OCC::SetEncryptionFlagApiJob(accountsState()->account(), QString(fileId)); - connect(job, &OCC::SetEncryptionFlagApiJob::jsonReceived, - [this, &fileId](const QJsonDocument& json, int httpResponse) { - Q_UNUSED(json); - qCInfo(lcAccountSettings) << "Encrypt Http Response" << httpResponse; - - // prepare and send the metadata to the folder - if (httpResponse == 200) { - /* TODO: Last preparations for today: - * Lock the folder - * Send the metadata - * Unlock the folder. - */ - auto lockJob = new LockEncryptFolderApiJob(accountsState()->account(), fileId); - //TODO: create the signals locked and error. - connect(lockJob, &LockEncryptFolderApiJob::jsonReceived, - [this, &fileId] (const QJsonDocument& json, int httpResponse) { - qCInfo(lcAccountSettings()) << "Client side encryption lock response"; - qCInfo(lcAccountSettings()) << json; - - if (httpResponse == 200) { - qCInfo(lcAccountSettings()) << "Locked Successfully"; - FolderMetadata emptyMetadata(accountsState()->account()); - - // Send the metadata. - // Currently only try to unlock the folder. - auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), fileId, QString("token here")); - connect(unlockJob, &UnlockEncryptFolderApiJob::jsonReceived, - [this, &fileId] (const QJsonDocument& json, int httpResponse) { - qCInfo(lcAccountSettings()) << "Unlock Folder response"; - qCInfo(lcAccountSettings()) << json; - qCInfo(lcAccountSettings()) << httpResponse; - }); - unlockJob->start(); - } else { - qCInfo(lcAccountSettings()) << "Problem locking."; - } - }); - lockJob->start(); - } - }); + auto job = new OCC::SetEncryptionFlagApiJob(accountsState()->account(), fileId); + connect(job, &OCC::SetEncryptionFlagApiJob::success, this, &AccountSettings::slotEncryptionFlagSuccess); + connect(job, &OCC::SetEncryptionFlagApiJob::error, this, &AccountSettings::slotEncryptionFlagError); job->start(); } diff --git a/src/gui/accountsettings.h b/src/gui/accountsettings.h index 074b93bb2..1fe885a84 100644 --- a/src/gui/accountsettings.h +++ b/src/gui/accountsettings.h @@ -92,6 +92,10 @@ protected slots: void slotFolderListClicked(const QModelIndex &indx); void doExpand(); void slotLinkActivated(const QString &link); + void slotEncryptionFlagSuccess(const QByteArray &folderId); + void slotEncryptionFlagError(const QByteArray &folderId, int httpReturnCode); + void slotLockFolderSuccess(const QByteArray& folderId, const QByteArray& token); + void slotLockFolderError(const QByteArray &folderId); private: void showConnectionLabel(const QString &message, diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 2e7ab7d7d..3e5ef7069 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -608,7 +608,7 @@ bool StorePrivateKeyApiJob::finished() emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); } -SetEncryptionFlagApiJob::SetEncryptionFlagApiJob(const AccountPtr& account, const QString& fileId, QObject* parent) +SetEncryptionFlagApiJob::SetEncryptionFlagApiJob(const AccountPtr& account, const QByteArray& fileId, QObject* parent) : AbstractNetworkJob(account, baseUrl() + QStringLiteral("encrypted/") + fileId, parent), _fileId(fileId) { } @@ -618,10 +618,6 @@ void SetEncryptionFlagApiJob::start() QNetworkRequest req; req.setRawHeader("OCS-APIREQUEST", "true"); QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) - }; - url.setQueryItems(params); qCInfo(lcCseJob()) << "marking the file with id" << _fileId << "as encrypted"; sendRequest("PUT", url, req); @@ -631,12 +627,13 @@ void SetEncryptionFlagApiJob::start() bool SetEncryptionFlagApiJob::finished() { int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (retCode != 200) + qCInfo(lcCse()) << "Encryption Flag Return" << reply()->readAll(); + if (retCode == 200) { + emit success(_fileId); + } else { qCInfo(lcCseJob()) << "Setting the encrypted flag failed with" << path() << errorString() << retCode; - - QJsonParseError error; - auto json = QJsonDocument::fromJson(reply()->readAll(), &error); - emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); + emit error(_fileId, retCode); + } } /* Test metdata: diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 6959110c1..de3180dc9 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -145,7 +145,7 @@ class OWNCLOUDSYNC_EXPORT SetEncryptionFlagApiJob : public AbstractNetworkJob { Q_OBJECT public: - explicit SetEncryptionFlagApiJob(const AccountPtr &account, const QString& fileId, QObject *parent = 0); + explicit SetEncryptionFlagApiJob(const AccountPtr &account, const QByteArray& fileId, QObject *parent = 0); public slots: void start() override; @@ -154,15 +154,11 @@ protected: bool finished() override; signals: + void success(const QByteArray fileId); + void error(const QByteArray fileId, int httpReturnCode); - /** - * @brief jsonReceived - signal to report the json answer from ocs - * @param json - the parsed json document - * @param statusCode - the OCS status code: 200 for success - */ - void jsonReceived(const QJsonDocument &json, int statusCode); private: - QString _fileId; + QByteArray _fileId; }; /* Generates the Metadata for the folder */ From b53003792f998c4b31124cbad232e068aaf49918 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 3 Nov 2017 12:34:30 +0100 Subject: [PATCH 078/262] [CSE] Removed lambdas for Metadata / Lock / Unlock --- src/gui/accountsettings.cpp | 54 ++++++++++++---------------- src/gui/accountsettings.h | 4 ++- src/libsync/clientsideencryption.cpp | 37 +++++++++++-------- src/libsync/clientsideencryption.h | 36 ++++++++----------- 4 files changed, 63 insertions(+), 68 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index fb0d94060..c269a3af3 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -257,36 +257,11 @@ void AccountSettings::doExpand() void AccountSettings::slotEncryptionFlagSuccess(const QByteArray& fileId) { - /* TODO: Last preparations for today: - * Lock the folder - * Send the metadata - * Unlock the folder. - */ auto lockJob = new LockEncryptFolderApiJob(accountsState()->account(), fileId); - //TODO: create the signals locked and error. - connect(lockJob, &LockEncryptFolderApiJob::jsonReceived, - [this, &fileId] (const QJsonDocument& json, int httpResponse) { - qCInfo(lcAccountSettings()) << "Client side encryption lock response"; - qCInfo(lcAccountSettings()) << json; - - if (httpResponse == 200) { - qCInfo(lcAccountSettings()) << "Locked Successfully"; - FolderMetadata emptyMetadata(accountsState()->account()); - - // Send the metadata. - // Currently only try to unlock the folder. - auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), fileId, QString("token here")); - connect(unlockJob, &UnlockEncryptFolderApiJob::jsonReceived, - [this, &fileId] (const QJsonDocument& json, int httpResponse) { - qCInfo(lcAccountSettings()) << "Unlock Folder response"; - qCInfo(lcAccountSettings()) << json; - qCInfo(lcAccountSettings()) << httpResponse; - }); - unlockJob->start(); - } else { - qCInfo(lcAccountSettings()) << "Problem locking."; - } - }); + connect(lockJob, &LockEncryptFolderApiJob::success, + this, &AccountSettings::slotLockFolderSuccess); + connect(lockJob, &LockEncryptFolderApiJob::error, + this, &AccountSettings::slotLockFolderError); lockJob->start(); } @@ -297,14 +272,31 @@ void AccountSettings::slotEncryptionFlagError(const QByteArray& fileId, int http void AccountSettings::slotLockFolderSuccess(const QByteArray& fileId, const QByteArray &token) { + qCInfo(lcAccountSettings()) << "Locked Successfully"; + FolderMetadata emptyMetadata(accountsState()->account()); + auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), fileId, token); + connect(unlockJob, &UnlockEncryptFolderApiJob::success, + this, &AccountSettings::slotUnlockFolderSuccess); + connect(unlockJob, &UnlockEncryptFolderApiJob::error, + this, &AccountSettings::slotUnlockFolderError); + + unlockJob->start(); } -void AccountSettings::slotLockFolderError(const QByteArray& fileId) +void AccountSettings::slotLockFolderError(const QByteArray& fileId, int httpErrorCode) { - + qCInfo(lcAccountSettings()) << "Locking error" << httpErrorCode; } +void AccountSettings::slotUnlockFolderError(const QByteArray& fileId, int httpErrorCode) +{ + qCInfo(lcAccountSettings()) << "Unlocking error!"; +} +void AccountSettings::slotUnlockFolderSuccess(const QByteArray& fileId) +{ + qCInfo(lcAccountSettings()) << "Unlocking success!"; +} void AccountSettings::slotMarkSubfolderEncrpted(const QByteArray& fileId) { //TODO: No good, we are three levels down in signal/slot hell. diff --git a/src/gui/accountsettings.h b/src/gui/accountsettings.h index 1fe885a84..bd6f1d946 100644 --- a/src/gui/accountsettings.h +++ b/src/gui/accountsettings.h @@ -95,7 +95,9 @@ protected slots: void slotEncryptionFlagSuccess(const QByteArray &folderId); void slotEncryptionFlagError(const QByteArray &folderId, int httpReturnCode); void slotLockFolderSuccess(const QByteArray& folderId, const QByteArray& token); - void slotLockFolderError(const QByteArray &folderId); + void slotLockFolderError(const QByteArray &folderId, int httpReturnCode); + void slotUnlockFolderSuccess(const QByteArray& folderId); + void slotUnlockFolderError(const QByteArray& folderId, int httpReturnCode); private: void showConnectionLabel(const QString &message, diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 3e5ef7069..59bfa6547 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1020,7 +1020,7 @@ void FolderMetadata::setupEmptyMetadata() { } -LockEncryptFolderApiJob::LockEncryptFolderApiJob(const AccountPtr& account, const QString& fileId, QObject* parent) +LockEncryptFolderApiJob::LockEncryptFolderApiJob(const AccountPtr& account, const QByteArray& fileId, QObject* parent) : AbstractNetworkJob(account, baseUrl() + QStringLiteral("lock/") + fileId, parent), _fileId(fileId) { } @@ -1043,19 +1043,25 @@ void LockEncryptFolderApiJob::start() bool LockEncryptFolderApiJob::finished() { int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (retCode != 200) + if (retCode != 200) { qCInfo(lcCseJob()) << "error locking file" << path() << errorString() << retCode; + emit error(_fileId, retCode); + return true; + } QJsonParseError error; auto json = QJsonDocument::fromJson(reply()->readAll(), &error); - emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); + + qCInfo(lcCse()) << "got json:" << json; + //TODO: Parse the token and submit. + emit success(_fileId, "TOKEN"); } UnlockEncryptFolderApiJob::UnlockEncryptFolderApiJob(const AccountPtr& account, - const QString& fileId, - const QString& token, + const QByteArray& fileId, + const QByteArray& token, QObject* parent) : AbstractNetworkJob(account, baseUrl() + QStringLiteral("lock/") + fileId, parent), _fileId(fileId), _token(token) { @@ -1068,10 +1074,12 @@ void UnlockEncryptFolderApiJob::start() QUrl url = Utility::concatUrlPath(account()->url(), path()); QList> params = { qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")), - qMakePair(QString::fromLatin1("token"), _token) }; url.setQueryItems(params); + // TODO: This should be a data parameter. + // qMakePair(QString::fromLatin1("token"), _token) + qCInfo(lcCseJob()) << "unlocking the folder with id" << _fileId << "as encrypted"; sendRequest("DELETE", url, req); AbstractNetworkJob::start(); @@ -1080,16 +1088,17 @@ void UnlockEncryptFolderApiJob::start() bool UnlockEncryptFolderApiJob::finished() { int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (retCode != 200) + if (retCode != 200) { qCInfo(lcCseJob()) << "error unlocking file" << path() << errorString() << retCode; - - QJsonParseError error; - auto json = QJsonDocument::fromJson(reply()->readAll(), &error); - emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); + emit error(_fileId, retCode); + return true; + } + emit success(_fileId); + return true; } StoreMetaDataApiJob::StoreMetaDataApiJob(const AccountPtr& account, - const QString& fileId, + const QByteArray& fileId, const QByteArray& b64Metadata, QObject* parent) : AbstractNetworkJob(account, baseUrl() + QStringLiteral("meta-data/") + fileId, parent), _fileId(fileId), _b64Metadata(b64Metadata) @@ -1127,8 +1136,8 @@ bool StoreMetaDataApiJob::finished() } GetMetadataApiJob::GetMetadataApiJob(const AccountPtr& account, - const QString& fileId, - QObject* parent) + const QByteArray& fileId, + QObject* parent) : AbstractNetworkJob(account, baseUrl() + QStringLiteral("meta-data/") + fileId, parent), _fileId(fileId) { } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index de3180dc9..04f54f562 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -204,7 +204,7 @@ class OWNCLOUDSYNC_EXPORT LockEncryptFolderApiJob : public AbstractNetworkJob { Q_OBJECT public: - explicit LockEncryptFolderApiJob(const AccountPtr &account, const QString& fileId, QObject *parent = 0); + explicit LockEncryptFolderApiJob(const AccountPtr &account, const QByteArray& fileId, QObject *parent = 0); public slots: void start() override; @@ -213,15 +213,11 @@ protected: bool finished() override; signals: + void success(const QByteArray& fileId, const QByteArray& token); + void error(const QByteArray& fileId, int httpdErrorCode); - /** - * @brief jsonReceived - signal to report the json answer from ocs - * @param json - the parsed json document - * @param statusCode - the OCS status code: 200 for success - */ - void jsonReceived(const QJsonDocument &json, int statusCode); private: - QString _fileId; + QByteArray _fileId; }; class OWNCLOUDSYNC_EXPORT UnlockEncryptFolderApiJob : public AbstractNetworkJob @@ -230,8 +226,8 @@ class OWNCLOUDSYNC_EXPORT UnlockEncryptFolderApiJob : public AbstractNetworkJob public: explicit UnlockEncryptFolderApiJob ( const AccountPtr &account, - const QString& fileId, - const QString& token, + const QByteArray& fileId, + const QByteArray& token, QObject *parent = 0); public slots: @@ -241,16 +237,12 @@ protected: bool finished() override; signals: + void success(const QByteArray& fileId); + void error(const QByteArray& fileId, int httpReturnCode); - /** - * @brief jsonReceived - signal to report the json answer from ocs - * @param json - the parsed json document - * @param statusCode - the OCS status code: 200 for success - */ - void jsonReceived(const QJsonDocument &json, int statusCode); private: - QString _fileId; - QString _token; + QByteArray _fileId; + QByteArray _token; }; class OWNCLOUDSYNC_EXPORT StoreMetaDataApiJob : public AbstractNetworkJob @@ -259,7 +251,7 @@ class OWNCLOUDSYNC_EXPORT StoreMetaDataApiJob : public AbstractNetworkJob public: explicit StoreMetaDataApiJob ( const AccountPtr &account, - const QString& fileId, + const QByteArray& fileId, const QByteArray& b64Metadata, QObject *parent = 0); @@ -273,7 +265,7 @@ signals: void jsonReceived(const QJsonDocument &json, int statusCode); private: - QString _fileId; + QByteArray _fileId; QByteArray _b64Metadata; }; @@ -283,7 +275,7 @@ class OWNCLOUDSYNC_EXPORT GetMetadataApiJob : public AbstractNetworkJob public: explicit GetMetadataApiJob ( const AccountPtr &account, - const QString& fileId, + const QByteArray& fileId, QObject *parent = 0); public slots: @@ -296,7 +288,7 @@ signals: void jsonReceived(const QJsonDocument &json, int statusCode); private: - QString _fileId; + QByteArray _fileId; }; } // namespace OCC From ea9e2135a1662033ab3eec30fcb4d41c85ed6ede Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 3 Nov 2017 14:28:13 +0100 Subject: [PATCH 079/262] [CSE] Remove uneeded comments. --- src/gui/accountsettings.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index c269a3af3..8451c6b9e 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -299,9 +299,6 @@ void AccountSettings::slotUnlockFolderSuccess(const QByteArray& fileId) } void AccountSettings::slotMarkSubfolderEncrpted(const QByteArray& fileId) { - //TODO: No good, we are three levels down in signal/slot hell. - // as soon as this works, rework this function to be way less - // deep. auto job = new OCC::SetEncryptionFlagApiJob(accountsState()->account(), fileId); connect(job, &OCC::SetEncryptionFlagApiJob::success, this, &AccountSettings::slotEncryptionFlagSuccess); connect(job, &OCC::SetEncryptionFlagApiJob::error, this, &AccountSettings::slotEncryptionFlagError); From 5514f14e881794ed9ab780537090716a84798848 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 3 Nov 2017 15:12:12 +0100 Subject: [PATCH 080/262] [CSE] Get and send the lock - token --- src/libsync/clientsideencryption.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 59bfa6547..84d80d348 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1051,11 +1051,12 @@ bool LockEncryptFolderApiJob::finished() QJsonParseError error; auto json = QJsonDocument::fromJson(reply()->readAll(), &error); - - qCInfo(lcCse()) << "got json:" << json; + auto obj = json.object().toVariantMap(); + auto token = obj["ocs"].toMap()["data"].toMap()["token"].toByteArray(); + qCInfo(lcCse()) << "got json:" << token; //TODO: Parse the token and submit. - emit success(_fileId, "TOKEN"); + emit success(_fileId, token); } From 832cbef8e721138817020ab64be51d6d6a77a5fe Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 3 Nov 2017 15:43:57 +0100 Subject: [PATCH 081/262] [CMAKE] Bump minimum version to 3.2 Removes a lot of warnings. --- CMakeLists.txt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a8edf7283..f862e8b10 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,9 +1,6 @@ -cmake_minimum_required(VERSION 2.6) -cmake_policy(VERSION 2.8.0) -if(POLICY CMP0020) - cmake_policy(SET CMP0020 NEW) -endif() +cmake_minimum_required(VERSION 3.2) +cmake_policy(VERSION 3.2.0) project(client) From 6ad68520455680da5777f010f7caff69e30b64db Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 3 Nov 2017 16:59:39 +0100 Subject: [PATCH 082/262] [CSE] Cleanup - remove commented metadata. --- src/libsync/clientsideencryption.cpp | 38 ---------------------------- 1 file changed, 38 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 84d80d348..7aa3e331e 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -636,44 +636,6 @@ bool SetEncryptionFlagApiJob::finished() } } -/* Test metdata: -{ - // Metadata about the share - "metadata": { - // Encryption algorithm: RSA/ECB/OAEPWithSHA-256AndMGF1Padding, encrypted via private/public key (asymmetric) - "metadataKeys": { - "0": "OLDESTMETADATAKEY", - "2": "…", - "3": "NEWESTMETADATAKEY" - }, - // Encryption algorithm: AES/GCM/NoPadding (128 bit key size) with metadata key from above (symmetric) - "sharing": { - // Name of recipients as well as public keys of the recipients - "recipient": { - "recipient1@example.com": "PUBLIC KEY", - "recipient2@example.com": "PUBLIC KEY" - }, - }, - "version": 1 - }, - // A JSON blob referencing all files - "files": { - "ia7OEEEyXMoRa1QWQk8r": { - // Encryption algorithm: AES/GCM/NoPadding (128 bit key size) with metadata key from above (symmetric) - "encrypted": { - "key": "jtboLmgGR1OQf2uneqCVHpklQLlIwWL5TXAQ0keK", - "filename": "/foo/test.txt", - "mimetype": "plain/text", - "version": 1 - }, - "initializationVector": "+mHu52HyZq+pAAIN", - "authenticationTag": "GCM authentication tag", - "metadataKey": 1 - } - } -} -*/ - //TODO: Create an actuall encryption here. auto metadataKeyEnc(const QByteArray& data) -> QByteArray { From be9cd358d49cd3efcfbe60eb4b3feabd74b9c289 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 3 Nov 2017 17:00:28 +0100 Subject: [PATCH 083/262] [CSE] Bypass Qt DELETE Bug It appears that Qt implementation of the DELETE http request does not send bodyData, and we need that for Nextcloud. Currently I changed the http request on the server side to accept a POST instead of a DELETE, so I can actually develop. Also, I already poked the Qt developers that did this code. --- src/gui/accountsettings.cpp | 5 ++--- src/libsync/clientsideencryption.cpp | 22 +++++++++++++--------- src/libsync/clientsideencryption.h | 1 + 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 8451c6b9e..d16bae32b 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -272,9 +272,8 @@ void AccountSettings::slotEncryptionFlagError(const QByteArray& fileId, int http void AccountSettings::slotLockFolderSuccess(const QByteArray& fileId, const QByteArray &token) { - qCInfo(lcAccountSettings()) << "Locked Successfully"; - FolderMetadata emptyMetadata(accountsState()->account()); - +// FolderMetadata emptyMetadata(accountsState()->account()); + qCInfo(lcAccountSettings()) << "Folder Locked Successfully" << fileId << token; auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), fileId, token); connect(unlockJob, &UnlockEncryptFolderApiJob::success, this, &AccountSettings::slotUnlockFolderSuccess); diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 7aa3e331e..e12d6a250 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1026,7 +1026,7 @@ UnlockEncryptFolderApiJob::UnlockEncryptFolderApiJob(const AccountPtr& account, const QByteArray& fileId, const QByteArray& token, QObject* parent) -: AbstractNetworkJob(account, baseUrl() + QStringLiteral("lock/") + fileId, parent), _fileId(fileId), _token(token) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("unlock/") + fileId, parent), _fileId(fileId), _token(token) { } @@ -1035,17 +1035,20 @@ void UnlockEncryptFolderApiJob::start() QNetworkRequest req; req.setRawHeader("OCS-APIREQUEST", "true"); QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")), - }; - url.setQueryItems(params); - // TODO: This should be a data parameter. - // qMakePair(QString::fromLatin1("token"), _token) + QByteArray bufferData("token=" + _token); + _tokenBuf = new QBuffer(); + _tokenBuf->setData(bufferData); + + qCInfo(lcCseJob()) << "================"; + qCInfo(lcCseJob()) << "unlocking the folder with id" << _fileId << "with token" << _token; + qCInfo(lcCseJob()) << url; + qCInfo(lcCseJob()) << bufferData; + qCInfo(lcCseJob()) << "==================="; + sendRequest("POST", url, req, _tokenBuf); - qCInfo(lcCseJob()) << "unlocking the folder with id" << _fileId << "as encrypted"; - sendRequest("DELETE", url, req); AbstractNetworkJob::start(); + qCInfo(lcCseJob()) << "Starting the request to unlock."; } bool UnlockEncryptFolderApiJob::finished() @@ -1053,6 +1056,7 @@ bool UnlockEncryptFolderApiJob::finished() int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); if (retCode != 200) { qCInfo(lcCseJob()) << "error unlocking file" << path() << errorString() << retCode; + qCInfo(lcCseJob()) << "Full Error Log" << reply()->readAll(); emit error(_fileId, retCode); return true; } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 04f54f562..a5484cfdf 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -243,6 +243,7 @@ signals: private: QByteArray _fileId; QByteArray _token; + QBuffer *_tokenBuf; }; class OWNCLOUDSYNC_EXPORT StoreMetaDataApiJob : public AbstractNetworkJob From 131fd4e48331f50a5f8c9dc2dcd49e7f08e6ca78 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Fri, 3 Nov 2017 17:58:27 +0100 Subject: [PATCH 084/262] Fix encryptJSON and descryptJSON Now working with tag Basically we called EVP_*Update to much which resulted in weird output. Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 75 ++++++++++++++++++---------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index e12d6a250..2dc15c5a1 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -84,7 +84,7 @@ namespace { /* Initialise key and IV */ if(1 != EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv)) { - qCInfo(lcCse()) << "Error initializing encryption"; + qCInfo(lcCse()) << "Error setting key and iv encryption"; handleErrors(); } @@ -820,13 +820,26 @@ std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const st qCInfo(lcCse()) << "Coult not create encryption context, aborting."; exit(1); } + + if(!EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { + qCInfo(lcCse()) << "Error initializing encryption aborting."; + exit(1); + } + EVP_CIPHER_CTX_set_padding(ctx, 0); + + unsigned char *iv = (unsigned char *)"0123456789012345"; + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL)) { + qCInfo(lcCse()) << "Could not set IV length, aborting."; + exit(1); + } + auto key = (const unsigned char*) pass.c_str(); - int err = EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, key, iv); + int err = EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv); if (err != 1) { - qCInfo(lcCse()) << "Error initializing the encryption, aborting."; + qCInfo(lcCse()) << "Error setting key and IV, aborting."; exit(1); } @@ -839,13 +852,11 @@ std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const st qCInfo(lcCse()) << "Metadata to be encrypted" << metadata; qCInfo(lcCse()) << "Metadata length: " << metadataLen; - err = EVP_EncryptUpdate(ctx, NULL, &outLen, metadataPtr, metadataLen); - if (err != 1) { - qCInfo(lcCse()) << "Error getting the size of the output text, aborting."; - exit(1); - } - //+16 since we append the tag - auto out = (unsigned char *) OPENSSL_malloc(outLen + 16); + /* + * max len is metadtaLen + blocksize (16 bytes) -1 + * Add 16 bytes for the tag + */ + auto out = (unsigned char *) OPENSSL_malloc(metadataLen + 16 + 16 - 1); err = EVP_EncryptUpdate(ctx, out, &outLen, metadataPtr, metadataLen); if (err != 1) { @@ -874,6 +885,8 @@ std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const st memcpy(out + totalOutputLen, tag, 16); totalOutputLen += 16; + qCInfo(lcCse()) << "Final output length (with tag): " << totalOutputLen; + // Transform the encrypted data into base64. const auto raw = QByteArray((const char*) out, totalOutputLen); const auto b64 = raw.toBase64(); @@ -903,24 +916,34 @@ std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetada qCInfo(lcCse()) << "Coult not create decryptioncontext, aborting."; exit(1); } + + /* Initialise the decryption operation. */ + if(!EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { + qCInfo(lcCse()) << "Error initialializing the decryption, aborting."; + exit(1); + } + EVP_CIPHER_CTX_set_padding(ctx, 0); unsigned char *iv = (unsigned char *)"0123456789012345"; + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL)) { + qCInfo(lcCse()) << "Could not set IV length, aborting."; + exit(1); + } + auto key = (const unsigned char*) pass.c_str(); - int err = EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, key, iv); + int err = EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv); if (err != 1) { - qCInfo(lcCse()) << "Error initializing the decryption, aborting."; + qCInfo(lcCse()) << "Error setting the key and iv, aborting."; exit(1); } int outlen = 0; - err = EVP_DecryptUpdate(ctx, NULL, &outlen, in, inlen); - if (err != 1) { - qCInfo(lcCse()) << "Error retrieving the length of the decryption text, aborting."; - exit(1); - } - auto out = (unsigned char *) OPENSSL_malloc(outlen); + /* + * Max outlen is inlen + blocksize (16 bytes) + */ + auto out = (unsigned char *) OPENSSL_malloc(inlen + 16); err = EVP_DecryptUpdate(ctx, out, &outlen, in, inlen); if (err != 1) { qCInfo(lcCse()) << "Error decrypting the json blob, aborting."; @@ -931,15 +954,18 @@ std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetada /* Set expected tag value. Works in OpenSSL 1.0.1d and later */ if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag)) { - qCInfo(lcCse()) << "Error decrypting the json blob, aborting."; + qCInfo(lcCse()) << "Error setting the tag, aborting."; exit(1); } -// // err = EVP_DecryptFinal_ex(ctx, out + outlen, &outlen); -// // if (err != 1) { -// // qCInfo(lcCse()) << "Error finalyzing the decryption, aborting."; -// // exit(1); -// // } + qCInfo(lcCse()) << "Tag: " << tag; + + int f_len = outlen; + err = EVP_DecryptFinal_ex(ctx, out + outlen, &f_len); + if (err != 1) { + qCInfo(lcCse()) << "Error finalyzing the decryption, aborting."; + exit(1); + } qCInfo(lcCse()) << "Decryption finalized."; const auto ret = std::string((char*) out, outlen); @@ -955,7 +981,6 @@ void FolderMetadata::setupEmptyMetadata() { json recepient = {"recipient", {}}; auto b64String = encryptMetadataKeys(metadataKeyObj); - auto sharingEncrypted = encryptJsonObject(recepient, newMetadataPass); auto sharingDecrypted = decryptJsonObject(sharingEncrypted, newMetadataPass); From 7290cf281301bb1ea5caee9af04633d4769851cd Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 6 Nov 2017 20:57:50 +0100 Subject: [PATCH 085/262] [CSE] Adjust the CN accordingly to the server --- src/libsync/clientsideencryption.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 2dc15c5a1..3c33a5402 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -311,12 +311,15 @@ void ClientSideEncryption::generateKeyPair() QString ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) { // OpenSSL expects const char. + auto cnArray = _account->davUser().toLocal8Bit(); + qCInfo(lcCse()) << "Getting the following array for the account Id" << cnArray; + auto certParams = std::map{ {"C", "DE"}, {"ST", "Baden-Wuerttemberg"}, {"L", "Stuttgart"}, {"O","Nextcloud"}, - {"CN", "www.nextcloud.com"} + {"CN", cnArray.constData()} }; int ret = 0; From ee4a848d9a1d1e0657a67a66a9ecb86d116d56a4 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 6 Nov 2017 20:58:30 +0100 Subject: [PATCH 086/262] [CSE] Correctly Unlock the Folder Wireshark is love, and life. --- src/libsync/clientsideencryption.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 3c33a5402..2203033b7 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1054,7 +1054,7 @@ UnlockEncryptFolderApiJob::UnlockEncryptFolderApiJob(const AccountPtr& account, const QByteArray& fileId, const QByteArray& token, QObject* parent) -: AbstractNetworkJob(account, baseUrl() + QStringLiteral("unlock/") + fileId, parent), _fileId(fileId), _token(token) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("lock/") + fileId, parent), _fileId(fileId), _token(token) { } @@ -1062,18 +1062,16 @@ void UnlockEncryptFolderApiJob::start() { QNetworkRequest req; req.setRawHeader("OCS-APIREQUEST", "true"); - QUrl url = Utility::concatUrlPath(account()->url(), path()); + req.setRawHeader("token", _token); - QByteArray bufferData("token=" + _token); - _tokenBuf = new QBuffer(); - _tokenBuf->setData(bufferData); + QUrl url = Utility::concatUrlPath(account()->url(), path()); qCInfo(lcCseJob()) << "================"; qCInfo(lcCseJob()) << "unlocking the folder with id" << _fileId << "with token" << _token; qCInfo(lcCseJob()) << url; - qCInfo(lcCseJob()) << bufferData; + qCInfo(lcCseJob()) << _token; qCInfo(lcCseJob()) << "==================="; - sendRequest("POST", url, req, _tokenBuf); + sendRequest("DELETE", url, req); AbstractNetworkJob::start(); qCInfo(lcCseJob()) << "Starting the request to unlock."; From 685ceacace8d489143a56177d0c45bcd6994b723 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sat, 11 Nov 2017 16:25:38 +0100 Subject: [PATCH 087/262] [CSE] Memleaks --- src/libsync/clientsideencryption.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 2203033b7..f9e95fe52 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -570,6 +570,7 @@ bool SignPublicKeyApiJob::finished() QJsonParseError error; auto json = QJsonDocument::fromJson(reply()->readAll(), &error); emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); + return true; } @@ -609,6 +610,7 @@ bool StorePrivateKeyApiJob::finished() QJsonParseError error; auto json = QJsonDocument::fromJson(reply()->readAll(), &error); emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); + return true; } SetEncryptionFlagApiJob::SetEncryptionFlagApiJob(const AccountPtr& account, const QByteArray& fileId, QObject* parent) @@ -637,6 +639,7 @@ bool SetEncryptionFlagApiJob::finished() qCInfo(lcCseJob()) << "Setting the encrypted flag failed with" << path() << errorString() << retCode; emit error(_fileId, retCode); } + return true; } //TODO: Create an actuall encryption here. @@ -1047,6 +1050,7 @@ bool LockEncryptFolderApiJob::finished() //TODO: Parse the token and submit. emit success(_fileId, token); + return true; } @@ -1065,12 +1069,6 @@ void UnlockEncryptFolderApiJob::start() req.setRawHeader("token", _token); QUrl url = Utility::concatUrlPath(account()->url(), path()); - - qCInfo(lcCseJob()) << "================"; - qCInfo(lcCseJob()) << "unlocking the folder with id" << _fileId << "with token" << _token; - qCInfo(lcCseJob()) << url; - qCInfo(lcCseJob()) << _token; - qCInfo(lcCseJob()) << "==================="; sendRequest("DELETE", url, req); AbstractNetworkJob::start(); @@ -1126,6 +1124,7 @@ bool StoreMetaDataApiJob::finished() QJsonParseError error; auto json = QJsonDocument::fromJson(reply()->readAll(), &error); emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); + return true; } GetMetadataApiJob::GetMetadataApiJob(const AccountPtr& account, @@ -1159,5 +1158,6 @@ bool GetMetadataApiJob::finished() QJsonParseError error; auto json = QJsonDocument::fromJson(reply()->readAll(), &error); emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); + return true; } } From 19d64e630802ab9781924e4956298d962bb606ef Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 12 Nov 2017 12:55:12 +0100 Subject: [PATCH 088/262] [CSE] Remove warnings --- src/libsync/clientsideencryption.cpp | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index f9e95fe52..78d451a1d 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -31,9 +31,9 @@ namespace OCC { Q_LOGGING_CATEGORY(lcCse, "sync.clientsideencryption", QtInfoMsg) -Q_LOGGING_CATEGORY(lcSignPublicKeyApiJob, "sync.networkjob.sendcsr", QtInfoMsg); -Q_LOGGING_CATEGORY(lcStorePrivateKeyApiJob, "sync.networkjob.storeprivatekey", QtInfoMsg); -Q_LOGGING_CATEGORY(lcCseJob, "sync.networkjob.clientsideencrypt", QtInfoMsg); +Q_LOGGING_CATEGORY(lcSignPublicKeyApiJob, "sync.networkjob.sendcsr", QtInfoMsg) +Q_LOGGING_CATEGORY(lcStorePrivateKeyApiJob, "sync.networkjob.storeprivatekey", QtInfoMsg) +Q_LOGGING_CATEGORY(lcCseJob, "sync.networkjob.clientsideencrypt", QtInfoMsg) QString baseUrl(){ return QStringLiteral("ocs/v2.php/apps/end_to_end_encryption/api/v1/"); @@ -461,15 +461,12 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) //: Hardcoed IV, really bad. unsigned char *fakepass = (unsigned char*) "qwertyuiasdfghjkzxcvbnm,qwertyui"; unsigned char *iv = (unsigned char *)"0123456789012345"; - unsigned char encryptTag[16]; - const char *encryptTest = "a quick brown fox jumps over the lazy dog"; // TODO: Find a way to unsigned char cryptedText[128]; - unsigned char decryptedText[128]; - unsigned char tag[16]; - int cryptedText_len = encrypt( + unsigned char tag[16]; + int cryptedText_len = encrypt( (unsigned char*) encryptTest, //unsigned char *plaintext, strlen(encryptTest), // int plaintext_len, fakepass, // unsigned char *key, @@ -477,7 +474,7 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) cryptedText, // unsigned char *ciphertext, tag // unsigned char *tag ); - + Q_UNUSED(cryptedText_len); //TODO: Fix this. /* qCInfo(lcCse()) << "Encrypted Text" << QByteArray( (const char*) cryptedText, cryptedText_len); int decryptedText_len = decrypt( @@ -497,6 +494,7 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) auto job = new StorePrivateKeyApiJob(_account, baseUrl() + "private-key", this); job->setPrivateKey(QByteArray((const char*) cryptedText, 128)); connect(job, &StorePrivateKeyApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { + Q_UNUSED(doc); switch(retCode) { case 200: qCInfo(lcCse()) << "Store private key working as expected."; @@ -519,7 +517,8 @@ void ClientSideEncryption::getPublicKeyFromServer() qCInfo(lcCse()) << "Retrieving public key from server"; auto job = new JsonApiJob(_account, baseUrl() + "public-key", this); connect(job, &JsonApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { - switch(retCode) { + Q_UNUSED(doc); + switch(retCode) { case 404: // no public key qCInfo(lcCse()) << "No public key on the server"; generateKeyPair(); @@ -1153,7 +1152,7 @@ bool GetMetadataApiJob::finished() { int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); if (retCode != 200) - qCInfo(lcCseJob()) << "error requesting the metadata the metadata" << path() << errorString() << retCode; + qCInfo(lcCseJob()) << "error requesting the metadata" << path() << errorString() << retCode; QJsonParseError error; auto json = QJsonDocument::fromJson(reply()->readAll(), &error); From 4755b8c8a3c29f842d52d6cb2e368eeb226e85d4 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 12 Nov 2017 13:03:52 +0100 Subject: [PATCH 089/262] [CSE] More warnings --- src/libsync/clientsideencryption.cpp | 3 +-- src/libsync/clientsideencryption.h | 2 +- src/libsync/networkjobs.cpp | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 78d451a1d..d4347b95a 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -308,7 +308,7 @@ void ClientSideEncryption::generateKeyPair() qCInfo(lcCse()) << "Keys generated correctly, sending to server."; } -QString ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) +void ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) { // OpenSSL expects const char. auto cnArray = _account->davUser().toLocal8Bit(); @@ -395,7 +395,6 @@ QString ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) free_all: X509_REQ_free(x509_req); BIO_free_all(out); - return ""; } void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index a5484cfdf..b2b406140 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -30,7 +30,7 @@ public: bool hasPrivateKey() const; bool hasPublicKey() const; void generateKeyPair(); - QString generateCSR(EVP_PKEY *keyPair); + void generateCSR(EVP_PKEY *keyPair); void getPrivateKeyFromServer(); void getPublicKeyFromServer(); void encryptPrivateKey(EVP_PKEY *keyPair); diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index c2cd851cc..fa7b5e551 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -946,6 +946,7 @@ bool DeleteApiJob::finished() const auto replyData = QString::fromUtf8(reply()->readAll()); qCInfo(lcJsonApiJob()) << "TMX Delete Job" << replyData; + return true; } } // namespace OCC From 22a2ab8999aa730e39c748c3f8010fc35a9edd77 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 13 Nov 2017 16:46:30 +0100 Subject: [PATCH 090/262] [CSE] Start to send the metadata to the server --- src/gui/accountsettings.cpp | 22 +++++++++++++++++++--- src/gui/accountsettings.h | 2 ++ src/libsync/clientsideencryption.cpp | 26 +++++++++++--------------- src/libsync/clientsideencryption.h | 4 +++- 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index d16bae32b..b913fbf53 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -272,9 +272,15 @@ void AccountSettings::slotEncryptionFlagError(const QByteArray& fileId, int http void AccountSettings::slotLockFolderSuccess(const QByteArray& fileId, const QByteArray &token) { -// FolderMetadata emptyMetadata(accountsState()->account()); - qCInfo(lcAccountSettings()) << "Folder Locked Successfully" << fileId << token; - auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), fileId, token); + FolderMetadata emptyMetadata(accountsState()->account()); + + auto storeMetadataJob = new StoreMetaDataApiJob(accountsState()->account(), fileId, emptyMetadata.encryptedMetadata()); + connect(storeMetadataJob, &StoreMetaDataApiJob::success, + this, &AccountSettings::slotUploadMetadataSuccess); + connect(storeMetadataJob, &StoreMetaDataApiJob::error, + this, &AccountSettings::slotUpdateMetadataError); + + auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), fileId, token); connect(unlockJob, &UnlockEncryptFolderApiJob::success, this, &AccountSettings::slotUnlockFolderSuccess); connect(unlockJob, &UnlockEncryptFolderApiJob::error, @@ -283,6 +289,16 @@ void AccountSettings::slotLockFolderSuccess(const QByteArray& fileId, const QByt unlockJob->start(); } +void AccountSettings::slotUploadMetadataSuccess(const QByteArray& folderId) +{ + +} + +void AccountSettings::slotUpdateMetadataError(const QByteArray& folderId, int httpReturnCode) +{ + +} + void AccountSettings::slotLockFolderError(const QByteArray& fileId, int httpErrorCode) { qCInfo(lcAccountSettings()) << "Locking error" << httpErrorCode; diff --git a/src/gui/accountsettings.h b/src/gui/accountsettings.h index bd6f1d946..405fc8b62 100644 --- a/src/gui/accountsettings.h +++ b/src/gui/accountsettings.h @@ -98,6 +98,8 @@ protected slots: void slotLockFolderError(const QByteArray &folderId, int httpReturnCode); void slotUnlockFolderSuccess(const QByteArray& folderId); void slotUnlockFolderError(const QByteArray& folderId, int httpReturnCode); + void slotUploadMetadataSuccess(const QByteArray& folderId); + void slotUpdateMetadataError(const QByteArray& folderId, int httpReturnCode); private: void showConnectionLabel(const QString &message, diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index d4347b95a..0ea8424dc 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -651,7 +651,7 @@ auto metadataKeyDec(const QByteArray& data) -> QByteArray return data; } -FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata) : _account(account) +FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata) : _account(account), _metadata(metadata) { if (metadata.isEmpty()) { qCInfo(lcCse()) << "Setupping Empty Metadata"; @@ -986,13 +986,6 @@ void FolderMetadata::setupEmptyMetadata() { auto b64String = encryptMetadataKeys(metadataKeyObj); auto sharingEncrypted = encryptJsonObject(recepient, newMetadataPass); - auto sharingDecrypted = decryptJsonObject(sharingEncrypted, newMetadataPass); - - qCInfo(lcCse()) << "====================="; - qCInfo(lcCse()) << "Original Json blob:" << recepient.dump(); - qCInfo(lcCse()) << "encrypted json blob:" << sharingEncrypted; - qCInfo(lcCse()) << "decrypted json blob:" << sharingDecrypted; - qCInfo(lcCse()) << "===================="; json m = { {"metadata", { @@ -1006,10 +999,12 @@ void FolderMetadata::setupEmptyMetadata() { std::string result = m.dump(); QString output = QString::fromStdString(result); - - qCInfo(lcCse()) << "Current Output" << output; + _metadata = output.toLocal8Bit(); } +QByteArray FolderMetadata::encryptedMetadata() { + return _metadata; +} LockEncryptFolderApiJob::LockEncryptFolderApiJob(const AccountPtr& account, const QByteArray& fileId, QObject* parent) : AbstractNetworkJob(account, baseUrl() + QStringLiteral("lock/") + fileId, parent), _fileId(fileId) @@ -1116,12 +1111,13 @@ void StoreMetaDataApiJob::start() bool StoreMetaDataApiJob::finished() { int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (retCode != 200) - qCInfo(lcCseJob()) << "error sending the metadata" << path() << errorString() << retCode; + if (retCode != 200) { + qCInfo(lcCseJob()) << "error sending the metadata" << path() << errorString() << retCode; + emit error(_fileId, retCode); + } - QJsonParseError error; - auto json = QJsonDocument::fromJson(reply()->readAll(), &error); - emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); + qCInfo(lcCseJob()) << "Metadata submited to the server successfully"; + emit success(_fileId); return true; } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index b2b406140..01b70a277 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -198,6 +198,7 @@ private: QVector _files; QVector _metadataKeys; AccountPtr _account; + QByteArray _metadata; }; class OWNCLOUDSYNC_EXPORT LockEncryptFolderApiJob : public AbstractNetworkJob @@ -263,7 +264,8 @@ protected: bool finished() override; signals: - void jsonReceived(const QJsonDocument &json, int statusCode); + void success(const QByteArray& fileId); + void error(const QByteArray& fileId, int httpReturnCode); private: QByteArray _fileId; From e32fd58578bb254a47b83f3607b76a76e9b274e7 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 13 Nov 2017 17:03:40 +0100 Subject: [PATCH 091/262] [CSE] handle success and failure of metadata upload. --- src/gui/accountsettings.cpp | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index b913fbf53..3df852492 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -279,24 +279,28 @@ void AccountSettings::slotLockFolderSuccess(const QByteArray& fileId, const QByt this, &AccountSettings::slotUploadMetadataSuccess); connect(storeMetadataJob, &StoreMetaDataApiJob::error, this, &AccountSettings::slotUpdateMetadataError); - - auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), fileId, token); - connect(unlockJob, &UnlockEncryptFolderApiJob::success, - this, &AccountSettings::slotUnlockFolderSuccess); - connect(unlockJob, &UnlockEncryptFolderApiJob::error, - this, &AccountSettings::slotUnlockFolderError); - - unlockJob->start(); } void AccountSettings::slotUploadMetadataSuccess(const QByteArray& folderId) { - + const QByteArray token; // fakeToken, just to compile. + auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), folderId, token); + connect(unlockJob, &UnlockEncryptFolderApiJob::success, + this, &AccountSettings::slotUnlockFolderSuccess); + connect(unlockJob, &UnlockEncryptFolderApiJob::error, + this, &AccountSettings::slotUnlockFolderError); + unlockJob->start(); } void AccountSettings::slotUpdateMetadataError(const QByteArray& folderId, int httpReturnCode) { - + const QByteArray token; // fakeToken, just to compile. + auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), folderId, token); + connect(unlockJob, &UnlockEncryptFolderApiJob::success, + this, &AccountSettings::slotUnlockFolderSuccess); + connect(unlockJob, &UnlockEncryptFolderApiJob::error, + this, &AccountSettings::slotUnlockFolderError); + unlockJob->start(); } void AccountSettings::slotLockFolderError(const QByteArray& fileId, int httpErrorCode) From 6351c01ee7c3ef43c32e725124b7a557bb41a514 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 13 Nov 2017 17:04:02 +0100 Subject: [PATCH 092/262] [CSE] Remember the token for folders --- src/libsync/clientsideencryption.cpp | 11 +++++++++++ src/libsync/clientsideencryption.h | 3 +++ 2 files changed, 14 insertions(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 0ea8424dc..bf913cb62 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -397,6 +397,17 @@ free_all: BIO_free_all(out); } +void ClientSideEncryption::setTokenForFolder(const QByteArray& folderId, const QByteArray& token) +{ + _folder2token[folderId] = token; +} + +QByteArray ClientSideEncryption::tokenForFolder(const QByteArray& folderId) const +{ + Q_ASSERT(_folder2token.contains(folderId)); + return _folder2token[folderId]; +} + void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) { // Write the Private File to a BIO diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 01b70a277..b8dc386af 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -34,6 +34,8 @@ public: void getPrivateKeyFromServer(); void getPublicKeyFromServer(); void encryptPrivateKey(EVP_PKEY *keyPair); + void setTokenForFolder(const QByteArray& folder, const QByteArray& token); + QByteArray tokenForFolder(const QByteArray& folder) const; signals: void initializationFinished(); @@ -41,6 +43,7 @@ signals: private: OCC::AccountPtr _account; bool isInitialized = false; + QMap _folder2token; }; /* From a0f0e5617ae4612d4fc974c430a0b1ef42e93bb5 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 13 Nov 2017 17:22:09 +0100 Subject: [PATCH 093/262] [CSE] Move the e2e object to the Account This is important as a lot of the code would start to rely in direct access to the client side encryption and there are different keys for different accounts. --- src/libsync/account.cpp | 9 +++++++++ src/libsync/account.h | 4 ++++ src/libsync/connectionvalidator.cpp | 5 ++--- src/libsync/connectionvalidator.h | 1 - 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/libsync/account.cpp b/src/libsync/account.cpp index d8b957e16..4bf97572a 100644 --- a/src/libsync/account.cpp +++ b/src/libsync/account.cpp @@ -50,9 +50,18 @@ AccountPtr Account::create() { AccountPtr acc = AccountPtr(new Account); acc->setSharedThis(acc); + + //TODO: This probably needs to have a better + // coupling, but it should work for now. + acc->e2e().setAccount(acc); return acc; } +ClientSideEncryption& Account::e2e() +{ + return _e2e; +} + Account::~Account() { } diff --git a/src/libsync/account.h b/src/libsync/account.h index f172e5476..e538b3689 100644 --- a/src/libsync/account.h +++ b/src/libsync/account.h @@ -226,6 +226,8 @@ public: /// Called by network jobs on credential errors, emits invalidCredentials() void handleInvalidCredentials(); + ClientSideEncryption& e2e(); + public slots: /// Used when forgetting credentials void clearQNAMCache(); @@ -283,6 +285,8 @@ private: static QString _configFileName; QString _davPath; // defaults to value from theme, might be overwritten in brandings + ClientSideEncryption _e2e; + friend class AccountManager; }; } diff --git a/src/libsync/connectionvalidator.cpp b/src/libsync/connectionvalidator.cpp index f4bd69360..afdcb3ab2 100644 --- a/src/libsync/connectionvalidator.cpp +++ b/src/libsync/connectionvalidator.cpp @@ -326,9 +326,8 @@ void ConnectionValidator::slotUserFetched(const QJsonDocument &json) void ConnectionValidator::slotAvatarImage(const QImage &img) { _account->setAvatar(img); - cse.setAccount(_account); - connect(&cse, &ClientSideEncryption::initializationFinished, this, &ConnectionValidator::reportConnected); - cse.initialize(); + connect(&_account->e2e(), &ClientSideEncryption::initializationFinished, this, &ConnectionValidator::reportConnected); + _account->e2e().initialize(); } void ConnectionValidator::reportConnected() { diff --git a/src/libsync/connectionvalidator.h b/src/libsync/connectionvalidator.h index b9dfda3ed..e6489c25c 100644 --- a/src/libsync/connectionvalidator.h +++ b/src/libsync/connectionvalidator.h @@ -144,7 +144,6 @@ private: QStringList _errors; AccountPtr _account; bool _isCheckingServerAndAuth; - ClientSideEncryption cse; }; } From 8adfc28de73bff0f9e69a581212b220347439f0b Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 13 Nov 2017 17:54:19 +0100 Subject: [PATCH 094/262] [CSE] Store lock-token and retrieve later --- src/gui/accountsettings.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 3df852492..9faf9eea0 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -272,18 +272,21 @@ void AccountSettings::slotEncryptionFlagError(const QByteArray& fileId, int http void AccountSettings::slotLockFolderSuccess(const QByteArray& fileId, const QByteArray &token) { - FolderMetadata emptyMetadata(accountsState()->account()); + accountsState()->account()->e2e().setTokenForFolder(fileId, token); + FolderMetadata emptyMetadata(accountsState()->account()); auto storeMetadataJob = new StoreMetaDataApiJob(accountsState()->account(), fileId, emptyMetadata.encryptedMetadata()); connect(storeMetadataJob, &StoreMetaDataApiJob::success, this, &AccountSettings::slotUploadMetadataSuccess); connect(storeMetadataJob, &StoreMetaDataApiJob::error, this, &AccountSettings::slotUpdateMetadataError); + + storeMetadataJob->start(); } void AccountSettings::slotUploadMetadataSuccess(const QByteArray& folderId) { - const QByteArray token; // fakeToken, just to compile. + const auto token = accountsState()->account()->e2e().tokenForFolder(folderId); auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), folderId, token); connect(unlockJob, &UnlockEncryptFolderApiJob::success, this, &AccountSettings::slotUnlockFolderSuccess); @@ -294,7 +297,7 @@ void AccountSettings::slotUploadMetadataSuccess(const QByteArray& folderId) void AccountSettings::slotUpdateMetadataError(const QByteArray& folderId, int httpReturnCode) { - const QByteArray token; // fakeToken, just to compile. + const auto token = accountsState()->account()->e2e().tokenForFolder(folderId); auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), folderId, token); connect(unlockJob, &UnlockEncryptFolderApiJob::success, this, &AccountSettings::slotUnlockFolderSuccess); From 14aeb6921b118ed85ed6e629f059ac7ce3b9de84 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 13 Nov 2017 18:15:08 +0100 Subject: [PATCH 095/262] [CSE] Fix invalid memory access --- src/libsync/clientsideencryption.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index bf913cb62..d22fc311b 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1111,11 +1111,11 @@ void StoreMetaDataApiJob::start() url.setQueryItems(params); QByteArray data = QByteArray("metaData=") + _b64Metadata; - QBuffer buffer; - buffer.setData(data); + auto buffer = new QBuffer(this); + buffer->setData(data); qCInfo(lcCseJob()) << "sending the metadata for the fileId" << _fileId << "as encrypted"; - sendRequest("POST", url, req, &buffer); + sendRequest("POST", url, req, buffer); AbstractNetworkJob::start(); } From 9394fe6e84c1496d216a24e191b1ef9baae784e7 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Thu, 16 Nov 2017 10:45:57 +0100 Subject: [PATCH 096/262] Use openssl functions to get random words Signed-off-by: Roeland Jago Douma --- src/libsync/wordlist.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/libsync/wordlist.cpp b/src/libsync/wordlist.cpp index 245089bc7..032210d88 100644 --- a/src/libsync/wordlist.cpp +++ b/src/libsync/wordlist.cpp @@ -1,10 +1,25 @@ #include "wordlist.h" +#include namespace OCC { namespace WordList { #include +int getRandomNumber(int max) { + unsigned char d[8]; + RAND_bytes(d, 8); + + unsigned int num = 0; + + for (int i = 0; i < 8; i++) { + num = num << 8; + num += d[i]; + } + + return num % max; +} + QStringList getRandomWords(int nr) { QStringList wordList = { @@ -2060,7 +2075,7 @@ QStringList getRandomWords(int nr) QStringList randomWords; while(randomWords.size() != nr) { - QString currWord = wordList.at(rand() % wordList.size()); + QString currWord = wordList.at(getRandomNumber(wordList.size())); if (!randomWords.contains(currWord)) { randomWords.append(currWord); } From d31aa7836a140e8443938c313b1374d63d59ad51 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 20 Nov 2017 21:38:17 +0100 Subject: [PATCH 097/262] [CSE] Add job to find what folders are encrypted. This still needs to be correctly setuped in the call chain. The job returns a QVariantMap with the folder-webdav-url and the encrypted status. --- src/gui/accountsettings.cpp | 5 ++ src/libsync/clientsideencryption.cpp | 101 +++++++++++++++++++++++++++ src/libsync/clientsideencryption.h | 32 +++++++++ src/libsync/networkjobs.cpp | 1 + 4 files changed, 139 insertions(+) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 9faf9eea0..ff1643184 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -360,6 +360,11 @@ void AccountSettings::slotSubfolderContextMenuRequested(const QModelIndex& index slotMarkSubfolderDecrypted(fileId); }); + ac = menu.addAction(tr("Fetch Enc")); + connect(ac, &QAction::triggered, [this]{ + accountsState()->account()->e2e().fetchFolderEncryptedStatus(); + }); + } menu.exec(QCursor::pos()); } diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index d22fc311b..1fa39b246 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -18,6 +18,9 @@ #include #include #include +#include +#include +#include #include "wordlist.h" @@ -544,6 +547,23 @@ void ClientSideEncryption::getPublicKeyFromServer() job->start(); } +void ClientSideEncryption::fetchFolderEncryptedStatus() { + auto getEncryptedStatus = new GetFolderEncryptStatus(_account); + getEncryptedStatus ->start(); +} + +void ClientSideEncryption::folderEncryptedStatusFetched(const QVariantMap& result) +{ + qDebug() << "Retrieved correctly the encrypted status of the folders." << result; +} + +void ClientSideEncryption::folderEncryptedStatusError(QNetworkReply *reply) +{ + qDebug() << "Failed to retrieve the status of the folders."; + if (reply) { + qDebug() << reply->errorString(); + } +} SignPublicKeyApiJob::SignPublicKeyApiJob(const AccountPtr& account, const QString& path, QObject* parent) : AbstractNetworkJob(account, path, parent) @@ -1165,4 +1185,85 @@ bool GetMetadataApiJob::finished() emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); return true; } + +GetFolderEncryptStatus::GetFolderEncryptStatus(const AccountPtr& account, QObject *parent) + : OCC::AbstractNetworkJob(account, QStringLiteral("remote.php/webdav"), parent) +{ +} + +void GetFolderEncryptStatus::start() +{ + QNetworkRequest req; + req.setPriority(QNetworkRequest::HighPriority); + req.setRawHeader("OCS-APIREQUEST", "true"); + req.setRawHeader("Content-Type", "application/xml"); + + QByteArray xml = " "; + QBuffer *buf = new QBuffer(this); + buf->setData(xml); + buf->open(QIODevice::ReadOnly); + sendRequest("PROPFIND", Utility::concatUrlPath(account()->url(), path()), req, buf); + + AbstractNetworkJob::start(); +} + +bool GetFolderEncryptStatus::finished() +{ + qCInfo(lcCse()) << "GetFolderEncryptStatus of" << reply()->request().url() << "finished with status" + << reply()->error() + << (reply()->error() == QNetworkReply::NoError ? QLatin1String("") : errorString()); + + int http_result_code = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + + if (http_result_code == 207) { + // Parse DAV response + QXmlStreamReader reader(reply()); + reader.addExtraNamespaceDeclaration(QXmlStreamNamespaceDeclaration("d", "DAV:")); + + /* Example Xml + + + + /remote.php/webdav/ + + + 0 + + HTTP/1.1 200 OK + + + + */ + + QString currFile; + int currEncryptedStatus = -1; + QMap folderStatus; + while (!reader.atEnd()) { + auto type = reader.readNext(); + if (type == QXmlStreamReader::StartElement) { + if (reader.name() == QLatin1String("href")) { + currFile = reader.readElementText(QXmlStreamReader::SkipChildElements); + } + if (reader.name() == QLatin1String("is-encrypted")) { + currEncryptedStatus = (bool) reader.readElementText(QXmlStreamReader::SkipChildElements).toInt(); + } + } + + if (!currFile.isEmpty() && currEncryptedStatus != -1) { + folderStatus.insert(currFile, currEncryptedStatus); + currFile.clear(); + currEncryptedStatus = -1; + } + } + + emit encryptStatusReceived(folderStatus); + } else { + qCWarning(lcCse()) << "*not* successful, http result code is" << http_result_code + << (http_result_code == 302 ? reply()->header(QNetworkRequest::LocationHeader).toString() : QLatin1String("")); + emit encryptStatusError(http_result_code); + // emit finishedWithError(reply()); + } + return true; +} + } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index b8dc386af..41d353619 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -37,13 +37,25 @@ public: void setTokenForFolder(const QByteArray& folder, const QByteArray& token); QByteArray tokenForFolder(const QByteArray& folder) const; + //TODO: Perhaps mode this to FolderStatusModel + // (as it makes sense, but it increase the chance + // of conflicts). + void fetchFolderEncryptedStatus(); + +private slots: + void folderEncryptedStatusFetched(const QVariantMap &values); + void folderEncryptedStatusError(QNetworkReply *reply = 0); + signals: void initializationFinished(); private: OCC::AccountPtr _account; bool isInitialized = false; + + //TODO: Save this on disk. QMap _folder2token; + QMap _folder2encryptedStatus; }; /* @@ -297,5 +309,25 @@ private: QByteArray _fileId; }; +/* I cant use the propfind network job because it defaults to the + * wrong dav url. + */ +class OWNCLOUDSYNC_EXPORT GetFolderEncryptStatus : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit GetFolderEncryptStatus (const AccountPtr &account, QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + void encryptStatusReceived(const QMap folderMetadata2EncryptionStatus); + void encryptStatusError(int statusCode); +}; + } // namespace OCC #endif diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index d4fe65f8f..e1bcb68d2 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -567,6 +567,7 @@ void PropfindJob::start() buf->setData(xml); buf->open(QIODevice::ReadOnly); sendRequest("PROPFIND", makeDavUrl(path()), req, buf); + AbstractNetworkJob::start(); } From a118419f0c36ab7152345cc10135dd7080207988 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 21 Nov 2017 09:55:44 +0100 Subject: [PATCH 098/262] [CSE] buildfix, use the correct type. --- src/libsync/clientsideencryption.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 41d353619..6ee471a68 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -325,7 +325,7 @@ protected: bool finished() override; signals: - void encryptStatusReceived(const QMap folderMetadata2EncryptionStatus); + void encryptStatusReceived(const QMap folderMetadata2EncryptionStatus); void encryptStatusError(int statusCode); }; From 57e0d7abcb1b5258d979907f72e17255f8b11aab Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 23 Nov 2017 16:54:45 +0100 Subject: [PATCH 099/262] [CSE] Fix pointer to e2e Qt expects things to be pointers even if they are stack variables. --- src/gui/accountsettings.cpp | 6 +++--- src/libsync/account.cpp | 7 ++++--- src/libsync/account.h | 2 +- src/libsync/connectionvalidator.cpp | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index ff1643184..44c965bd3 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -272,7 +272,7 @@ void AccountSettings::slotEncryptionFlagError(const QByteArray& fileId, int http void AccountSettings::slotLockFolderSuccess(const QByteArray& fileId, const QByteArray &token) { - accountsState()->account()->e2e().setTokenForFolder(fileId, token); + accountsState()->account()->e2e()->setTokenForFolder(fileId, token); FolderMetadata emptyMetadata(accountsState()->account()); auto storeMetadataJob = new StoreMetaDataApiJob(accountsState()->account(), fileId, emptyMetadata.encryptedMetadata()); @@ -286,7 +286,7 @@ void AccountSettings::slotLockFolderSuccess(const QByteArray& fileId, const QByt void AccountSettings::slotUploadMetadataSuccess(const QByteArray& folderId) { - const auto token = accountsState()->account()->e2e().tokenForFolder(folderId); + const auto token = accountsState()->account()->e2e()->tokenForFolder(folderId); auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), folderId, token); connect(unlockJob, &UnlockEncryptFolderApiJob::success, this, &AccountSettings::slotUnlockFolderSuccess); @@ -297,7 +297,7 @@ void AccountSettings::slotUploadMetadataSuccess(const QByteArray& folderId) void AccountSettings::slotUpdateMetadataError(const QByteArray& folderId, int httpReturnCode) { - const auto token = accountsState()->account()->e2e().tokenForFolder(folderId); + const auto token = accountsState()->account()->e2e()->tokenForFolder(folderId); auto unlockJob = new UnlockEncryptFolderApiJob(accountsState()->account(), folderId, token); connect(unlockJob, &UnlockEncryptFolderApiJob::success, this, &AccountSettings::slotUnlockFolderSuccess); diff --git a/src/libsync/account.cpp b/src/libsync/account.cpp index 50070a17a..58149db9c 100644 --- a/src/libsync/account.cpp +++ b/src/libsync/account.cpp @@ -53,13 +53,14 @@ AccountPtr Account::create() //TODO: This probably needs to have a better // coupling, but it should work for now. - acc->e2e().setAccount(acc); + acc->e2e()->setAccount(acc); return acc; } -ClientSideEncryption& Account::e2e() +ClientSideEncryption* Account::e2e() { - return _e2e; + // Qt expects everything in the connect to be a pointer, so return a pointer. + return &_e2e; } Account::~Account() diff --git a/src/libsync/account.h b/src/libsync/account.h index 5138129a0..af60e8b52 100644 --- a/src/libsync/account.h +++ b/src/libsync/account.h @@ -229,7 +229,7 @@ public: /// Called by network jobs on credential errors, emits invalidCredentials() void handleInvalidCredentials(); - ClientSideEncryption& e2e(); + ClientSideEncryption* e2e(); public slots: /// Used when forgetting credentials diff --git a/src/libsync/connectionvalidator.cpp b/src/libsync/connectionvalidator.cpp index 4ff269eb7..4ecf75738 100644 --- a/src/libsync/connectionvalidator.cpp +++ b/src/libsync/connectionvalidator.cpp @@ -350,8 +350,8 @@ void ConnectionValidator::slotUserFetched(const QJsonDocument &json) void ConnectionValidator::slotAvatarImage(const QImage &img) { _account->setAvatar(img); - connect(&_account->e2e(), &ClientSideEncryption::initializationFinished, this, &ConnectionValidator::reportConnected); - _account->e2e().initialize(); + connect(_account->e2e(), &ClientSideEncryption::initializationFinished, this, &ConnectionValidator::reportConnected); + _account->e2e()->initialize(); } void ConnectionValidator::reportConnected() { From 45d932365335c393d89d4bc95bdea52f0f007d81 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 23 Nov 2017 16:55:12 +0100 Subject: [PATCH 100/262] [CSE] Update encryption status when folder is updated --- src/gui/accountsettings.cpp | 6 ------ src/gui/folderstatusmodel.cpp | 8 ++++++++ src/libsync/clientsideencryption.cpp | 18 +++++++++++------- src/libsync/clientsideencryption.h | 6 +++--- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 44c965bd3..bb0837f24 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -359,12 +359,6 @@ void AccountSettings::slotSubfolderContextMenuRequested(const QModelIndex& index connect(ac, &QAction::triggered, [this, &fileId] { slotMarkSubfolderDecrypted(fileId); }); - - ac = menu.addAction(tr("Fetch Enc")); - connect(ac, &QAction::triggered, [this]{ - accountsState()->account()->e2e().fetchFolderEncryptedStatus(); - }); - } menu.exec(QCursor::pos()); } diff --git a/src/gui/folderstatusmodel.cpp b/src/gui/folderstatusmodel.cpp index d3a06f4f5..28616d2f6 100644 --- a/src/gui/folderstatusmodel.cpp +++ b/src/gui/folderstatusmodel.cpp @@ -46,6 +46,7 @@ FolderStatusModel::FolderStatusModel(QObject *parent) , _accountState(0) , _dirty(false) { + } FolderStatusModel::~FolderStatusModel() @@ -553,6 +554,13 @@ void FolderStatusModel::fetchMore(const QModelIndex &parent) } path += info->_path; } + + //TODO: This is the correct place, but this doesn't seems to be the right + // Way to call fetchFolderEncryptedStatus. + if (_accountState->account()->capabilities().clientSideEncryptionAvaliable()) { + _accountState->account()->e2e()->fetchFolderEncryptedStatus(); + } + LsColJob *job = new LsColJob(_accountState->account(), path, this); job->setProperties(QList() << "resourcetype" << "http://owncloud.org/ns:size" diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 1fa39b246..4d47e8dc9 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -548,21 +548,25 @@ void ClientSideEncryption::getPublicKeyFromServer() } void ClientSideEncryption::fetchFolderEncryptedStatus() { + _refreshingEncryptionStatus = true; auto getEncryptedStatus = new GetFolderEncryptStatus(_account); - getEncryptedStatus ->start(); + connect(getEncryptedStatus, &GetFolderEncryptStatus::encryptStatusReceived, + this, &ClientSideEncryption::folderEncryptedStatusFetched); + connect(getEncryptedStatus, &GetFolderEncryptStatus::encryptStatusError, + this, &ClientSideEncryption::folderEncryptedStatusError); + getEncryptedStatus->start(); } -void ClientSideEncryption::folderEncryptedStatusFetched(const QVariantMap& result) +void ClientSideEncryption::folderEncryptedStatusFetched(const QMap& result) { + _refreshingEncryptionStatus = false; qDebug() << "Retrieved correctly the encrypted status of the folders." << result; } -void ClientSideEncryption::folderEncryptedStatusError(QNetworkReply *reply) +void ClientSideEncryption::folderEncryptedStatusError(int error) { - qDebug() << "Failed to retrieve the status of the folders."; - if (reply) { - qDebug() << reply->errorString(); - } + _refreshingEncryptionStatus = false; + qDebug() << "Failed to retrieve the status of the folders." << error; } SignPublicKeyApiJob::SignPublicKeyApiJob(const AccountPtr& account, const QString& path, QObject* parent) diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 6ee471a68..2e697f729 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -43,8 +43,8 @@ public: void fetchFolderEncryptedStatus(); private slots: - void folderEncryptedStatusFetched(const QVariantMap &values); - void folderEncryptedStatusError(QNetworkReply *reply = 0); + void folderEncryptedStatusFetched(const QMap &values); + void folderEncryptedStatusError(int error); signals: void initializationFinished(); @@ -52,7 +52,7 @@ signals: private: OCC::AccountPtr _account; bool isInitialized = false; - + bool _refreshingEncryptionStatus = false; //TODO: Save this on disk. QMap _folder2token; QMap _folder2encryptedStatus; From 6d145a676b072227d12ce5d6e2d0171eb5c5e035 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Fri, 24 Nov 2017 20:58:20 +0100 Subject: [PATCH 101/262] Added EncryptionHelper This is to move generic encryption methods out of the main code and into small helper functions. So we don't scatter the encryption code all over the place. Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 78 ++++++++++++++++++---------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 4d47e8dc9..6e048e140 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -202,6 +202,53 @@ namespace { } } +class EncryptionHelper { +public: + static QVector generatePassword(QString wordlist, QVector &salt); +}; + +QVector EncryptionHelper::generatePassword(QString wordlist, QVector &salt) { + qCInfo(lcCse()) << "Start encryption key generation!"; + + // TODO generate salt + const unsigned char *_salt = (unsigned char *)"$4$YmBjm3hk$Qb74D5IUYwghUmzsMqeNFx5z0/8$"; + const int saltLen = 40; + + for (int i = 0; i < saltLen; i++) { + salt.append(_salt[i]); + } + + const int iterationCount = 1024; + const int keyStrength = 256; + const int keyLength = keyStrength/8; + + unsigned char secretKey[keyLength]; + + int ret = PKCS5_PBKDF2_HMAC_SHA1( + wordlist.toLocal8Bit().constData(), // const char *password, + wordlist.size(), // int password length, + salt.constData(), // const unsigned char *salt, + salt.size(), // int saltlen, + iterationCount, // int iterations, + keyLength, // int keylen, + secretKey // unsigned char *out + ); + + if (ret != 1) { + qCInfo(lcCse()) << "Failed to generate encryption key"; + // Error out? + } + + qCInfo(lcCse()) << "Encryption key generated!"; + + QVector result; + for (int i = 0; i < keyLength; i++) { + result.append(secretKey[i]); + } + + return result; +} + ClientSideEncryption::ClientSideEncryption() { } @@ -416,19 +463,10 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) // Write the Private File to a BIO // Retrieve the BIO contents, and encrypt it. // Send the encrypted key to the server. - // I have no idea what I'm doing. - - using ucharp = unsigned char *; - const char *salt = "$4$YmBjm3hk$Qb74D5IUYwghUmzsMqeNFx5z0/8$"; - const int saltLen = 40; - const int iterationCount = 1024; - const int keyStrength = 256; BIO* bio = BIO_new(BIO_s_mem()); QString passPhrase = WordList::getUnifiedString(WordList::getRandomWords(12)); - const char* passPhrasePtr = qPrintable(passPhrase); - qCInfo(lcCse()) << "Passphrase Generated:"; - qCInfo(lcCse()) << passPhrase; + qCInfo(lcCse()) << "Passphrase Generated:" << passPhrase; // Extract the Private key from the key pair. PEM_write_bio_PrivateKey(bio, keyPair, NULL, NULL, 0, 0, NULL); @@ -447,24 +485,8 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) qCInfo(lcCse()) << "Private Key Extracted"; qCInfo(lcCse()) << output; - /* Jesus. the OpenSSL docs do not help at all. - * This PKCS5_PBKDF2_HMAC_SHA1 call will generate - * a new password from the password that was submited. - */ - unsigned char secretKey[keyStrength]; - - ret = PKCS5_PBKDF2_HMAC_SHA1( - passPhrasePtr, // const char *password, - passPhrase.size(), // int password length, - (ucharp) salt, // const unsigned char *salt, - saltLen, // int saltlen, - iterationCount, // int iterations, - keyStrength, // int keylen, - secretKey // unsigned char *out - ); - qCInfo(lcCse()) << "Return of the PKCS5" << ret; - qCInfo(lcCse()) << "Result String" << secretKey; - + QVector salt; + auto secretKey = EncryptionHelper::generatePassword(passPhrase, salt); /** * NOTES: the key + iv have to be base64 encoded in the metadata. From 12adff76e2a9ba0e108d33013504c0203c908e70 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Fri, 24 Nov 2017 22:10:28 +0100 Subject: [PATCH 102/262] Properly decrypt private key and send it to the server Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 152 ++++++++++++++++++--------- 1 file changed, 103 insertions(+), 49 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 6e048e140..34923d5b9 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -204,19 +204,37 @@ namespace { class EncryptionHelper { public: - static QVector generatePassword(QString wordlist, QVector &salt); + static QByteArray generateRandom(int size); + static QByteArray generatePassword(QString wordlist, QByteArray &salt); + static QByteArray encryptPrivateKey( + const QByteArray key, + const QByteArray privateKey + ); }; -QVector EncryptionHelper::generatePassword(QString wordlist, QVector &salt) { +QByteArray EncryptionHelper::generateRandom(int size) { + unsigned char *tmp = (unsigned char *)malloc(sizeof(unsigned char) * size); + + int ret = RAND_bytes(tmp, size); + if (ret != 1) { + qCInfo(lcCse()) << "Random byte generation failed!"; + // Error out? + } + + QByteArray result((const char *)tmp, size); + free(tmp); + + return result; +} + +QByteArray EncryptionHelper::generatePassword(QString wordlist, QByteArray &salt) { qCInfo(lcCse()) << "Start encryption key generation!"; // TODO generate salt const unsigned char *_salt = (unsigned char *)"$4$YmBjm3hk$Qb74D5IUYwghUmzsMqeNFx5z0/8$"; const int saltLen = 40; - for (int i = 0; i < saltLen; i++) { - salt.append(_salt[i]); - } + salt.append((const char *)_salt, saltLen); const int iterationCount = 1024; const int keyStrength = 256; @@ -227,7 +245,7 @@ QVector EncryptionHelper::generatePassword(QString wordlist, QVec int ret = PKCS5_PBKDF2_HMAC_SHA1( wordlist.toLocal8Bit().constData(), // const char *password, wordlist.size(), // int password length, - salt.constData(), // const unsigned char *salt, + (const unsigned char *)salt.constData(), // const unsigned char *salt, salt.size(), // int saltlen, iterationCount, // int iterations, keyLength, // int keylen, @@ -241,11 +259,83 @@ QVector EncryptionHelper::generatePassword(QString wordlist, QVec qCInfo(lcCse()) << "Encryption key generated!"; - QVector result; - for (int i = 0; i < keyLength; i++) { - result.append(secretKey[i]); + QByteArray result((const char *)secretKey, keyLength); + return result; +} + +QByteArray EncryptionHelper::encryptPrivateKey( + const QByteArray key, + const QByteArray privateKey + ) { + + QByteArray iv = generateRandom(12); + + EVP_CIPHER_CTX *ctx; + /* Create and initialise the context */ + if(!(ctx = EVP_CIPHER_CTX_new())) { + qCInfo(lcCse()) << "Error creating cipher"; + handleErrors(); } + /* Initialise the decryption operation. */ + if(!EVP_EncryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, NULL, NULL)) { + qCInfo(lcCse()) << "Error initializing context with aes_256"; + handleErrors(); + } + + // No padding + EVP_CIPHER_CTX_set_padding(ctx, 0); + + /* Set IV length. */ + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, iv.size(), NULL)) { + qCInfo(lcCse()) << "Error setting iv length"; + handleErrors(); + } + + /* Initialise key and IV */ + if(!EVP_EncryptInit_ex(ctx, NULL, NULL, (unsigned char *)key.constData(), (unsigned char *)iv.constData())) { + qCInfo(lcCse()) << "Error initialising key and iv"; + handleErrors(); + } + + // We write the base64 encoded private key + QByteArray privateKeyB64 = privateKey.toBase64(); + + // Make sure we have enough room in the cipher text + unsigned char *ctext = (unsigned char *)malloc(sizeof(unsigned char) * (privateKeyB64.size() + 32)); + + // Do the actual encryption + int len = 0; + if(!EVP_EncryptUpdate(ctx, ctext, &len, (unsigned char *)privateKeyB64.constData(), privateKeyB64.size())) { + qCInfo(lcCse()) << "Error encrypting"; + handleErrors(); + } + + int clen = len; + + /* Finalise the encryption. Normally ciphertext bytes may be written at + * this stage, but this does not occur in GCM mode + */ + if(1 != EVP_EncryptFinal_ex(ctx, ctext + len, &len)) { + qCInfo(lcCse()) << "Error finalizing encryption"; + handleErrors(); + } + clen += len; + + /* Get the tag */ + unsigned char *tag = (unsigned char *)calloc(sizeof(unsigned char), 16); + if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag)) { + qCInfo(lcCse()) << "Error getting the tag"; + handleErrors(); + } + + QByteArray cipherTXT((char *)ctext, clen); + cipherTXT.append((char *)tag, 16); + + QByteArray result = cipherTXT.toBase64(); + result += "fA=="; + result += iv.toBase64(); + return result; } @@ -485,49 +575,13 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) qCInfo(lcCse()) << "Private Key Extracted"; qCInfo(lcCse()) << output; - QVector salt; + QByteArray salt; auto secretKey = EncryptionHelper::generatePassword(passPhrase, salt); + auto cryptedText = EncryptionHelper::encryptPrivateKey(secretKey, output.toLocal8Bit()); - /** - * NOTES: the key + iv have to be base64 encoded in the metadata. - */ - - //: FIRST TRY A SILLY PHRASE. - //: Hardcoed IV, really bad. - unsigned char *fakepass = (unsigned char*) "qwertyuiasdfghjkzxcvbnm,qwertyui"; - unsigned char *iv = (unsigned char *)"0123456789012345"; - - const char *encryptTest = "a quick brown fox jumps over the lazy dog"; - // TODO: Find a way to - unsigned char cryptedText[128]; - unsigned char tag[16]; - int cryptedText_len = encrypt( - (unsigned char*) encryptTest, //unsigned char *plaintext, - strlen(encryptTest), // int plaintext_len, - fakepass, // unsigned char *key, - iv, // unsigned char *iv, - cryptedText, // unsigned char *ciphertext, - tag // unsigned char *tag - ); - Q_UNUSED(cryptedText_len); //TODO: Fix this. -/* - qCInfo(lcCse()) << "Encrypted Text" << QByteArray( (const char*) cryptedText, cryptedText_len); - int decryptedText_len = decrypt( - cryptedText, //unsigned char *ciphertext, - cryptedText_len, //int ciphertext_len, - NULL, //unsigned char *aad, - 0, //int aad_len, - tag, //unsigned char *tag, - fakepass, //unsigned char *key, - iv, //unsigned char *iv, - decryptedText //unsigned char *plaintext - ); - qCInfo(lcCse()) << "Decrypted Text" << QByteArray( (const char*) decryptedText, decryptedText_len); -*/ - -// Pretend that the private key is actually encrypted and send it to the server. + // Send private key to the server auto job = new StorePrivateKeyApiJob(_account, baseUrl() + "private-key", this); - job->setPrivateKey(QByteArray((const char*) cryptedText, 128)); + job->setPrivateKey(QByteArray(cryptedText)); connect(job, &StorePrivateKeyApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { Q_UNUSED(doc); switch(retCode) { From e0fbdfe17530c04aad21ccb42105ecbaf8ff39e0 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Sat, 25 Nov 2017 15:50:47 +0100 Subject: [PATCH 103/262] Remove obsolete encryption functions Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 148 --------------------------- 1 file changed, 148 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 34923d5b9..9a29e5d5d 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -52,154 +52,6 @@ namespace { ERR_print_errors_fp(stdout); // This line is not printing anything. fflush(stdout); } - - int encrypt(unsigned char *plaintext, - int plaintext_len, - unsigned char *key, - unsigned char *iv, - unsigned char *ciphertext, - unsigned char *tag) - { - EVP_CIPHER_CTX *ctx; - int len; - int ciphertext_len; - - /* Create and initialise the context */ - if(!(ctx = EVP_CIPHER_CTX_new())) { - qCInfo(lcCse()) << "Error creating the Cipher."; - handleErrors(); - } - - /* Initialise the encryption operation. */ - if(1 != EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { - qCInfo(lcCse()) << "Error initializing the context with aes_256"; - handleErrors(); - } - - // We don't do padding - EVP_CIPHER_CTX_set_padding(ctx, 0); - - /* Set IV length to 16 bytes */ - if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL)) { - qCInfo(lcCse()) << "Error setting the iv length to 16 bytes. "; - handleErrors(); - } - - /* Initialise key and IV */ - if(1 != EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv)) { - qCInfo(lcCse()) << "Error setting key and iv encryption"; - handleErrors(); - } - - /* Provide the message to be encrypted, and obtain the encrypted output. - * EVP_EncryptUpdate can be called multiple times if necessary - */ - if(1 != EVP_EncryptUpdate(ctx, ciphertext, &len, plaintext, plaintext_len)) { - qCInfo(lcCse()) << "Error encrypting the cipher ext"; // Current error is here. - handleErrors(); - } - ciphertext_len = len; - - /* Finalise the encryption. Normally ciphertext bytes may be written at - * this stage, but this does not occur in GCM mode - */ - if(1 != EVP_EncryptFinal_ex(ctx, ciphertext + len, &len)) { - qCInfo(lcCse()) << "Error finalizing the encryption"; - handleErrors(); - } - ciphertext_len += len; - - /* Get the tag */ - if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag)) { - qCInfo(lcCse()) << "Error Retrieving the tag"; - handleErrors(); - } - - /* Add tag to cypher text to be compatible with the Android implementation */ - memcpy(ciphertext + ciphertext_len, tag, 16); - ciphertext_len += 16; - - /* Clean up */ - EVP_CIPHER_CTX_free(ctx); - - return ciphertext_len; - } - - int decrypt(unsigned char *ciphertext, - int ciphertext_len, - unsigned char *tag, - unsigned char *key, - unsigned char *iv, - unsigned char *plaintext) - { - EVP_CIPHER_CTX *ctx; - int len; - int plaintext_len; - int ret; - - /* Create and initialise the context */ - if(!(ctx = EVP_CIPHER_CTX_new())) { - qCInfo(lcCse()) << "Error Initializing the decrypt context"; - handleErrors(); - } - - /* Initialise the decryption operation. */ - if(!EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { - qCInfo(lcCse()) << "Error initializing the decryption context"; - handleErrors(); - } - - /* Set IV length to 16 bytes */ - if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL)) { - qCInfo(lcCse()) << "Error seting th iv length for the decrypt context"; - handleErrors(); - } - - /* Initialise key and IV */ - if(!EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv)) { - qCInfo(lcCse()) << "Error setting the key and iv for decryption"; - handleErrors(); - } - - /* Provide the message to be decrypted, and obtain the plaintext output. - * EVP_DecryptUpdate can be called multiple times if necessary - * - * Do not try to decrypt the last 16 bytes. The tag is appended by Android. - * So we ignore the last 16 bytes. - */ - if(!EVP_DecryptUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len - 16)) { - qCInfo(lcCse()) << "Error decrypting the text"; - handleErrors(); - } - plaintext_len = len; - - /* Set expected tag value. Works in OpenSSL 1.0.1d and later */ - if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag)) { - qCInfo(lcCse()) << "Error setting the tag on the decrupt context"; - handleErrors(); - } - - /* Finalise the decryption. A positive return value indicates success, - * anything else is a failure - the plaintext is not trustworthy. - */ - ret = EVP_DecryptFinal_ex(ctx, plaintext + len, &len); - - /* Clean up */ - EVP_CIPHER_CTX_free(ctx); - - if(ret > 0) - { - /* Success */ - plaintext_len += len; - return plaintext_len; - } - else - { - qCInfo(lcCse()) << "Error finalizing the decrypt"; - /* Verify failed */ - return -1; - } - } } class EncryptionHelper { From 9cbe795045111aecbdf49642966a08bdf9db829a Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Sat, 25 Nov 2017 21:43:15 +0100 Subject: [PATCH 104/262] Move more encryption functions to encryption helper Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 304 +++++++++++++-------------- src/libsync/clientsideencryption.h | 4 +- 2 files changed, 149 insertions(+), 159 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 9a29e5d5d..be113e6c6 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -62,6 +62,9 @@ public: const QByteArray key, const QByteArray privateKey ); + static QByteArray encryptStringSymmetric(const QByteArray key, const QByteArray data); + static QByteArray decryptStringSymmetric(const QByteArray key, const QByteArray data); + static QByteArray encryptStringAsymmetric(EVP_PKEY *key, const QByteArray data); }; QByteArray EncryptionHelper::generateRandom(int size) { @@ -191,6 +194,136 @@ QByteArray EncryptionHelper::encryptPrivateKey( return result; } +QByteArray EncryptionHelper::decryptStringSymmetric(const QByteArray key, const QByteArray data) { + +} + +QByteArray EncryptionHelper::encryptStringSymmetric(const QByteArray key, const QByteArray data) { + QByteArray iv = generateRandom(16); + + EVP_CIPHER_CTX *ctx; + /* Create and initialise the context */ + if(!(ctx = EVP_CIPHER_CTX_new())) { + qCInfo(lcCse()) << "Error creating cipher"; + handleErrors(); + } + + /* Initialise the decryption operation. */ + if(!EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { + qCInfo(lcCse()) << "Error initializing context with aes_256"; + handleErrors(); + } + + // No padding + EVP_CIPHER_CTX_set_padding(ctx, 0); + + /* Set IV length. */ + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, iv.size(), NULL)) { + qCInfo(lcCse()) << "Error setting iv length"; + handleErrors(); + } + + /* Initialise key and IV */ + if(!EVP_EncryptInit_ex(ctx, NULL, NULL, (unsigned char *)key.constData(), (unsigned char *)iv.constData())) { + qCInfo(lcCse()) << "Error initialising key and iv"; + handleErrors(); + } + + // We write the data base64 encoded + QByteArray dataB64 = data.toBase64(); + + // Make sure we have enough room in the cipher text + unsigned char *ctext = (unsigned char *)malloc(sizeof(unsigned char) * (dataB64.size() + 16)); + + // Do the actual encryption + int len = 0; + if(!EVP_EncryptUpdate(ctx, ctext, &len, (unsigned char *)dataB64.constData(), dataB64.size())) { + qCInfo(lcCse()) << "Error encrypting"; + handleErrors(); + } + + int clen = len; + + /* Finalise the encryption. Normally ciphertext bytes may be written at + * this stage, but this does not occur in GCM mode + */ + if(1 != EVP_EncryptFinal_ex(ctx, ctext + len, &len)) { + qCInfo(lcCse()) << "Error finalizing encryption"; + handleErrors(); + } + clen += len; + + /* Get the tag */ + unsigned char *tag = (unsigned char *)calloc(sizeof(unsigned char), 16); + if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag)) { + qCInfo(lcCse()) << "Error getting the tag"; + handleErrors(); + } + + QByteArray cipherTXT((char *)ctext, clen); + cipherTXT.append((char *)tag, 16); + + QByteArray result = cipherTXT.toBase64(); + result += "fA=="; + result += iv.toBase64(); + + return result; +} + +QByteArray EncryptionHelper::encryptStringAsymmetric(EVP_PKEY *key, const QByteArray data) { + int err = -1; + + auto ctx = EVP_PKEY_CTX_new(key, ENGINE_get_default_RSA()); + if (!ctx) { + qCInfo(lcCse()) << "Could not initialize the pkey context."; + exit(1); + } + + if (EVP_PKEY_encrypt_init(ctx) != 1) { + qCInfo(lcCse()) << "Error initilaizing the encryption."; + exit(1); + } + + if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) <= 0) { + qCInfo(lcCse()) << "Error setting the encryption padding."; + exit(1); + } + + if (EVP_PKEY_CTX_set_rsa_oaep_md(ctx, EVP_sha256()) <= 0) { + qCInfo(lcCse()) << "Error setting OAEP SHA 256"; + exit(1); + } + + if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, EVP_sha256()) <= 0) { + qCInfo(lcCse()) << "Error setting MGF1 padding"; + exit(1); + } + + size_t outLen = 0; + if (EVP_PKEY_encrypt(ctx, NULL, &outLen, (unsigned char *)data.constData(), data.size()) != 1) { + qCInfo(lcCse()) << "Error retrieving the size of the encrypted data"; + exit(1); + } else { + qCInfo(lcCse()) << "Encrption Length:" << outLen; + } + + unsigned char *out = (uchar*) OPENSSL_malloc(outLen); + if (!out) { + qCInfo(lcCse()) << "Error requesting memory for the encrypted contents"; + exit(1); + } + + if (EVP_PKEY_encrypt(ctx, out, &outLen, (unsigned char *)data.constData(), data.size()) != 1) { + qCInfo(lcCse()) << "Could not encrypt key." << err; + exit(1); + } + + // Transform the encrypted data into base64. + QByteArray raw((const char*) out, outLen); + qCInfo(lcCse()) << raw.toBase64(); + return raw.toBase64(); +} + ClientSideEncryption::ClientSideEncryption() { } @@ -625,19 +758,7 @@ FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata) : } // RSA/ECB/OAEPWithSHA-256AndMGF1Padding using private / public key. -std::string FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKeys) const { - std::string metadata = metadataKeys.dump(); - unsigned char *metadataPtr = (unsigned char *) metadata.c_str(); - size_t metadataPtrLen = metadata.size(); - - int err = -1; - const int rsaOeapPadding = 1; - EVP_PKEY *key = nullptr; - - /*TODO: Verify if we need to setup a RSA engine. - * by default it's RSA OEAP */ - ENGINE *eng = nullptr; - +QByteArray FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKeys) const { auto path = publicKeyPath(_account); const char *pathC = qPrintable(path); @@ -647,68 +768,16 @@ std::string FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKe exit(1); } - key = PEM_read_PUBKEY(pkeyFile, NULL, NULL, NULL); + EVP_PKEY *key = PEM_read_PUBKEY(pkeyFile, NULL, NULL, NULL); - auto ctx = EVP_PKEY_CTX_new(key, eng); - if (!ctx) { - qCInfo(lcCse()) << "Could not initialize the pkey context."; - exit(1); - } + auto data = QByteArray::fromStdString(metadataKeys.dump()); + auto ret = EncryptionHelper::encryptStringAsymmetric(key, data); - err = EVP_PKEY_encrypt_init(ctx); - if (err <= 0) { - qCInfo(lcCse()) << "Error initilaizing the encryption."; - exit(1); - } + EVP_PKEY_free(key); - err = EVP_PKEY_CTX_set_rsa_padding(ctx, rsaOeapPadding); - if (err <= 0) { - qCInfo(lcCse()) << "Error setting the encryption padding."; - exit(1); - } - - size_t outLen = 0; - err = EVP_PKEY_encrypt(ctx, NULL, &outLen, metadataPtr, metadataPtrLen); - if (err <= 0) { - qCInfo(lcCse()) << "Error retrieving the size of the encrypted data"; - exit(1); - } else { - qCInfo(lcCse()) << "Encrption Length:" << outLen; - } - - unsigned char *out = (uchar*) OPENSSL_malloc(outLen); - if (!out) { - qCInfo(lcCse()) << "Error requesting memory for the encrypted contents"; - exit(1); - } - - err = EVP_PKEY_encrypt(ctx, out, &outLen, metadataPtr, metadataPtrLen); - if (err <= 0) { - qCInfo(lcCse()) << "Could not encrypt key." << err; - exit(1); - } - - // Transform the encrypted data into base64. - const auto raw = QByteArray((const char*) out, outLen); - const auto b64 = raw.toBase64(); - const auto ret = std::string(b64.constData(), b64.length()); - - qCInfo(lcCse()) << raw.toBase64(); return ret; } -std::string FolderMetadata::genMetadataPass() const { - const char* charmap = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - const size_t charmapLength = strlen(charmap); - const int bytes = 16; - std::string result; - result.reserve(bytes); - generate_n(back_inserter(result), bytes, [&](){ - return charmap[rand() % charmapLength]; - }); - return result; -} - std::string FolderMetadata::decryptMetadataKeys(const std::string& encryptedMetadata) const { qCInfo(lcCse()) << "Starting to decrypt the metadata key"; @@ -781,86 +850,9 @@ std::string FolderMetadata::decryptMetadataKeys(const std::string& encryptedMeta } // AES/GCM/NoPadding (128 bit key size) -std::string FolderMetadata::encryptJsonObject(const nlohmann::json& obj,const std::string& pass) const { - EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); - if (!ctx) { - qCInfo(lcCse()) << "Coult not create encryption context, aborting."; - exit(1); - } - - if(!EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { - qCInfo(lcCse()) << "Error initializing encryption aborting."; - exit(1); - } - - EVP_CIPHER_CTX_set_padding(ctx, 0); - - - - unsigned char *iv = (unsigned char *)"0123456789012345"; - if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL)) { - qCInfo(lcCse()) << "Could not set IV length, aborting."; - exit(1); - } - - auto key = (const unsigned char*) pass.c_str(); - int err = EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv); - if (err != 1) { - qCInfo(lcCse()) << "Error setting key and IV, aborting."; - exit(1); - } - - std::string metadata = obj.dump(); - int metadataLen = metadata.size(); - int outLen = 0; - - unsigned char *metadataPtr = (unsigned char *) metadata.c_str(); - - qCInfo(lcCse()) << "Metadata to be encrypted" << metadata; - qCInfo(lcCse()) << "Metadata length: " << metadataLen; - - /* - * max len is metadtaLen + blocksize (16 bytes) -1 - * Add 16 bytes for the tag - */ - auto out = (unsigned char *) OPENSSL_malloc(metadataLen + 16 + 16 - 1); - - err = EVP_EncryptUpdate(ctx, out, &outLen, metadataPtr, metadataLen); - if (err != 1) { - qCInfo(lcCse()) << "Error encrypting the metadata, aborting."; - exit(1); - } - qCInfo(lcCse()) << "Successfully encrypted the internal json blob."; - int totalOutputLen = outLen; - - qCInfo(lcCse()) << "Current output length: " << totalOutputLen; - err = EVP_EncryptFinal_ex(ctx, out + outLen, &outLen); - if (err != 1) { - qCInfo(lcCse()) << "Error finalyzing the encryption."; - } - totalOutputLen += outLen; - qCInfo(lcCse()) << "Final output length: " << totalOutputLen; - - /* Get the tag */ - unsigned char tag[16]; - if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag)) { - qCInfo(lcCse()) << "Error Retrieving the tag"; - handleErrors(); - } - - /* Add tag to cypher text to be compatible with the Android implementation */ - memcpy(out + totalOutputLen, tag, 16); - totalOutputLen += 16; - - qCInfo(lcCse()) << "Final output length (with tag): " << totalOutputLen; - - // Transform the encrypted data into base64. - const auto raw = QByteArray((const char*) out, totalOutputLen); - const auto b64 = raw.toBase64(); - const auto ret = std::string(b64.constData(), b64.length()); - - qCInfo(lcCse()) << raw.toBase64(); - return ret; +QByteArray FolderMetadata::encryptJsonObject(const nlohmann::json& obj, const QByteArray pass) const { + auto data = QByteArray::fromStdString(obj.dump()); + return EncryptionHelper::encryptStringSymmetric(pass, data); } std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetadata, const std::string& pass) const @@ -941,10 +933,10 @@ std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetada void FolderMetadata::setupEmptyMetadata() { using namespace nlohmann; - std::string newMetadataPass = genMetadataPass(); + QByteArray newMetadataPass = EncryptionHelper::generateRandom(16); qCInfo(lcCse()) << "Key Generated for the Metadata" << newMetadataPass; - json metadataKeyObj = {"0", newMetadataPass}; + json metadataKeyObj = {"0", newMetadataPass.toStdString()}; json recepient = {"recipient", {}}; auto b64String = encryptMetadataKeys(metadataKeyObj); @@ -952,17 +944,15 @@ void FolderMetadata::setupEmptyMetadata() { json m = { {"metadata", { - {"metadataKeys", b64String}, - {"sharing", sharingEncrypted}, + {"metadataKeys", b64String.toStdString()}, + {"sharing", sharingEncrypted.toStdString()}, {"version",1} }}, - {"files", { + {"files", { }} }; - std::string result = m.dump(); - QString output = QString::fromStdString(result); - _metadata = output.toLocal8Bit(); + _metadata = QByteArray::fromStdString(m.dump()); } QByteArray FolderMetadata::encryptedMetadata() { diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 2e697f729..8d641478c 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -203,11 +203,11 @@ private: * to ease the port to Nlohmann Json API */ void setupEmptyMetadata(); - std::string encryptMetadataKeys(const nlohmann::json& metadataKeys) const; + QByteArray encryptMetadataKeys(const nlohmann::json& metadataKeys) const; std::string decryptMetadataKeys(const std::string& encryptedMetadataKeys) const; std::string genMetadataPass() const; - std::string encryptJsonObject(const nlohmann::json& obj, const std::string& pass) const; + QByteArray encryptJsonObject(const nlohmann::json& obj, const QByteArray pass) const; std::string decryptJsonObject(const std::string& encryptedJsonBlob, const std::string& pass) const; QVector _files; From e3050f7456992c1765aa4e123ab3fecc4ecef2ca Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 27 Nov 2017 15:21:29 +0100 Subject: [PATCH 105/262] [CSE] Pass by reference, return by value. Also, Do not create variables in the heap to change it's value via reference, prefer an aggregation value. use a Typedef to fully specify what you want in return. --- src/libsync/clientsideencryption.cpp | 39 +++++++++++++++------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index be113e6c6..396bf3e21 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -56,15 +56,18 @@ namespace { class EncryptionHelper { public: + using Password = QByteArray; + using Salt = QByteArray; + static QByteArray generateRandom(int size); - static QByteArray generatePassword(QString wordlist, QByteArray &salt); + static QPair generatePassword(const QString &wordlist); static QByteArray encryptPrivateKey( - const QByteArray key, - const QByteArray privateKey + const QByteArray& key, + const QByteArray& privateKey ); - static QByteArray encryptStringSymmetric(const QByteArray key, const QByteArray data); - static QByteArray decryptStringSymmetric(const QByteArray key, const QByteArray data); - static QByteArray encryptStringAsymmetric(EVP_PKEY *key, const QByteArray data); + static QByteArray encryptStringSymmetric(const QByteArray& key, const QByteArray& data); + static QByteArray decryptStringSymmetric(const QByteArray& key, const QByteArray& data); + static QByteArray encryptStringAsymmetric(EVP_PKEY *key, const QByteArray& data); }; QByteArray EncryptionHelper::generateRandom(int size) { @@ -82,14 +85,14 @@ QByteArray EncryptionHelper::generateRandom(int size) { return result; } -QByteArray EncryptionHelper::generatePassword(QString wordlist, QByteArray &salt) { +QPair EncryptionHelper::generatePassword(const QString& wordlist) { qCInfo(lcCse()) << "Start encryption key generation!"; // TODO generate salt const unsigned char *_salt = (unsigned char *)"$4$YmBjm3hk$Qb74D5IUYwghUmzsMqeNFx5z0/8$"; const int saltLen = 40; - salt.append((const char *)_salt, saltLen); + QByteArray salt((const char *)_salt, saltLen); const int iterationCount = 1024; const int keyStrength = 256; @@ -114,13 +117,13 @@ QByteArray EncryptionHelper::generatePassword(QString wordlist, QByteArray &salt qCInfo(lcCse()) << "Encryption key generated!"; - QByteArray result((const char *)secretKey, keyLength); - return result; + QByteArray password((const char *)secretKey, keyLength); + return {password, salt}; } QByteArray EncryptionHelper::encryptPrivateKey( - const QByteArray key, - const QByteArray privateKey + const QByteArray& key, + const QByteArray& privateKey ) { QByteArray iv = generateRandom(12); @@ -194,11 +197,11 @@ QByteArray EncryptionHelper::encryptPrivateKey( return result; } -QByteArray EncryptionHelper::decryptStringSymmetric(const QByteArray key, const QByteArray data) { +QByteArray EncryptionHelper::decryptStringSymmetric(const QByteArray& key, const QByteArray& data) { } -QByteArray EncryptionHelper::encryptStringSymmetric(const QByteArray key, const QByteArray data) { +QByteArray EncryptionHelper::encryptStringSymmetric(const QByteArray& key, const QByteArray& data) { QByteArray iv = generateRandom(16); EVP_CIPHER_CTX *ctx; @@ -270,7 +273,7 @@ QByteArray EncryptionHelper::encryptStringSymmetric(const QByteArray key, const return result; } -QByteArray EncryptionHelper::encryptStringAsymmetric(EVP_PKEY *key, const QByteArray data) { +QByteArray EncryptionHelper::encryptStringAsymmetric(EVP_PKEY *key, const QByteArray& data) { int err = -1; auto ctx = EVP_PKEY_CTX_new(key, ENGINE_get_default_RSA()); @@ -560,9 +563,9 @@ void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) qCInfo(lcCse()) << "Private Key Extracted"; qCInfo(lcCse()) << output; - QByteArray salt; - auto secretKey = EncryptionHelper::generatePassword(passPhrase, salt); - auto cryptedText = EncryptionHelper::encryptPrivateKey(secretKey, output.toLocal8Bit()); + /*TODO: C++17: auto [secretKey, salt]. */ + auto secretKey = EncryptionHelper::generatePassword(passPhrase); + auto cryptedText = EncryptionHelper::encryptPrivateKey(secretKey.first, output.toLocal8Bit()); // Send private key to the server auto job = new StorePrivateKeyApiJob(_account, baseUrl() + "private-key", this); From 74c4d27ea095819077aedf3585f564723163fbb7 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Mon, 27 Nov 2017 09:27:31 +0100 Subject: [PATCH 106/262] Indent Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.h | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 8d641478c..fb9603db5 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -34,17 +34,17 @@ public: void getPrivateKeyFromServer(); void getPublicKeyFromServer(); void encryptPrivateKey(EVP_PKEY *keyPair); - void setTokenForFolder(const QByteArray& folder, const QByteArray& token); - QByteArray tokenForFolder(const QByteArray& folder) const; + void setTokenForFolder(const QByteArray& folder, const QByteArray& token); + QByteArray tokenForFolder(const QByteArray& folder) const; - //TODO: Perhaps mode this to FolderStatusModel - // (as it makes sense, but it increase the chance - // of conflicts). - void fetchFolderEncryptedStatus(); + //TODO: Perhaps mode this to FolderStatusModel + // (as it makes sense, but it increase the chance + // of conflicts). + void fetchFolderEncryptedStatus(); private slots: - void folderEncryptedStatusFetched(const QMap &values); - void folderEncryptedStatusError(int error); + void folderEncryptedStatusFetched(const QMap &values); + void folderEncryptedStatusError(int error); signals: void initializationFinished(); @@ -52,10 +52,10 @@ signals: private: OCC::AccountPtr _account; bool isInitialized = false; - bool _refreshingEncryptionStatus = false; - //TODO: Save this on disk. - QMap _folder2token; - QMap _folder2encryptedStatus; + bool _refreshingEncryptionStatus = false; + //TODO: Save this on disk. + QMap _folder2token; + QMap _folder2encryptedStatus; }; /* @@ -213,7 +213,7 @@ private: QVector _files; QVector _metadataKeys; AccountPtr _account; - QByteArray _metadata; + QByteArray _metadata; }; class OWNCLOUDSYNC_EXPORT LockEncryptFolderApiJob : public AbstractNetworkJob From ffb9f69cf6037847601c1472f35437909787d4c0 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Mon, 27 Nov 2017 16:11:21 +0100 Subject: [PATCH 107/262] Start with moving data to the keychain * Check for cert + privateKey in keychain * Work with QSslKey and QSslCertificate * Abstract reading the BIO's a bit more Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 203 +++++++++++++++------------ src/libsync/clientsideencryption.h | 24 +++- 2 files changed, 138 insertions(+), 89 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 396bf3e21..8939e0985 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -2,6 +2,8 @@ #include "account.h" #include "capabilities.h" #include "networkjobs.h" +#include "theme.h" +#include "creds/abstractcredentials.h" #include #include @@ -22,6 +24,8 @@ #include #include +#include + #include "wordlist.h" QDebug operator<<(QDebug out, const std::string& str) @@ -30,6 +34,8 @@ QDebug operator<<(QDebug out, const std::string& str) return out; } +using namespace QKeychain; + namespace OCC { @@ -56,8 +62,8 @@ namespace { class EncryptionHelper { public: - using Password = QByteArray; - using Salt = QByteArray; + using Password = QByteArray; + using Salt = QByteArray; static QByteArray generateRandom(int size); static QPair generatePassword(const QString &wordlist); @@ -68,6 +74,7 @@ public: static QByteArray encryptStringSymmetric(const QByteArray& key, const QByteArray& data); static QByteArray decryptStringSymmetric(const QByteArray& key, const QByteArray& data); static QByteArray encryptStringAsymmetric(EVP_PKEY *key, const QByteArray& data); + static QByteArray BIO2ByteArray(BIO *b); }; QByteArray EncryptionHelper::generateRandom(int size) { @@ -327,6 +334,17 @@ QByteArray EncryptionHelper::encryptStringAsymmetric(EVP_PKEY *key, const QByteA return raw.toBase64(); } +QByteArray EncryptionHelper::BIO2ByteArray(BIO *b) { + int pending = BIO_ctrl_pending(b); + char *tmp = (char *)calloc(pending+1, sizeof(char)); + BIO_read(b, tmp, pending); + + QByteArray res(tmp, pending); + free(tmp); + + return res; +} + ClientSideEncryption::ClientSideEncryption() { } @@ -344,14 +362,77 @@ void ClientSideEncryption::initialize() emit initializationFinished(); } - if (hasPrivateKey() && hasPublicKey()) { - qCInfo(lcCse()) << "Public and private keys already downloaded"; - emit initializationFinished(); + fetchFromKeyChain(); +} + +void ClientSideEncryption::fetchFromKeyChain() { + const QString kck = AbstractCredentials::keychainKey( + _account->url().toString(), + _account->credentials()->user() + "_e2e-pub", + _account->id() + ); + + ReadPasswordJob *job = new ReadPasswordJob(Theme::instance()->appName()); + job->setInsecureFallback(false); + job->setKey(kck); + connect(job, &ReadPasswordJob::finished, this, &ClientSideEncryption::publicKeyFetched); + job->start(); +} + +void ClientSideEncryption::publicKeyFetched(Job *incoming) { + ReadPasswordJob *readJob = static_cast(incoming); + + // Error or no valid public key error out + if (readJob->error() != NoError || readJob->binaryData().length() == 0) { + getPublicKeyFromServer(); + return; } - getPublicKeyFromServer(); + _certificate = QSslCertificate(readJob->binaryData(), QSsl::Pem); + + if (_certificate.isNull()) { + getPublicKeyFromServer(); + return; + } + + qCInfo(lcCse()) << "Public key fetched from keychain"; + + const QString kck = AbstractCredentials::keychainKey( + _account->url().toString(), + _account->credentials()->user() + "_e2e-private", + _account->id() + ); + + ReadPasswordJob *job = new ReadPasswordJob(Theme::instance()->appName()); + job->setInsecureFallback(false); + job->setKey(kck); + connect(job, &ReadPasswordJob::finished, this, &ClientSideEncryption::privateKeyFetched); + job->start(); } +void ClientSideEncryption::privateKeyFetched(Job *incoming) { + ReadPasswordJob *readJob = static_cast(incoming); + + // Error or no valid public key error out + if (readJob->error() != NoError || readJob->binaryData().length() == 0) { + _certificate = QSslCertificate(); + getPublicKeyFromServer(); + return; + } + + _privateKey = QSslKey(readJob->binaryData(), QSsl::Rsa, QSsl::Pem, QSsl::PrivateKey); + + if (_privateKey.isNull()) { + getPrivateKeyFromServer(); + return; + } + + qCInfo(lcCse()) << "Private key fetched from keychain"; + + emit initializationFinished(); +} + + QString publicKeyPath(AccountPtr account) { return baseDirectory() + account->displayName() + ".pub"; @@ -402,33 +483,13 @@ void ClientSideEncryption::generateKeyPair() qCInfo(lcCse()) << "Key correctly generated"; qCInfo(lcCse()) << "Storing keys locally"; - QDir dir; - if (!dir.mkpath(baseDirectory())) { - qCInfo(lcCse()) << "Could not create the folder for the keys."; + BIO *privKey = BIO_new(BIO_s_mem()); + if (PEM_write_bio_PrivateKey(privKey, localKeyPair, NULL, NULL, 0, NULL, NULL) <= 0) { + qCInfo(lcCse()) << "Could not read private key from bio."; return; } - - auto privKeyPath = privateKeyPath(_account).toLocal8Bit(); - auto pubKeyPath = publicKeyPath(_account).toLocal8Bit(); - FILE *privKeyFile = fopen(privKeyPath.constData(), "w"); - FILE *pubKeyFile = fopen(pubKeyPath.constData(), "w"); - - qCInfo(lcCse()) << "Private key filename" << privKeyPath; - qCInfo(lcCse()) << "Public key filename" << pubKeyPath; - - //TODO: Verify if the key needs to be stored with a Cipher and Pass. - if (!PEM_write_PrivateKey(privKeyFile, localKeyPair, NULL, NULL, 0, 0, NULL)) { - qCInfo(lcCse()) << "Could not write the private key to a file."; - return; - } - - if (!PEM_write_PUBKEY(pubKeyFile, localKeyPair)) { - qCInfo(lcCse()) << "Could not write the public key to a file."; - return; - } - - fclose(privKeyFile); - fclose(pubKeyFile); + QByteArray key = EncryptionHelper::BIO2ByteArray(privKey); + _privateKey = QSslKey(key, QSsl::Rsa, QSsl::Pem, QSsl::PrivateKey); generateCSR(localKeyPair); @@ -454,9 +515,6 @@ void ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) int nVersion = 1; X509_REQ *x509_req = nullptr; - auto out = BIO_new(BIO_s_mem()); - QByteArray output; - char data[80]; SignPublicKeyApiJob *job = nullptr; // 2. set version of x509 req @@ -471,31 +529,30 @@ void ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) ret = X509_NAME_add_entry_by_txt(x509_name, v.first, MBSTRING_ASC, (ucharp) v.second, -1, -1, 0); if (ret != 1) { qCInfo(lcCse()) << "Error Generating the Certificate while adding" << v.first << v.second; - goto free_all; + X509_REQ_free(x509_req); + return; } } ret = X509_REQ_set_pubkey(x509_req, keyPair); if (ret != 1){ qCInfo(lcCse()) << "Error setting the public key on the csr"; - goto free_all; + X509_REQ_free(x509_req); + return; } ret = X509_REQ_sign(x509_req, keyPair, EVP_sha1()); // return x509_req->signature->length if (ret <= 0){ qCInfo(lcCse()) << "Error setting the public key on the csr"; - goto free_all; + X509_REQ_free(x509_req); + return; } + BIO *out = BIO_new(BIO_s_mem()); ret = PEM_write_bio_X509_REQ(out, x509_req); - do { - ret = BIO_gets(out, data, 80); - output += data; - if (output.endsWith("-----END CERTIFICATE REQUEST-----")) { - output = output.trimmed(); - break; - } - } while (ret > 0 ); + QByteArray output = EncryptionHelper::BIO2ByteArray(out); + BIO_free(out); + EVP_PKEY_free(keyPair); qCInfo(lcCse()) << "Returning the certificate"; qCInfo(lcCse()) << output; @@ -503,26 +560,16 @@ void ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) job = new SignPublicKeyApiJob(_account, baseUrl() + "public-key", this); job->setCsr(output); - connect(job, &SignPublicKeyApiJob::jsonReceived, [this, keyPair](const QJsonDocument& json, int retCode) { + connect(job, &SignPublicKeyApiJob::jsonReceived, [this](const QJsonDocument& json, int retCode) { if (retCode == 200) { - auto caps = json.object().value("ocs").toObject().value("data").toObject().value("public-key").toString(); - qCInfo(lcCse()) << "Public Key Returned" << caps; - QFile file(publicKeyPath(_account) + ".sign"); - if (file.open(QIODevice::WriteOnly)) { - QTextStream s(&file); - s << caps; - } - file.close(); - qCInfo(lcCse()) << "public key saved, Encrypting Private Key."; - encryptPrivateKey(keyPair); + QString cert = json.object().value("ocs").toObject().value("data").toObject().value("public-key").toString(); + _certificate = QSslCertificate(cert.toLocal8Bit(), QSsl::Pem); + qCInfo(lcCse()) << "Certificate saved, Encrypting Private Key."; + encryptPrivateKey(); } qCInfo(lcCse()) << retCode; }); job->start(); - -free_all: - X509_REQ_free(x509_req); - BIO_free_all(out); } void ClientSideEncryption::setTokenForFolder(const QByteArray& folderId, const QByteArray& token) @@ -536,45 +583,27 @@ QByteArray ClientSideEncryption::tokenForFolder(const QByteArray& folderId) cons return _folder2token[folderId]; } -void ClientSideEncryption::encryptPrivateKey(EVP_PKEY *keyPair) +void ClientSideEncryption::encryptPrivateKey() { - // Write the Private File to a BIO - // Retrieve the BIO contents, and encrypt it. - // Send the encrypted key to the server. - BIO* bio = BIO_new(BIO_s_mem()); + QStringList list = WordList::getRandomWords(12); + _mnemonic = list.join(' '); - QString passPhrase = WordList::getUnifiedString(WordList::getRandomWords(12)); + QString passPhrase = list.join(QString()); qCInfo(lcCse()) << "Passphrase Generated:" << passPhrase; - // Extract the Private key from the key pair. - PEM_write_bio_PrivateKey(bio, keyPair, NULL, NULL, 0, 0, NULL); - char data[80]; - QString output; - int ret = 0; - do { - ret = BIO_gets(bio, data, 80); - output += data; - if (output.endsWith("-----END PRIVATE KEY-----")) { - output = output.trimmed(); - break; - } - } while (ret > 0 ); - - qCInfo(lcCse()) << "Private Key Extracted"; - qCInfo(lcCse()) << output; - - /*TODO: C++17: auto [secretKey, salt]. */ + /*TODO: C++17: auto [secretKey, salt]. */ auto secretKey = EncryptionHelper::generatePassword(passPhrase); - auto cryptedText = EncryptionHelper::encryptPrivateKey(secretKey.first, output.toLocal8Bit()); + auto cryptedText = EncryptionHelper::encryptPrivateKey(secretKey.first, _privateKey.toPem()); // Send private key to the server auto job = new StorePrivateKeyApiJob(_account, baseUrl() + "private-key", this); - job->setPrivateKey(QByteArray(cryptedText)); + job->setPrivateKey(cryptedText); connect(job, &StorePrivateKeyApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { Q_UNUSED(doc); switch(retCode) { case 200: - qCInfo(lcCse()) << "Store private key working as expected."; + qCInfo(lcCse()) << "Private key stored encrypted on server."; + //TODO Save keys + mnemonic to keychain! emit initializationFinished(); break; default: diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index fb9603db5..3a648cce7 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include @@ -14,6 +16,12 @@ #include +namespace QKeychain { +class Job; +class WritePasswordJob; +class ReadPasswordJob; +} + namespace OCC { QString baseUrl(); @@ -33,7 +41,7 @@ public: void generateCSR(EVP_PKEY *keyPair); void getPrivateKeyFromServer(); void getPublicKeyFromServer(); - void encryptPrivateKey(EVP_PKEY *keyPair); + void encryptPrivateKey(); void setTokenForFolder(const QByteArray& folder, const QByteArray& token); QByteArray tokenForFolder(const QByteArray& folder) const; @@ -46,16 +54,28 @@ private slots: void folderEncryptedStatusFetched(const QMap &values); void folderEncryptedStatusError(int error); + void publicKeyFetched(QKeychain::Job *incoming); + void privateKeyFetched(QKeychain::Job *incoming); + signals: void initializationFinished(); private: - OCC::AccountPtr _account; + void fetchFromKeyChain(); + + void writePrivateKey(QByteArray data); + void writeCertificate(QByteArray data); + + AccountPtr _account; bool isInitialized = false; bool _refreshingEncryptionStatus = false; //TODO: Save this on disk. QMap _folder2token; QMap _folder2encryptedStatus; + + QSslKey _privateKey; + QSslCertificate _certificate; + QString _mnemonic; }; /* From d2d2df4c751ffec9f1ec26fbe144692f870b41d1 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 27 Nov 2017 21:06:38 +0100 Subject: [PATCH 108/262] [CSE] Try to find the webdav url of a folder. --- src/gui/accountsettings.cpp | 24 ++++++++++++++++++------ src/libsync/clientsideencryption.cpp | 7 +++++++ src/libsync/clientsideencryption.h | 3 ++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index bb0837f24..6eb234393 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -347,17 +347,29 @@ void AccountSettings::slotSubfolderContextMenuRequested(const QModelIndex& index if (!QFile::exists(fileName)) { ac->setEnabled(false); } - auto fileId = _model->data(index, FolderStatusModel::FileIdRole).toByteArray(); + auto info = _model->infoForIndex(index); + // jesus, nothing here returns me the webdav url + qDebug() << "path" << info->_folder->path(); + qDebug() << "remote" << info->_folder->remotePath(); + qDebug() << "clean" << info->_folder->cleanPath(); + qDebug() << "url" << info->_folder->remoteUrl(); + qDebug() << "info path" << info->_path; + qDebug() << "name" << info->_name; + qDebug() << _model->data(index, OCC::FolderStatusDelegate::FolderPathRole).toString(); + qDebug() << _model->data(index, OCC::FolderStatusDelegate::FolderSecondPathRole).toString(); + + auto acc = _accountState->account(); + acc->e2e()->printWebdavFolders(); + if (acc->capabilities().clientSideEncryptionAvaliable()) { - if (accountsState()->account()->capabilities().clientSideEncryptionAvaliable()) { ac = menu.addAction(tr("Encrypt")); - connect(ac, &QAction::triggered, [this, &fileId] { - slotMarkSubfolderEncrpted(fileId); + connect(ac, &QAction::triggered, [this, &info] { + slotMarkSubfolderEncrpted(info->_fileId); }); ac = menu.addAction(tr("Decrypt")); - connect(ac, &QAction::triggered, [this, &fileId] { - slotMarkSubfolderDecrypted(fileId); + connect(ac, &QAction::triggered, [this, &info] { + slotMarkSubfolderDecrypted(info->_fileId); }); } menu.exec(QCursor::pos()); diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 8939e0985..fedf2eb46 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -653,6 +653,7 @@ void ClientSideEncryption::fetchFolderEncryptedStatus() { void ClientSideEncryption::folderEncryptedStatusFetched(const QMap& result) { _refreshingEncryptionStatus = false; + _folder2encryptedStatus = result; qDebug() << "Retrieved correctly the encrypted status of the folders." << result; } @@ -1220,4 +1221,10 @@ bool GetFolderEncryptStatus::finished() return true; } +void ClientSideEncryption::printWebdavFolders() +{ + for(const auto folder : _folder2encryptedStatus.keys()) { + qDebug() << folder << _folder2encryptedStatus[folder]; + } +} } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 3a648cce7..34b139d93 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -49,6 +49,7 @@ public: // (as it makes sense, but it increase the chance // of conflicts). void fetchFolderEncryptedStatus(); + void printWebdavFolders(); private slots: void folderEncryptedStatusFetched(const QMap &values); @@ -71,7 +72,7 @@ private: bool _refreshingEncryptionStatus = false; //TODO: Save this on disk. QMap _folder2token; - QMap _folder2encryptedStatus; + QMap _folder2encryptedStatus; QSslKey _privateKey; QSslCertificate _certificate; From dd903d447f74b59d1094a223c8f52fe005ef579f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 27 Nov 2017 21:09:13 +0100 Subject: [PATCH 109/262] [CSE] Store the encryption status without the webdav url The Folder information on the desktop client doesn't knows about the webdav layout aparently. --- src/libsync/clientsideencryption.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index fedf2eb46..a0de1ea14 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1205,7 +1205,7 @@ bool GetFolderEncryptStatus::finished() } if (!currFile.isEmpty() && currEncryptedStatus != -1) { - folderStatus.insert(currFile, currEncryptedStatus); + folderStatus.insert(currFile.remove("/remote.php/webdav/"), currEncryptedStatus); currFile.clear(); currEncryptedStatus = -1; } From 47b5cd0fbbb32ecf4a15bde30b6b6ac95459b1b7 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 27 Nov 2017 21:19:54 +0100 Subject: [PATCH 110/262] [CSE] Shows correctly Encrypt / Decrypt in the menu --- src/gui/accountsettings.cpp | 28 ++++++++-------------------- src/libsync/clientsideencryption.cpp | 11 ++++++----- src/libsync/clientsideencryption.h | 4 +++- 3 files changed, 17 insertions(+), 26 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 6eb234393..46d3c7f38 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -348,29 +348,17 @@ void AccountSettings::slotSubfolderContextMenuRequested(const QModelIndex& index ac->setEnabled(false); } auto info = _model->infoForIndex(index); - // jesus, nothing here returns me the webdav url - qDebug() << "path" << info->_folder->path(); - qDebug() << "remote" << info->_folder->remotePath(); - qDebug() << "clean" << info->_folder->cleanPath(); - qDebug() << "url" << info->_folder->remoteUrl(); - qDebug() << "info path" << info->_path; - qDebug() << "name" << info->_name; - qDebug() << _model->data(index, OCC::FolderStatusDelegate::FolderPathRole).toString(); - qDebug() << _model->data(index, OCC::FolderStatusDelegate::FolderSecondPathRole).toString(); - auto acc = _accountState->account(); - acc->e2e()->printWebdavFolders(); + if (acc->capabilities().clientSideEncryptionAvaliable()) { + bool isEncrypted = acc->e2e()->isFolderEncrypted(info->_path); + ac = menu.addAction( isEncrypted ? tr("Decrypt") : tr("Encrypt")); - ac = menu.addAction(tr("Encrypt")); - connect(ac, &QAction::triggered, [this, &info] { - slotMarkSubfolderEncrpted(info->_fileId); - }); - - ac = menu.addAction(tr("Decrypt")); - connect(ac, &QAction::triggered, [this, &info] { - slotMarkSubfolderDecrypted(info->_fileId); - }); + if (not isEncrypted) { + connect(ac, &QAction::triggered, [this, &info] { slotMarkSubfolderEncrpted(info->_fileId); }); + } else { + connect(ac, &QAction::triggered, [this, &info] { slotMarkSubfolderDecrypted(info->_fileId); }); + } } menu.exec(QCursor::pos()); } diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index a0de1ea14..3aff711a6 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1221,10 +1221,11 @@ bool GetFolderEncryptStatus::finished() return true; } -void ClientSideEncryption::printWebdavFolders() -{ - for(const auto folder : _folder2encryptedStatus.keys()) { - qDebug() << folder << _folder2encryptedStatus[folder]; - } +bool ClientSideEncryption::isFolderEncrypted(const QString& path) { + auto it = _folder2encryptedStatus.find(path); + if (it == _folder2encryptedStatus.end()) + return false; + return (*it); } + } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 34b139d93..b806a41f4 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -49,7 +49,9 @@ public: // (as it makes sense, but it increase the chance // of conflicts). void fetchFolderEncryptedStatus(); - void printWebdavFolders(); + + // to be used together with FolderStatusModel::FolderInfo::_path. + bool isFolderEncrypted(const QString& path); private slots: void folderEncryptedStatusFetched(const QMap &values); From c4d3d0987d902ec366d00d4425d96a1ea917d044 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 27 Nov 2017 22:20:24 +0100 Subject: [PATCH 111/262] [CSE] Find the start of the upload --- src/libsync/propagateupload.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 6de6316af..f3685bbab 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -166,6 +166,10 @@ void PropagateUploadFileCommon::setDeleteExisting(bool enabled) void PropagateUploadFileCommon::start() { + qDebug() << "Starting to upload a file."; + qDebug() << _item->_file; + qDebug() << _item->_fileId; + if (propagator()->_abortRequested.fetchAndAddRelaxed(0)) { return; } From 0f60deb043ce9647731786a5bf0f769e994cb5ee Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Tue, 28 Nov 2017 12:36:35 +0100 Subject: [PATCH 112/262] Store and retrieve keys in keychain * Store privatekey, certificate and mnemonic in keychain * Retrieve private + public key from server - ask for mnemonic to decrypt private key Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 388 +++++++++++++++++++++++++-- src/libsync/clientsideencryption.h | 13 +- 2 files changed, 374 insertions(+), 27 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 3aff711a6..ab2bca7eb 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -23,6 +23,8 @@ #include #include #include +#include +#include #include @@ -71,9 +73,22 @@ public: const QByteArray& key, const QByteArray& privateKey ); - static QByteArray encryptStringSymmetric(const QByteArray& key, const QByteArray& data); - static QByteArray decryptStringSymmetric(const QByteArray& key, const QByteArray& data); - static QByteArray encryptStringAsymmetric(EVP_PKEY *key, const QByteArray& data); + static QByteArray decryptPrivateKey( + const QByteArray& key, + const QByteArray& data + ); + static QByteArray encryptStringSymmetric( + const QByteArray& key, + const QByteArray& data + ); + static QByteArray decryptStringSymmetric( + const QByteArray& key, + const QByteArray& data + ); + static QByteArray encryptStringAsymmetric( + EVP_PKEY *key, + const QByteArray& data + ); static QByteArray BIO2ByteArray(BIO *b); }; @@ -99,7 +114,7 @@ QPair EncryptionHelper::gene const unsigned char *_salt = (unsigned char *)"$4$YmBjm3hk$Qb74D5IUYwghUmzsMqeNFx5z0/8$"; const int saltLen = 40; - QByteArray salt((const char *)_salt, saltLen); + QByteArray salt((const char *)_salt, saltLen); const int iterationCount = 1024; const int keyStrength = 256; @@ -204,8 +219,182 @@ QByteArray EncryptionHelper::encryptPrivateKey( return result; } -QByteArray EncryptionHelper::decryptStringSymmetric(const QByteArray& key, const QByteArray& data) { +QByteArray EncryptionHelper::decryptPrivateKey(const QByteArray& key, const QByteArray& data) { + qCInfo(lcCse()) << "decryptStringSymmetric key: " << key; + qCInfo(lcCse()) << "decryptStringSymmetric data: " << data; + int sep = data.indexOf("fA=="); + qCInfo(lcCse()) << "sep at" << sep; + + QByteArray cipherTXT64 = data.left(sep); + QByteArray ivB64 = data.right(data.size() - sep - 4); + + qCInfo(lcCse()) << "decryptStringSymmetric cipherTXT: " << cipherTXT64; + qCInfo(lcCse()) << "decryptStringSymmetric IV: " << ivB64; + + QByteArray cipherTXT = QByteArray::fromBase64(cipherTXT64); + QByteArray iv = QByteArray::fromBase64(ivB64); + + QByteArray tag = cipherTXT.right(16); + cipherTXT.chop(16); + + // Init + EVP_CIPHER_CTX *ctx; + + /* Create and initialise the context */ + if(!(ctx = EVP_CIPHER_CTX_new())) { + qCInfo(lcCse()) << "Error creating cipher"; + return QByteArray(); + } + + /* Initialise the decryption operation. */ + if(!EVP_DecryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, NULL, NULL)) { + qCInfo(lcCse()) << "Error initialising context with aes 256"; + EVP_CIPHER_CTX_free(ctx); + return QByteArray(); + } + + /* Set IV length. Not necessary if this is 12 bytes (96 bits) */ + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, iv.size(), NULL)) { + qCInfo(lcCse()) << "Error setting IV size"; + EVP_CIPHER_CTX_free(ctx); + return QByteArray(); + } + + /* Initialise key and IV */ + if(!EVP_DecryptInit_ex(ctx, NULL, NULL, (unsigned char *)key.constData(), (unsigned char *)iv.constData())) { + qCInfo(lcCse()) << "Error initialising key and iv"; + EVP_CIPHER_CTX_free(ctx); + return QByteArray(); + } + + unsigned char *ptext = (unsigned char *)calloc(cipherTXT.size() + 16, sizeof(unsigned char)); + int plen; + + /* Provide the message to be decrypted, and obtain the plaintext output. + * EVP_DecryptUpdate can be called multiple times if necessary + */ + if(!EVP_DecryptUpdate(ctx, ptext, &plen, (unsigned char *)cipherTXT.constData(), cipherTXT.size())) { + qCInfo(lcCse()) << "Could not decrypt"; + EVP_CIPHER_CTX_free(ctx); + free(ptext); + return QByteArray(); + } + + /* Set expected tag value. Works in OpenSSL 1.0.1d and later */ + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, tag.size(), (unsigned char *)tag.constData())) { + qCInfo(lcCse()) << "Could not set tag"; + EVP_CIPHER_CTX_free(ctx); + free(ptext); + return QByteArray(); + } + + /* Finalise the decryption. A positive return value indicates success, + * anything else is a failure - the plaintext is not trustworthy. + */ + int len = plen; + if (EVP_DecryptFinal_ex(ctx, ptext + plen, &len) == 0) { + qCInfo(lcCse()) << "Tag did not match!"; + EVP_CIPHER_CTX_free(ctx); + free(ptext); + return QByteArray(); + } + + QByteArray result((char *)ptext, plen); + + free(ptext); + EVP_CIPHER_CTX_free(ctx); + + return QByteArray::fromBase64(result); +} + +QByteArray EncryptionHelper::decryptStringSymmetric(const QByteArray& key, const QByteArray& data) { + qCInfo(lcCse()) << "decryptStringSymmetric key: " << key; + qCInfo(lcCse()) << "decryptStringSymmetric data: " << data; + + int sep = data.indexOf("fA=="); + qCInfo(lcCse()) << "sep at" << sep; + + QByteArray cipherTXT64 = data.left(sep); + QByteArray ivB64 = data.right(data.size() - sep - 4); + + qCInfo(lcCse()) << "decryptStringSymmetric cipherTXT: " << cipherTXT64; + qCInfo(lcCse()) << "decryptStringSymmetric IV: " << ivB64; + + QByteArray cipherTXT = QByteArray::fromBase64(cipherTXT64); + QByteArray iv = QByteArray::fromBase64(ivB64); + + QByteArray tag = cipherTXT.right(16); + cipherTXT.chop(16); + + // Init + EVP_CIPHER_CTX *ctx; + + /* Create and initialise the context */ + if(!(ctx = EVP_CIPHER_CTX_new())) { + qCInfo(lcCse()) << "Error creating cipher"; + return QByteArray(); + } + + /* Initialise the decryption operation. */ + if(!EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { + qCInfo(lcCse()) << "Error initialising context with aes 128"; + EVP_CIPHER_CTX_free(ctx); + return QByteArray(); + } + + /* Set IV length. Not necessary if this is 12 bytes (96 bits) */ + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, iv.size(), NULL)) { + qCInfo(lcCse()) << "Error setting IV size"; + EVP_CIPHER_CTX_free(ctx); + return QByteArray(); + } + + /* Initialise key and IV */ + if(!EVP_DecryptInit_ex(ctx, NULL, NULL, (unsigned char *)key.constData(), (unsigned char *)iv.constData())) { + qCInfo(lcCse()) << "Error initialising key and iv"; + EVP_CIPHER_CTX_free(ctx); + return QByteArray(); + } + + unsigned char *ptext = (unsigned char *)calloc(cipherTXT.size() + 16, sizeof(unsigned char)); + int plen; + + /* Provide the message to be decrypted, and obtain the plaintext output. + * EVP_DecryptUpdate can be called multiple times if necessary + */ + if(!EVP_DecryptUpdate(ctx, ptext, &plen, (unsigned char *)cipherTXT.constData(), cipherTXT.size())) { + qCInfo(lcCse()) << "Could not decrypt"; + EVP_CIPHER_CTX_free(ctx); + free(ptext); + return QByteArray(); + } + + /* Set expected tag value. Works in OpenSSL 1.0.1d and later */ + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, tag.size(), (unsigned char *)tag.constData())) { + qCInfo(lcCse()) << "Could not set tag"; + EVP_CIPHER_CTX_free(ctx); + free(ptext); + return QByteArray(); + } + + /* Finalise the decryption. A positive return value indicates success, + * anything else is a failure - the plaintext is not trustworthy. + */ + int len = plen; + if (EVP_DecryptFinal_ex(ctx, ptext + plen, &len) == 0) { + qCInfo(lcCse()) << "Tag did not match!"; + EVP_CIPHER_CTX_free(ctx); + free(ptext); + return QByteArray(); + } + + QByteArray result((char *)ptext, plen); + + free(ptext); + EVP_CIPHER_CTX_free(ctx); + + return QByteArray(); } QByteArray EncryptionHelper::encryptStringSymmetric(const QByteArray& key, const QByteArray& data) { @@ -220,7 +409,7 @@ QByteArray EncryptionHelper::encryptStringSymmetric(const QByteArray& key, const /* Initialise the decryption operation. */ if(!EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { - qCInfo(lcCse()) << "Error initializing context with aes_256"; + qCInfo(lcCse()) << "Error initializing context with aes_128"; handleErrors(); } @@ -368,7 +557,7 @@ void ClientSideEncryption::initialize() void ClientSideEncryption::fetchFromKeyChain() { const QString kck = AbstractCredentials::keychainKey( _account->url().toString(), - _account->credentials()->user() + "_e2e-pub", + _account->credentials()->user() + "_e2e-certificate", _account->id() ); @@ -395,6 +584,8 @@ void ClientSideEncryption::publicKeyFetched(Job *incoming) { return; } + _publicKey = _certificate.publicKey(); + qCInfo(lcCse()) << "Public key fetched from keychain"; const QString kck = AbstractCredentials::keychainKey( @@ -416,6 +607,7 @@ void ClientSideEncryption::privateKeyFetched(Job *incoming) { // Error or no valid public key error out if (readJob->error() != NoError || readJob->binaryData().length() == 0) { _certificate = QSslCertificate(); + _publicKey = QSslKey(); getPublicKeyFromServer(); return; } @@ -429,9 +621,92 @@ void ClientSideEncryption::privateKeyFetched(Job *incoming) { qCInfo(lcCse()) << "Private key fetched from keychain"; + const QString kck = AbstractCredentials::keychainKey( + _account->url().toString(), + _account->credentials()->user() + "_e2e-mnemonic", + _account->id() + ); + + ReadPasswordJob *job = new ReadPasswordJob(Theme::instance()->appName()); + job->setInsecureFallback(false); + job->setKey(kck); + connect(job, &ReadPasswordJob::finished, this, &ClientSideEncryption::mnemonicKeyFetched); + job->start(); +} + +void ClientSideEncryption::mnemonicKeyFetched(QKeychain::Job *incoming) { + ReadPasswordJob *readJob = static_cast(incoming); + + // Error or no valid public key error out + if (readJob->error() != NoError || readJob->textData().length() == 0) { + _certificate = QSslCertificate(); + _publicKey = QSslKey(); + _privateKey = QSslKey(); + getPublicKeyFromServer(); + return; + } + + _mnemonic = readJob->textData(); + + qCInfo(lcCse()) << "Mnemonic key fetched from keychain"; + emit initializationFinished(); } +void ClientSideEncryption::writePrivateKey() { + const QString kck = AbstractCredentials::keychainKey( + _account->url().toString(), + _account->credentials()->user() + "_e2e-private", + _account->id() + ); + + WritePasswordJob *job = new WritePasswordJob(Theme::instance()->appName()); + job->setInsecureFallback(false); + job->setKey(kck); + job->setBinaryData(_privateKey.toPem()); + connect(job, &WritePasswordJob::finished, [this](Job *incoming) { + Q_UNUSED(incoming); + qCInfo(lcCse()) << "Private key stored in keychain"; + }); + job->start(); +} + +void ClientSideEncryption::writeCertificate() { + const QString kck = AbstractCredentials::keychainKey( + _account->url().toString(), + _account->credentials()->user() + "_e2e-certificate", + _account->id() + ); + + WritePasswordJob *job = new WritePasswordJob(Theme::instance()->appName()); + job->setInsecureFallback(false); + job->setKey(kck); + job->setBinaryData(_certificate.toPem()); + connect(job, &WritePasswordJob::finished, [this](Job *incoming) { + Q_UNUSED(incoming); + qCInfo(lcCse()) << "Certificate stored in keychain"; + }); + job->start(); +} + +void ClientSideEncryption::writeMnemonic() { + const QString kck = AbstractCredentials::keychainKey( + _account->url().toString(), + _account->credentials()->user() + "_e2e-mnemonic", + _account->id() + ); + + WritePasswordJob *job = new WritePasswordJob(Theme::instance()->appName()); + job->setInsecureFallback(false); + job->setKey(kck); + job->setTextData(_mnemonic); + connect(job, &WritePasswordJob::finished, [this](Job *incoming) { + Q_UNUSED(incoming); + qCInfo(lcCse()) << "Mnemonic stored in keychain"; + }); + job->start(); +} + QString publicKeyPath(AccountPtr account) { @@ -491,10 +766,8 @@ void ClientSideEncryption::generateKeyPair() QByteArray key = EncryptionHelper::BIO2ByteArray(privKey); _privateKey = QSslKey(key, QSsl::Rsa, QSsl::Pem, QSsl::PrivateKey); - generateCSR(localKeyPair); - - //TODO: Send to server. qCInfo(lcCse()) << "Keys generated correctly, sending to server."; + generateCSR(localKeyPair); } void ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) @@ -587,6 +860,7 @@ void ClientSideEncryption::encryptPrivateKey() { QStringList list = WordList::getRandomWords(12); _mnemonic = list.join(' '); + qCInfo(lcCse()) << "mnemonic Generated:" << _mnemonic; QString passPhrase = list.join(QString()); qCInfo(lcCse()) << "Passphrase Generated:" << passPhrase; @@ -603,7 +877,9 @@ void ClientSideEncryption::encryptPrivateKey() switch(retCode) { case 200: qCInfo(lcCse()) << "Private key stored encrypted on server."; - //TODO Save keys + mnemonic to keychain! + writePrivateKey(); + writeCertificate(); + writeMnemonic(); emit initializationFinished(); break; default: @@ -613,9 +889,75 @@ void ClientSideEncryption::encryptPrivateKey() job->start(); } +void ClientSideEncryption::decryptPrivateKey(const QByteArray &key) { + QString msg = tr("Please enter your end to end encryption passphrase:
" + "
" + "User: %2
" + "Account: %3
") + .arg(Utility::escape(_account->credentials()->user()), + Utility::escape(_account->displayName())); + + QInputDialog dialog; + dialog.setWindowTitle(tr("Enter E2E passphrase")); + dialog.setLabelText(msg); + dialog.setTextEchoMode(QLineEdit::Normal); + + QString prev; + + while(true) { + if (!prev.isEmpty()) { + dialog.setTextValue(prev); + } + bool ok = dialog.exec(); + if (ok) { + qCInfo(lcCse()) << "Got mnemonic:" << dialog.textValue(); + prev = dialog.textValue(); + + _mnemonic = prev; + QString mnemonic = prev.split(" ").join(QString()); + qCInfo(lcCse()) << "mnemonic:" << mnemonic; + auto pass = EncryptionHelper::generatePassword(mnemonic); + qCInfo(lcCse()) << "Generated key:" << pass.first; + + QByteArray privateKey = EncryptionHelper::decryptPrivateKey(pass.first, key); + _privateKey = QSslKey(privateKey, QSsl::Rsa, QSsl::Pem, QSsl::PrivateKey); + + qCInfo(lcCse()) << "Private key: " << _privateKey.toPem(); + + if (!_privateKey.isNull()) { + writePrivateKey(); + writeCertificate(); + writeMnemonic(); + break; + } + } else { + _mnemonic = QString(); + _privateKey = QSslKey(); + qCInfo(lcCse()) << "Cancelled"; + break; + } + } + + emit initializationFinished(); +} + void ClientSideEncryption::getPrivateKeyFromServer() { - qCInfo(lcCse()) << "Trying to store the private key on the server."; + qCInfo(lcCse()) << "Retrieving private key from server"; + auto job = new JsonApiJob(_account, baseUrl() + "private-key", this); + connect(job, &JsonApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { + if (retCode == 200) { + QString key = doc.object()["ocs"].toObject()["data"].toObject()["private-key"].toString(); + qCInfo(lcCse()) << key; + qCInfo(lcCse()) << "Found private key, lets decrypt it!"; + decryptPrivateKey(key.toLocal8Bit()); + } else if (retCode == 404) { + qCInfo(lcCse()) << "No private key on the server: setup is incomplete."; + } else { + qCInfo(lcCse()) << "Error while requesting public key: " << retCode; + } + }); + job->start(); } void ClientSideEncryption::getPublicKeyFromServer() @@ -623,19 +965,19 @@ void ClientSideEncryption::getPublicKeyFromServer() qCInfo(lcCse()) << "Retrieving public key from server"; auto job = new JsonApiJob(_account, baseUrl() + "public-key", this); connect(job, &JsonApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { - Q_UNUSED(doc); - switch(retCode) { - case 404: // no public key + if (retCode == 200) { + QString publicKey = doc.object()["ocs"].toObject()["data"].toObject()["public-keys"].toObject()[_account->davUser()].toString(); + _certificate = QSslCertificate(publicKey.toLocal8Bit(), QSsl::Pem); + _publicKey = _certificate.publicKey(); + qCInfo(lcCse()) << publicKey; + qCInfo(lcCse()) << "Found Public key, requesting Private Key."; + getPrivateKeyFromServer(); + } else if (retCode == 404) { qCInfo(lcCse()) << "No public key on the server"; generateKeyPair(); - break; - case 400: // internal error - qCInfo(lcCse()) << "Internal server error while requesting the public key, encryption aborted."; - break; - case 200: // ok - qCInfo(lcCse()) << "Found Public key, requesting Private Key."; - break; - } + } else { + qCInfo(lcCse()) << "Error while requesting public key: " << retCode; + } }); job->start(); } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index b806a41f4..580818703 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -39,8 +39,6 @@ public: bool hasPublicKey() const; void generateKeyPair(); void generateCSR(EVP_PKEY *keyPair); - void getPrivateKeyFromServer(); - void getPublicKeyFromServer(); void encryptPrivateKey(); void setTokenForFolder(const QByteArray& folder, const QByteArray& token); QByteArray tokenForFolder(const QByteArray& folder) const; @@ -59,15 +57,21 @@ private slots: void publicKeyFetched(QKeychain::Job *incoming); void privateKeyFetched(QKeychain::Job *incoming); + void mnemonicKeyFetched(QKeychain::Job *incoming); signals: void initializationFinished(); private: + void getPrivateKeyFromServer(); + void getPublicKeyFromServer(); + void decryptPrivateKey(const QByteArray &key); + void fetchFromKeyChain(); - void writePrivateKey(QByteArray data); - void writeCertificate(QByteArray data); + void writePrivateKey(); + void writeCertificate(); + void writeMnemonic(); AccountPtr _account; bool isInitialized = false; @@ -77,6 +81,7 @@ private: QMap _folder2encryptedStatus; QSslKey _privateKey; + QSslKey _publicKey; QSslCertificate _certificate; QString _mnemonic; }; From f609336bfcb8101176f47fb7af6405cfc34964c0 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 28 Nov 2017 21:28:06 +0100 Subject: [PATCH 113/262] Split file to upload from the file on disk They can be conceptually equal - I can upload the file on disk, and that's what I do right now. But if we want to accept filters in the future, filters that change the file on disk like shrinking an image, the current information used is wrong and we need a way to separate those. This patch introduces a new struct that holds the *actual* file that will be uploaded, be it a temporary one or the original file. --- src/libsync/propagateupload.cpp | 47 ++++++++++++++++++++++----------- src/libsync/propagateupload.h | 13 +++++++++ 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 6de6316af..b2bc198a8 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -163,27 +163,36 @@ void PropagateUploadFileCommon::setDeleteExisting(bool enabled) _deleteExisting = enabled; } - void PropagateUploadFileCommon::start() { + /* Currently the File to upload has the same information as the SyncFileItem, + * but the idea is to make a separation between the local file and the file that's + * being uploaded, in our current case they are the same, but perhaps we can apply + * some filters in the future like 'resize' for pictures and so on. + * + * this by no means is a finished job, but a first step. + */ + _fileToUpload._file = _item->_file; + _fileToUpload._size = _item->_size; + if (propagator()->_abortRequested.fetchAndAddRelaxed(0)) { return; } // Check if the specific file can be accessed - if (propagator()->hasCaseClashAccessibilityProblem(_item->_file)) { + if (propagator()->hasCaseClashAccessibilityProblem(_fileToUpload._file)) { done(SyncFileItem::NormalError, tr("File %1 cannot be uploaded because another file with the same name, differing only in case, exists").arg(QDir::toNativeSeparators(_item->_file))); return; } // Check if we believe that the upload will fail due to remote quota limits const quint64 quotaGuess = propagator()->_folderQuota.value( - QFileInfo(_item->_file).path(), std::numeric_limits::max()); - if (_item->_size > quotaGuess) { + QFileInfo(_fileToUpload._file).path(), std::numeric_limits::max()); + if (_fileToUpload._size > quotaGuess) { // Necessary for blacklisting logic _item->_httpErrorCode = 507; emit propagator()->insufficientRemoteStorage(); - done(SyncFileItem::DetailError, tr("Upload of %1 exceeds the quota for the folder").arg(Utility::octetsToString(_item->_size))); + done(SyncFileItem::DetailError, tr("Upload of %1 exceeds the quota for the folder").arg(Utility::octetsToString(_fileToUpload._size))); return; } @@ -194,7 +203,7 @@ void PropagateUploadFileCommon::start() } auto job = new DeleteJob(propagator()->account(), - propagator()->_remoteFolder + _item->_file, + propagator()->_remoteFolder + _fileToUpload._file, this); _jobs.append(job); connect(job, &DeleteJob::finishedSignal, this, &PropagateUploadFileCommon::slotComputeContentChecksum); @@ -208,10 +217,12 @@ void PropagateUploadFileCommon::slotComputeContentChecksum() return; } - const QString filePath = propagator()->getFilePath(_item->_file); + const QString filePath = propagator()->getFilePath(_fileToUpload._file); // remember the modtime before checksumming to be able to detect a file - // change during the checksum calculation + // change during the checksum calculation - This goes inside of the _item->_file + // and not the _fileToUpload because we are checking the original file, not there + // probably temporary one. _item->_modtime = FileSystem::getModTime(filePath); #ifdef WITH_TESTING @@ -221,6 +232,9 @@ void PropagateUploadFileCommon::slotComputeContentChecksum() QByteArray checksumType = contentChecksumType(); // Maybe the discovery already computed the checksum? + // Should I compute the checksum of the original (_item->_file) + // or the maybe-modified? (_fileToUpload._file) ? + QByteArray existingChecksumType, existingChecksum; parseChecksumHeader(_item->_checksumHeader, &existingChecksumType, &existingChecksum); if (existingChecksumType == checksumType) { @@ -285,7 +299,7 @@ void PropagateUploadFileCommon::slotStartUpload(const QByteArray &transmissionCh _item->_checksumHeader = _transmissionChecksumHeader; } - const QString fullFilePath = propagator()->getFilePath(_item->_file); + const QString fullFilePath = propagator()->getFilePath(_fileToUpload._file); if (!FileSystem::fileExists(fullFilePath)) { done(SyncFileItem::SoftError, tr("File Removed")); @@ -307,7 +321,7 @@ void PropagateUploadFileCommon::slotStartUpload(const QByteArray &transmissionCh } quint64 fileSize = FileSystem::getSize(fullFilePath); - _item->_size = fileSize; + _fileToUpload._size = fileSize; // But skip the file if the mtime is too close to 'now'! // That usually indicates a file that is still being changed @@ -536,17 +550,20 @@ void PropagateUploadFileCommon::commonErrorHandling(AbstractNetworkJob *job) // Insufficient remote storage. if (_item->_httpErrorCode == 507) { // Update the quota expectation + /* store the quota for the real local file using the information + * on the file to upload, that could have been modified by + * filters or something. */ const auto path = QFileInfo(_item->_file).path(); auto quotaIt = propagator()->_folderQuota.find(path); if (quotaIt != propagator()->_folderQuota.end()) { - quotaIt.value() = qMin(quotaIt.value(), _item->_size - 1); + quotaIt.value() = qMin(quotaIt.value(), _fileToUpload._size - 1); } else { - propagator()->_folderQuota[path] = _item->_size - 1; + propagator()->_folderQuota[path] = _fileToUpload._size - 1; } // Set up the error status = SyncFileItem::DetailError; - errorString = tr("Upload of %1 exceeds the quota for the folder").arg(Utility::octetsToString(_item->_size)); + errorString = tr("Upload of %1 exceeds the quota for the folder").arg(Utility::octetsToString(_fileToUpload._size)); emit propagator()->insufficientRemoteStorage(); } @@ -615,9 +632,9 @@ void PropagateUploadFileCommon::finalize() // Update the quota, if known auto quotaIt = propagator()->_folderQuota.find(QFileInfo(_item->_file).path()); if (quotaIt != propagator()->_folderQuota.end()) - quotaIt.value() -= _item->_size; + quotaIt.value() -= _fileToUpload._size; - // Update the database entry + // Update the database entry - use the local file, not the temporary one. if (!propagator()->_journal->setFileRecord(_item->toSyncJournalFileRecordWithInode(propagator()->getFilePath(_item->_file)))) { done(SyncFileItem::FatalError, tr("Error writing metadata to the database")); return; diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index c156190b5..32bc1950c 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -212,6 +212,19 @@ protected: bool _deleteExisting BITFIELD(1); quint64 _abortCount; /// Keep track of number of aborted items + /* This is a minified version of the SyncFileItem, + * that holds only the specifics about the file that's + * being uploaded. + * + * This is needed if we wanna apply changes on the file + * that's being uploaded while keeping the original on disk. + */ + struct UploadFileInfo { + QString _file; /// I'm still unsure if I should use a SyncFilePtr here. + quint64 _size; + }; + UploadFileInfo _fileToUpload; + // measure the performance of checksum calc and upload #ifdef WITH_TESTING Utility::StopWatch _stopWatch; From a623b14209a967a7f52344b3379b9a78317d808f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 28 Nov 2017 22:17:29 +0100 Subject: [PATCH 114/262] [uploadImprovements] use the new struct info in PropagateUploadV1 Add a new member for the UploadFileInfo in PropagateUploadCommon to hold the full file path - as it can change if we use a temporary file to upload. Adapt propagateuploadv1 to use the new calls. --- src/libsync/owncloudpropagator.h | 1 + src/libsync/propagateupload.cpp | 1 + src/libsync/propagateupload.h | 1 + src/libsync/propagateuploadv1.cpp | 24 +++++++++++++++--------- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/libsync/owncloudpropagator.h b/src/libsync/owncloudpropagator.h index 9655e1ac4..bc91df848 100644 --- a/src/libsync/owncloudpropagator.h +++ b/src/libsync/owncloudpropagator.h @@ -437,6 +437,7 @@ public: */ bool hasCaseClashAccessibilityProblem(const QString &relfile); + /* returns the local file path for the given tmp_file_name */ QString getFilePath(const QString &tmp_file_name) const; PropagateItemJob *createJob(const SyncFileItemPtr &item); diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index b2bc198a8..619e2ccbe 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -174,6 +174,7 @@ void PropagateUploadFileCommon::start() */ _fileToUpload._file = _item->_file; _fileToUpload._size = _item->_size; + _fileToUpload._path = propagator()->getFilePath(_fileToUpload._file); if (propagator()->_abortRequested.fetchAndAddRelaxed(0)) { return; diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index 32bc1950c..6a1ff2001 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -222,6 +222,7 @@ protected: struct UploadFileInfo { QString _file; /// I'm still unsure if I should use a SyncFilePtr here. quint64 _size; + QString _path; /// the full path on disk. }; UploadFileInfo _fileToUpload; diff --git a/src/libsync/propagateuploadv1.cpp b/src/libsync/propagateuploadv1.cpp index 8ebdd3786..c8c212c06 100644 --- a/src/libsync/propagateuploadv1.cpp +++ b/src/libsync/propagateuploadv1.cpp @@ -37,9 +37,9 @@ namespace OCC { void PropagateUploadFileV1::doStartUpload() { - _chunkCount = std::ceil(_item->_size / double(chunkSize())); + _chunkCount = std::ceil(_fileToUpload._size / double(chunkSize())); _startChunk = 0; - _transferId = qrand() ^ _item->_modtime ^ (_item->_size << 16); + _transferId = qrand() ^ _item->_modtime ^ (_fileToUpload._size << 16); const SyncJournalDb::UploadInfo progressInfo = propagator()->_journal->getUploadInfo(_item->_file); @@ -68,12 +68,12 @@ void PropagateUploadFileV1::startNextChunk() // is sent last. return; } - quint64 fileSize = _item->_size; + quint64 fileSize = _fileToUpload._size; auto headers = PropagateUploadFileCommon::headers(); headers["OC-Total-Length"] = QByteArray::number(fileSize); headers["OC-Chunk-Size"] = QByteArray::number(quint64(chunkSize())); - QString path = _item->_file; + QString path = _fileToUpload._file; UploadDevice *device = new UploadDevice(&propagator()->_bandwidthManager); qint64 chunkStart = 0; @@ -108,7 +108,7 @@ void PropagateUploadFileV1::startNextChunk() headers[checkSumHeaderC] = _transmissionChecksumHeader; } - const QString fileName = propagator()->getFilePath(_item->_file); + const QString fileName = _fileToUpload._path; if (!device->prepareAndOpen(fileName, chunkStart, currentChunkSize)) { qCWarning(lcPropagateUpload) << "Could not prepare upload device: " << device->errorString(); @@ -153,7 +153,6 @@ void PropagateUploadFileV1::startNextChunk() } } - if (_currentChunk + _startChunk >= _chunkCount - 1) { // Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that // https://github.com/owncloud/core/issues/11106 @@ -221,9 +220,16 @@ void PropagateUploadFileV1::slotPutFinished() QByteArray etag = getEtagFromReply(job->reply()); bool finished = etag.length() > 0; - // Check if the file still exists + /* Check if the file still exists, + * but we could be operating in a temporary file, so check both if + * the file to upload is different than the file on disk + */ + const QString fileToUploadPath = _fileToUpload._path; const QString fullFilePath(propagator()->getFilePath(_item->_file)); - if (!FileSystem::fileExists(fullFilePath)) { + bool fileExists = fileToUploadPath == fullFilePath ? FileSystem::fileExists(fullFilePath) + : (FileSystem::fileExists(fileToUploadPath) && FileSystem::fileExists(fullFilePath)); + + if (!fileExists) { if (!finished) { abortWithError(SyncFileItem::SoftError, tr("The local file was removed during sync.")); return; @@ -232,7 +238,7 @@ void PropagateUploadFileV1::slotPutFinished() } } - // Check whether the file changed since discovery. + // Check whether the file changed since discovery. the file check here is the original and not the temprary. if (!FileSystem::verifyFileUnchanged(fullFilePath, _item->_size, _item->_modtime)) { propagator()->_anotherSyncNeeded = true; if (!finished) { From a64bd6059b5f12ac29bcc0afe2282b5ba730a0bc Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 28 Nov 2017 22:22:37 +0100 Subject: [PATCH 115/262] [uploadImprovements] fix struct hole --- src/libsync/propagateupload.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index 6a1ff2001..4adf8b007 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -221,8 +221,8 @@ protected: */ struct UploadFileInfo { QString _file; /// I'm still unsure if I should use a SyncFilePtr here. - quint64 _size; QString _path; /// the full path on disk. + quint64 _size; }; UploadFileInfo _fileToUpload; From 35eb071874c83a18d3101d6df1bf888f396a8d7e Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 28 Nov 2017 22:58:52 +0100 Subject: [PATCH 116/262] [uploadImprovements] use the new struct on PropagateUploadNg use the new struct on PropagateUploadNg to make the algorithm more future proof. --- src/libsync/propagateuploadng.cpp | 34 +++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/libsync/propagateuploadng.cpp b/src/libsync/propagateuploadng.cpp index fa16a4abe..f176f2818 100644 --- a/src/libsync/propagateuploadng.cpp +++ b/src/libsync/propagateuploadng.cpp @@ -137,12 +137,12 @@ void PropagateUploadFileNG::slotPropfindFinished() ++_currentChunk; } - if (_sent > _item->_size) { + if (_sent > _fileToUpload._size) { // Normally this can't happen because the size is xor'ed with the transfer id, and it is // therefore impossible that there is more data on the server than on the file. qCCritical(lcPropagateUpload) << "Inconsistency while resuming " << _item->_file << ": the size on the server (" << _sent << ") is bigger than the size of the file (" - << _item->_size << ")"; + << _fileToUpload._size << ")"; startNewUpload(); return; } @@ -220,7 +220,7 @@ void PropagateUploadFileNG::slotDeleteJobFinished() void PropagateUploadFileNG::startNewUpload() { ASSERT(propagator()->_activeJobList.count(this) == 1); - _transferId = qrand() ^ _item->_modtime ^ (_item->_size << 16) ^ qHash(_item->_file); + _transferId = qrand() ^ _item->_modtime ^ (_fileToUpload._size << 16) ^ qHash(_fileToUpload._file); _sent = 0; _currentChunk = 0; @@ -230,10 +230,14 @@ void PropagateUploadFileNG::startNewUpload() pi._valid = true; pi._transferid = _transferId; pi._modtime = _item->_modtime; + + // Journal should store the actual file propagator()->_journal->setUploadInfo(_item->_file, pi); propagator()->_journal->commit("Upload info"); QMap headers; - headers["OC-Total-Length"] = QByteArray::number(_item->_size); + + // But we should send the temporary (or something) one. + headers["OC-Total-Length"] = QByteArray::number(_fileToUpload._size); auto job = new MkColJob(propagator()->account(), chunkUrl(), headers, this); connect(job, SIGNAL(finished(QNetworkReply::NetworkError)), @@ -264,7 +268,7 @@ void PropagateUploadFileNG::startNextChunk() if (propagator()->_abortRequested.fetchAndAddRelaxed(0)) return; - quint64 fileSize = _item->_size; + quint64 fileSize = _fileToUpload._size; ENFORCE(fileSize >= _sent, "Sent data exceeds file size"); // prevent situation that chunk size is bigger then required one to send @@ -274,8 +278,9 @@ void PropagateUploadFileNG::startNextChunk() Q_ASSERT(_jobs.isEmpty()); // There should be no running job anymore _finished = true; // Finish with a MOVE + // If we changed the file name, we must store the changed filename in the remote folder, not the original one. QString destination = QDir::cleanPath(propagator()->account()->url().path() + QLatin1Char('/') - + propagator()->account()->davPath() + propagator()->_remoteFolder + _item->_file); + + propagator()->account()->davPath() + propagator()->_remoteFolder + _fileToUpload._file); auto headers = PropagateUploadFileCommon::headers(); // "If-Match applies to the source, but we are interested in comparing the etag of the destination @@ -300,7 +305,7 @@ void PropagateUploadFileNG::startNextChunk() } auto device = new UploadDevice(&propagator()->_bandwidthManager); - const QString fileName = propagator()->getFilePath(_item->_file); + const QString fileName = _fileToUpload._path; if (!device->prepareAndOpen(fileName, _sent, _currentChunkSize)) { qCWarning(lcPropagateUpload) << "Could not prepare upload device: " << device->errorString(); @@ -357,7 +362,7 @@ void PropagateUploadFileNG::slotPutFinished() return; } - ENFORCE(_sent <= _item->_size, "can't send more than size"); + ENFORCE(_sent <= _fileToUpload._size, "can't send more than size"); // Adjust the chunk size for the time taken. // @@ -390,11 +395,18 @@ void PropagateUploadFileNG::slotPutFinished() << propagator()->_chunkSize << "bytes"; } - bool finished = _sent == _item->_size; + bool finished = _sent == _fileToUpload._size; // Check if the file still exists + /* Check if the file still exists, + * but we could be operating in a temporary file, so check both if + * the file to upload is different than the file on disk + */ + const QString fileToUploadPath = _fileToUpload._path; const QString fullFilePath(propagator()->getFilePath(_item->_file)); - if (!FileSystem::fileExists(fullFilePath)) { + bool fileExists = fileToUploadPath == fullFilePath ? FileSystem::fileExists(fullFilePath) + : (FileSystem::fileExists(fileToUploadPath) && FileSystem::fileExists(fullFilePath)); + if (!fileExists) { if (!finished) { abortWithError(SyncFileItem::SoftError, tr("The local file was removed during sync.")); return; @@ -403,7 +415,7 @@ void PropagateUploadFileNG::slotPutFinished() } } - // Check whether the file changed since discovery. + // Check whether the file changed since discovery - this acts on the original file. if (!FileSystem::verifyFileUnchanged(fullFilePath, _item->_size, _item->_modtime)) { propagator()->_anotherSyncNeeded = true; if (!finished) { From 387737d90876d12c8d54fb38841368b937669dc0 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 4 Dec 2017 21:33:43 +0100 Subject: [PATCH 117/262] [CSE] Start to create the upload logic. --- src/libsync/propagateupload.cpp | 26 +++++++++++++++++++++++--- src/libsync/propagateupload.h | 4 +++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 79de3a9ba..26d34cebf 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -176,10 +176,30 @@ void PropagateUploadFileCommon::start() * * this by no means is a finished job, but a first step. */ - _fileToUpload._file = _item->_file; - _fileToUpload._size = _item->_size; - _fileToUpload._path = propagator()->getFilePath(_fileToUpload._file); + if (propagator()->account()->capabilities().clientSideEncryptionAvaliable()) { + qDebug() << "Uploading to Remote Folder: " << propagator()->_remoteFolder; + qDebug() << "Uploading from Local Dir" << propagator()->_localDir; + qDebug() << "Local File" << _item->_file; + qDebug() << QDir::cleanPath(propagator()->account()->url().path() + QLatin1Char('/') + + propagator()->account()->davPath() + propagator()->_remoteFolder + _item->_file); + //TODO: Those shall die. + _fileToUpload._file = _item->_file; + _fileToUpload._size = _item->_size; + _fileToUpload._path = propagator()->getFilePath(_fileToUpload._file); + } else { + _fileToUpload._file = _item->_file; + _fileToUpload._size = _item->_size; + _fileToUpload._path = propagator()->getFilePath(_fileToUpload._file); + startUploadRawFile(); + } +} +void PropageteUploadFileCommon::startUploadEncryptedFile() +{ + +} + +void PropagateUploadFileCommon::startUploadRawFile() { if (propagator()->_abortRequested.fetchAndAddRelaxed(0)) { return; } diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index 4adf8b007..b0e576a24 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -250,8 +250,10 @@ public: */ void setDeleteExisting(bool enabled); + /* start should setup the file, path and size that will be send to the server */ void start() Q_DECL_OVERRIDE; - + void startUploadEncryptedFile(); + void startUploadRawFile(); bool isLikelyFinishedQuickly() Q_DECL_OVERRIDE { return _item->_size < propagator()->smallFileSize(); } private slots: From 6f35cbff6bb5dad94efe8ef30524b1d013f29189 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 4 Dec 2017 22:27:13 +0100 Subject: [PATCH 118/262] [CSE] Fetch the Folder information for encryption status --- src/libsync/clientsideencryption.h | 3 -- src/libsync/propagateupload.cpp | 60 +++++++++++++++++++++++------- src/libsync/propagateupload.h | 6 ++- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 580818703..3873ba414 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -43,9 +43,6 @@ public: void setTokenForFolder(const QByteArray& folder, const QByteArray& token); QByteArray tokenForFolder(const QByteArray& folder) const; - //TODO: Perhaps mode this to FolderStatusModel - // (as it makes sense, but it increase the chance - // of conflicts). void fetchFolderEncryptedStatus(); // to be used together with FolderStatusModel::FolderInfo::_path. diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 26d34cebf..ab5640519 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -26,6 +26,7 @@ #include "syncengine.h" #include "propagateremotedelete.h" #include "common/asserts.h" +#include "networkjobs.h" #include #include @@ -177,29 +178,60 @@ void PropagateUploadFileCommon::start() * this by no means is a finished job, but a first step. */ if (propagator()->account()->capabilities().clientSideEncryptionAvaliable()) { - qDebug() << "Uploading to Remote Folder: " << propagator()->_remoteFolder; - qDebug() << "Uploading from Local Dir" << propagator()->_localDir; - qDebug() << "Local File" << _item->_file; - qDebug() << QDir::cleanPath(propagator()->account()->url().path() + QLatin1Char('/') - + propagator()->account()->davPath() + propagator()->_remoteFolder + _item->_file); - - //TODO: Those shall die. + /* If the file is in a encrypted-enabled nextcloud instance, we need to + * do the long road: Fetch the folder status of the encrypted bit, + * if it's encrypted, find the ID of the folder. + * lock the folder using it's id. + * download the metadata + * update the metadata + * upload the file + * upload the metadata + * unlock the folder. + * + * If the folder is unencrypted we just follow the old way. + */ + auto getEncryptedStatus = new GetFolderEncryptStatus(propagator()->account()); + connect(getEncryptedStatus, &GetFolderEncryptStatus::encryptStatusReceived, + this, &PropagateUploadFileCommon::slotFolderEncryptedStatusFetched); + connect(getEncryptedStatus, &GetFolderEncryptStatus::encryptStatusError, + this, &PropagateUploadFileCommon::slotFolderEncryptedStatusError); + getEncryptedStatus->start(); + } else { _fileToUpload._file = _item->_file; _fileToUpload._size = _item->_size; _fileToUpload._path = propagator()->getFilePath(_fileToUpload._file); - } else { - _fileToUpload._file = _item->_file; - _fileToUpload._size = _item->_size; - _fileToUpload._path = propagator()->getFilePath(_fileToUpload._file); - startUploadRawFile(); + startUploadFile(); } } -void PropageteUploadFileCommon::startUploadEncryptedFile() + +void PropagateUploadFileCommon::slotFolderEncryptedStatusFetched(const QMap& result) +{ + qDebug() << "####################################"; + qDebug() << "Encrypted Status Result by folder:"; + for(const auto& path : result.keys()) { + qDebug() << result[path] << path; + } + qDebug() << "Uploading to Remote Folder: " << propagator()->_remoteFolder; + qDebug() << "Uploading from Local Dir" << propagator()->_localDir; + qDebug() << "Local File" << _item->_file; + qDebug() << QDir::cleanPath(propagator()->account()->url().path() + QLatin1Char('/') + + propagator()->account()->davPath() + propagator()->_remoteFolder + _item->_file); + qDebug() << "###################################"; + qDebug() << "Retrieved correctly the encrypted status of the folders." << result; +} + +void PropagateUploadFileCommon::slotFolderEncryptedStatusError(int error) +{ + qDebug() << "Failed to retrieve the status of the folders." << error; +} + + +void PropagateUploadFileCommon::startUploadEncryptedFile() { } -void PropagateUploadFileCommon::startUploadRawFile() { +void PropagateUploadFileCommon::startUploadFile() { if (propagator()->_abortRequested.fetchAndAddRelaxed(0)) { return; } diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index b0e576a24..10f459381 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -253,7 +253,7 @@ public: /* start should setup the file, path and size that will be send to the server */ void start() Q_DECL_OVERRIDE; void startUploadEncryptedFile(); - void startUploadRawFile(); + void startUploadFile(); bool isLikelyFinishedQuickly() Q_DECL_OVERRIDE { return _item->_size < propagator()->smallFileSize(); } private slots: @@ -278,6 +278,10 @@ private slots: void slotReplyAbortFinished(); void slotPollFinished(); + // Encryption Stuff + void slotFolderEncryptedStatusFetched(const QMap& result); + void slotFolderEncryptedStatusError(int error); + protected: /** * Prepares the abort e.g. connects proper signals and slots From 66aecb96269702977db537d858dace61fb79fb76 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 4 Dec 2017 22:45:10 +0100 Subject: [PATCH 119/262] [CSE] Handle normal uploads in CSE enabled instances --- src/libsync/propagateupload.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index ab5640519..4f84fa839 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -33,6 +33,8 @@ #include #include #include +#include + #include #include @@ -216,8 +218,22 @@ void PropagateUploadFileCommon::slotFolderEncryptedStatusFetched(const QMap_file; qDebug() << QDir::cleanPath(propagator()->account()->url().path() + QLatin1Char('/') + propagator()->account()->davPath() + propagator()->_remoteFolder + _item->_file); + QFileInfo fileInfo(_item->_file); + QString currFilePath = fileInfo.path(); + if (!currFilePath.endsWith(QDir::separator())) + currFilePath += QDir::separator(); + qDebug() << "###################################"; qDebug() << "Retrieved correctly the encrypted status of the folders." << result; + if (result[currFilePath] == true) { + qDebug() << "Uploading to an encrypted folder. do the thing."; + } else { + _fileToUpload._file = _item->_file; + _fileToUpload._size = _item->_size; + _fileToUpload._path = propagator()->getFilePath(_fileToUpload._file); + startUploadFile(); + qDebug() << "Uploading to a folder that's not encrypted. call the default uploader."; + } } void PropagateUploadFileCommon::slotFolderEncryptedStatusError(int error) From 863e86138f1f9ed16b6097a82ed0eeb87106b7e6 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 7 Dec 2017 17:32:35 +0100 Subject: [PATCH 120/262] [CSE] Pass the folder to the GetEncryptionStatus Sometimes we are only interested in folders. --- src/libsync/clientsideencryption.cpp | 9 +++++---- src/libsync/clientsideencryption.h | 4 +++- src/libsync/propagateupload.cpp | 6 +++++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index ab2bca7eb..7da5f1859 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -984,7 +984,7 @@ void ClientSideEncryption::getPublicKeyFromServer() void ClientSideEncryption::fetchFolderEncryptedStatus() { _refreshingEncryptionStatus = true; - auto getEncryptedStatus = new GetFolderEncryptStatus(_account); + auto getEncryptedStatus = new GetFolderEncryptStatus(_account, QString()); connect(getEncryptedStatus, &GetFolderEncryptStatus::encryptStatusReceived, this, &ClientSideEncryption::folderEncryptedStatusFetched); connect(getEncryptedStatus, &GetFolderEncryptStatus::encryptStatusError, @@ -1483,8 +1483,8 @@ bool GetMetadataApiJob::finished() return true; } -GetFolderEncryptStatus::GetFolderEncryptStatus(const AccountPtr& account, QObject *parent) - : OCC::AbstractNetworkJob(account, QStringLiteral("remote.php/webdav"), parent) +GetFolderEncryptStatus::GetFolderEncryptStatus(const AccountPtr& account, const QString& folder, QObject *parent) + : OCC::AbstractNetworkJob(account, QStringLiteral("remote.php/webdav"), parent), _folder(folder) { } @@ -1499,7 +1499,8 @@ void GetFolderEncryptStatus::start() QBuffer *buf = new QBuffer(this); buf->setData(xml); buf->open(QIODevice::ReadOnly); - sendRequest("PROPFIND", Utility::concatUrlPath(account()->url(), path()), req, buf); + QString tmpPath = path() + (!_folder.isEmpty() ? "/" + _folder : QString()); + sendRequest("PROPFIND", Utility::concatUrlPath(account()->url(), tmpPath), req, buf); AbstractNetworkJob::start(); } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 3873ba414..ffeed550f 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -341,7 +341,7 @@ class OWNCLOUDSYNC_EXPORT GetFolderEncryptStatus : public AbstractNetworkJob { Q_OBJECT public: - explicit GetFolderEncryptStatus (const AccountPtr &account, QObject *parent = 0); + explicit GetFolderEncryptStatus (const AccountPtr &account, const QString& folder, QObject *parent = 0); public slots: void start() override; @@ -352,6 +352,8 @@ protected: signals: void encryptStatusReceived(const QMap folderMetadata2EncryptionStatus); void encryptStatusError(int statusCode); +private: + QString _folder; }; } // namespace OCC diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 4f84fa839..0631db984 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -192,7 +192,11 @@ void PropagateUploadFileCommon::start() * * If the folder is unencrypted we just follow the old way. */ - auto getEncryptedStatus = new GetFolderEncryptStatus(propagator()->account()); + QFileInfo info(_item->_file); + + auto getEncryptedStatus = new GetFolderEncryptStatus(propagator()->account(), + info.path()); + connect(getEncryptedStatus, &GetFolderEncryptStatus::encryptStatusReceived, this, &PropagateUploadFileCommon::slotFolderEncryptedStatusFetched); connect(getEncryptedStatus, &GetFolderEncryptStatus::encryptStatusError, From 7fe4dd2163a612414e6ac53cc108c7251273a643 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 7 Dec 2017 18:06:55 +0100 Subject: [PATCH 121/262] [CSE] Renane GetFolderEncryptStatus to GetFolderEncryptStatusJob it's a network job after all --- src/libsync/clientsideencryption.cpp | 12 ++++++------ src/libsync/clientsideencryption.h | 4 ++-- src/libsync/propagateupload.cpp | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 7da5f1859..8f777f5ef 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -984,10 +984,10 @@ void ClientSideEncryption::getPublicKeyFromServer() void ClientSideEncryption::fetchFolderEncryptedStatus() { _refreshingEncryptionStatus = true; - auto getEncryptedStatus = new GetFolderEncryptStatus(_account, QString()); - connect(getEncryptedStatus, &GetFolderEncryptStatus::encryptStatusReceived, + auto getEncryptedStatus = new GetFolderEncryptStatusJob(_account, QString()); + connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusReceived, this, &ClientSideEncryption::folderEncryptedStatusFetched); - connect(getEncryptedStatus, &GetFolderEncryptStatus::encryptStatusError, + connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusError, this, &ClientSideEncryption::folderEncryptedStatusError); getEncryptedStatus->start(); } @@ -1483,12 +1483,12 @@ bool GetMetadataApiJob::finished() return true; } -GetFolderEncryptStatus::GetFolderEncryptStatus(const AccountPtr& account, const QString& folder, QObject *parent) +GetFolderEncryptStatusJob::GetFolderEncryptStatusJob(const AccountPtr& account, const QString& folder, QObject *parent) : OCC::AbstractNetworkJob(account, QStringLiteral("remote.php/webdav"), parent), _folder(folder) { } -void GetFolderEncryptStatus::start() +void GetFolderEncryptStatusJob::start() { QNetworkRequest req; req.setPriority(QNetworkRequest::HighPriority); @@ -1505,7 +1505,7 @@ void GetFolderEncryptStatus::start() AbstractNetworkJob::start(); } -bool GetFolderEncryptStatus::finished() +bool GetFolderEncryptStatusJob::finished() { qCInfo(lcCse()) << "GetFolderEncryptStatus of" << reply()->request().url() << "finished with status" << reply()->error() diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index ffeed550f..f2d931971 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -337,11 +337,11 @@ private: /* I cant use the propfind network job because it defaults to the * wrong dav url. */ -class OWNCLOUDSYNC_EXPORT GetFolderEncryptStatus : public AbstractNetworkJob +class OWNCLOUDSYNC_EXPORT GetFolderEncryptStatusJob : public AbstractNetworkJob { Q_OBJECT public: - explicit GetFolderEncryptStatus (const AccountPtr &account, const QString& folder, QObject *parent = 0); + explicit GetFolderEncryptStatusJob (const AccountPtr &account, const QString& folder, QObject *parent = 0); public slots: void start() override; diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 0631db984..fd0c224ae 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -194,12 +194,12 @@ void PropagateUploadFileCommon::start() */ QFileInfo info(_item->_file); - auto getEncryptedStatus = new GetFolderEncryptStatus(propagator()->account(), + auto getEncryptedStatus = new GetFolderEncryptStatusJob(propagator()->account(), info.path()); - connect(getEncryptedStatus, &GetFolderEncryptStatus::encryptStatusReceived, + connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusReceived, this, &PropagateUploadFileCommon::slotFolderEncryptedStatusFetched); - connect(getEncryptedStatus, &GetFolderEncryptStatus::encryptStatusError, + connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusError, this, &PropagateUploadFileCommon::slotFolderEncryptedStatusError); getEncryptedStatus->start(); } else { From 19120fde9fb6207d9a290239bef3d72fda027a7f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 7 Dec 2017 18:10:14 +0100 Subject: [PATCH 122/262] [CSE] Don't query for files on GetFolderEncryptStatusJob The request for folders whas also replying for files. --- src/libsync/clientsideencryption.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 8f777f5ef..27df5a130 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1540,7 +1540,10 @@ bool GetFolderEncryptStatusJob::finished() auto type = reader.readNext(); if (type == QXmlStreamReader::StartElement) { if (reader.name() == QLatin1String("href")) { - currFile = reader.readElementText(QXmlStreamReader::SkipChildElements); + // If the current file is not a folder, ignore it. + currFile = reader.readElementText(QXmlStreamReader::SkipChildElements); + if (!currFile.endsWith('/')) + continue; } if (reader.name() == QLatin1String("is-encrypted")) { currEncryptedStatus = (bool) reader.readElementText(QXmlStreamReader::SkipChildElements).toInt(); From 893ca66af814745e890b65ce2133c0aac2d2fbdc Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 7 Dec 2017 18:12:25 +0100 Subject: [PATCH 123/262] [CSE] Fix indentation Parts of the code are indented by spaces, other parts by tabs. This needs to run in the whole codebase. --- src/libsync/clientsideencryption.cpp | 56 ++++++++++++++-------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 27df5a130..e95aaca2b 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1513,48 +1513,48 @@ bool GetFolderEncryptStatusJob::finished() int http_result_code = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (http_result_code == 207) { + if (http_result_code == 207) { // Parse DAV response QXmlStreamReader reader(reply()); reader.addExtraNamespaceDeclaration(QXmlStreamNamespaceDeclaration("d", "DAV:")); - /* Example Xml - - - - /remote.php/webdav/ - - - 0 - - HTTP/1.1 200 OK - - - - */ + /* Example Xml + + + + /remote.php/webdav/ + + + 0 + + HTTP/1.1 200 OK + + + + */ - QString currFile; - int currEncryptedStatus = -1; - QMap folderStatus; + QString currFile; + int currEncryptedStatus = -1; + QMap folderStatus; while (!reader.atEnd()) { auto type = reader.readNext(); if (type == QXmlStreamReader::StartElement) { - if (reader.name() == QLatin1String("href")) { - // If the current file is not a folder, ignore it. + if (reader.name() == QLatin1String("href")) { + // If the current file is not a folder, ignore it. currFile = reader.readElementText(QXmlStreamReader::SkipChildElements); if (!currFile.endsWith('/')) continue; - } + } if (reader.name() == QLatin1String("is-encrypted")) { - currEncryptedStatus = (bool) reader.readElementText(QXmlStreamReader::SkipChildElements).toInt(); - } + currEncryptedStatus = (bool) reader.readElementText(QXmlStreamReader::SkipChildElements).toInt(); + } } if (!currFile.isEmpty() && currEncryptedStatus != -1) { - folderStatus.insert(currFile.remove("/remote.php/webdav/"), currEncryptedStatus); - currFile.clear(); - currEncryptedStatus = -1; - } + folderStatus.insert(currFile.remove("/remote.php/webdav/"), currEncryptedStatus); + currFile.clear(); + currEncryptedStatus = -1; + } } emit encryptStatusReceived(folderStatus); @@ -1562,7 +1562,7 @@ bool GetFolderEncryptStatusJob::finished() qCWarning(lcCse()) << "*not* successful, http result code is" << http_result_code << (http_result_code == 302 ? reply()->header(QNetworkRequest::LocationHeader).toString() : QLatin1String("")); emit encryptStatusError(http_result_code); - // emit finishedWithError(reply()); + // emit finishedWithError(reply()); } return true; } From 4878e824e5de723607c8091faf94933de38eb595 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 7 Dec 2017 19:04:12 +0100 Subject: [PATCH 124/262] [CSE] Fix reading encrypted status of folders. --- src/libsync/clientsideencryption.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index e95aaca2b..5e0b7ea79 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1541,17 +1541,18 @@ bool GetFolderEncryptStatusJob::finished() if (type == QXmlStreamReader::StartElement) { if (reader.name() == QLatin1String("href")) { // If the current file is not a folder, ignore it. - currFile = reader.readElementText(QXmlStreamReader::SkipChildElements); + currFile = reader.readElementText(QXmlStreamReader::SkipChildElements).remove("/remote.php/webdav/"); if (!currFile.endsWith('/')) - continue; + currFile.clear(); + currEncryptedStatus = -1; } - if (reader.name() == QLatin1String("is-encrypted")) { + if (not currFile.isEmpty() && reader.name() == QLatin1String("is-encrypted")) { currEncryptedStatus = (bool) reader.readElementText(QXmlStreamReader::SkipChildElements).toInt(); } } if (!currFile.isEmpty() && currEncryptedStatus != -1) { - folderStatus.insert(currFile.remove("/remote.php/webdav/"), currEncryptedStatus); + folderStatus.insert(currFile, currEncryptedStatus); currFile.clear(); currEncryptedStatus = -1; } From 76916b6c48d560cb159ec44427bad8ac5b9b6638 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 7 Dec 2017 19:39:30 +0100 Subject: [PATCH 125/262] [CSE] Request the Folder ID. --- src/libsync/propagateupload.cpp | 19 ++++++++++++++++++- src/libsync/propagateupload.h | 2 ++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index fd0c224ae..1ff67d577 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -229,8 +229,15 @@ void PropagateUploadFileCommon::slotFolderEncryptedStatusFetched(const QMap_file); + LsColJob *job = new LsColJob(propagator()->account(), info.path(), this); + job->setProperties({"resourcetype", "http://owncloud.org/ns:fileid"}); + connect(job, &LsColJob::directoryListingSubfolders, this, &PropagateUploadFileCommon::slotFolderEncryptedIdReceived); + connect(job, &LsColJob::finishedWithError, this, &PropagateUploadFileCommon::slotFolderEncryptedIdError); + job->start(); } else { _fileToUpload._file = _item->_file; _fileToUpload._size = _item->_size; @@ -240,6 +247,16 @@ void PropagateUploadFileCommon::slotFolderEncryptedStatusFetched(const QMap& result); void slotFolderEncryptedStatusError(int error); + void slotFolderEncryptedIdReceived(const QStringList &list); + void slotFolderEncryptedIdError(QNetworkReply *r); protected: /** From 9f678652f80b587851dce1d999658aed99bb43c3 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 7 Dec 2017 20:52:54 +0100 Subject: [PATCH 126/262] [CSE] Lock the folder to upload a file. --- src/libsync/propagateupload.cpp | 19 ++++++++++++++++++- src/libsync/propagateupload.h | 3 +++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 1ff67d577..e1d1c6794 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -27,6 +27,7 @@ #include "propagateremotedelete.h" #include "common/asserts.h" #include "networkjobs.h" +#include "clientsideencryption.h" #include #include @@ -249,7 +250,23 @@ void PropagateUploadFileCommon::slotFolderEncryptedStatusFetched(const QMap(sender()); + const auto& folderInfo = job->_folderInfos.value(list.first()); + auto *lockJob = new LockEncryptFolderApiJob(propagator()->account(), list.first().toLocal8Bit(), this); + connect(lockJob, &LockEncryptFolderApiJob::success, this, &PropagateUploadFileCommon::slotFolderLockedSuccessfully); + connect(lockJob, &LockEncryptFolderApiJob::error, this, &PropagateUploadFileCommon::slotFolderLockedError); + lockJob->start(); +} + +void PropagateUploadFileCommon::slotFolderLockedSuccessfully(const QByteArray& fileId, const QByteArray& token) +{ + qDebug() << "Folder" << fileId << "Locked Successfully for Upload"; +} + +void PropagateUploadFileCommon::slotFolderLockedError(const QByteArray& fileId, int httpErrorCode) +{ + qDebug() << "Folder" << fileId << "Coundn't be locked."; } void PropagateUploadFileCommon::slotFolderEncryptedIdError(QNetworkReply *r) diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index 0ef8a876a..7e36b166e 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -283,6 +283,9 @@ private slots: void slotFolderEncryptedStatusError(int error); void slotFolderEncryptedIdReceived(const QStringList &list); void slotFolderEncryptedIdError(QNetworkReply *r); + void slotFolderLockedSuccessfully(const QByteArray& fileId, const QByteArray& token); + void slotFolderLockedError(const QByteArray& fileId, int httpErrorCode); + protected: /** From ccd8cff4db5a97a780721ff16c4721f4b00688af Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 7 Dec 2017 21:31:20 +0100 Subject: [PATCH 127/262] [CSE] Try to lock repeteadly the folder if needed. The upload is made in an event loop with more than one upload at the same time, this confuses the hell out of the folder locking mechanism. We need to lock the folder and ask the other trials to try again in a few seconds in the future to give time for the uploader to actually upload the current file that's locking the folder. --- src/libsync/propagateupload.cpp | 29 ++++++++++++++++++++++++++--- src/libsync/propagateupload.h | 4 ++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index e1d1c6794..d0a611ddc 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -248,12 +248,26 @@ void PropagateUploadFileCommon::slotFolderEncryptedStatusFetched(const QMap fail. + * the 'loop': / + * slotFolderEncryptedIdReceived -> slotTryLock -> lockError -> stillTime? -> slotTryLock + * \ + * -> success. + */ + void PropagateUploadFileCommon::slotFolderEncryptedIdReceived(const QStringList &list) { - // Got the ID, Try to Lock the Folder!. auto job = qobject_cast(sender()); const auto& folderInfo = job->_folderInfos.value(list.first()); - auto *lockJob = new LockEncryptFolderApiJob(propagator()->account(), list.first().toLocal8Bit(), this); + _folderLockFirstTry.start(); + slotTryLock(folderInfo.fileId); +} + +void PropagateUploadFileCommon::slotTryLock(const QByteArray& fileId) +{ + auto *lockJob = new LockEncryptFolderApiJob(propagator()->account(), fileId, this); connect(lockJob, &LockEncryptFolderApiJob::success, this, &PropagateUploadFileCommon::slotFolderLockedSuccessfully); connect(lockJob, &LockEncryptFolderApiJob::error, this, &PropagateUploadFileCommon::slotFolderLockedError); lockJob->start(); @@ -266,7 +280,16 @@ void PropagateUploadFileCommon::slotFolderLockedSuccessfully(const QByteArray& f void PropagateUploadFileCommon::slotFolderLockedError(const QByteArray& fileId, int httpErrorCode) { - qDebug() << "Folder" << fileId << "Coundn't be locked."; + // Add a counter or Something, this will enter in a loop. + QTimer::singleShot(1000, this, [this, fileId]{ + if (_folderLockFirstTry.elapsed() > /* one minute */ 60000) { + qDebug() << "One minute passed, ignoring more attemps to lock the folder."; + return; + } + slotTryLock(fileId); + }); + + qDebug() << "Folder" << fileId << "Coundn't be locked."; } void PropagateUploadFileCommon::slotFolderEncryptedIdError(QNetworkReply *r) diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index 7e36b166e..e6f2edaa2 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -285,7 +285,11 @@ private slots: void slotFolderEncryptedIdError(QNetworkReply *r); void slotFolderLockedSuccessfully(const QByteArray& fileId, const QByteArray& token); void slotFolderLockedError(const QByteArray& fileId, int httpErrorCode); + void slotTryLock(const QByteArray& fileId); +// Private Encryption Stuff +private: + QElapsedTimer _folderLockFirstTry; protected: /** From 70d562668c0e7c5abff1381bb8063bc8b7700c1e Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 7 Dec 2017 21:51:42 +0100 Subject: [PATCH 128/262] [CSE] Add handling for side-by-side uploads. --- src/libsync/propagateupload.cpp | 17 ++++++++++++++--- src/libsync/propagateupload.h | 1 + 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index d0a611ddc..183bcb039 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -276,13 +276,24 @@ void PropagateUploadFileCommon::slotTryLock(const QByteArray& fileId) void PropagateUploadFileCommon::slotFolderLockedSuccessfully(const QByteArray& fileId, const QByteArray& token) { qDebug() << "Folder" << fileId << "Locked Successfully for Upload"; + // Should I use a mutex here? + _currentLockingInProgress = true; } void PropagateUploadFileCommon::slotFolderLockedError(const QByteArray& fileId, int httpErrorCode) { - // Add a counter or Something, this will enter in a loop. - QTimer::singleShot(1000, this, [this, fileId]{ - if (_folderLockFirstTry.elapsed() > /* one minute */ 60000) { + /* try to call the lock from 5 to 5 seconds + and fail if it's more than 5 minutes. */ + QTimer::singleShot(5000, this, [this, fileId]{ + if (!_currentLockingInProgress) { + qDebug() << "Error locking the folder while no other update is locking it up."; + qDebug() << "Perhaps another client locked it."; + qDebug() << "Abort"; + return; + } + + // Perhaps I should remove the elapsed timer if the lock is from this client? + if (_folderLockFirstTry.elapsed() > /* five minutes */ 1000 * 60 * 5 ) { qDebug() << "One minute passed, ignoring more attemps to lock the folder."; return; } diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index e6f2edaa2..4c625f661 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -290,6 +290,7 @@ private slots: // Private Encryption Stuff private: QElapsedTimer _folderLockFirstTry; + bool _currentLockingInProgress; protected: /** From 6745777ca3be1b12f5fd28f519c86062f375a56c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 8 Dec 2017 10:19:56 +0100 Subject: [PATCH 129/262] [CSE] Save folder id and token for the lock So we can unlock it later. - perhaps we should store this on the disk or something, to make it harder to lose. --- src/libsync/propagateupload.cpp | 2 ++ src/libsync/propagateupload.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 183bcb039..049b62156 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -278,6 +278,8 @@ void PropagateUploadFileCommon::slotFolderLockedSuccessfully(const QByteArray& f qDebug() << "Folder" << fileId << "Locked Successfully for Upload"; // Should I use a mutex here? _currentLockingInProgress = true; + _folderToken = token; + _folderId = fileId; } void PropagateUploadFileCommon::slotFolderLockedError(const QByteArray& fileId, int httpErrorCode) diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index 4c625f661..3fc636fbb 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -291,6 +291,8 @@ private slots: private: QElapsedTimer _folderLockFirstTry; bool _currentLockingInProgress; + QByteArray _folderToken; + QByteArray _folderId; protected: /** From a08a32ceca9bf2596d6e8bd302f783417e89e969 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 8 Dec 2017 10:32:05 +0100 Subject: [PATCH 130/262] [CSE] Fetch the metadata for the upload. --- src/libsync/propagateupload.cpp | 10 ++++++++++ src/libsync/propagateupload.h | 1 + 2 files changed, 11 insertions(+) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 049b62156..f8eab1392 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -280,6 +280,16 @@ void PropagateUploadFileCommon::slotFolderLockedSuccessfully(const QByteArray& f _currentLockingInProgress = true; _folderToken = token; _folderId = fileId; + + auto job = new GetMetadataApiJob(propagator()->account(), _folderId); + connect(job, &GetMetadataApiJob::jsonReceived, + this, &PropagateUploadFileCommon::slotFolderEncriptedMetadataReceived); + job->start(); +} + +void PropagateUploadFileCommon::slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode) +{ + qDebug() << "Metadata Received" << json.toVariant(); } void PropagateUploadFileCommon::slotFolderLockedError(const QByteArray& fileId, int httpErrorCode) diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index 3fc636fbb..2dbc14416 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -286,6 +286,7 @@ private slots: void slotFolderLockedSuccessfully(const QByteArray& fileId, const QByteArray& token); void slotFolderLockedError(const QByteArray& fileId, int httpErrorCode); void slotTryLock(const QByteArray& fileId); + void slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode); // Private Encryption Stuff private: From 0a83d3e743b917075aeb7d90a02d148a2af8739c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 8 Dec 2017 11:24:22 +0100 Subject: [PATCH 131/262] [CSE] Fix reading the public key for the metadata This broke when we started to use QSslKey and the Qt Keychain framework. --- src/libsync/clientsideencryption.cpp | 12 ++++-------- src/libsync/clientsideencryption.h | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 5e0b7ea79..f96e97c03 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1134,16 +1134,12 @@ FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata) : // RSA/ECB/OAEPWithSHA-256AndMGF1Padding using private / public key. QByteArray FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKeys) const { - auto path = publicKeyPath(_account); - const char *pathC = qPrintable(path); - FILE* pkeyFile = fopen(pathC, "r"); - if (!pkeyFile) { - qCInfo(lcCse()) << "Could not open the public key"; - exit(1); - } + BIO *publicKeyBio = BIO_new(BIO_s_mem()); + QByteArray publicKeyPem = _account->e2e()->_publicKey.toPem(); + BIO_write(publicKeyBio, publicKeyPem.constData(), publicKeyPem.size()); - EVP_PKEY *key = PEM_read_PUBKEY(pkeyFile, NULL, NULL, NULL); + EVP_PKEY *key = PEM_read_bio_PUBKEY(publicKeyBio, NULL, NULL, NULL); auto data = QByteArray::fromStdString(metadataKeys.dump()); auto ret = EncryptionHelper::encryptStringAsymmetric(key, data); diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index f2d931971..c43dfb8cb 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -42,7 +42,6 @@ public: void encryptPrivateKey(); void setTokenForFolder(const QByteArray& folder, const QByteArray& token); QByteArray tokenForFolder(const QByteArray& folder) const; - void fetchFolderEncryptedStatus(); // to be used together with FolderStatusModel::FolderInfo::_path. @@ -77,6 +76,7 @@ private: QMap _folder2token; QMap _folder2encryptedStatus; +public: QSslKey _privateKey; QSslKey _publicKey; QSslCertificate _certificate; From e6f835d138a4a9734209106bdd07f3156393ab90 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Dec 2017 11:26:54 +0100 Subject: [PATCH 132/262] [CSE] Unlock the folder after a lock This makes sure the folder will be unlocked for next tests --- src/libsync/propagateupload.cpp | 14 ++++++++++++++ src/libsync/propagateupload.h | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index f8eab1392..f735e1a6d 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -290,6 +290,20 @@ void PropagateUploadFileCommon::slotFolderLockedSuccessfully(const QByteArray& f void PropagateUploadFileCommon::slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode) { qDebug() << "Metadata Received" << json.toVariant(); + auto *unlockJob = new UnlockEncryptFolderApiJob(propagator()->account(), _folderId, _folderToken, this); + connect(unlockJob, &UnlockEncryptFolderApiJob::success, this, &PropagateUploadFileCommon::slotUnlockEncryptedFolderSuccess); + connect(unlockJob, &UnlockEncryptFolderApiJob::error, this, &PropagateUploadFileCommon::slotUnlockEncryptedFolderError); + unlockJob->start(); +} + +void PropagateUploadFileCommon::slotUnlockEncryptedFolderSuccess(const QByteArray& fileId) +{ + qDebug() << "Unlock Job worked for folder " << fileId; +} + +void PropagateUploadFileCommon::slotUnlockEncryptedFolderError(const QByteArray& fileId, int httpStatusCode) +{ + qDebug() << "There was an error unlocking " << fileId << httpStatusCode; } void PropagateUploadFileCommon::slotFolderLockedError(const QByteArray& fileId, int httpErrorCode) diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index 2dbc14416..e7cb2626d 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -287,7 +287,8 @@ private slots: void slotFolderLockedError(const QByteArray& fileId, int httpErrorCode); void slotTryLock(const QByteArray& fileId); void slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode); - + void slotUnlockEncryptedFolderSuccess(const QByteArray& fileId); + void slotUnlockEncryptedFolderError(const QByteArray& fileId, int httpReturnCode); // Private Encryption Stuff private: QElapsedTimer _folderLockFirstTry; From 0a58ea76e52b9a7113fde1a1b317f454e79bcced Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Dec 2017 15:35:53 +0100 Subject: [PATCH 133/262] [CSE] Start the work with an existing metadata --- src/libsync/clientsideencryption.cpp | 32 +++++++++++++++++++++++++++- src/libsync/clientsideencryption.h | 2 ++ src/libsync/propagateupload.cpp | 3 +++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index f96e97c03..08b0a50f3 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1128,10 +1128,40 @@ FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata) : qCInfo(lcCse()) << "Setupping Empty Metadata"; setupEmptyMetadata(); } else { - qCInfo(lcCse()) << "Metadata already exists, deal with it later."; + qCInfo(lcCse()) << "Setting up existing metadata"; + setupExistingMetadata(); } } +/* + "{\n \"meta-data\": \"{\\\"files\\\":null,\\\"metadata\\\":{\\\"metadataKeys\\\":\\\"VTgqEKn8QBNCu5XtqeTg vmqG56j9uQ96wZUHamqilS32AMGKMO3Spu6F /jP3F5aNq66r InABxwaDq8YsuuqXPngQ0GCM3RQf /1/T427c/pFTye2bpD8v5Hi VwEjuEPNeTLoZ/YJg/0PDeeF7J5YdSiMb2UMiEJXH zAFnS2FqCCZBZdj8afnyomvxO6etvveRzIxs/JjR4SQS69AR/vJG4P/oyPDt y7Md EicMzKaV6evO2wcJzy8XM6T5rHibhw5veavSDfHrw8nrsSwU 4u7r6y rR4tajGSm6vg6pKXCBubd6ZCOvXDTSueJbWZkWP81bYxs9TPvWydTA==\\\",\\\"sharing\\\":\\\"BDLzU0ZDA1ajP4HmRfQS0/etaPBzn6t5/LFZePWXXHn/nm4nV6mGww==fA==K0oYOZuVLYr4FxDAmh7mRA==\\\",\\\"version\\\":1}}\"\n}\n" +*/ + +void FolderMetadata::setupExistingMetadata() +{ + /* This is the json response from the server, it contains two extra objects that we are *not* interested. + * ocs and data. + */ + std::string byteArray(_metadata.constData(), _metadata.length()); + + nlohmann::json j = nlohmann::json::parse(byteArray); + + // The metadata is being retrieved as a string stored in a json. + // This *seems* to be broken - the strung us nit base64 encoded, + // I'm currently unsure if this is error on my side or in the server implementation. + auto metaData = nlohmann::json::parse(j["ocs"]["data"]["meta-data"].get()); + + qDebug() << "######################################333"; + qDebug() << " EXisting Metadata"; + qDebug() << _metadata; + qDebug() << metaData.dump(4); + for (nlohmann::json::iterator it = metaData.begin(); it != metaData.end(); ++it) { + std::cout << it.key() << " : " << it.value() << std::endl; + } + qDebug() << "##########################################"; + +} + // RSA/ECB/OAEPWithSHA-256AndMGF1Padding using private / public key. QByteArray FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKeys) const { diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index c43dfb8cb..8cce02916 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -228,6 +228,8 @@ private: * to ease the port to Nlohmann Json API */ void setupEmptyMetadata(); + void setupExistingMetadata(); + QByteArray encryptMetadataKeys(const nlohmann::json& metadataKeys) const; std::string decryptMetadataKeys(const std::string& encryptedMetadataKeys) const; diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index f735e1a6d..b3ff1d398 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -290,6 +290,9 @@ void PropagateUploadFileCommon::slotFolderLockedSuccessfully(const QByteArray& f void PropagateUploadFileCommon::slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode) { qDebug() << "Metadata Received" << json.toVariant(); + + FolderMetadata metaData(propagator()->account(), json.toJson(QJsonDocument::Compact)); + auto *unlockJob = new UnlockEncryptFolderApiJob(propagator()->account(), _folderId, _folderToken, this); connect(unlockJob, &UnlockEncryptFolderApiJob::success, this, &PropagateUploadFileCommon::slotUnlockEncryptedFolderSuccess); connect(unlockJob, &UnlockEncryptFolderApiJob::error, this, &PropagateUploadFileCommon::slotUnlockEncryptedFolderError); From ce37235cc4c2e5e6bdd0721d1e7f2eb52bbe6c9f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Dec 2017 16:09:31 +0100 Subject: [PATCH 134/262] [CSE] Retrieve the Private Key from the KeyChain This fixes one thing and exposes a bug. the MetadataKeys are not being correctly unencrypted. --- src/libsync/clientsideencryption.cpp | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 08b0a50f3..694988767 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1143,23 +1143,30 @@ void FolderMetadata::setupExistingMetadata() * ocs and data. */ std::string byteArray(_metadata.constData(), _metadata.length()); - nlohmann::json j = nlohmann::json::parse(byteArray); // The metadata is being retrieved as a string stored in a json. // This *seems* to be broken - the strung us nit base64 encoded, // I'm currently unsure if this is error on my side or in the server implementation. - auto metaData = nlohmann::json::parse(j["ocs"]["data"]["meta-data"].get()); + // And because inside of the meta-data there's an object called metadata, without '-' + // make it really different. + auto meta_Data = nlohmann::json::parse(j["ocs"]["data"]["meta-data"].get()); qDebug() << "######################################333"; qDebug() << " EXisting Metadata"; qDebug() << _metadata; - qDebug() << metaData.dump(4); - for (nlohmann::json::iterator it = metaData.begin(); it != metaData.end(); ++it) { + qDebug() << meta_Data.dump(4); + for (nlohmann::json::iterator it = meta_Data.begin(); it != meta_Data.end(); ++it) { std::cout << it.key() << " : " << it.value() << std::endl; } qDebug() << "##########################################"; + // This is the encrypted metadata string. + std::string encrypted_metadata_keys = meta_Data["metadata"]["metadataKeys"]; + std::string decrypted_metadata_keys = decryptMetadataKeys(encrypted_metadata_keys); + qDebug() << encrypted_metadata_keys; + qDebug() << decrypted_metadata_keys; + } // RSA/ECB/OAEPWithSHA-256AndMGF1Padding using private / public key. @@ -1186,10 +1193,11 @@ std::string FolderMetadata::decryptMetadataKeys(const std::string& encryptedMeta size_t outlen = 0; int err = -1; - auto path = privateKeyPath(_account); - auto pathC = qPrintable(path); - auto pkeyFile = fopen(pathC, "r"); - auto key = PEM_read_PrivateKey(pkeyFile, NULL, NULL, NULL); + BIO *privateKeyBio = BIO_new(BIO_s_mem()); + QByteArray publicKeyPem = _account->e2e()->_privateKey.toPem(); + BIO_write(privateKeyBio, publicKeyPem.constData(), publicKeyPem.size()); + EVP_PKEY *key = PEM_read_bio_PrivateKey(privateKeyBio, NULL, NULL, NULL); + if (!key) { qCInfo(lcCse()) << "Error reading private key"; } @@ -1341,6 +1349,9 @@ void FolderMetadata::setupEmptyMetadata() { json recepient = {"recipient", {}}; auto b64String = encryptMetadataKeys(metadataKeyObj); + qCInfo(lcCse()) << "ENCRYPTED METADATA KEY" << b64String; + auto db64String = decryptMetadataKeys(std::string(b64String.constData(), b64String.size())); + qCInfo(lcCse()) << "DECRYPTED METADATA KEY" << b64String; auto sharingEncrypted = encryptJsonObject(recepient, newMetadataPass); json m = { From f6f078d1ee5ebd0a51b0142851a3127143277277 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Dec 2017 16:15:05 +0100 Subject: [PATCH 135/262] [CSE] Rename Variable --- src/libsync/clientsideencryption.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 694988767..2ff4aa687 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1194,8 +1194,8 @@ std::string FolderMetadata::decryptMetadataKeys(const std::string& encryptedMeta int err = -1; BIO *privateKeyBio = BIO_new(BIO_s_mem()); - QByteArray publicKeyPem = _account->e2e()->_privateKey.toPem(); - BIO_write(privateKeyBio, publicKeyPem.constData(), publicKeyPem.size()); + QByteArray privateKeyPem = _account->e2e()->_privateKey.toPem(); + BIO_write(privateKeyBio, privateKeyPem.constData(), privateKeyPem.size()); EVP_PKEY *key = PEM_read_bio_PrivateKey(privateKeyBio, NULL, NULL, NULL); if (!key) { From 2127b2629dddb027f623f82205bac651b898fea8 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Dec 2017 19:36:47 +0100 Subject: [PATCH 136/262] [CSE] Move encryption related network jobs to its own file Move encryption related network jobs to it's own file, the original file was starting to be just way too big. --- src/gui/accountsettings.cpp | 1 + src/libsync/CMakeLists.txt | 1 + src/libsync/clientsideencryption.cpp | 339 +-------------------- src/libsync/clientsideencryption.h | 233 -------------- src/libsync/clientsideencryptionjobs.cpp | 368 +++++++++++++++++++++++ src/libsync/clientsideencryptionjobs.h | 251 ++++++++++++++++ src/libsync/propagateupload.cpp | 1 + 7 files changed, 623 insertions(+), 571 deletions(-) create mode 100644 src/libsync/clientsideencryptionjobs.cpp create mode 100644 src/libsync/clientsideencryptionjobs.h diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index b312bbd5a..44aa230c2 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -33,6 +33,7 @@ #include "creds/httpcredentialsgui.h" #include "tooltipupdater.h" #include "filesystem.h" +#include "clientsideencryptionjobs.h" #include diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index f9dd52e92..2e0db0c94 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -60,6 +60,7 @@ set(libsync_SRCS syncresult.cpp theme.cpp clientsideencryption.cpp + clientsideencryptionjobs.cpp creds/dummycredentials.cpp creds/abstractcredentials.cpp creds/credentialscommon.cpp diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 2ff4aa687..37847bf9d 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -2,6 +2,7 @@ #include "account.h" #include "capabilities.h" #include "networkjobs.h" +#include "clientsideencryptionjobs.h" #include "theme.h" #include "creds/abstractcredentials.h" @@ -42,9 +43,6 @@ namespace OCC { Q_LOGGING_CATEGORY(lcCse, "sync.clientsideencryption", QtInfoMsg) -Q_LOGGING_CATEGORY(lcSignPublicKeyApiJob, "sync.networkjob.sendcsr", QtInfoMsg) -Q_LOGGING_CATEGORY(lcStorePrivateKeyApiJob, "sync.networkjob.storeprivatekey", QtInfoMsg) -Q_LOGGING_CATEGORY(lcCseJob, "sync.networkjob.clientsideencrypt", QtInfoMsg) QString baseUrl(){ return QStringLiteral("ocs/v2.php/apps/end_to_end_encryption/api/v1/"); @@ -1005,111 +1003,7 @@ void ClientSideEncryption::folderEncryptedStatusError(int error) qDebug() << "Failed to retrieve the status of the folders." << error; } -SignPublicKeyApiJob::SignPublicKeyApiJob(const AccountPtr& account, const QString& path, QObject* parent) -: AbstractNetworkJob(account, path, parent) -{ -} -void SignPublicKeyApiJob::setCsr(const QByteArray& csr) -{ - QByteArray data = "csr="; - data += QUrl::toPercentEncoding(csr); - _csr.setData(data); -} - -void SignPublicKeyApiJob::start() -{ - QNetworkRequest req; - req.setRawHeader("OCS-APIREQUEST", "true"); - QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) - }; - url.setQueryItems(params); - - qCInfo(lcSignPublicKeyApiJob) << "Sending the CSR" << _csr.data(); - sendRequest("POST", url, req, &_csr); - AbstractNetworkJob::start(); -} - -bool SignPublicKeyApiJob::finished() -{ - qCInfo(lcStorePrivateKeyApiJob()) << "Sending CSR ended with" << path() << errorString() << reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute); - - QJsonParseError error; - auto json = QJsonDocument::fromJson(reply()->readAll(), &error); - emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); - return true; -} - - -StorePrivateKeyApiJob::StorePrivateKeyApiJob(const AccountPtr& account, const QString& path, QObject* parent) -: AbstractNetworkJob(account, path, parent) -{ -} - -void StorePrivateKeyApiJob::setPrivateKey(const QByteArray& privKey) -{ - QByteArray data = "privateKey="; - data += QUrl::toPercentEncoding(privKey); - _privKey.setData(data); -} - -void StorePrivateKeyApiJob::start() -{ - QNetworkRequest req; - req.setRawHeader("OCS-APIREQUEST", "true"); - QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) - }; - url.setQueryItems(params); - - qCInfo(lcStorePrivateKeyApiJob) << "Sending the private key" << _privKey.data(); - sendRequest("POST", url, req, &_privKey); - AbstractNetworkJob::start(); -} - -bool StorePrivateKeyApiJob::finished() -{ - int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (retCode != 200) - qCInfo(lcStorePrivateKeyApiJob()) << "Sending private key ended with" << path() << errorString() << retCode; - - QJsonParseError error; - auto json = QJsonDocument::fromJson(reply()->readAll(), &error); - emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); - return true; -} - -SetEncryptionFlagApiJob::SetEncryptionFlagApiJob(const AccountPtr& account, const QByteArray& fileId, QObject* parent) -: AbstractNetworkJob(account, baseUrl() + QStringLiteral("encrypted/") + fileId, parent), _fileId(fileId) -{ -} - -void SetEncryptionFlagApiJob::start() -{ - QNetworkRequest req; - req.setRawHeader("OCS-APIREQUEST", "true"); - QUrl url = Utility::concatUrlPath(account()->url(), path()); - - qCInfo(lcCseJob()) << "marking the file with id" << _fileId << "as encrypted"; - sendRequest("PUT", url, req); - AbstractNetworkJob::start(); -} - -bool SetEncryptionFlagApiJob::finished() -{ - int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - qCInfo(lcCse()) << "Encryption Flag Return" << reply()->readAll(); - if (retCode == 200) { - emit success(_fileId); - } else { - qCInfo(lcCseJob()) << "Setting the encrypted flag failed with" << path() << errorString() << retCode; - emit error(_fileId, retCode); - } - return true; -} //TODO: Create an actuall encryption here. auto metadataKeyEnc(const QByteArray& data) -> QByteArray @@ -1371,239 +1265,8 @@ QByteArray FolderMetadata::encryptedMetadata() { return _metadata; } -LockEncryptFolderApiJob::LockEncryptFolderApiJob(const AccountPtr& account, const QByteArray& fileId, QObject* parent) -: AbstractNetworkJob(account, baseUrl() + QStringLiteral("lock/") + fileId, parent), _fileId(fileId) -{ -} - -void LockEncryptFolderApiJob::start() -{ - QNetworkRequest req; - req.setRawHeader("OCS-APIREQUEST", "true"); - QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) - }; - url.setQueryItems(params); - - qCInfo(lcCseJob()) << "locking the folder with id" << _fileId << "as encrypted"; - sendRequest("POST", url, req); - AbstractNetworkJob::start(); -} - -bool LockEncryptFolderApiJob::finished() -{ - int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (retCode != 200) { - qCInfo(lcCseJob()) << "error locking file" << path() << errorString() << retCode; - emit error(_fileId, retCode); - return true; - } - - QJsonParseError error; - auto json = QJsonDocument::fromJson(reply()->readAll(), &error); - auto obj = json.object().toVariantMap(); - auto token = obj["ocs"].toMap()["data"].toMap()["token"].toByteArray(); - qCInfo(lcCse()) << "got json:" << token; - - //TODO: Parse the token and submit. - emit success(_fileId, token); - return true; -} -UnlockEncryptFolderApiJob::UnlockEncryptFolderApiJob(const AccountPtr& account, - const QByteArray& fileId, - const QByteArray& token, - QObject* parent) -: AbstractNetworkJob(account, baseUrl() + QStringLiteral("lock/") + fileId, parent), _fileId(fileId), _token(token) -{ -} - -void UnlockEncryptFolderApiJob::start() -{ - QNetworkRequest req; - req.setRawHeader("OCS-APIREQUEST", "true"); - req.setRawHeader("token", _token); - - QUrl url = Utility::concatUrlPath(account()->url(), path()); - sendRequest("DELETE", url, req); - - AbstractNetworkJob::start(); - qCInfo(lcCseJob()) << "Starting the request to unlock."; -} - -bool UnlockEncryptFolderApiJob::finished() -{ - int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (retCode != 200) { - qCInfo(lcCseJob()) << "error unlocking file" << path() << errorString() << retCode; - qCInfo(lcCseJob()) << "Full Error Log" << reply()->readAll(); - emit error(_fileId, retCode); - return true; - } - emit success(_fileId); - return true; -} - -StoreMetaDataApiJob::StoreMetaDataApiJob(const AccountPtr& account, - const QByteArray& fileId, - const QByteArray& b64Metadata, - QObject* parent) -: AbstractNetworkJob(account, baseUrl() + QStringLiteral("meta-data/") + fileId, parent), _fileId(fileId), _b64Metadata(b64Metadata) -{ -} - -void StoreMetaDataApiJob::start() -{ - QNetworkRequest req; - req.setRawHeader("OCS-APIREQUEST", "true"); - QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")), - }; - url.setQueryItems(params); - - QByteArray data = QByteArray("metaData=") + _b64Metadata; - auto buffer = new QBuffer(this); - buffer->setData(data); - - qCInfo(lcCseJob()) << "sending the metadata for the fileId" << _fileId << "as encrypted"; - sendRequest("POST", url, req, buffer); - AbstractNetworkJob::start(); -} - -bool StoreMetaDataApiJob::finished() -{ - int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (retCode != 200) { - qCInfo(lcCseJob()) << "error sending the metadata" << path() << errorString() << retCode; - emit error(_fileId, retCode); - } - - qCInfo(lcCseJob()) << "Metadata submited to the server successfully"; - emit success(_fileId); - return true; -} - -GetMetadataApiJob::GetMetadataApiJob(const AccountPtr& account, - const QByteArray& fileId, - QObject* parent) -: AbstractNetworkJob(account, baseUrl() + QStringLiteral("meta-data/") + fileId, parent), _fileId(fileId) -{ -} - -void GetMetadataApiJob::start() -{ - QNetworkRequest req; - req.setRawHeader("OCS-APIREQUEST", "true"); - QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")), - }; - url.setQueryItems(params); - - qCInfo(lcCseJob()) << "Requesting the metadata for the fileId" << _fileId << "as encrypted"; - sendRequest("GET", url, req); - AbstractNetworkJob::start(); -} - -bool GetMetadataApiJob::finished() -{ - int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (retCode != 200) - qCInfo(lcCseJob()) << "error requesting the metadata" << path() << errorString() << retCode; - - QJsonParseError error; - auto json = QJsonDocument::fromJson(reply()->readAll(), &error); - emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); - return true; -} - -GetFolderEncryptStatusJob::GetFolderEncryptStatusJob(const AccountPtr& account, const QString& folder, QObject *parent) - : OCC::AbstractNetworkJob(account, QStringLiteral("remote.php/webdav"), parent), _folder(folder) -{ -} - -void GetFolderEncryptStatusJob::start() -{ - QNetworkRequest req; - req.setPriority(QNetworkRequest::HighPriority); - req.setRawHeader("OCS-APIREQUEST", "true"); - req.setRawHeader("Content-Type", "application/xml"); - - QByteArray xml = " "; - QBuffer *buf = new QBuffer(this); - buf->setData(xml); - buf->open(QIODevice::ReadOnly); - QString tmpPath = path() + (!_folder.isEmpty() ? "/" + _folder : QString()); - sendRequest("PROPFIND", Utility::concatUrlPath(account()->url(), tmpPath), req, buf); - - AbstractNetworkJob::start(); -} - -bool GetFolderEncryptStatusJob::finished() -{ - qCInfo(lcCse()) << "GetFolderEncryptStatus of" << reply()->request().url() << "finished with status" - << reply()->error() - << (reply()->error() == QNetworkReply::NoError ? QLatin1String("") : errorString()); - - int http_result_code = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - - if (http_result_code == 207) { - // Parse DAV response - QXmlStreamReader reader(reply()); - reader.addExtraNamespaceDeclaration(QXmlStreamNamespaceDeclaration("d", "DAV:")); - - /* Example Xml - - - - /remote.php/webdav/ - - - 0 - - HTTP/1.1 200 OK - - - - */ - - QString currFile; - int currEncryptedStatus = -1; - QMap folderStatus; - while (!reader.atEnd()) { - auto type = reader.readNext(); - if (type == QXmlStreamReader::StartElement) { - if (reader.name() == QLatin1String("href")) { - // If the current file is not a folder, ignore it. - currFile = reader.readElementText(QXmlStreamReader::SkipChildElements).remove("/remote.php/webdav/"); - if (!currFile.endsWith('/')) - currFile.clear(); - currEncryptedStatus = -1; - } - if (not currFile.isEmpty() && reader.name() == QLatin1String("is-encrypted")) { - currEncryptedStatus = (bool) reader.readElementText(QXmlStreamReader::SkipChildElements).toInt(); - } - } - - if (!currFile.isEmpty() && currEncryptedStatus != -1) { - folderStatus.insert(currFile, currEncryptedStatus); - currFile.clear(); - currEncryptedStatus = -1; - } - } - - emit encryptStatusReceived(folderStatus); - } else { - qCWarning(lcCse()) << "*not* successful, http result code is" << http_result_code - << (http_result_code == 302 ? reply()->header(QNetworkRequest::LocationHeader).toString() : QLatin1String("")); - emit encryptStatusError(http_result_code); - // emit finishedWithError(reply()); - } - return true; -} bool ClientSideEncryption::isFolderEncrypted(const QString& path) { auto it = _folder2encryptedStatus.find(path); diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 8cce02916..6f60c247e 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -83,124 +83,6 @@ public: QString _mnemonic; }; -/* - * @brief Job to sigh the CSR that return JSON - * - * To be used like this: - * \code - * _job = new SignPublicKeyApiJob(account, QLatin1String("ocs/v1.php/foo/bar"), this); - * _job->setCsr( csr ); - * connect(_job...); - * _job->start(); - * \encode - * - * @ingroup libsync - */ -class OWNCLOUDSYNC_EXPORT SignPublicKeyApiJob : public AbstractNetworkJob -{ - Q_OBJECT -public: - explicit SignPublicKeyApiJob(const AccountPtr &account, const QString &path, QObject *parent = 0); - - /** - * @brief setCsr - the CSR with the public key. - * This function needs to be called before start() obviously. - */ - void setCsr(const QByteArray& csr); - -public slots: - void start() override; - -protected: - bool finished() override; -signals: - - /** - * @brief jsonReceived - signal to report the json answer from ocs - * @param json - the parsed json document - * @param statusCode - the OCS status code: 100 (!) for success - */ - void jsonReceived(const QJsonDocument &json, int statusCode); - -private: - QBuffer _csr; -}; - -/* - * @brief Job to upload the PrivateKey that return JSON - * - * To be used like this: - * \code - * _job = new StorePrivateKeyApiJob(account, QLatin1String("ocs/v1.php/foo/bar"), this); - * _job->setPrivateKey( privKey ); - * connect(_job...); - * _job->start(); - * \encode - * - * @ingroup libsync - */ -class OWNCLOUDSYNC_EXPORT StorePrivateKeyApiJob : public AbstractNetworkJob -{ - Q_OBJECT -public: - explicit StorePrivateKeyApiJob(const AccountPtr &account, const QString &path, QObject *parent = 0); - - /** - * @brief setCsr - the CSR with the public key. - * This function needs to be called before start() obviously. - */ - void setPrivateKey(const QByteArray& privateKey); - -public slots: - void start() override; - -protected: - bool finished() override; -signals: - - /** - * @brief jsonReceived - signal to report the json answer from ocs - * @param json - the parsed json document - * @param statusCode - the OCS status code: 100 (!) for success - */ - void jsonReceived(const QJsonDocument &json, int statusCode); - -private: - QBuffer _privKey; -}; - -/* - * @brief Job to mark a folder as encrypted JSON - * - * To be used like this: - * \code - * _job = new SetEncryptionFlagApiJob(account, 2, this); - * connect(_job...); - * _job->start(); - * \encode - * - * @ingroup libsync - */ -class OWNCLOUDSYNC_EXPORT SetEncryptionFlagApiJob : public AbstractNetworkJob -{ - Q_OBJECT -public: - explicit SetEncryptionFlagApiJob(const AccountPtr &account, const QByteArray& fileId, QObject *parent = 0); - -public slots: - void start() override; - -protected: - bool finished() override; - -signals: - void success(const QByteArray fileId); - void error(const QByteArray fileId, int httpReturnCode); - -private: - QByteArray _fileId; -}; - /* Generates the Metadata for the folder */ struct EncryptedFile { QByteArray encryptionKey; @@ -213,8 +95,6 @@ struct EncryptedFile { int metadataKey; }; - - class FolderMetadata { public: FolderMetadata(AccountPtr account, const QByteArray& metadata = QByteArray()); @@ -243,120 +123,7 @@ private: QByteArray _metadata; }; -class OWNCLOUDSYNC_EXPORT LockEncryptFolderApiJob : public AbstractNetworkJob -{ - Q_OBJECT -public: - explicit LockEncryptFolderApiJob(const AccountPtr &account, const QByteArray& fileId, QObject *parent = 0); -public slots: - void start() override; - -protected: - bool finished() override; - -signals: - void success(const QByteArray& fileId, const QByteArray& token); - void error(const QByteArray& fileId, int httpdErrorCode); - -private: - QByteArray _fileId; -}; - -class OWNCLOUDSYNC_EXPORT UnlockEncryptFolderApiJob : public AbstractNetworkJob -{ - Q_OBJECT -public: - explicit UnlockEncryptFolderApiJob ( - const AccountPtr &account, - const QByteArray& fileId, - const QByteArray& token, - QObject *parent = 0); - -public slots: - void start() override; - -protected: - bool finished() override; - -signals: - void success(const QByteArray& fileId); - void error(const QByteArray& fileId, int httpReturnCode); - -private: - QByteArray _fileId; - QByteArray _token; - QBuffer *_tokenBuf; -}; - -class OWNCLOUDSYNC_EXPORT StoreMetaDataApiJob : public AbstractNetworkJob -{ - Q_OBJECT -public: - explicit StoreMetaDataApiJob ( - const AccountPtr &account, - const QByteArray& fileId, - const QByteArray& b64Metadata, - QObject *parent = 0); - -public slots: - void start() override; - -protected: - bool finished() override; - -signals: - void success(const QByteArray& fileId); - void error(const QByteArray& fileId, int httpReturnCode); - -private: - QByteArray _fileId; - QByteArray _b64Metadata; -}; - -class OWNCLOUDSYNC_EXPORT GetMetadataApiJob : public AbstractNetworkJob -{ - Q_OBJECT -public: - explicit GetMetadataApiJob ( - const AccountPtr &account, - const QByteArray& fileId, - QObject *parent = 0); - -public slots: - void start() override; - -protected: - bool finished() override; - -signals: - void jsonReceived(const QJsonDocument &json, int statusCode); - -private: - QByteArray _fileId; -}; - -/* I cant use the propfind network job because it defaults to the - * wrong dav url. - */ -class OWNCLOUDSYNC_EXPORT GetFolderEncryptStatusJob : public AbstractNetworkJob -{ - Q_OBJECT -public: - explicit GetFolderEncryptStatusJob (const AccountPtr &account, const QString& folder, QObject *parent = 0); - -public slots: - void start() override; - -protected: - bool finished() override; - -signals: - void encryptStatusReceived(const QMap folderMetadata2EncryptionStatus); - void encryptStatusError(int statusCode); -private: - QString _folder; -}; } // namespace OCC #endif diff --git a/src/libsync/clientsideencryptionjobs.cpp b/src/libsync/clientsideencryptionjobs.cpp new file mode 100644 index 000000000..9e2a6a601 --- /dev/null +++ b/src/libsync/clientsideencryptionjobs.cpp @@ -0,0 +1,368 @@ +#include "clientsideencryptionjobs.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "clientsideencryption.h" +#include "account.h" +#include "capabilities.h" +#include "networkjobs.h" +#include "clientsideencryptionjobs.h" +#include "theme.h" +#include "creds/abstractcredentials.h" + +Q_LOGGING_CATEGORY(lcSignPublicKeyApiJob, "sync.networkjob.sendcsr", QtInfoMsg) +Q_LOGGING_CATEGORY(lcStorePrivateKeyApiJob, "sync.networkjob.storeprivatekey", QtInfoMsg) +Q_LOGGING_CATEGORY(lcCseJob, "sync.networkjob.clientsideencrypt", QtInfoMsg) + +namespace OCC { + +GetFolderEncryptStatusJob::GetFolderEncryptStatusJob(const AccountPtr& account, const QString& folder, QObject *parent) + : OCC::AbstractNetworkJob(account, QStringLiteral("remote.php/webdav"), parent), _folder(folder) +{ +} + +void GetFolderEncryptStatusJob::start() +{ + QNetworkRequest req; + req.setPriority(QNetworkRequest::HighPriority); + req.setRawHeader("OCS-APIREQUEST", "true"); + req.setRawHeader("Content-Type", "application/xml"); + + QByteArray xml = " "; + QBuffer *buf = new QBuffer(this); + buf->setData(xml); + buf->open(QIODevice::ReadOnly); + QString tmpPath = path() + (!_folder.isEmpty() ? "/" + _folder : QString()); + sendRequest("PROPFIND", Utility::concatUrlPath(account()->url(), tmpPath), req, buf); + + AbstractNetworkJob::start(); +} + +bool GetFolderEncryptStatusJob::finished() +{ + qCInfo(lcCseJob()) << "GetFolderEncryptStatus of" << reply()->request().url() << "finished with status" + << reply()->error() + << (reply()->error() == QNetworkReply::NoError ? QLatin1String("") : errorString()); + + int http_result_code = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + + if (http_result_code == 207) { + // Parse DAV response + QXmlStreamReader reader(reply()); + reader.addExtraNamespaceDeclaration(QXmlStreamNamespaceDeclaration("d", "DAV:")); + + /* Example Xml + + + + /remote.php/webdav/ + + + 0 + + HTTP/1.1 200 OK + + + + */ + + QString currFile; + int currEncryptedStatus = -1; + QMap folderStatus; + while (!reader.atEnd()) { + auto type = reader.readNext(); + if (type == QXmlStreamReader::StartElement) { + if (reader.name() == QLatin1String("href")) { + // If the current file is not a folder, ignore it. + currFile = reader.readElementText(QXmlStreamReader::SkipChildElements).remove("/remote.php/webdav/"); + if (!currFile.endsWith('/')) + currFile.clear(); + currEncryptedStatus = -1; + } + if (not currFile.isEmpty() && reader.name() == QLatin1String("is-encrypted")) { + currEncryptedStatus = (bool) reader.readElementText(QXmlStreamReader::SkipChildElements).toInt(); + } + } + + if (!currFile.isEmpty() && currEncryptedStatus != -1) { + folderStatus.insert(currFile, currEncryptedStatus); + currFile.clear(); + currEncryptedStatus = -1; + } + } + + emit encryptStatusReceived(folderStatus); + } else { + qCWarning(lcCseJob()) << "*not* successful, http result code is" << http_result_code + << (http_result_code == 302 ? reply()->header(QNetworkRequest::LocationHeader).toString() : QLatin1String("")); + emit encryptStatusError(http_result_code); + // emit finishedWithError(reply()); + } + return true; +} + + +GetMetadataApiJob::GetMetadataApiJob(const AccountPtr& account, + const QByteArray& fileId, + QObject* parent) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("meta-data/") + fileId, parent), _fileId(fileId) +{ +} + +void GetMetadataApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")), + }; + url.setQueryItems(params); + + qCInfo(lcCseJob()) << "Requesting the metadata for the fileId" << _fileId << "as encrypted"; + sendRequest("GET", url, req); + AbstractNetworkJob::start(); +} + +bool GetMetadataApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) + qCInfo(lcCseJob()) << "error requesting the metadata" << path() << errorString() << retCode; + + QJsonParseError error; + auto json = QJsonDocument::fromJson(reply()->readAll(), &error); + emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); + return true; +} + +StoreMetaDataApiJob::StoreMetaDataApiJob(const AccountPtr& account, + const QByteArray& fileId, + const QByteArray& b64Metadata, + QObject* parent) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("meta-data/") + fileId, parent), _fileId(fileId), _b64Metadata(b64Metadata) +{ +} + +void StoreMetaDataApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")), + }; + url.setQueryItems(params); + + QByteArray data = QByteArray("metaData=") + _b64Metadata; + auto buffer = new QBuffer(this); + buffer->setData(data); + + qCInfo(lcCseJob()) << "sending the metadata for the fileId" << _fileId << "as encrypted"; + sendRequest("POST", url, req, buffer); + AbstractNetworkJob::start(); +} + +bool StoreMetaDataApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) { + qCInfo(lcCseJob()) << "error sending the metadata" << path() << errorString() << retCode; + emit error(_fileId, retCode); + } + + qCInfo(lcCseJob()) << "Metadata submited to the server successfully"; + emit success(_fileId); + return true; +} + + +UnlockEncryptFolderApiJob::UnlockEncryptFolderApiJob(const AccountPtr& account, + const QByteArray& fileId, + const QByteArray& token, + QObject* parent) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("lock/") + fileId, parent), _fileId(fileId), _token(token) +{ +} + +void UnlockEncryptFolderApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + req.setRawHeader("token", _token); + + QUrl url = Utility::concatUrlPath(account()->url(), path()); + sendRequest("DELETE", url, req); + + AbstractNetworkJob::start(); + qCInfo(lcCseJob()) << "Starting the request to unlock."; +} + +bool UnlockEncryptFolderApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) { + qCInfo(lcCseJob()) << "error unlocking file" << path() << errorString() << retCode; + qCInfo(lcCseJob()) << "Full Error Log" << reply()->readAll(); + emit error(_fileId, retCode); + return true; + } + emit success(_fileId); + return true; +} + + +LockEncryptFolderApiJob::LockEncryptFolderApiJob(const AccountPtr& account, const QByteArray& fileId, QObject* parent) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("lock/") + fileId, parent), _fileId(fileId) +{ +} + +void LockEncryptFolderApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) + }; + url.setQueryItems(params); + + qCInfo(lcCseJob()) << "locking the folder with id" << _fileId << "as encrypted"; + sendRequest("POST", url, req); + AbstractNetworkJob::start(); +} + +bool LockEncryptFolderApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) { + qCInfo(lcCseJob()) << "error locking file" << path() << errorString() << retCode; + emit error(_fileId, retCode); + return true; + } + + QJsonParseError error; + auto json = QJsonDocument::fromJson(reply()->readAll(), &error); + auto obj = json.object().toVariantMap(); + auto token = obj["ocs"].toMap()["data"].toMap()["token"].toByteArray(); + qCInfo(lcCseJob()) << "got json:" << token; + + //TODO: Parse the token and submit. + emit success(_fileId, token); + return true; +} + +SetEncryptionFlagApiJob::SetEncryptionFlagApiJob(const AccountPtr& account, const QByteArray& fileId, QObject* parent) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("encrypted/") + fileId, parent), _fileId(fileId) +{ +} + +void SetEncryptionFlagApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + + qCInfo(lcCseJob()) << "marking the file with id" << _fileId << "as encrypted"; + sendRequest("PUT", url, req); + AbstractNetworkJob::start(); +} + +bool SetEncryptionFlagApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + qCInfo(lcCseJob()) << "Encryption Flag Return" << reply()->readAll(); + if (retCode == 200) { + emit success(_fileId); + } else { + qCInfo(lcCseJob()) << "Setting the encrypted flag failed with" << path() << errorString() << retCode; + emit error(_fileId, retCode); + } + return true; +} + +StorePrivateKeyApiJob::StorePrivateKeyApiJob(const AccountPtr& account, const QString& path, QObject* parent) +: AbstractNetworkJob(account, path, parent) +{ +} + +void StorePrivateKeyApiJob::setPrivateKey(const QByteArray& privKey) +{ + QByteArray data = "privateKey="; + data += QUrl::toPercentEncoding(privKey); + _privKey.setData(data); +} + +void StorePrivateKeyApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) + }; + url.setQueryItems(params); + + qCInfo(lcStorePrivateKeyApiJob) << "Sending the private key" << _privKey.data(); + sendRequest("POST", url, req, &_privKey); + AbstractNetworkJob::start(); +} + +bool StorePrivateKeyApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) + qCInfo(lcStorePrivateKeyApiJob()) << "Sending private key ended with" << path() << errorString() << retCode; + + QJsonParseError error; + auto json = QJsonDocument::fromJson(reply()->readAll(), &error); + emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); + return true; +} + +SignPublicKeyApiJob::SignPublicKeyApiJob(const AccountPtr& account, const QString& path, QObject* parent) +: AbstractNetworkJob(account, path, parent) +{ +} + +void SignPublicKeyApiJob::setCsr(const QByteArray& csr) +{ + QByteArray data = "csr="; + data += QUrl::toPercentEncoding(csr); + _csr.setData(data); +} + +void SignPublicKeyApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + QUrl url = Utility::concatUrlPath(account()->url(), path()); + QList> params = { + qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) + }; + url.setQueryItems(params); + + qCInfo(lcSignPublicKeyApiJob) << "Sending the CSR" << _csr.data(); + sendRequest("POST", url, req, &_csr); + AbstractNetworkJob::start(); +} + +bool SignPublicKeyApiJob::finished() +{ + qCInfo(lcStorePrivateKeyApiJob()) << "Sending CSR ended with" << path() << errorString() << reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute); + + QJsonParseError error; + auto json = QJsonDocument::fromJson(reply()->readAll(), &error); + emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); + return true; +} + +} diff --git a/src/libsync/clientsideencryptionjobs.h b/src/libsync/clientsideencryptionjobs.h new file mode 100644 index 000000000..2cacbbe25 --- /dev/null +++ b/src/libsync/clientsideencryptionjobs.h @@ -0,0 +1,251 @@ +#ifndef CLIENTSIDEENCRYPTIONJOBS_H +#define CLIENTSIDEENCRYPTIONJOBS_H + +#include "networkjobs.h" +#include "accountfwd.h" +#include +#include + +namespace OCC { +/* Here are all of the network jobs for the client side encryption. + * anything that goes thru the server and expects a response, is here. + */ + +/* + * @brief Job to sigh the CSR that return JSON + * + * To be used like this: + * \code + * _job = new SignPublicKeyApiJob(account, QLatin1String("ocs/v1.php/foo/bar"), this); + * _job->setCsr( csr ); + * connect(_job...); + * _job->start(); + * \encode + * + * @ingroup libsync + */ +class OWNCLOUDSYNC_EXPORT SignPublicKeyApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit SignPublicKeyApiJob(const AccountPtr &account, const QString &path, QObject *parent = 0); + + /** + * @brief setCsr - the CSR with the public key. + * This function needs to be called before start() obviously. + */ + void setCsr(const QByteArray& csr); + +public slots: + void start() override; + +protected: + bool finished() override; +signals: + + /** + * @brief jsonReceived - signal to report the json answer from ocs + * @param json - the parsed json document + * @param statusCode - the OCS status code: 100 (!) for success + */ + void jsonReceived(const QJsonDocument &json, int statusCode); + +private: + QBuffer _csr; +}; + +/* + * @brief Job to upload the PrivateKey that return JSON + * + * To be used like this: + * \code + * _job = new StorePrivateKeyApiJob(account, QLatin1String("ocs/v1.php/foo/bar"), this); + * _job->setPrivateKey( privKey ); + * connect(_job...); + * _job->start(); + * \encode + * + * @ingroup libsync + */ +class OWNCLOUDSYNC_EXPORT StorePrivateKeyApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit StorePrivateKeyApiJob(const AccountPtr &account, const QString &path, QObject *parent = 0); + + /** + * @brief setCsr - the CSR with the public key. + * This function needs to be called before start() obviously. + */ + void setPrivateKey(const QByteArray& privateKey); + +public slots: + void start() override; + +protected: + bool finished() override; +signals: + + /** + * @brief jsonReceived - signal to report the json answer from ocs + * @param json - the parsed json document + * @param statusCode - the OCS status code: 100 (!) for success + */ + void jsonReceived(const QJsonDocument &json, int statusCode); + +private: + QBuffer _privKey; +}; + + +/* + * @brief Job to mark a folder as encrypted JSON + * + * To be used like this: + * \code + * _job = new SetEncryptionFlagApiJob(account, 2, this); + * connect(_job...); + * _job->start(); + * \encode + * + * @ingroup libsync + */ +class OWNCLOUDSYNC_EXPORT SetEncryptionFlagApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit SetEncryptionFlagApiJob(const AccountPtr &account, const QByteArray& fileId, QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + void success(const QByteArray fileId); + void error(const QByteArray fileId, int httpReturnCode); + +private: + QByteArray _fileId; +}; + +class OWNCLOUDSYNC_EXPORT LockEncryptFolderApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit LockEncryptFolderApiJob(const AccountPtr &account, const QByteArray& fileId, QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + void success(const QByteArray& fileId, const QByteArray& token); + void error(const QByteArray& fileId, int httpdErrorCode); + +private: + QByteArray _fileId; +}; + + +class OWNCLOUDSYNC_EXPORT UnlockEncryptFolderApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit UnlockEncryptFolderApiJob ( + const AccountPtr &account, + const QByteArray& fileId, + const QByteArray& token, + QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + void success(const QByteArray& fileId); + void error(const QByteArray& fileId, int httpReturnCode); + +private: + QByteArray _fileId; + QByteArray _token; + QBuffer *_tokenBuf; +}; + + +class OWNCLOUDSYNC_EXPORT StoreMetaDataApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit StoreMetaDataApiJob ( + const AccountPtr &account, + const QByteArray& fileId, + const QByteArray& b64Metadata, + QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + void success(const QByteArray& fileId); + void error(const QByteArray& fileId, int httpReturnCode); + +private: + QByteArray _fileId; + QByteArray _b64Metadata; +}; + +class OWNCLOUDSYNC_EXPORT GetMetadataApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit GetMetadataApiJob ( + const AccountPtr &account, + const QByteArray& fileId, + QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + void jsonReceived(const QJsonDocument &json, int statusCode); + +private: + QByteArray _fileId; +}; + +/* I cant use the propfind network job because it defaults to the + * wrong dav url. + */ +class OWNCLOUDSYNC_EXPORT GetFolderEncryptStatusJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit GetFolderEncryptStatusJob (const AccountPtr &account, const QString& folder, QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + void encryptStatusReceived(const QMap folderMetadata2EncryptionStatus); + void encryptStatusError(int statusCode); +private: + QString _folder; +}; + +} +#endif diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index b3ff1d398..1b9f3c9aa 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -28,6 +28,7 @@ #include "common/asserts.h" #include "networkjobs.h" #include "clientsideencryption.h" +#include "clientsideencryptionjobs.h" #include #include From aca298ca6492728a9852b677126163e3e3232201 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Dec 2017 20:14:31 +0100 Subject: [PATCH 137/262] [CSE] Be explicit that async encryption uses publicKey --- src/libsync/clientsideencryption.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 37847bf9d..88007806f 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -84,7 +84,7 @@ public: const QByteArray& data ); static QByteArray encryptStringAsymmetric( - EVP_PKEY *key, + EVP_PKEY *publicKey, const QByteArray& data ); static QByteArray BIO2ByteArray(BIO *b); @@ -467,10 +467,10 @@ QByteArray EncryptionHelper::encryptStringSymmetric(const QByteArray& key, const return result; } -QByteArray EncryptionHelper::encryptStringAsymmetric(EVP_PKEY *key, const QByteArray& data) { +QByteArray EncryptionHelper::encryptStringAsymmetric(EVP_PKEY *publicKey, const QByteArray& data) { int err = -1; - auto ctx = EVP_PKEY_CTX_new(key, ENGINE_get_default_RSA()); + auto ctx = EVP_PKEY_CTX_new(publicKey, ENGINE_get_default_RSA()); if (!ctx) { qCInfo(lcCse()) << "Could not initialize the pkey context."; exit(1); @@ -1027,10 +1027,6 @@ FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata) : } } -/* - "{\n \"meta-data\": \"{\\\"files\\\":null,\\\"metadata\\\":{\\\"metadataKeys\\\":\\\"VTgqEKn8QBNCu5XtqeTg vmqG56j9uQ96wZUHamqilS32AMGKMO3Spu6F /jP3F5aNq66r InABxwaDq8YsuuqXPngQ0GCM3RQf /1/T427c/pFTye2bpD8v5Hi VwEjuEPNeTLoZ/YJg/0PDeeF7J5YdSiMb2UMiEJXH zAFnS2FqCCZBZdj8afnyomvxO6etvveRzIxs/JjR4SQS69AR/vJG4P/oyPDt y7Md EicMzKaV6evO2wcJzy8XM6T5rHibhw5veavSDfHrw8nrsSwU 4u7r6y rR4tajGSm6vg6pKXCBubd6ZCOvXDTSueJbWZkWP81bYxs9TPvWydTA==\\\",\\\"sharing\\\":\\\"BDLzU0ZDA1ajP4HmRfQS0/etaPBzn6t5/LFZePWXXHn/nm4nV6mGww==fA==K0oYOZuVLYr4FxDAmh7mRA==\\\",\\\"version\\\":1}}\"\n}\n" -*/ - void FolderMetadata::setupExistingMetadata() { /* This is the json response from the server, it contains two extra objects that we are *not* interested. @@ -1070,12 +1066,12 @@ QByteArray FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKey QByteArray publicKeyPem = _account->e2e()->_publicKey.toPem(); BIO_write(publicKeyBio, publicKeyPem.constData(), publicKeyPem.size()); - EVP_PKEY *key = PEM_read_bio_PUBKEY(publicKeyBio, NULL, NULL, NULL); + EVP_PKEY *publicKey = PEM_read_bio_PUBKEY(publicKeyBio, NULL, NULL, NULL); auto data = QByteArray::fromStdString(metadataKeys.dump()); - auto ret = EncryptionHelper::encryptStringAsymmetric(key, data); + auto ret = EncryptionHelper::encryptStringAsymmetric(publicKey, data); - EVP_PKEY_free(key); + EVP_PKEY_free(publicKey); return ret; } From 159c0e138fb37ca9fc3d4602a6c7aec227e6c20f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Dec 2017 21:23:02 +0100 Subject: [PATCH 138/262] [CSE] Add a DecryptAsync method Also, cleanup a bit of the old calls. One particular difference is that I used to set padding to 1 and it should actually be PKCS1_OAEP_PADDING --- src/libsync/clientsideencryption.cpp | 133 ++++++++++++++------------- 1 file changed, 70 insertions(+), 63 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 88007806f..116855057 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -83,10 +83,17 @@ public: const QByteArray& key, const QByteArray& data ); + + //TODO: change those two EVP_PKEY into QSslKey. static QByteArray encryptStringAsymmetric( EVP_PKEY *publicKey, const QByteArray& data ); + static QByteArray decryptStringAsymmetric( + EVP_PKEY *privateKey, + const QByteArray& data + ); + static QByteArray BIO2ByteArray(BIO *b); }; @@ -467,6 +474,65 @@ QByteArray EncryptionHelper::encryptStringSymmetric(const QByteArray& key, const return result; } +QByteArray EncryptionHelper::decryptStringAsymmetric(EVP_PKEY *privateKey, const QByteArray& data) { + int err = -1; + + auto ctx = EVP_PKEY_CTX_new(privateKey, ENGINE_get_default_RSA()); + if (!ctx) { + qCInfo(lcCse()) << "Could not create the PKEY context."; + exit(1); + } + + err = EVP_PKEY_decrypt_init(ctx); + if (err <= 0) { + qCInfo(lcCse()) << "Could not init the decryption of the metadata"; + exit(1); + } + + if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) <= 0) { + qCInfo(lcCse()) << "Error setting the encryption padding."; + exit(1); + } + + if (EVP_PKEY_CTX_set_rsa_oaep_md(ctx, EVP_sha256()) <= 0) { + qCInfo(lcCse()) << "Error setting OAEP SHA 256"; + exit(1); + } + + if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, EVP_sha256()) <= 0) { + qCInfo(lcCse()) << "Error setting MGF1 padding"; + exit(1); + } + + size_t outlen = 0; + err = EVP_PKEY_decrypt(ctx, NULL, &outlen, (unsigned char *)data.constData(), data.size()); + if (err <= 0) { + qCInfo(lcCse()) << "Could not determine the buffer length"; + exit(1); + } else { + qCInfo(lcCse()) << "Size of output is: " << outlen; + } + + unsigned char *out = (unsigned char *) OPENSSL_malloc(outlen); + if (!out) { + qCInfo(lcCse()) << "Could not alloc space for the decrypted metadata"; + exit(1); + } + + if (EVP_PKEY_decrypt(ctx, out, &outlen, (unsigned char *)data.constData(), data.size()) <= 0) { + qCInfo(lcCse()) << "Could not decrypt the metadata"; + exit(1); + } + + qCInfo(lcCse()) << "Metadata decrypted successfully"; + const auto ret = std::string((char*) out, outlen); + + // Transform the encrypted data into base64. + QByteArray raw((const char*) out, outlen); + qCInfo(lcCse()) << raw; + return raw; +} + QByteArray EncryptionHelper::encryptStringAsymmetric(EVP_PKEY *publicKey, const QByteArray& data) { int err = -1; @@ -1065,7 +1131,6 @@ QByteArray FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKey BIO *publicKeyBio = BIO_new(BIO_s_mem()); QByteArray publicKeyPem = _account->e2e()->_publicKey.toPem(); BIO_write(publicKeyBio, publicKeyPem.constData(), publicKeyPem.size()); - EVP_PKEY *publicKey = PEM_read_bio_PUBKEY(publicKeyBio, NULL, NULL, NULL); auto data = QByteArray::fromStdString(metadataKeys.dump()); @@ -1076,75 +1141,17 @@ QByteArray FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKey return ret; } +//TODO: Change this from std::string to QByteArray. std::string FolderMetadata::decryptMetadataKeys(const std::string& encryptedMetadata) const { - qCInfo(lcCse()) << "Starting to decrypt the metadata key"; - unsigned char *out = nullptr; - size_t outlen = 0; - int err = -1; - BIO *privateKeyBio = BIO_new(BIO_s_mem()); QByteArray privateKeyPem = _account->e2e()->_privateKey.toPem(); BIO_write(privateKeyBio, privateKeyPem.constData(), privateKeyPem.size()); EVP_PKEY *key = PEM_read_bio_PrivateKey(privateKeyBio, NULL, NULL, NULL); - if (!key) { - qCInfo(lcCse()) << "Error reading private key"; - } - - // Data is base64 encoded. - qCInfo(lcCse()) << "encryptedMetadata" << encryptedMetadata; - auto raw = QByteArray(encryptedMetadata.c_str(), encryptedMetadata.length()); - auto b64d = QByteArray::fromBase64(raw); - auto in = (unsigned char *) b64d.constData(); - size_t inlen = b64d.length(); - - qCInfo(lcCse()) << "Encrypted metadata length: " << inlen; - - /* NB: assumes key in, inlen are already set up - * and that key is an RSA private key - */ - auto ctx = EVP_PKEY_CTX_new(key, nullptr); - if (!ctx) { - qCInfo(lcCse()) << "Could not create the PKEY context."; - exit(1); - } - - err = EVP_PKEY_decrypt_init(ctx); - if (err <= 0) { - qCInfo(lcCse()) << "Could not init the decryption of the metadata"; - exit(1); - } - - err = EVP_PKEY_CTX_set_rsa_padding(ctx, 1); //TODO: Make this a class variable. - if (err <= 0) { - qCInfo(lcCse()) << "Could not set the RSA padding"; - exit(1); - } - - err = EVP_PKEY_decrypt(ctx, NULL, &outlen, in, inlen); - if (err <= 0) { - qCInfo(lcCse()) << "Could not determine the buffer length"; - exit(1); - } else { - qCInfo(lcCse()) << "Size of output is: " << outlen; - } - - out = (unsigned char *) OPENSSL_malloc(outlen); - if (!out) { - qCInfo(lcCse()) << "Could not alloc space for the decrypted metadata"; - exit(1); - } - - err = EVP_PKEY_decrypt(ctx, out, &outlen, in, inlen); - if (err <= 0) { - qCInfo(lcCse()) << "Could not decrypt the metadata"; - exit(1); - } - - qCInfo(lcCse()) << "Metadata decrypted successfully"; - const auto ret = std::string((char*) out, outlen); - + auto data = QByteArray::fromStdString(encryptedMetadata); + auto decryptedData = EncryptionHelper::decryptStringAsymmetric(key, data); + std::string ret(decryptedData.constData(), decryptedData.length()); return ret; } From ea3558faf656ca3fe3f54a61e1c7cf77efda7739 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Dec 2017 21:29:06 +0100 Subject: [PATCH 139/262] [CSE] Fix strings --- src/libsync/clientsideencryption.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 116855057..38e8f7cbf 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -520,14 +520,13 @@ QByteArray EncryptionHelper::decryptStringAsymmetric(EVP_PKEY *privateKey, const } if (EVP_PKEY_decrypt(ctx, out, &outlen, (unsigned char *)data.constData(), data.size()) <= 0) { - qCInfo(lcCse()) << "Could not decrypt the metadata"; + qCInfo(lcCse()) << "Could not decrypt the data."; exit(1); + } else { + qCInfo(lcCse()) << "data decrypted successfully"; } - qCInfo(lcCse()) << "Metadata decrypted successfully"; const auto ret = std::string((char*) out, outlen); - - // Transform the encrypted data into base64. QByteArray raw((const char*) out, outlen); qCInfo(lcCse()) << raw; return raw; From 25734afd396fbdec240790042f04433fd6ca8bbc Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Tue, 12 Dec 2017 21:42:37 +0100 Subject: [PATCH 140/262] First base64 decode Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 38e8f7cbf..4bfb9dc1d 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -477,6 +477,8 @@ QByteArray EncryptionHelper::encryptStringSymmetric(const QByteArray& key, const QByteArray EncryptionHelper::decryptStringAsymmetric(EVP_PKEY *privateKey, const QByteArray& data) { int err = -1; + const QByteArray rawData = QByteArray::fromBase64(data); + auto ctx = EVP_PKEY_CTX_new(privateKey, ENGINE_get_default_RSA()); if (!ctx) { qCInfo(lcCse()) << "Could not create the PKEY context."; @@ -505,7 +507,7 @@ QByteArray EncryptionHelper::decryptStringAsymmetric(EVP_PKEY *privateKey, const } size_t outlen = 0; - err = EVP_PKEY_decrypt(ctx, NULL, &outlen, (unsigned char *)data.constData(), data.size()); + err = EVP_PKEY_decrypt(ctx, NULL, &outlen, (unsigned char *)rawData.constData(), rawData.size()); if (err <= 0) { qCInfo(lcCse()) << "Could not determine the buffer length"; exit(1); @@ -519,7 +521,7 @@ QByteArray EncryptionHelper::decryptStringAsymmetric(EVP_PKEY *privateKey, const exit(1); } - if (EVP_PKEY_decrypt(ctx, out, &outlen, (unsigned char *)data.constData(), data.size()) <= 0) { + if (EVP_PKEY_decrypt(ctx, out, &outlen, (unsigned char *)rawData.constData(), rawData.size()) <= 0) { qCInfo(lcCse()) << "Could not decrypt the data."; exit(1); } else { From 9916583ffab2d2c8c41179f3dac5cd721b131af5 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 12 Dec 2017 21:50:30 +0100 Subject: [PATCH 141/262] [CSE] More verbose output --- src/libsync/clientsideencryption.cpp | 31 +++++++++++++++++++--------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 4bfb9dc1d..98b788a17 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -43,6 +43,7 @@ namespace OCC { Q_LOGGING_CATEGORY(lcCse, "sync.clientsideencryption", QtInfoMsg) +Q_LOGGING_CATEGORY(lcCseDecryption, "e2e", QtInfoMsg) QString baseUrl(){ return QStringLiteral("ocs/v2.php/apps/end_to_end_encryption/api/v1/"); @@ -479,53 +480,63 @@ QByteArray EncryptionHelper::decryptStringAsymmetric(EVP_PKEY *privateKey, const const QByteArray rawData = QByteArray::fromBase64(data); + qCInfo(lcCseDecryption()) << "Start to work the decryption."; auto ctx = EVP_PKEY_CTX_new(privateKey, ENGINE_get_default_RSA()); if (!ctx) { - qCInfo(lcCse()) << "Could not create the PKEY context."; + qCInfo(lcCseDecryption()) << "Could not create the PKEY context."; + handleErrors(); exit(1); } err = EVP_PKEY_decrypt_init(ctx); if (err <= 0) { - qCInfo(lcCse()) << "Could not init the decryption of the metadata"; + qCInfo(lcCseDecryption()) << "Could not init the decryption of the metadata"; + handleErrors(); exit(1); } if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) <= 0) { - qCInfo(lcCse()) << "Error setting the encryption padding."; + qCInfo(lcCseDecryption()) << "Error setting the encryption padding."; + handleErrors(); exit(1); } if (EVP_PKEY_CTX_set_rsa_oaep_md(ctx, EVP_sha256()) <= 0) { - qCInfo(lcCse()) << "Error setting OAEP SHA 256"; + qCInfo(lcCseDecryption()) << "Error setting OAEP SHA 256"; + handleErrors(); exit(1); } if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, EVP_sha256()) <= 0) { - qCInfo(lcCse()) << "Error setting MGF1 padding"; + qCInfo(lcCseDecryption()) << "Error setting MGF1 padding"; + handleErrors(); exit(1); } size_t outlen = 0; err = EVP_PKEY_decrypt(ctx, NULL, &outlen, (unsigned char *)rawData.constData(), rawData.size()); if (err <= 0) { - qCInfo(lcCse()) << "Could not determine the buffer length"; + qCInfo(lcCseDecryption()) << "Could not determine the buffer length"; + handleErrors(); exit(1); } else { - qCInfo(lcCse()) << "Size of output is: " << outlen; + qCInfo(lcCseDecryption()) << "Size of output is: " << outlen; + qCInfo(lcCseDecryption()) << "Size of data is: " << data.size(); } unsigned char *out = (unsigned char *) OPENSSL_malloc(outlen); if (!out) { - qCInfo(lcCse()) << "Could not alloc space for the decrypted metadata"; + qCInfo(lcCseDecryption()) << "Could not alloc space for the decrypted metadata"; + handleErrors(); exit(1); } if (EVP_PKEY_decrypt(ctx, out, &outlen, (unsigned char *)rawData.constData(), rawData.size()) <= 0) { - qCInfo(lcCse()) << "Could not decrypt the data."; + qCInfo(lcCseDecryption()) << "Could not decrypt the data."; + ERR_print_errors_fp(stdout); // This line is not printing anything. exit(1); } else { - qCInfo(lcCse()) << "data decrypted successfully"; + qCInfo(lcCseDecryption()) << "data decrypted successfully"; } const auto ret = std::string((char*) out, outlen); From c592871f9469e4798a18fd3fced82d9b843b84cd Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 13 Dec 2017 16:37:52 +0100 Subject: [PATCH 142/262] [CSE] Do not handle b64 inside of the decrypt function the decrypt function should deal with the raw data always. --- src/libsync/clientsideencryption.cpp | 21 +++++++++++---------- src/libsync/clientsideencryption.h | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 98b788a17..a5ad5a54b 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -478,8 +478,6 @@ QByteArray EncryptionHelper::encryptStringSymmetric(const QByteArray& key, const QByteArray EncryptionHelper::decryptStringAsymmetric(EVP_PKEY *privateKey, const QByteArray& data) { int err = -1; - const QByteArray rawData = QByteArray::fromBase64(data); - qCInfo(lcCseDecryption()) << "Start to work the decryption."; auto ctx = EVP_PKEY_CTX_new(privateKey, ENGINE_get_default_RSA()); if (!ctx) { @@ -514,7 +512,7 @@ QByteArray EncryptionHelper::decryptStringAsymmetric(EVP_PKEY *privateKey, const } size_t outlen = 0; - err = EVP_PKEY_decrypt(ctx, NULL, &outlen, (unsigned char *)rawData.constData(), rawData.size()); + err = EVP_PKEY_decrypt(ctx, NULL, &outlen, (unsigned char *)data.constData(), data.size()); if (err <= 0) { qCInfo(lcCseDecryption()) << "Could not determine the buffer length"; handleErrors(); @@ -531,7 +529,7 @@ QByteArray EncryptionHelper::decryptStringAsymmetric(EVP_PKEY *privateKey, const exit(1); } - if (EVP_PKEY_decrypt(ctx, out, &outlen, (unsigned char *)rawData.constData(), rawData.size()) <= 0) { + if (EVP_PKEY_decrypt(ctx, out, &outlen, (unsigned char *)data.constData(), data.size()) <= 0) { qCInfo(lcCseDecryption()) << "Could not decrypt the data."; ERR_print_errors_fp(stdout); // This line is not printing anything. exit(1); @@ -1129,9 +1127,12 @@ void FolderMetadata::setupExistingMetadata() } qDebug() << "##########################################"; - // This is the encrypted metadata string. + // base 64 encrypted metadata. three transformations for it seems quite wrong. std::string encrypted_metadata_keys = meta_Data["metadata"]["metadataKeys"]; - std::string decrypted_metadata_keys = decryptMetadataKeys(encrypted_metadata_keys); + QByteArray data = QByteArray::fromStdString(encrypted_metadata_keys); + QByteArray rawData = QByteArray::fromBase64(data); + + std::string decrypted_metadata_keys = decryptMetadataKeys(rawData); qDebug() << encrypted_metadata_keys; qDebug() << decrypted_metadata_keys; @@ -1154,15 +1155,15 @@ QByteArray FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKey } //TODO: Change this from std::string to QByteArray. -std::string FolderMetadata::decryptMetadataKeys(const std::string& encryptedMetadata) const +std::string FolderMetadata::decryptMetadataKeys(const QByteArray& encryptedMetadatab64) const { BIO *privateKeyBio = BIO_new(BIO_s_mem()); QByteArray privateKeyPem = _account->e2e()->_privateKey.toPem(); BIO_write(privateKeyBio, privateKeyPem.constData(), privateKeyPem.size()); EVP_PKEY *key = PEM_read_bio_PrivateKey(privateKeyBio, NULL, NULL, NULL); - auto data = QByteArray::fromStdString(encryptedMetadata); - auto decryptedData = EncryptionHelper::decryptStringAsymmetric(key, data); + auto rawData = QByteArray::fromBase64(encryptedMetadatab64); + auto decryptedData = EncryptionHelper::decryptStringAsymmetric(key, rawData); std::string ret(decryptedData.constData(), decryptedData.length()); return ret; } @@ -1259,7 +1260,7 @@ void FolderMetadata::setupEmptyMetadata() { auto b64String = encryptMetadataKeys(metadataKeyObj); qCInfo(lcCse()) << "ENCRYPTED METADATA KEY" << b64String; - auto db64String = decryptMetadataKeys(std::string(b64String.constData(), b64String.size())); + auto db64String = decryptMetadataKeys(b64String); qCInfo(lcCse()) << "DECRYPTED METADATA KEY" << b64String; auto sharingEncrypted = encryptJsonObject(recepient, newMetadataPass); diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 6f60c247e..6591afe68 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -111,7 +111,7 @@ private: void setupExistingMetadata(); QByteArray encryptMetadataKeys(const nlohmann::json& metadataKeys) const; - std::string decryptMetadataKeys(const std::string& encryptedMetadataKeys) const; + std::string decryptMetadataKeys(const QByteArray& encryptedMetadataKeysb64) const; std::string genMetadataPass() const; QByteArray encryptJsonObject(const nlohmann::json& obj, const QByteArray pass) const; From 82c07236c1368cc0fedd71936f0d89b19d64070f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 14 Dec 2017 14:39:07 +0100 Subject: [PATCH 143/262] [CSE] Add new method "infoForFolderId" we need to query the model for the subfolderinfo, but we don't have the model index, we have the folder id. this returns it. --- src/gui/folderstatusmodel.cpp | 12 ++++++++++++ src/gui/folderstatusmodel.h | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/gui/folderstatusmodel.cpp b/src/gui/folderstatusmodel.cpp index 28616d2f6..0bf3a1f9e 100644 --- a/src/gui/folderstatusmodel.cpp +++ b/src/gui/folderstatusmodel.cpp @@ -396,6 +396,18 @@ FolderStatusModel::SubFolderInfo *FolderStatusModel::infoForIndex(const QModelIn } } +FolderStatusModel::SubFolderInfo *FolderStatusModel::infoForFileId(const QByteArray& fileId) const +{ + for(int i = 0, end = _folders.size(); i < end; i++) { + auto *info = const_cast(&_folders[i]); + if (info->_fileId == fileId) { + return info; + } + } + + return nullptr; +} + QModelIndex FolderStatusModel::indexForPath(Folder *f, const QString &path) const { if (!f) { diff --git a/src/gui/folderstatusmodel.h b/src/gui/folderstatusmodel.h index 4b6a82915..d981617a3 100644 --- a/src/gui/folderstatusmodel.h +++ b/src/gui/folderstatusmodel.h @@ -120,7 +120,7 @@ public: FetchLabel }; ItemType classify(const QModelIndex &index) const; SubFolderInfo *infoForIndex(const QModelIndex &index) const; - + SubFolderInfo *infoForFileId(const QByteArray &fileId) const; // If the selective sync check boxes were changed bool isDirty() { return _dirty; } From be3afd1291b079082a3774834078b2b8acf49502 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Thu, 14 Dec 2017 16:54:56 +0100 Subject: [PATCH 144/262] Add FileEncryptJob and FileDecryptJob This handles encryption and decryption of files. Just create the job and start off. Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 185 +++++++++++++++++++++++++++ src/libsync/clientsideencryption.h | 38 ++++++ 2 files changed, 223 insertions(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index a5ad5a54b..cf358eb4c 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include @@ -1291,4 +1292,188 @@ bool ClientSideEncryption::isFolderEncrypted(const QString& path) { return (*it); } + +FileEncryptionJob::FileEncryptionJob(QByteArray &key, QByteArray &iv, QFile *input, QFile *output, QObject *parent) + : QObject(parent), + _key(key), + _iv(iv), + _input(input), + _output(output) +{ +} + +void FileEncryptionJob::start() +{ + _input->open(QIODevice::ReadOnly); + _output->open(QIODevice::WriteOnly); + + // Init + EVP_CIPHER_CTX *ctx; + + /* Create and initialise the context */ + if(!(ctx = EVP_CIPHER_CTX_new())) { + qCInfo(lcCse()) << "Could not create context"; + exit(-1); + } + + /* Initialise the decryption operation. */ + if(!EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { + qCInfo(lcCse()) << "Could not init cipher"; + exit(-1); + } + + EVP_CIPHER_CTX_set_padding(ctx, 0); + + /* Set IV length. */ + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, _iv.size(), NULL)) { + qCInfo(lcCse()) << "Could not set iv length"; + exit(-1); + } + + /* Initialise key and IV */ + if(!EVP_EncryptInit_ex(ctx, NULL, NULL, (const unsigned char *)_key.constData(), (const unsigned char *)_iv.constData())) { + qCInfo(lcCse()) << "Could not set key and iv"; + exit(-1); + } + + unsigned char *out = (unsigned char *)malloc(sizeof(unsigned char) * (1024 + 16 -1)); + int len = 0; + int total_len = 0; + + while(!_input->atEnd()) { + QByteArray data = _input->read(1024); + + if (data.size() == 0) { + qCInfo(lcCse()) << "Could not read data from file"; + exit(-1); + } + + if(!EVP_EncryptUpdate(ctx, out, &len, (unsigned char *)data.constData(), data.size())) { + qCInfo(lcCse()) << "Could not encrypt"; + exit(-1); + } + + _output->write((char *)out, len); + total_len += len; + } + + if(1 != EVP_EncryptFinal_ex(ctx, out, &len)) { + qCInfo(lcCse()) << "Could finalize encryption"; + exit(-1); + } + _output->write((char *)out, len); + total_len += len; + + /* Get the tag */ + unsigned char *tag = (unsigned char *)malloc(sizeof(unsigned char) * 16); + if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag)) { + qCInfo(lcCse()) << "Could not get tag"; + exit(-1); + } + + _output->write((char *)tag, 16); + + free(out); + free(tag); + EVP_CIPHER_CTX_free(ctx); + + _input->close(); + _output->close(); + + emit finished(_output); +} + +FileDecryptionJob::FileDecryptionJob(QByteArray &key, QByteArray &iv, QFile *input, QFile *output, QObject *parent) + : QObject(parent), + _key(key), + _iv(iv), + _input(input), + _output(output) +{ +} + +void FileDecryptionJob::start() +{ + _input->open(QIODevice::ReadOnly); + _output->open(QIODevice::WriteOnly); + + // Init + EVP_CIPHER_CTX *ctx; + + /* Create and initialise the context */ + if(!(ctx = EVP_CIPHER_CTX_new())) { + qCInfo(lcCse()) << "Could not create context"; + exit(-1); + } + + /* Initialise the decryption operation. */ + if(!EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { + qCInfo(lcCse()) << "Could not init cipher"; + exit(-1); + } + + EVP_CIPHER_CTX_set_padding(ctx, 0); + + /* Set IV length. */ + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, _iv.size(), NULL)) { + qCInfo(lcCse()) << "Could not set iv length"; + exit(-1); + } + + /* Initialise key and IV */ + if(!EVP_DecryptInit_ex(ctx, NULL, NULL, (const unsigned char *)_key.constData(), (const unsigned char *)_iv.constData())) { + qCInfo(lcCse()) << "Could not set key and iv"; + exit(-1); + } + + qint64 size = _input->size() - 16; + + unsigned char *out = (unsigned char *)malloc(sizeof(unsigned char) * (1024 + 16 -1)); + int len = 0; + + while(_input->pos() < size) { + + int toRead = size - _input->pos(); + if (toRead > 1024) { + toRead = 1024; + } + + QByteArray data = _input->read(toRead); + + if (data.size() == 0) { + qCInfo(lcCse()) << "Could not read data from file"; + exit(-1); + } + + if(!EVP_DecryptUpdate(ctx, out, &len, (unsigned char *)data.constData(), data.size())) { + qCInfo(lcCse()) << "Could not decrypt"; + exit(-1); + } + + _output->write((char *)out, len); + } + + QByteArray tag = _input->read(16); + + /* Set expected tag value. Works in OpenSSL 1.0.1d and later */ + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, tag.size(), (unsigned char *)tag.constData())) { + qCInfo(lcCse()) << "Could not set expected tag"; + exit(-1); + } + + if(1 != EVP_DecryptFinal_ex(ctx, out, &len)) { + qCInfo(lcCse()) << "Could finalize decryption"; + exit(-1); + } + _output->write((char *)out, len); + + free(out); + EVP_CIPHER_CTX_free(ctx); + + _input->close(); + _output->close(); + + emit finished(_output); +} + } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 6591afe68..de15f3799 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -123,6 +124,43 @@ private: QByteArray _metadata; }; +class FileEncryptionJob : public QObject +{ + Q_OBJECT +public: + FileEncryptionJob(QByteArray &key, QByteArray &iv, QFile *input, QFile *output, QObject *parent = 0); + +public slots: + void start(); + +signals: + void finished(QFile *output); + +private: + QByteArray _key; + QByteArray _iv; + QPointer _input; + QPointer _output; +}; + +class FileDecryptionJob : public QObject +{ + Q_OBJECT +public: + FileDecryptionJob(QByteArray &key, QByteArray &iv, QFile *input, QFile *output, QObject *parent = 0); + +public slots: + void start(); + +signals: + void finished(QFile *output); + +private: + QByteArray _key; + QByteArray _iv; + QPointer _input; + QPointer _output; +}; } // namespace OCC From 69a201a62dc702345a640f730c174f7444c3d4c2 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Thu, 14 Dec 2017 23:11:07 +0100 Subject: [PATCH 145/262] Percentencode the json we send so special chars don't cause trouble The + sign was lost in transfer. Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryptionjobs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryptionjobs.cpp b/src/libsync/clientsideencryptionjobs.cpp index 9e2a6a601..272ae45f2 100644 --- a/src/libsync/clientsideencryptionjobs.cpp +++ b/src/libsync/clientsideencryptionjobs.cpp @@ -162,7 +162,7 @@ void StoreMetaDataApiJob::start() }; url.setQueryItems(params); - QByteArray data = QByteArray("metaData=") + _b64Metadata; + QByteArray data = QByteArray("metaData=") + QUrl::toPercentEncoding(_b64Metadata); auto buffer = new QBuffer(this); buffer->setData(data); From 99b438159140a5586aad8d0b1a7d96d81fb1f20e Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Thu, 14 Dec 2017 23:11:40 +0100 Subject: [PATCH 146/262] Don't try to decrypt twice The DecryptMetaDataKeys already base64 decoded Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index cf358eb4c..857645d66 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1131,9 +1131,8 @@ void FolderMetadata::setupExistingMetadata() // base 64 encrypted metadata. three transformations for it seems quite wrong. std::string encrypted_metadata_keys = meta_Data["metadata"]["metadataKeys"]; QByteArray data = QByteArray::fromStdString(encrypted_metadata_keys); - QByteArray rawData = QByteArray::fromBase64(data); - std::string decrypted_metadata_keys = decryptMetadataKeys(rawData); + std::string decrypted_metadata_keys = decryptMetadataKeys(data); qDebug() << encrypted_metadata_keys; qDebug() << decrypted_metadata_keys; From 43332d3ac79398d13909369dd73ff161994150e6 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 15 Dec 2017 14:00:42 +0100 Subject: [PATCH 147/262] [CSE] Properly update UI status to encrypted / decrypted --- src/gui/accountsettings.cpp | 16 +++++++++++++++- src/libsync/clientsideencryption.cpp | 5 +++++ src/libsync/clientsideencryption.h | 1 + src/libsync/networkjobs.cpp | 1 + 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 44aa230c2..0e1031dd5 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -258,6 +258,11 @@ void AccountSettings::doExpand() void AccountSettings::slotEncryptionFlagSuccess(const QByteArray& fileId) { + if (auto info = _model->infoForFileId(fileId)) { + accountsState()->account()->e2e()->setFolderEncryptedStatus(info->_path, true); + } else { + qCInfo(lcAccountSettings()) << "Could not get information from the current folder."; + } auto lockJob = new LockEncryptFolderApiJob(accountsState()->account(), fileId); connect(lockJob, &LockEncryptFolderApiJob::success, this, &AccountSettings::slotLockFolderSuccess); @@ -332,7 +337,16 @@ void AccountSettings::slotMarkSubfolderDecrypted(const QByteArray& fileId) { auto job = new OCC::DeleteApiJob(accountsState()->account(), "ocs/v2.php/apps/end_to_end_encryption/api/v1/encrypted/" + QString(fileId)); - connect(job, &OCC::DeleteApiJob::result, [](int httpResponse) { + connect(job, &OCC::DeleteApiJob::result, [this, &fileId](int httpResponse) { + if (httpResponse == 200) { + if (auto info = _model->infoForFileId(fileId)) { + accountsState()->account()->e2e()->setFolderEncryptedStatus(info->_path, false); + } else { + qCInfo(lcAccountSettings()) << "Could not get information for the current path."; + } + } else { + qCInfo(lcAccountSettings()) << "Response different than 200, cannot set folder to false."; + } qCInfo(lcAccountSettings) << "Decrypt Http Response" << httpResponse; }); job->start(); diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index cf358eb4c..d08c4a75b 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -676,6 +676,11 @@ void ClientSideEncryption::publicKeyFetched(Job *incoming) { job->start(); } +void ClientSideEncryption::setFolderEncryptedStatus(const QString& folder, bool status) +{ + _folder2encryptedStatus[folder] = status; +} + void ClientSideEncryption::privateKeyFetched(Job *incoming) { ReadPasswordJob *readJob = static_cast(incoming); diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index de15f3799..870c07d53 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -47,6 +47,7 @@ public: // to be used together with FolderStatusModel::FolderInfo::_path. bool isFolderEncrypted(const QString& path); + void setFolderEncryptedStatus(const QString& path, bool status); private slots: void folderEncryptedStatusFetched(const QMap &values); diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index 805861a3b..4dddd659d 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -962,6 +962,7 @@ bool DeleteApiJob::finished() const auto replyData = QString::fromUtf8(reply()->readAll()); qCInfo(lcJsonApiJob()) << "TMX Delete Job" << replyData; + emit result(statusCode); return true; } From 5607e27f20adcd87370a9e4a06773ebb1ddf0ce7 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 15 Dec 2017 14:25:57 +0100 Subject: [PATCH 148/262] [CSE] Actually save the http response result --- src/libsync/clientsideencryption.cpp | 5 +---- src/libsync/networkjobs.cpp | 9 +++++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index d08c4a75b..8943c51c3 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -678,6 +678,7 @@ void ClientSideEncryption::publicKeyFetched(Job *incoming) { void ClientSideEncryption::setFolderEncryptedStatus(const QString& folder, bool status) { + qDebug() << "Setting folder" << folder << "as encrypted" << status; _folder2encryptedStatus[folder] = status; } @@ -1287,9 +1288,6 @@ QByteArray FolderMetadata::encryptedMetadata() { return _metadata; } - - - bool ClientSideEncryption::isFolderEncrypted(const QString& path) { auto it = _folder2encryptedStatus.find(path); if (it == _folder2encryptedStatus.end()) @@ -1297,7 +1295,6 @@ bool ClientSideEncryption::isFolderEncrypted(const QString& path) { return (*it); } - FileEncryptionJob::FileEncryptionJob(QByteArray &key, QByteArray &iv, QFile *input, QFile *output, QObject *parent) : QObject(parent), _key(key), diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index 4dddd659d..01c1b0901 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -952,17 +952,18 @@ bool DeleteApiJob::finished() << reply()->error() << (reply()->error() == QNetworkReply::NoError ? QLatin1String("") : errorString()); - int statusCode = 0; + int httpStatus = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (reply()->error() != QNetworkReply::NoError) { - qCWarning(lcJsonApiJob) << "Network error: " << path() << errorString() << reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute); - emit result(statusCode); + qCWarning(lcJsonApiJob) << "Network error: " << path() << errorString() << httpStatus; + emit result(httpStatus); return true; } const auto replyData = QString::fromUtf8(reply()->readAll()); qCInfo(lcJsonApiJob()) << "TMX Delete Job" << replyData; - emit result(statusCode); + emit result(httpStatus); return true; } From 250c8ff1bf30c826e679c432acc8c68a683541c3 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 15 Dec 2017 15:02:43 +0100 Subject: [PATCH 149/262] [CSE] Add job to delete metadata This is needed for the correct functioning of the Set Folder as Decrypted --- src/libsync/clientsideencryptionjobs.cpp | 33 ++++++++++++++++++++++++ src/libsync/clientsideencryptionjobs.h | 23 +++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/src/libsync/clientsideencryptionjobs.cpp b/src/libsync/clientsideencryptionjobs.cpp index 9e2a6a601..ba1cdcfb7 100644 --- a/src/libsync/clientsideencryptionjobs.cpp +++ b/src/libsync/clientsideencryptionjobs.cpp @@ -220,6 +220,39 @@ bool UnlockEncryptFolderApiJob::finished() } + +DeleteMetadataApiJob::DeleteMetadataApiJob(const AccountPtr& account, + const QByteArray& fileId, + QObject* parent) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("encrypted/") + fileId, parent), _fileId(fileId) +{ +} + +void DeleteMetadataApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + + QUrl url = Utility::concatUrlPath(account()->url(), path()); + sendRequest("DELETE", url, req); + + AbstractNetworkJob::start(); + qCInfo(lcCseJob()) << "Starting the request to remove the metadata."; +} + +bool DeleteMetadataApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) { + qCInfo(lcCseJob()) << "error removing metadata for" << path() << errorString() << retCode; + qCInfo(lcCseJob()) << "Full Error Log" << reply()->readAll(); + emit error(_fileId, retCode); + return true; + } + emit success(_fileId); + return true; +} + LockEncryptFolderApiJob::LockEncryptFolderApiJob(const AccountPtr& account, const QByteArray& fileId, QObject* parent) : AbstractNetworkJob(account, baseUrl() + QStringLiteral("lock/") + fileId, parent), _fileId(fileId) { diff --git a/src/libsync/clientsideencryptionjobs.h b/src/libsync/clientsideencryptionjobs.h index 2cacbbe25..021dcbc17 100644 --- a/src/libsync/clientsideencryptionjobs.h +++ b/src/libsync/clientsideencryptionjobs.h @@ -225,6 +225,29 @@ private: QByteArray _fileId; }; +class OWNCLOUDSYNC_EXPORT DeleteMetadataApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit DeleteMetadataApiJob ( + const AccountPtr &account, + const QByteArray& fileId, + QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + void success(const QByteArray& fileId); + void error(const QByteArray& fileId, int httpErrorCode); + +private: + QByteArray _fileId; +}; + /* I cant use the propfind network job because it defaults to the * wrong dav url. */ From ce41e3e42ff3c3cbf3f639e9f57dbe08717dac42 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 15 Dec 2017 23:32:41 +0100 Subject: [PATCH 150/262] [CSE] Work the Set Decrypted Bit --- src/gui/accountsettings.cpp | 106 +++++++++++++++++++++++++++++------- src/gui/accountsettings.h | 16 +++++- 2 files changed, 101 insertions(+), 21 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 0e1031dd5..921423b6b 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -265,9 +265,9 @@ void AccountSettings::slotEncryptionFlagSuccess(const QByteArray& fileId) } auto lockJob = new LockEncryptFolderApiJob(accountsState()->account(), fileId); connect(lockJob, &LockEncryptFolderApiJob::success, - this, &AccountSettings::slotLockFolderSuccess); + this, &AccountSettings::slotLockForEncryptionSuccess); connect(lockJob, &LockEncryptFolderApiJob::error, - this, &AccountSettings::slotLockFolderError); + this, &AccountSettings::slotLockForEncryptionError); lockJob->start(); } @@ -276,7 +276,7 @@ void AccountSettings::slotEncryptionFlagError(const QByteArray& fileId, int http qDebug() << "Error on the encryption flag"; } -void AccountSettings::slotLockFolderSuccess(const QByteArray& fileId, const QByteArray &token) +void AccountSettings::slotLockForEncryptionSuccess(const QByteArray& fileId, const QByteArray &token) { accountsState()->account()->e2e()->setTokenForFolder(fileId, token); @@ -312,7 +312,7 @@ void AccountSettings::slotUpdateMetadataError(const QByteArray& folderId, int ht unlockJob->start(); } -void AccountSettings::slotLockFolderError(const QByteArray& fileId, int httpErrorCode) +void AccountSettings::slotLockForEncryptionError(const QByteArray& fileId, int httpErrorCode) { qCInfo(lcAccountSettings()) << "Locking error" << httpErrorCode; } @@ -333,23 +333,91 @@ void AccountSettings::slotMarkSubfolderEncrpted(const QByteArray& fileId) job->start(); } + +// Order: +// 1 - Lock folder, +// 2 - Delete Metadata, +// 3 - Unlock Folder, +// 4 - Mark as Decrypted. void AccountSettings::slotMarkSubfolderDecrypted(const QByteArray& fileId) { - auto job = new OCC::DeleteApiJob(accountsState()->account(), - "ocs/v2.php/apps/end_to_end_encryption/api/v1/encrypted/" + QString(fileId)); - connect(job, &OCC::DeleteApiJob::result, [this, &fileId](int httpResponse) { - if (httpResponse == 200) { - if (auto info = _model->infoForFileId(fileId)) { - accountsState()->account()->e2e()->setFolderEncryptedStatus(info->_path, false); - } else { - qCInfo(lcAccountSettings()) << "Could not get information for the current path."; - } - } else { - qCInfo(lcAccountSettings()) << "Response different than 200, cannot set folder to false."; - } - qCInfo(lcAccountSettings) << "Decrypt Http Response" << httpResponse; - }); - job->start(); + qDebug() << "Starting to mark as decrypted"; + qDebug() << "Locking the folder"; + auto lockJob = new LockEncryptFolderApiJob(accountsState()->account(), fileId); + connect(lockJob, &LockEncryptFolderApiJob::success, + this, &AccountSettings::slotLockForDecryptionSuccess); + connect(lockJob, &LockEncryptFolderApiJob::error, + this, &AccountSettings::slotLockForDecryptionError); + lockJob->start(); +} + +void AccountSettings::slotLockForDecryptionSuccess(const QByteArray& fileId, const QByteArray& token) +{ + qDebug() << "Locking success, trying to delete the metadata"; + accountsState()->account()->e2e()->setTokenForFolder(fileId, token); + auto job = new DeleteMetadataApiJob(accountsState()->account(), fileId); + connect(job, &DeleteMetadataApiJob::success, + this, &AccountSettings::slotDeleteMetadataSuccess); + connect(job, &DeleteMetadataApiJob::error, + this, &AccountSettings::slotDeleteMetadataError); + job->start(); +} + +void AccountSettings::slotDeleteMetadataSuccess(const QByteArray& fileId) +{ + qDebug() << "Metadata successfully deleted, unlocking the folder"; + auto token = accountsState()->account()->e2e()->tokenForFolder(fileId); + auto job = new UnlockEncryptFolderApiJob(accountsState()->account(), fileId, token); + connect(job, &UnlockEncryptFolderApiJob::success, + this, &AccountSettings::slotUnlockForDecryptionSuccess); + connect(job, &UnlockEncryptFolderApiJob::error, + this, &AccountSettings::slotUnlockForDecryptionError); + job->start(); +} + +void AccountSettings::slotUnlockForDecryptionSuccess(const QByteArray& fileId) +{ + qDebug() << "Unlocked the folder successfully, removing the encrypted bit."; + auto job = new OCC::DeleteApiJob(accountsState()->account(), + "ocs/v2.php/apps/end_to_end_encryption/api/v1/encrypted/" + QString(fileId)); + + // This Delete ApiJob is different than all other jobs used here, sigh. + connect(job, &OCC::DeleteApiJob::result, [this, &fileId](int httpResponse) { + if (httpResponse == 200) { + slotDecryptionFlagSuccess(fileId); + } else { + slotDecryptionFlagError(fileId, httpResponse); + } + }); + job->start(); +} + +void AccountSettings::slotDecryptionFlagSuccess(const QByteArray& fileId) +{ + if (auto info = _model->infoForFileId(fileId)) { + accountsState()->account()->e2e()->setFolderEncryptedStatus(info->_path, false); + } else { + qCInfo(lcAccountSettings()) << "Could not get information for the current path."; + } +} + +void AccountSettings::slotDecryptionFlagError(const QByteArray& fileID, int httpReturnCode) +{ + qDebug() << "Error Setting the Decryption Flag"; +} + +void AccountSettings::slotUnlockForDecryptionError(const QByteArray& fileId, int httpReturnCode) +{ + +} + +void AccountSettings::slotDeleteMetadataError(const QByteArray& fileId, int httpReturnCode) +{ + qDebug() << "Error deleting the metadata"; +} +void AccountSettings::slotLockForDecryptionError(const QByteArray& fileId, int httpReturnCode) +{ + qDebug() << "Error Locking for decryption"; } void AccountSettings::slotSubfolderContextMenuRequested(const QModelIndex& index, const QPoint& pos) diff --git a/src/gui/accountsettings.h b/src/gui/accountsettings.h index 405fc8b62..798b867f0 100644 --- a/src/gui/accountsettings.h +++ b/src/gui/accountsettings.h @@ -92,15 +92,27 @@ protected slots: void slotFolderListClicked(const QModelIndex &indx); void doExpand(); void slotLinkActivated(const QString &link); + + // Encryption Related Stuff. void slotEncryptionFlagSuccess(const QByteArray &folderId); void slotEncryptionFlagError(const QByteArray &folderId, int httpReturnCode); - void slotLockFolderSuccess(const QByteArray& folderId, const QByteArray& token); - void slotLockFolderError(const QByteArray &folderId, int httpReturnCode); + void slotLockForEncryptionSuccess(const QByteArray& folderId, const QByteArray& token); + void slotLockForEncryptionError(const QByteArray &folderId, int httpReturnCode); void slotUnlockFolderSuccess(const QByteArray& folderId); void slotUnlockFolderError(const QByteArray& folderId, int httpReturnCode); void slotUploadMetadataSuccess(const QByteArray& folderId); void slotUpdateMetadataError(const QByteArray& folderId, int httpReturnCode); + // Remove Encryotion Bit. + void slotLockForDecryptionSuccess(const QByteArray& folderId, const QByteArray& token); + void slotLockForDecryptionError(const QByteArray& folderId, int httpReturnCode); + void slotDeleteMetadataSuccess(const QByteArray& folderId); + void slotDeleteMetadataError(const QByteArray& folderId, int httpReturnCode); + void slotUnlockForDecryptionSuccess(const QByteArray& folderId); + void slotUnlockForDecryptionError(const QByteArray& folderId, int httpReturnCode); + void slotDecryptionFlagSuccess(const QByteArray& folderId); + void slotDecryptionFlagError(const QByteArray& folderId, int httpReturnCode); + private: void showConnectionLabel(const QString &message, QStringList errors = QStringList()); From 7ef2489f1d2c00b4a58ec2eb027320dfe4ba0ac2 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 17 Dec 2017 19:14:34 +0100 Subject: [PATCH 151/262] [CSE] Fix infoForFileId We need to traverse the path recursivelly. --- src/gui/accountsettings.cpp | 2 +- src/gui/folderstatusmodel.cpp | 18 +++++++++++++++--- src/gui/folderstatusmodel.h | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 921423b6b..cdcaa8f37 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -408,7 +408,7 @@ void AccountSettings::slotDecryptionFlagError(const QByteArray& fileID, int http void AccountSettings::slotUnlockForDecryptionError(const QByteArray& fileId, int httpReturnCode) { - + qDebug() << "Error unlocking folder after decryption"; } void AccountSettings::slotDeleteMetadataError(const QByteArray& fileId, int httpReturnCode) diff --git a/src/gui/folderstatusmodel.cpp b/src/gui/folderstatusmodel.cpp index 6e68ff21e..e530b08c4 100644 --- a/src/gui/folderstatusmodel.cpp +++ b/src/gui/folderstatusmodel.cpp @@ -396,12 +396,24 @@ FolderStatusModel::SubFolderInfo *FolderStatusModel::infoForIndex(const QModelIn } } -FolderStatusModel::SubFolderInfo *FolderStatusModel::infoForFileId(const QByteArray& fileId) const + +/* Recursivelly traverse the file info looking for the id */ +FolderStatusModel::SubFolderInfo *FolderStatusModel::infoForFileId(const QByteArray& fileId, SubFolderInfo* info) const { - for(int i = 0, end = _folders.size(); i < end; i++) { - auto *info = const_cast(&_folders[i]); + qDebug() << "Looking for " << fileId; + qDebug() << "Current size of the folders array" << _folders.size(); + + // We are in the root folder, start. + const QVector& infoVec = info ? info->_subs : _folders; + for(int i = 0, end = infoVec.size(); i < end; i++) { + auto *info = const_cast(&infoVec[i]); + qDebug() << "Current file id " << info->_fileId; if (info->_fileId == fileId) { return info; + } else if (info->_subs.size()) { + if (auto *subInfo = infoForFileId(fileId, info)) { + return subInfo; + } } } diff --git a/src/gui/folderstatusmodel.h b/src/gui/folderstatusmodel.h index d981617a3..4de1451f9 100644 --- a/src/gui/folderstatusmodel.h +++ b/src/gui/folderstatusmodel.h @@ -120,7 +120,7 @@ public: FetchLabel }; ItemType classify(const QModelIndex &index) const; SubFolderInfo *infoForIndex(const QModelIndex &index) const; - SubFolderInfo *infoForFileId(const QByteArray &fileId) const; + SubFolderInfo *infoForFileId(const QByteArray &fileId, SubFolderInfo *info = nullptr) const; // If the selective sync check boxes were changed bool isDirty() { return _dirty; } From 23f832caf457e49408125b771cb580a80acd2d62 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 17 Dec 2017 19:19:44 +0100 Subject: [PATCH 152/262] [CSE] Code Cleanup --- src/gui/folderstatusmodel.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/gui/folderstatusmodel.cpp b/src/gui/folderstatusmodel.cpp index e530b08c4..e6f36811b 100644 --- a/src/gui/folderstatusmodel.cpp +++ b/src/gui/folderstatusmodel.cpp @@ -396,18 +396,12 @@ FolderStatusModel::SubFolderInfo *FolderStatusModel::infoForIndex(const QModelIn } } - /* Recursivelly traverse the file info looking for the id */ FolderStatusModel::SubFolderInfo *FolderStatusModel::infoForFileId(const QByteArray& fileId, SubFolderInfo* info) const { - qDebug() << "Looking for " << fileId; - qDebug() << "Current size of the folders array" << _folders.size(); - - // We are in the root folder, start. const QVector& infoVec = info ? info->_subs : _folders; for(int i = 0, end = infoVec.size(); i < end; i++) { auto *info = const_cast(&infoVec[i]); - qDebug() << "Current file id " << info->_fileId; if (info->_fileId == fileId) { return info; } else if (info->_subs.size()) { From b9cdc8e922e44971c928b42318cd919ae3ba5b4c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 17 Dec 2017 21:19:32 +0100 Subject: [PATCH 153/262] [CSE] Fix the url to delete the metadata --- src/libsync/clientsideencryptionjobs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryptionjobs.cpp b/src/libsync/clientsideencryptionjobs.cpp index 6519e73a2..cdb25a900 100644 --- a/src/libsync/clientsideencryptionjobs.cpp +++ b/src/libsync/clientsideencryptionjobs.cpp @@ -224,7 +224,7 @@ bool UnlockEncryptFolderApiJob::finished() DeleteMetadataApiJob::DeleteMetadataApiJob(const AccountPtr& account, const QByteArray& fileId, QObject* parent) -: AbstractNetworkJob(account, baseUrl() + QStringLiteral("encrypted/") + fileId, parent), _fileId(fileId) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("meta-data/") + fileId, parent), _fileId(fileId) { } From aa36cc4a5c2063724c3b6e621921c1f55656448f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 17 Dec 2017 22:38:43 +0100 Subject: [PATCH 154/262] [CSE] Reduce probability of conflicts when rebasing Creates a new file/class to handle upload, propagateuploadencrypted.h/cpp. This should have all the logic to deal with the encryption and upload. --- src/libsync/CMakeLists.txt | 1 + src/libsync/propagateupload.cpp | 179 ++--------------------- src/libsync/propagateupload.h | 21 +-- src/libsync/propagateuploadencrypted.cpp | 174 ++++++++++++++++++++++ src/libsync/propagateuploadencrypted.h | 64 ++++++++ 5 files changed, 255 insertions(+), 184 deletions(-) create mode 100644 src/libsync/propagateuploadencrypted.cpp create mode 100644 src/libsync/propagateuploadencrypted.h diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index 2e0db0c94..ede6e7a6a 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -53,6 +53,7 @@ set(libsync_SRCS propagateremotedelete.cpp propagateremotemove.cpp propagateremotemkdir.cpp + propagateuploadencrypted.cpp syncengine.cpp syncfileitem.cpp syncfilestatus.cpp diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 1b9f3c9aa..383ae98c2 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -14,6 +14,7 @@ #include "config.h" #include "propagateupload.h" +#include "propagateuploadencrypted.h" #include "owncloudpropagator_p.h" #include "networkjobs.h" #include "account.h" @@ -170,183 +171,31 @@ void PropagateUploadFileCommon::setDeleteExisting(bool enabled) void PropagateUploadFileCommon::start() { - qDebug() << "Starting to upload a file."; - qDebug() << _item->_file; - qDebug() << _item->_fileId; - - /* Currently the File to upload has the same information as the SyncFileItem, - * but the idea is to make a separation between the local file and the file that's - * being uploaded, in our current case they are the same, but perhaps we can apply - * some filters in the future like 'resize' for pictures and so on. - * - * this by no means is a finished job, but a first step. - */ if (propagator()->account()->capabilities().clientSideEncryptionAvaliable()) { - /* If the file is in a encrypted-enabled nextcloud instance, we need to - * do the long road: Fetch the folder status of the encrypted bit, - * if it's encrypted, find the ID of the folder. - * lock the folder using it's id. - * download the metadata - * update the metadata - * upload the file - * upload the metadata - * unlock the folder. - * - * If the folder is unencrypted we just follow the old way. - */ - QFileInfo info(_item->_file); - - auto getEncryptedStatus = new GetFolderEncryptStatusJob(propagator()->account(), - info.path()); - - connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusReceived, - this, &PropagateUploadFileCommon::slotFolderEncryptedStatusFetched); - connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusError, - this, &PropagateUploadFileCommon::slotFolderEncryptedStatusError); - getEncryptedStatus->start(); + auto *encryptedJob = new PropagateUploadEncrypted(propagator(), _item); + connect(encryptedJob, &PropagateUploadEncrypted::folerNotEncrypted, + this, &PropagateUploadFileCommon::setupUnencryptedFile); + connect(encryptedJob, &PropagateUploadEncrypted::finalized, + this, &PropagateUploadFileCommon::setupEncryptedFile); + connect(encryptedJob, &PropagateUploadEncrypted::error, + []{ qDebug() << "Error setting up encryption."; }); + encryptedJob->start(); } else { - _fileToUpload._file = _item->_file; - _fileToUpload._size = _item->_size; - _fileToUpload._path = propagator()->getFilePath(_fileToUpload._file); - startUploadFile(); + setupUnencryptedFile(); } } -void PropagateUploadFileCommon::slotFolderEncryptedStatusFetched(const QMap& result) +void PropagateUploadFileCommon::setupEncryptedFile() { - qDebug() << "####################################"; - qDebug() << "Encrypted Status Result by folder:"; - for(const auto& path : result.keys()) { - qDebug() << result[path] << path; - } - qDebug() << "Uploading to Remote Folder: " << propagator()->_remoteFolder; - qDebug() << "Uploading from Local Dir" << propagator()->_localDir; - qDebug() << "Local File" << _item->_file; - qDebug() << QDir::cleanPath(propagator()->account()->url().path() + QLatin1Char('/') - + propagator()->account()->davPath() + propagator()->_remoteFolder + _item->_file); - QFileInfo fileInfo(_item->_file); - QString currFilePath = fileInfo.path(); - if (!currFilePath.endsWith(QDir::separator())) - currFilePath += QDir::separator(); - qDebug() << "###################################"; - qDebug() << "Retrieved correctly the encrypted status of the folders." << result; +} - /* We are inside an encrypted folder, we need to find it's Id. */ - if (result[currFilePath] == true) { - QFileInfo info(_item->_file); - LsColJob *job = new LsColJob(propagator()->account(), info.path(), this); - job->setProperties({"resourcetype", "http://owncloud.org/ns:fileid"}); - connect(job, &LsColJob::directoryListingSubfolders, this, &PropagateUploadFileCommon::slotFolderEncryptedIdReceived); - connect(job, &LsColJob::finishedWithError, this, &PropagateUploadFileCommon::slotFolderEncryptedIdError); - job->start(); - } else { +void PropagateUploadFileCommon::setupUnencryptedFile() +{ _fileToUpload._file = _item->_file; _fileToUpload._size = _item->_size; _fileToUpload._path = propagator()->getFilePath(_fileToUpload._file); startUploadFile(); - qDebug() << "Uploading to a folder that's not encrypted. call the default uploader."; - } -} - -/* We try to lock a folder, if it's locked we try again in one second. - * if it's still locked we try again in one second. looping untill one minute. - * -> fail. - * the 'loop': / - * slotFolderEncryptedIdReceived -> slotTryLock -> lockError -> stillTime? -> slotTryLock - * \ - * -> success. - */ - -void PropagateUploadFileCommon::slotFolderEncryptedIdReceived(const QStringList &list) -{ - auto job = qobject_cast(sender()); - const auto& folderInfo = job->_folderInfos.value(list.first()); - _folderLockFirstTry.start(); - slotTryLock(folderInfo.fileId); -} - -void PropagateUploadFileCommon::slotTryLock(const QByteArray& fileId) -{ - auto *lockJob = new LockEncryptFolderApiJob(propagator()->account(), fileId, this); - connect(lockJob, &LockEncryptFolderApiJob::success, this, &PropagateUploadFileCommon::slotFolderLockedSuccessfully); - connect(lockJob, &LockEncryptFolderApiJob::error, this, &PropagateUploadFileCommon::slotFolderLockedError); - lockJob->start(); -} - -void PropagateUploadFileCommon::slotFolderLockedSuccessfully(const QByteArray& fileId, const QByteArray& token) -{ - qDebug() << "Folder" << fileId << "Locked Successfully for Upload"; - // Should I use a mutex here? - _currentLockingInProgress = true; - _folderToken = token; - _folderId = fileId; - - auto job = new GetMetadataApiJob(propagator()->account(), _folderId); - connect(job, &GetMetadataApiJob::jsonReceived, - this, &PropagateUploadFileCommon::slotFolderEncriptedMetadataReceived); - job->start(); -} - -void PropagateUploadFileCommon::slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode) -{ - qDebug() << "Metadata Received" << json.toVariant(); - - FolderMetadata metaData(propagator()->account(), json.toJson(QJsonDocument::Compact)); - - auto *unlockJob = new UnlockEncryptFolderApiJob(propagator()->account(), _folderId, _folderToken, this); - connect(unlockJob, &UnlockEncryptFolderApiJob::success, this, &PropagateUploadFileCommon::slotUnlockEncryptedFolderSuccess); - connect(unlockJob, &UnlockEncryptFolderApiJob::error, this, &PropagateUploadFileCommon::slotUnlockEncryptedFolderError); - unlockJob->start(); -} - -void PropagateUploadFileCommon::slotUnlockEncryptedFolderSuccess(const QByteArray& fileId) -{ - qDebug() << "Unlock Job worked for folder " << fileId; -} - -void PropagateUploadFileCommon::slotUnlockEncryptedFolderError(const QByteArray& fileId, int httpStatusCode) -{ - qDebug() << "There was an error unlocking " << fileId << httpStatusCode; -} - -void PropagateUploadFileCommon::slotFolderLockedError(const QByteArray& fileId, int httpErrorCode) -{ - /* try to call the lock from 5 to 5 seconds - and fail if it's more than 5 minutes. */ - QTimer::singleShot(5000, this, [this, fileId]{ - if (!_currentLockingInProgress) { - qDebug() << "Error locking the folder while no other update is locking it up."; - qDebug() << "Perhaps another client locked it."; - qDebug() << "Abort"; - return; - } - - // Perhaps I should remove the elapsed timer if the lock is from this client? - if (_folderLockFirstTry.elapsed() > /* five minutes */ 1000 * 60 * 5 ) { - qDebug() << "One minute passed, ignoring more attemps to lock the folder."; - return; - } - slotTryLock(fileId); - }); - - qDebug() << "Folder" << fileId << "Coundn't be locked."; -} - -void PropagateUploadFileCommon::slotFolderEncryptedIdError(QNetworkReply *r) -{ - qDebug() << "Error retrieving the Id of the encrypted folder."; -} - -void PropagateUploadFileCommon::slotFolderEncryptedStatusError(int error) -{ - qDebug() << "Failed to retrieve the status of the folders." << error; -} - - -void PropagateUploadFileCommon::startUploadEncryptedFile() -{ - } void PropagateUploadFileCommon::startUploadFile() { diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index e7cb2626d..adef62590 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -252,7 +252,8 @@ public: /* start should setup the file, path and size that will be send to the server */ void start() Q_DECL_OVERRIDE; - void startUploadEncryptedFile(); + void setupEncryptedFile(); + void setupUnencryptedFile(); void startUploadFile(); bool isLikelyFinishedQuickly() Q_DECL_OVERRIDE { return _item->_size < propagator()->smallFileSize(); } @@ -278,24 +279,6 @@ private slots: void slotReplyAbortFinished(); void slotPollFinished(); - // Encryption Stuff - void slotFolderEncryptedStatusFetched(const QMap& result); - void slotFolderEncryptedStatusError(int error); - void slotFolderEncryptedIdReceived(const QStringList &list); - void slotFolderEncryptedIdError(QNetworkReply *r); - void slotFolderLockedSuccessfully(const QByteArray& fileId, const QByteArray& token); - void slotFolderLockedError(const QByteArray& fileId, int httpErrorCode); - void slotTryLock(const QByteArray& fileId); - void slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode); - void slotUnlockEncryptedFolderSuccess(const QByteArray& fileId); - void slotUnlockEncryptedFolderError(const QByteArray& fileId, int httpReturnCode); -// Private Encryption Stuff -private: - QElapsedTimer _folderLockFirstTry; - bool _currentLockingInProgress; - QByteArray _folderToken; - QByteArray _folderId; - protected: /** * Prepares the abort e.g. connects proper signals and slots diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp new file mode 100644 index 000000000..413e142e3 --- /dev/null +++ b/src/libsync/propagateuploadencrypted.cpp @@ -0,0 +1,174 @@ +#include "propagateuploadencrypted.h" +#include "clientsideencryptionjobs.h" +#include "networkjobs.h" +#include "clientsideencryption.h" +#include "account.h" + +#include +#include +#include + +namespace OCC { + +PropagateUploadEncrypted::PropagateUploadEncrypted(OwncloudPropagator *propagator, SyncFileItemPtr item) +: _propagator(propagator), + _item(item) +{ +} + +void PropagateUploadEncrypted::start() +{ + /* If the file is in a encrypted-enabled nextcloud instance, we need to + * do the long road: Fetch the folder status of the encrypted bit, + * if it's encrypted, find the ID of the folder. + * lock the folder using it's id. + * download the metadata + * update the metadata + * upload the file + * upload the metadata + * unlock the folder. + * + * If the folder is unencrypted we just follow the old way. + */ + QFileInfo info(_item->_file); + + auto getEncryptedStatus = new GetFolderEncryptStatusJob(_propagator->account(), + info.path()); + + connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusReceived, + this, &PropagateUploadEncrypted::slotFolderEncryptedStatusFetched); + connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusError, + this, &PropagateUploadEncrypted::slotFolderEncryptedStatusError); + getEncryptedStatus->start(); +} + +void PropagateUploadEncrypted::slotFolderEncryptedStatusFetched(const QMap& result) +{ + qDebug() << "####################################"; + qDebug() << "Encrypted Status Result by folder:"; + for(const auto& path : result.keys()) { + qDebug() << result[path] << path; + } + qDebug() << "Uploading to Remote Folder: " << _propagator->_remoteFolder; + qDebug() << "Uploading from Local Dir" << _propagator->_localDir; + qDebug() << "Local File" << _item->_file; + qDebug() << QDir::cleanPath(_propagator->account()->url().path() + QLatin1Char('/') + + _propagator->account()->davPath() + _propagator->_remoteFolder + _item->_file); + QFileInfo fileInfo(_item->_file); + QString currFilePath = fileInfo.path(); + if (!currFilePath.endsWith(QDir::separator())) + currFilePath += QDir::separator(); + + qDebug() << "###################################"; + qDebug() << "Retrieved correctly the encrypted status of the folders." << result; + + /* We are inside an encrypted folder, we need to find it's Id. */ + if (result[currFilePath] == true) { + QFileInfo info(_item->_file); + LsColJob *job = new LsColJob(_propagator->account(), info.path(), this); + job->setProperties({"resourcetype", "http://owncloud.org/ns:fileid"}); + connect(job, &LsColJob::directoryListingSubfolders, this, &PropagateUploadEncrypted::slotFolderEncryptedIdReceived); + connect(job, &LsColJob::finishedWithError, this, &PropagateUploadEncrypted::slotFolderEncryptedIdError); + job->start(); + } else { + emit folerNotEncrypted(); + } +} + + + +/* We try to lock a folder, if it's locked we try again in one second. + * if it's still locked we try again in one second. looping untill one minute. + * -> fail. + * the 'loop': / + * slotFolderEncryptedIdReceived -> slotTryLock -> lockError -> stillTime? -> slotTryLock + * \ + * -> success. + */ + +void PropagateUploadEncrypted::slotFolderEncryptedIdReceived(const QStringList &list) +{ + auto job = qobject_cast(sender()); + const auto& folderInfo = job->_folderInfos.value(list.first()); + _folderLockFirstTry.start(); + slotTryLock(folderInfo.fileId); +} + +void PropagateUploadEncrypted::slotTryLock(const QByteArray& fileId) +{ + auto *lockJob = new LockEncryptFolderApiJob(_propagator->account(), fileId, this); + connect(lockJob, &LockEncryptFolderApiJob::success, this, &PropagateUploadEncrypted::slotFolderLockedSuccessfully); + connect(lockJob, &LockEncryptFolderApiJob::error, this, &PropagateUploadEncrypted::slotFolderLockedError); + lockJob->start(); +} + +void PropagateUploadEncrypted::slotFolderLockedSuccessfully(const QByteArray& fileId, const QByteArray& token) +{ + qDebug() << "Folder" << fileId << "Locked Successfully for Upload"; + // Should I use a mutex here? + _currentLockingInProgress = true; + _folderToken = token; + _folderId = fileId; + + auto job = new GetMetadataApiJob(_propagator->account(), _folderId); + connect(job, &GetMetadataApiJob::jsonReceived, + this, &PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived); + job->start(); +} + +void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode) +{ + qDebug() << "Metadata Received" << json.toVariant(); + + FolderMetadata metaData(_propagator->account(), json.toJson(QJsonDocument::Compact)); + + auto *unlockJob = new UnlockEncryptFolderApiJob(_propagator->account(), _folderId, _folderToken, this); + connect(unlockJob, &UnlockEncryptFolderApiJob::success, this, &PropagateUploadEncrypted::slotUnlockEncryptedFolderSuccess); + connect(unlockJob, &UnlockEncryptFolderApiJob::error, this, &PropagateUploadEncrypted::slotUnlockEncryptedFolderError); + unlockJob->start(); +} + +void PropagateUploadEncrypted::slotUnlockEncryptedFolderSuccess(const QByteArray& fileId) +{ + qDebug() << "Unlock Job worked for folder " << fileId; +} + +void PropagateUploadEncrypted::slotUnlockEncryptedFolderError(const QByteArray& fileId, int httpStatusCode) +{ + qDebug() << "There was an error unlocking " << fileId << httpStatusCode; +} + +void PropagateUploadEncrypted::slotFolderLockedError(const QByteArray& fileId, int httpErrorCode) +{ + /* try to call the lock from 5 to 5 seconds + and fail if it's more than 5 minutes. */ + QTimer::singleShot(5000, this, [this, fileId]{ + if (!_currentLockingInProgress) { + qDebug() << "Error locking the folder while no other update is locking it up."; + qDebug() << "Perhaps another client locked it."; + qDebug() << "Abort"; + return; + } + + // Perhaps I should remove the elapsed timer if the lock is from this client? + if (_folderLockFirstTry.elapsed() > /* five minutes */ 1000 * 60 * 5 ) { + qDebug() << "One minute passed, ignoring more attemps to lock the folder."; + return; + } + slotTryLock(fileId); + }); + + qDebug() << "Folder" << fileId << "Coundn't be locked."; +} + +void PropagateUploadEncrypted::slotFolderEncryptedIdError(QNetworkReply *r) +{ + qDebug() << "Error retrieving the Id of the encrypted folder."; +} + +void PropagateUploadEncrypted::slotFolderEncryptedStatusError(int error) +{ + qDebug() << "Failed to retrieve the status of the folders." << error; +} + +} // namespace OCC diff --git a/src/libsync/propagateuploadencrypted.h b/src/libsync/propagateuploadencrypted.h new file mode 100644 index 000000000..38d747e0b --- /dev/null +++ b/src/libsync/propagateuploadencrypted.h @@ -0,0 +1,64 @@ +#ifndef PROPAGATEUPLOADENCRYPTED_H +#define PROPAGATEUPLOADENCRYPTED_H + +#include +#include +#include +#include +#include +#include + +#include "owncloudpropagator.h" + +namespace OCC { +/* This class is used if the server supports end to end encryption. + * It will fire for *any* folder, encrypted or not, because when the + * client starts the upload request we don't know if the folder is + * encrypted on the server. + * + * emits: + * finalized() if the encrypted file is ready to be uploaded + * error() if there was an error with the encryption + * folerNotEncrypted() if the file is within a folder that's not encrypted. + * + */ +class PropagateUploadEncrypted : public QObject +{ + Q_OBJECT +public: + PropagateUploadEncrypted(OwncloudPropagator *propagator, SyncFileItemPtr item); + void start(); + +private slots: + void slotFolderEncryptedStatusFetched(const QMap& result); + void slotFolderEncryptedStatusError(int error); + void slotFolderEncryptedIdReceived(const QStringList &list); + void slotFolderEncryptedIdError(QNetworkReply *r); + void slotFolderLockedSuccessfully(const QByteArray& fileId, const QByteArray& token); + void slotFolderLockedError(const QByteArray& fileId, int httpErrorCode); + void slotTryLock(const QByteArray& fileId); + void slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode); + void slotUnlockEncryptedFolderSuccess(const QByteArray& fileId); + void slotUnlockEncryptedFolderError(const QByteArray& fileId, int httpReturnCode); + +signals: + // Emmited after the file is encrypted and everythign is setup. + void finalized(); + void error(); + + // Emited if the file is not in a encrypted folder. + void folerNotEncrypted(); + +private: + OwncloudPropagator *_propagator; + SyncFileItemPtr _item; + + QElapsedTimer _folderLockFirstTry; + bool _currentLockingInProgress; + QByteArray _folderToken; + QByteArray _folderId; +}; + + +} +#endif From 5609a1dbe8f3093a1377ff7289f6b231d165b53a Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 19 Dec 2017 17:02:23 +0100 Subject: [PATCH 155/262] [CSE] Remove old debug code so I can focus. --- src/libsync/propagateuploadencrypted.cpp | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 413e142e3..25d164420 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -30,8 +30,8 @@ void PropagateUploadEncrypted::start() * * If the folder is unencrypted we just follow the old way. */ + qDebug() << "Starting to send an encrypted file!"; QFileInfo info(_item->_file); - auto getEncryptedStatus = new GetFolderEncryptStatusJob(_propagator->account(), info.path()); @@ -44,26 +44,15 @@ void PropagateUploadEncrypted::start() void PropagateUploadEncrypted::slotFolderEncryptedStatusFetched(const QMap& result) { - qDebug() << "####################################"; - qDebug() << "Encrypted Status Result by folder:"; - for(const auto& path : result.keys()) { - qDebug() << result[path] << path; - } - qDebug() << "Uploading to Remote Folder: " << _propagator->_remoteFolder; - qDebug() << "Uploading from Local Dir" << _propagator->_localDir; - qDebug() << "Local File" << _item->_file; - qDebug() << QDir::cleanPath(_propagator->account()->url().path() + QLatin1Char('/') - + _propagator->account()->davPath() + _propagator->_remoteFolder + _item->_file); + qDebug() << "Encrypted Status Fetched"; QFileInfo fileInfo(_item->_file); QString currFilePath = fileInfo.path(); if (!currFilePath.endsWith(QDir::separator())) currFilePath += QDir::separator(); - qDebug() << "###################################"; - qDebug() << "Retrieved correctly the encrypted status of the folders." << result; - /* We are inside an encrypted folder, we need to find it's Id. */ if (result[currFilePath] == true) { + qDebug() << "Folder is encrypted, let's get the Id from it."; QFileInfo info(_item->_file); LsColJob *job = new LsColJob(_propagator->account(), info.path(), this); job->setProperties({"resourcetype", "http://owncloud.org/ns:fileid"}); @@ -71,6 +60,7 @@ void PropagateUploadEncrypted::slotFolderEncryptedStatusFetched(const QMapstart(); } else { + qDebug() << "Folder is not encrypted, getting back to default."; emit folerNotEncrypted(); } } @@ -88,6 +78,7 @@ void PropagateUploadEncrypted::slotFolderEncryptedStatusFetched(const QMap(sender()); const auto& folderInfo = job->_folderInfos.value(list.first()); _folderLockFirstTry.start(); @@ -104,7 +95,7 @@ void PropagateUploadEncrypted::slotTryLock(const QByteArray& fileId) void PropagateUploadEncrypted::slotFolderLockedSuccessfully(const QByteArray& fileId, const QByteArray& token) { - qDebug() << "Folder" << fileId << "Locked Successfully for Upload"; + qDebug() << "Folder" << fileId << "Locked Successfully for Upload, Fetching Metadata"; // Should I use a mutex here? _currentLockingInProgress = true; _folderToken = token; @@ -118,9 +109,10 @@ void PropagateUploadEncrypted::slotFolderLockedSuccessfully(const QByteArray& fi void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode) { - qDebug() << "Metadata Received" << json.toVariant(); + qDebug() << "Metadata Received, Preparing it for the new file." << json.toVariant(); FolderMetadata metaData(_propagator->account(), json.toJson(QJsonDocument::Compact)); + qDebug() << "Unlockign folder because I didn't finished the metadata yet."; auto *unlockJob = new UnlockEncryptFolderApiJob(_propagator->account(), _folderId, _folderToken, this); connect(unlockJob, &UnlockEncryptFolderApiJob::success, this, &PropagateUploadEncrypted::slotUnlockEncryptedFolderSuccess); From a02246dc65249acbf3e9f7e432ee1a8adaa585e1 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 19 Dec 2017 22:47:05 +0100 Subject: [PATCH 156/262] [CSE] Fix Metadata bugs p1 The RFC was misleading, I took the chance to fix a few inconsistencies regarding the QJsonDocument. --- src/libsync/clientsideencryption.cpp | 154 ++++++++++++++------------- src/libsync/clientsideencryption.h | 11 +- 2 files changed, 83 insertions(+), 82 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index b41cd06d7..9b4d3dd99 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -45,6 +45,7 @@ namespace OCC Q_LOGGING_CATEGORY(lcCse, "sync.clientsideencryption", QtInfoMsg) Q_LOGGING_CATEGORY(lcCseDecryption, "e2e", QtInfoMsg) +Q_LOGGING_CATEGORY(lcCseMetadata, "metadata", QtInfoMsg) QString baseUrl(){ return QStringLiteral("ocs/v2.php/apps/end_to_end_encryption/api/v1/"); @@ -1102,10 +1103,10 @@ auto metadataKeyDec(const QByteArray& data) -> QByteArray FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata) : _account(account), _metadata(metadata) { if (metadata.isEmpty()) { - qCInfo(lcCse()) << "Setupping Empty Metadata"; + qCInfo(lcCseMetadata()) << "Setupping Empty Metadata"; setupEmptyMetadata(); } else { - qCInfo(lcCse()) << "Setting up existing metadata"; + qCInfo(lcCseMetadata()) << "Setting up existing metadata"; setupExistingMetadata(); } } @@ -1115,95 +1116,80 @@ void FolderMetadata::setupExistingMetadata() /* This is the json response from the server, it contains two extra objects that we are *not* interested. * ocs and data. */ - std::string byteArray(_metadata.constData(), _metadata.length()); - nlohmann::json j = nlohmann::json::parse(byteArray); + //std::string byteArray(_metadata.constData(), _metadata.length()); + //nlohmann::json j = nlohmann::json::parse(byteArray); // The metadata is being retrieved as a string stored in a json. // This *seems* to be broken - the strung us nit base64 encoded, // I'm currently unsure if this is error on my side or in the server implementation. // And because inside of the meta-data there's an object called metadata, without '-' // make it really different. - auto meta_Data = nlohmann::json::parse(j["ocs"]["data"]["meta-data"].get()); + //auto meta_Data = nlohmann::json::parse(j["ocs"]["data"]["meta-data"].get()); - qDebug() << "######################################333"; - qDebug() << " EXisting Metadata"; - qDebug() << _metadata; - qDebug() << meta_Data.dump(4); - for (nlohmann::json::iterator it = meta_Data.begin(); it != meta_Data.end(); ++it) { - std::cout << it.key() << " : " << it.value() << std::endl; - } - qDebug() << "##########################################"; + //qCInfo(lcCseMetadata()) << "######################################333"; + //qCInfo(lcCseMetadata()) << " EXisting Metadata"; + //qCInfo(lcCseMetadata()) << _metadata; + //qCInfo(lcCseMetadata()) << meta_Data.dump(4); + //for (nlohmann::json::iterator it = meta_Data.begin(); it != meta_Data.end(); ++it) { + // std::cout << it.key() << " : " << it.value() << std::endl; + // } + //qCInfo(lcCseMetadata()) << "##########################################"; // base 64 encrypted metadata. three transformations for it seems quite wrong. - std::string encrypted_metadata_keys = meta_Data["metadata"]["metadataKeys"]; - QByteArray data = QByteArray::fromStdString(encrypted_metadata_keys); - - std::string decrypted_metadata_keys = decryptMetadataKeys(data); - qDebug() << encrypted_metadata_keys; - qDebug() << decrypted_metadata_keys; + //QByteArray encrypted_metadata_keys = meta_Data["metadata"]["metadataKeys"]; + //QByteArray decrypted_metadata_keys = decryptMetadataKey(encrypted_metadata_keys); + //qCInfo(lcCseMetadata()) << "Encrypted Key" << encrypted_metadata_keys; + //qCInfo(lcCseMetadata()) << "Decrypted Key" << decrypted_metadata_keys; } // RSA/ECB/OAEPWithSHA-256AndMGF1Padding using private / public key. -QByteArray FolderMetadata::encryptMetadataKeys(const nlohmann::json& metadataKeys) const { +QByteArray FolderMetadata::encryptMetadataKey(const QByteArray& data) const { BIO *publicKeyBio = BIO_new(BIO_s_mem()); QByteArray publicKeyPem = _account->e2e()->_publicKey.toPem(); BIO_write(publicKeyBio, publicKeyPem.constData(), publicKeyPem.size()); EVP_PKEY *publicKey = PEM_read_bio_PUBKEY(publicKeyBio, NULL, NULL, NULL); - auto data = QByteArray::fromStdString(metadataKeys.dump()); auto ret = EncryptionHelper::encryptStringAsymmetric(publicKey, data); - EVP_PKEY_free(publicKey); - - return ret; + return ret; // ret is already b64 } -//TODO: Change this from std::string to QByteArray. -std::string FolderMetadata::decryptMetadataKeys(const QByteArray& encryptedMetadatab64) const +QByteArray FolderMetadata::decryptMetadataKey(const QByteArray& encryptedMetadata) const { BIO *privateKeyBio = BIO_new(BIO_s_mem()); QByteArray privateKeyPem = _account->e2e()->_privateKey.toPem(); BIO_write(privateKeyBio, privateKeyPem.constData(), privateKeyPem.size()); EVP_PKEY *key = PEM_read_bio_PrivateKey(privateKeyBio, NULL, NULL, NULL); - auto rawData = QByteArray::fromBase64(encryptedMetadatab64); - auto decryptedData = EncryptionHelper::decryptStringAsymmetric(key, rawData); - std::string ret(decryptedData.constData(), decryptedData.length()); - return ret; + return EncryptionHelper::decryptStringAsymmetric( + key, QByteArray::fromBase64(encryptedMetadata)); } // AES/GCM/NoPadding (128 bit key size) -QByteArray FolderMetadata::encryptJsonObject(const nlohmann::json& obj, const QByteArray pass) const { - auto data = QByteArray::fromStdString(obj.dump()); - return EncryptionHelper::encryptStringSymmetric(pass, data); +QByteArray FolderMetadata::encryptJsonObject(const QByteArray& obj, const QByteArray pass) const { + return EncryptionHelper::encryptStringSymmetric(pass, obj); } -std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetadata, const std::string& pass) const +QByteArray FolderMetadata::decryptJsonObject(const QByteArray& encryptedMetadata, const QByteArray& pass) const { - // Jesus, should I encrypt here in 128kb chunks? - // perhaps. - - // Data is base64 encoded. - // TODO: Transform this bit into a function to remove duplicated code. - qCInfo(lcCse()) << "encryptedMetadata" << encryptedMetadata; - auto raw = QByteArray(encryptedMetadata.c_str(), encryptedMetadata.length()); - auto b64d = QByteArray::fromBase64(raw); - auto in = (unsigned char *) b64d.constData(); + qCInfo(lcCseMetadata()) << "encryptedMetadata" << encryptedMetadata; + auto raw = encryptedMetadata; + auto in = (unsigned char *) raw.constData(); // The tag is appended but it is not part of the cipher text - size_t inlen = b64d.length() - 16; + size_t inlen = raw.length() - 16; auto tag = in + inlen; EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); if (!ctx) { - qCInfo(lcCse()) << "Coult not create decryptioncontext, aborting."; + qCInfo(lcCseMetadata()) << "Coult not create decryptioncontext, aborting."; exit(1); } /* Initialise the decryption operation. */ if(!EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { - qCInfo(lcCse()) << "Error initialializing the decryption, aborting."; + qCInfo(lcCseMetadata()) << "Error initialializing the decryption, aborting."; exit(1); } @@ -1211,14 +1197,14 @@ std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetada unsigned char *iv = (unsigned char *)"0123456789012345"; if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL)) { - qCInfo(lcCse()) << "Could not set IV length, aborting."; + qCInfo(lcCseMetadata()) << "Could not set IV length, aborting."; exit(1); } - auto key = (const unsigned char*) pass.c_str(); + auto key = (const unsigned char*) pass.constData(); int err = EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv); if (err != 1) { - qCInfo(lcCse()) << "Error setting the key and iv, aborting."; + qCInfo(lcCseMetadata()) << "Error setting the key and iv, aborting."; exit(1); } @@ -1230,57 +1216,75 @@ std::string FolderMetadata::decryptJsonObject(const std::string& encryptedMetada auto out = (unsigned char *) OPENSSL_malloc(inlen + 16); err = EVP_DecryptUpdate(ctx, out, &outlen, in, inlen); if (err != 1) { - qCInfo(lcCse()) << "Error decrypting the json blob, aborting."; + qCInfo(lcCseMetadata()) << "Error decrypting the json blob, aborting."; exit(1); } - qCInfo(lcCse()) << "currently decrypted" << std::string( (char*) out, outlen); - qCInfo(lcCse()) << "Current decrypt length" << outlen; + qCInfo(lcCseMetadata()) << "currently decrypted" << std::string( (char*) out, outlen); + qCInfo(lcCseMetadata()) << "Current decrypt length" << outlen; /* Set expected tag value. Works in OpenSSL 1.0.1d and later */ if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag)) { - qCInfo(lcCse()) << "Error setting the tag, aborting."; + qCInfo(lcCseMetadata()) << "Error setting the tag, aborting."; exit(1); } - qCInfo(lcCse()) << "Tag: " << tag; + qCInfo(lcCseMetadata()) << "Tag: " << tag; int f_len = outlen; err = EVP_DecryptFinal_ex(ctx, out + outlen, &f_len); if (err != 1) { - qCInfo(lcCse()) << "Error finalyzing the decryption, aborting."; + qCInfo(lcCseMetadata()) << "Error finalyzing the decryption, aborting."; exit(1); } - qCInfo(lcCse()) << "Decryption finalized."; - const auto ret = std::string((char*) out, outlen); + qCInfo(lcCseMetadata()) << "Decryption finalized."; + const auto ret = QByteArray((char*) out, outlen); return ret; } void FolderMetadata::setupEmptyMetadata() { - using namespace nlohmann; + qDebug() << "Settint up empty metadata"; QByteArray newMetadataPass = EncryptionHelper::generateRandom(16); - qCInfo(lcCse()) << "Key Generated for the Metadata" << newMetadataPass; + QByteArray encryptedMetadataPass = encryptMetadataKey(newMetadataPass); + QByteArray decryptedMetadataPass = decryptMetadataKey(encryptedMetadataPass); - json metadataKeyObj = {"0", newMetadataPass.toStdString()}; - json recepient = {"recipient", {}}; + qCInfo(lcCseMetadata()) << "Key Generated for the Metadata" << newMetadataPass; + qCInfo(lcCseMetadata()) << "Key Encrypted:" << encryptedMetadataPass; + qCInfo(lcCseMetadata()) << "Key Decrypted:" << decryptedMetadataPass; - auto b64String = encryptMetadataKeys(metadataKeyObj); - qCInfo(lcCse()) << "ENCRYPTED METADATA KEY" << b64String; - auto db64String = decryptMetadataKeys(b64String); - qCInfo(lcCse()) << "DECRYPTED METADATA KEY" << b64String; - auto sharingEncrypted = encryptJsonObject(recepient, newMetadataPass); + QString publicKey = _account->e2e()->_publicKey.toPem().toBase64(); + QString displayName = _account->displayName(); - json m = { - {"metadata", { - {"metadataKeys", b64String.toStdString()}, - {"sharing", sharingEncrypted.toStdString()}, - {"version",1} - }}, - {"files", { - }} + QJsonObject recepient = {{ + displayName, publicKey + }}; + QJsonDocument recepientDoc; + recepientDoc.setObject(recepient); + + QString sharingEncrypted = encryptJsonObject( + recepientDoc.toJson(QJsonDocument::Compact), newMetadataPass); + + QJsonObject metadataKeys = { + {"0", QString(encryptedMetadataPass)} + }; + QJsonObject metadata = { + {"metadataKeys", metadataKeys}, + {"sharing", sharingEncrypted}, + {"version", 1} }; - _metadata = QByteArray::fromStdString(m.dump()); + QJsonObject files = { + }; + + QJsonObject metaObject = { + {"metadata", metadataKeys}, + {"files", files} + }; + + QJsonDocument internalMetadata; + internalMetadata.setObject(metaObject); + _metadata = internalMetadata.toJson(); + qDebug() << "Generated Json" << _metadata; } QByteArray FolderMetadata::encryptedMetadata() { diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 870c07d53..56236b795 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -15,8 +15,6 @@ #include "accountfwd.h" #include "networkjobs.h" -#include - namespace QKeychain { class Job; class WritePasswordJob; @@ -112,12 +110,11 @@ private: void setupEmptyMetadata(); void setupExistingMetadata(); - QByteArray encryptMetadataKeys(const nlohmann::json& metadataKeys) const; - std::string decryptMetadataKeys(const QByteArray& encryptedMetadataKeysb64) const; + QByteArray encryptMetadataKey(const QByteArray& metadataKey) const; + QByteArray decryptMetadataKey(const QByteArray& encryptedKey) const; - std::string genMetadataPass() const; - QByteArray encryptJsonObject(const nlohmann::json& obj, const QByteArray pass) const; - std::string decryptJsonObject(const std::string& encryptedJsonBlob, const std::string& pass) const; + QByteArray encryptJsonObject(const QByteArray& obj, const QByteArray pass) const; + QByteArray decryptJsonObject(const QByteArray& encryptedJsonBlob, const QByteArray& pass) const; QVector _files; QVector _metadataKeys; From 494ae31de94080795f0525bea68dd3e528f27a4a Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 19 Dec 2017 23:09:39 +0100 Subject: [PATCH 157/262] [CSE] Fix correct json object --- src/libsync/clientsideencryption.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 9b4d3dd99..a1aef8869 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1277,7 +1277,7 @@ void FolderMetadata::setupEmptyMetadata() { }; QJsonObject metaObject = { - {"metadata", metadataKeys}, + {"metadata", metadata}, {"files", files} }; From a1e6901ecc5da643aeec20ac1129217e5d7c6c2e Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 20 Dec 2017 00:20:27 +0100 Subject: [PATCH 158/262] [CSE] Decrypt correctly the metadata There's a bug that's returning an empty bytearray, need to look into that. --- src/libsync/clientsideencryption.cpp | 114 +++++++-------------------- src/libsync/clientsideencryption.h | 2 +- 2 files changed, 31 insertions(+), 85 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index a1aef8869..b33ebf85c 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1116,31 +1116,41 @@ void FolderMetadata::setupExistingMetadata() /* This is the json response from the server, it contains two extra objects that we are *not* interested. * ocs and data. */ - //std::string byteArray(_metadata.constData(), _metadata.length()); - //nlohmann::json j = nlohmann::json::parse(byteArray); + QJsonDocument doc = QJsonDocument::fromJson(_metadata); + qCInfo(lcCseMetadata()) << doc.toJson(QJsonDocument::Compact); // The metadata is being retrieved as a string stored in a json. - // This *seems* to be broken - the strung us nit base64 encoded, + // This *seems* to be broken but the RFC doesn't explicits how it wants. // I'm currently unsure if this is error on my side or in the server implementation. // And because inside of the meta-data there's an object called metadata, without '-' // make it really different. - //auto meta_Data = nlohmann::json::parse(j["ocs"]["data"]["meta-data"].get()); - //qCInfo(lcCseMetadata()) << "######################################333"; - //qCInfo(lcCseMetadata()) << " EXisting Metadata"; - //qCInfo(lcCseMetadata()) << _metadata; - //qCInfo(lcCseMetadata()) << meta_Data.dump(4); - //for (nlohmann::json::iterator it = meta_Data.begin(); it != meta_Data.end(); ++it) { - // std::cout << it.key() << " : " << it.value() << std::endl; - // } - //qCInfo(lcCseMetadata()) << "##########################################"; + QString metaDataStr = doc.object()["ocs"] + .toObject()["data"] + .toObject()["meta-data"] + .toString(); - // base 64 encrypted metadata. three transformations for it seems quite wrong. - //QByteArray encrypted_metadata_keys = meta_Data["metadata"]["metadataKeys"]; - //QByteArray decrypted_metadata_keys = decryptMetadataKey(encrypted_metadata_keys); - //qCInfo(lcCseMetadata()) << "Encrypted Key" << encrypted_metadata_keys; - //qCInfo(lcCseMetadata()) << "Decrypted Key" << decrypted_metadata_keys; + QJsonDocument metaDataDoc = QJsonDocument::fromJson(metaDataStr.toLocal8Bit()); + QJsonObject metadataObj = metaDataDoc.object()["metadata"].toObject(); + QJsonObject metadataKeys = metadataObj["metadataKeys"].toObject(); + QByteArray sharing = metadataObj["sharing"].toString().toLocal8Bit(); + QJsonDocument debugHelper; + debugHelper.setObject(metadataKeys); + qDebug() << "Keys: " << debugHelper.toJson(QJsonDocument::Compact); + + // Iterate over the document to store the keys. I'm unsure that the keys are in order, + // perhaps it's better to store a map instead of a vector, perhaps this just doesn't matter. + for(auto it = metadataKeys.constBegin(), end = metadataKeys.constEnd(); it != end; it++) { + QByteArray currB64Pass = it.value().toString().toLocal8Bit(); + QByteArray decryptedKey = decryptMetadataKey(currB64Pass); + _metadataKeys.push_back(decryptedKey); + } + + // Cool, We actually have the key, we can decrypt the rest of the metadata. + qDebug() << "Sharing: " << sharing; + QByteArray sharingDecrypted = decryptJsonObject(sharing, _metadataKeys.last()); + qDebug() << "Sharing Decrypted" << sharingDecrypted; } // RSA/ECB/OAEPWithSHA-256AndMGF1Padding using private / public key. @@ -1168,78 +1178,14 @@ QByteArray FolderMetadata::decryptMetadataKey(const QByteArray& encryptedMetadat } // AES/GCM/NoPadding (128 bit key size) -QByteArray FolderMetadata::encryptJsonObject(const QByteArray& obj, const QByteArray pass) const { +QByteArray FolderMetadata::encryptJsonObject(const QByteArray& obj, const QByteArray pass) const +{ return EncryptionHelper::encryptStringSymmetric(pass, obj); } QByteArray FolderMetadata::decryptJsonObject(const QByteArray& encryptedMetadata, const QByteArray& pass) const { - qCInfo(lcCseMetadata()) << "encryptedMetadata" << encryptedMetadata; - auto raw = encryptedMetadata; - auto in = (unsigned char *) raw.constData(); - // The tag is appended but it is not part of the cipher text - size_t inlen = raw.length() - 16; - auto tag = in + inlen; - - EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); - if (!ctx) { - qCInfo(lcCseMetadata()) << "Coult not create decryptioncontext, aborting."; - exit(1); - } - - /* Initialise the decryption operation. */ - if(!EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { - qCInfo(lcCseMetadata()) << "Error initialializing the decryption, aborting."; - exit(1); - } - - EVP_CIPHER_CTX_set_padding(ctx, 0); - - unsigned char *iv = (unsigned char *)"0123456789012345"; - if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 16, NULL)) { - qCInfo(lcCseMetadata()) << "Could not set IV length, aborting."; - exit(1); - } - - auto key = (const unsigned char*) pass.constData(); - int err = EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv); - if (err != 1) { - qCInfo(lcCseMetadata()) << "Error setting the key and iv, aborting."; - exit(1); - } - - int outlen = 0; - - /* - * Max outlen is inlen + blocksize (16 bytes) - */ - auto out = (unsigned char *) OPENSSL_malloc(inlen + 16); - err = EVP_DecryptUpdate(ctx, out, &outlen, in, inlen); - if (err != 1) { - qCInfo(lcCseMetadata()) << "Error decrypting the json blob, aborting."; - exit(1); - } - qCInfo(lcCseMetadata()) << "currently decrypted" << std::string( (char*) out, outlen); - qCInfo(lcCseMetadata()) << "Current decrypt length" << outlen; - - /* Set expected tag value. Works in OpenSSL 1.0.1d and later */ - if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag)) { - qCInfo(lcCseMetadata()) << "Error setting the tag, aborting."; - exit(1); - } - - qCInfo(lcCseMetadata()) << "Tag: " << tag; - - int f_len = outlen; - err = EVP_DecryptFinal_ex(ctx, out + outlen, &f_len); - if (err != 1) { - qCInfo(lcCseMetadata()) << "Error finalyzing the decryption, aborting."; - exit(1); - } - - qCInfo(lcCseMetadata()) << "Decryption finalized."; - const auto ret = QByteArray((char*) out, outlen); - return ret; + return EncryptionHelper::decryptStringSymmetric(pass, encryptedMetadata); } void FolderMetadata::setupEmptyMetadata() { diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 56236b795..8d7d78e22 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -117,7 +117,7 @@ private: QByteArray decryptJsonObject(const QByteArray& encryptedJsonBlob, const QByteArray& pass) const; QVector _files; - QVector _metadataKeys; + QVector _metadataKeys; AccountPtr _account; QByteArray _metadata; }; From d26ade18709e1cdbfc962cfb8e13a1c826bf54c2 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 20 Dec 2017 00:25:39 +0100 Subject: [PATCH 159/262] [CSE] Fix the result of the decryption However the text still looks like encrypted. --- src/libsync/clientsideencryption.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index b33ebf85c..b4d31cc8d 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -402,7 +402,7 @@ QByteArray EncryptionHelper::decryptStringSymmetric(const QByteArray& key, const free(ptext); EVP_CIPHER_CTX_free(ctx); - return QByteArray(); + return result; } QByteArray EncryptionHelper::encryptStringSymmetric(const QByteArray& key, const QByteArray& data) { From ab89231ca2087ab8f637a196db100452ea976f7f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 20 Dec 2017 00:25:56 +0100 Subject: [PATCH 160/262] [CSE] Remove unused debug --- src/libsync/clientsideencryption.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index b4d31cc8d..97352b43a 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1194,10 +1194,6 @@ void FolderMetadata::setupEmptyMetadata() { QByteArray encryptedMetadataPass = encryptMetadataKey(newMetadataPass); QByteArray decryptedMetadataPass = decryptMetadataKey(encryptedMetadataPass); - qCInfo(lcCseMetadata()) << "Key Generated for the Metadata" << newMetadataPass; - qCInfo(lcCseMetadata()) << "Key Encrypted:" << encryptedMetadataPass; - qCInfo(lcCseMetadata()) << "Key Decrypted:" << decryptedMetadataPass; - QString publicKey = _account->e2e()->_publicKey.toPem().toBase64(); QString displayName = _account->displayName(); From ee281963fead220ae558cf7ba4d1c01d11ae18b4 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Wed, 20 Dec 2017 10:22:35 +0100 Subject: [PATCH 161/262] Base64 magic: * metadata keys: bin -> b64 -> enc -> b64 * sharing metadat: string -> b64 -> enc -> b64 Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 97352b43a..6b195d66e 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1087,19 +1087,6 @@ void ClientSideEncryption::folderEncryptedStatusError(int error) qDebug() << "Failed to retrieve the status of the folders." << error; } - - -//TODO: Create an actuall encryption here. -auto metadataKeyEnc(const QByteArray& data) -> QByteArray -{ - return data; -} - -auto metadataKeyDec(const QByteArray& data) -> QByteArray -{ - return data; -} - FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata) : _account(account), _metadata(metadata) { if (metadata.isEmpty()) { @@ -1151,6 +1138,7 @@ void FolderMetadata::setupExistingMetadata() qDebug() << "Sharing: " << sharing; QByteArray sharingDecrypted = decryptJsonObject(sharing, _metadataKeys.last()); qDebug() << "Sharing Decrypted" << sharingDecrypted; + qDebug() << "Sharing B64 Decrypted" << QByteArray::fromBase64(sharingDecrypted); } // RSA/ECB/OAEPWithSHA-256AndMGF1Padding using private / public key. @@ -1161,7 +1149,8 @@ QByteArray FolderMetadata::encryptMetadataKey(const QByteArray& data) const { BIO_write(publicKeyBio, publicKeyPem.constData(), publicKeyPem.size()); EVP_PKEY *publicKey = PEM_read_bio_PUBKEY(publicKeyBio, NULL, NULL, NULL); - auto ret = EncryptionHelper::encryptStringAsymmetric(publicKey, data); + // The metadata key is binary so base64 encode it first + auto ret = EncryptionHelper::encryptStringAsymmetric(publicKey, data.toBase64()); EVP_PKEY_free(publicKey); return ret; // ret is already b64 } @@ -1173,8 +1162,13 @@ QByteArray FolderMetadata::decryptMetadataKey(const QByteArray& encryptedMetadat BIO_write(privateKeyBio, privateKeyPem.constData(), privateKeyPem.size()); EVP_PKEY *key = PEM_read_bio_PrivateKey(privateKeyBio, NULL, NULL, NULL); - return EncryptionHelper::decryptStringAsymmetric( - key, QByteArray::fromBase64(encryptedMetadata)); + // Also base64 decode the result + return QByteArray::fromBase64( + EncryptionHelper::decryptStringAsymmetric( + key, + QByteArray::fromBase64(encryptedMetadata) + ) + ); } // AES/GCM/NoPadding (128 bit key size) From a48a3e0accf8fcffde34f53d9b8fad7282931042 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Wed, 20 Dec 2017 13:41:13 +0100 Subject: [PATCH 162/262] Cleanup old functions Those functions had no use anymore since we store the key and cert in the keychain. Removed them so we don't use them by accident. Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 19 ++----------------- src/libsync/clientsideencryption.h | 4 ---- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 6b195d66e..c4412795f 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -51,10 +51,6 @@ QString baseUrl(){ return QStringLiteral("ocs/v2.php/apps/end_to_end_encryption/api/v1/"); } -QString baseDirectory() { - return QDir::homePath() + QStringLiteral("/.nextcloud-keys/"); -} - namespace { void handleErrors(void) { @@ -789,25 +785,14 @@ void ClientSideEncryption::writeMnemonic() { job->start(); } - -QString publicKeyPath(AccountPtr account) -{ - return baseDirectory() + account->displayName() + ".pub"; -} - -QString privateKeyPath(AccountPtr account) -{ - return baseDirectory() + account->displayName() + ".rsa"; -} - bool ClientSideEncryption::hasPrivateKey() const { - return QFileInfo(privateKeyPath(_account)).exists(); + return !_privateKey.isNull(); } bool ClientSideEncryption::hasPublicKey() const { - return QFileInfo(publicKeyPath(_account)).exists(); + return !_publicKey.isNull(); } void ClientSideEncryption::generateKeyPair() diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 8d7d78e22..01410ff3f 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -10,7 +10,6 @@ #include #include -#include #include "accountfwd.h" #include "networkjobs.h" @@ -24,9 +23,6 @@ class ReadPasswordJob; namespace OCC { QString baseUrl(); -QString baseDirectory(); -QString privateKeyPath(AccountPtr account); -QString publicKeyPath(AccountPtr account); class ClientSideEncryption : public QObject { Q_OBJECT From 4551bbe0e0ad81038708ae2c0e5e434cebe0984f Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Wed, 20 Dec 2017 15:35:23 +0100 Subject: [PATCH 163/262] Forget key + cert + mnemonic on account removal Signed-off-by: Roeland Jago Douma --- src/gui/accountmanager.cpp | 3 +++ src/libsync/account.h | 2 +- src/libsync/clientsideencryption.cpp | 38 +++++++++++++++++++++++----- src/libsync/clientsideencryption.h | 2 ++ 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/src/gui/accountmanager.cpp b/src/gui/accountmanager.cpp index 38a0ff185..15758cc8a 100644 --- a/src/gui/accountmanager.cpp +++ b/src/gui/accountmanager.cpp @@ -312,6 +312,9 @@ void AccountManager::deleteAccount(AccountState *account) auto settings = ConfigFile::settingsWithGroup(QLatin1String(accountsC)); settings->remove(account->account()->id()); + // Forget E2E keys + account->account()->e2e()->forgetSensitiveData(); + emit accountRemoved(account); } diff --git a/src/libsync/account.h b/src/libsync/account.h index 8d6774a4a..1bc58d115 100644 --- a/src/libsync/account.h +++ b/src/libsync/account.h @@ -234,7 +234,7 @@ public: /// Called by network jobs on credential errors, emits invalidCredentials() void handleInvalidCredentials(); - ClientSideEncryption* e2e(); + ClientSideEncryption* e2e(); public slots: /// Used when forgetting credentials diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index c4412795f..5553a8592 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -51,6 +51,12 @@ QString baseUrl(){ return QStringLiteral("ocs/v2.php/apps/end_to_end_encryption/api/v1/"); } +namespace { + const char e2e_cert[] = "_e2e-certificate"; + const char e2e_private[] = "_e2e-private"; + const char e2e_mnemonic[] = "_e2e-mnemonic"; +} // ns + namespace { void handleErrors(void) { @@ -629,7 +635,7 @@ void ClientSideEncryption::initialize() void ClientSideEncryption::fetchFromKeyChain() { const QString kck = AbstractCredentials::keychainKey( _account->url().toString(), - _account->credentials()->user() + "_e2e-certificate", + _account->credentials()->user() + e2e_cert, _account->id() ); @@ -662,7 +668,7 @@ void ClientSideEncryption::publicKeyFetched(Job *incoming) { const QString kck = AbstractCredentials::keychainKey( _account->url().toString(), - _account->credentials()->user() + "_e2e-private", + _account->credentials()->user() + e2e_private, _account->id() ); @@ -701,7 +707,7 @@ void ClientSideEncryption::privateKeyFetched(Job *incoming) { const QString kck = AbstractCredentials::keychainKey( _account->url().toString(), - _account->credentials()->user() + "_e2e-mnemonic", + _account->credentials()->user() + e2e_mnemonic, _account->id() ); @@ -734,7 +740,7 @@ void ClientSideEncryption::mnemonicKeyFetched(QKeychain::Job *incoming) { void ClientSideEncryption::writePrivateKey() { const QString kck = AbstractCredentials::keychainKey( _account->url().toString(), - _account->credentials()->user() + "_e2e-private", + _account->credentials()->user() + e2e_private, _account->id() ); @@ -752,7 +758,7 @@ void ClientSideEncryption::writePrivateKey() { void ClientSideEncryption::writeCertificate() { const QString kck = AbstractCredentials::keychainKey( _account->url().toString(), - _account->credentials()->user() + "_e2e-certificate", + _account->credentials()->user() + e2e_cert, _account->id() ); @@ -770,7 +776,7 @@ void ClientSideEncryption::writeCertificate() { void ClientSideEncryption::writeMnemonic() { const QString kck = AbstractCredentials::keychainKey( _account->url().toString(), - _account->credentials()->user() + "_e2e-mnemonic", + _account->credentials()->user() + e2e_mnemonic, _account->id() ); @@ -785,6 +791,26 @@ void ClientSideEncryption::writeMnemonic() { job->start(); } +void ClientSideEncryption::forgetSensitiveData() +{ + _privateKey = QSslKey(); + _certificate = QSslCertificate(); + _publicKey = QSslKey(); + _mnemonic = QString(); + + auto startDeleteJob = [this](QString user) { + DeletePasswordJob *job = new DeletePasswordJob(Theme::instance()->appName()); + job->setInsecureFallback(false); + job->setKey(AbstractCredentials::keychainKey(_account->url().toString(), user, _account->id())); + job->start(); + }; + + auto user = _account->credentials()->user(); + startDeleteJob(user + e2e_private); + startDeleteJob(user + e2e_cert); + startDeleteJob(user + e2e_mnemonic); +} + bool ClientSideEncryption::hasPrivateKey() const { return !_privateKey.isNull(); diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 01410ff3f..10123166d 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -43,6 +43,8 @@ public: bool isFolderEncrypted(const QString& path); void setFolderEncryptedStatus(const QString& path, bool status); + void forgetSensitiveData(); + private slots: void folderEncryptedStatusFetched(const QMap &values); void folderEncryptedStatusError(int error); From b924ad72824f8f0cc0cb7d0b82e4fbf5ae9969f6 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Wed, 20 Dec 2017 15:41:38 +0100 Subject: [PATCH 164/262] Lower mnemonic when generating password Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 5553a8592..b1c69cdcf 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -955,7 +955,7 @@ void ClientSideEncryption::encryptPrivateKey() _mnemonic = list.join(' '); qCInfo(lcCse()) << "mnemonic Generated:" << _mnemonic; - QString passPhrase = list.join(QString()); + QString passPhrase = list.join(QString()).toLower(); qCInfo(lcCse()) << "Passphrase Generated:" << passPhrase; /*TODO: C++17: auto [secretKey, salt]. */ @@ -1007,7 +1007,7 @@ void ClientSideEncryption::decryptPrivateKey(const QByteArray &key) { prev = dialog.textValue(); _mnemonic = prev; - QString mnemonic = prev.split(" ").join(QString()); + QString mnemonic = prev.split(" ").join(QString()).toLower(); qCInfo(lcCse()) << "mnemonic:" << mnemonic; auto pass = EncryptionHelper::generatePassword(mnemonic); qCInfo(lcCse()) << "Generated key:" << pass.first; From fc73ad747605cf8d924ebf999339b67bcb5f77ef Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Wed, 20 Dec 2017 16:07:33 +0100 Subject: [PATCH 165/262] Salt should not be hardcoded We append the salt (just like the IV) to the ciphertext of the private key. This means we also have to split it off properly. This breaks compartibility with currently stored keys on your server. Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 42 +++++++++++++++------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index b1c69cdcf..3b9fe038c 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -67,14 +67,12 @@ namespace { class EncryptionHelper { public: - using Password = QByteArray; - using Salt = QByteArray; - static QByteArray generateRandom(int size); - static QPair generatePassword(const QString &wordlist); + static QByteArray generatePassword(const QString &wordlist, const QByteArray& salt); static QByteArray encryptPrivateKey( const QByteArray& key, - const QByteArray& privateKey + const QByteArray& privateKey, + const QByteArray &salt ); static QByteArray decryptPrivateKey( const QByteArray& key, @@ -117,15 +115,9 @@ QByteArray EncryptionHelper::generateRandom(int size) { return result; } -QPair EncryptionHelper::generatePassword(const QString& wordlist) { +QByteArray EncryptionHelper::generatePassword(const QString& wordlist, const QByteArray& salt) { qCInfo(lcCse()) << "Start encryption key generation!"; - // TODO generate salt - const unsigned char *_salt = (unsigned char *)"$4$YmBjm3hk$Qb74D5IUYwghUmzsMqeNFx5z0/8$"; - const int saltLen = 40; - - QByteArray salt((const char *)_salt, saltLen); - const int iterationCount = 1024; const int keyStrength = 256; const int keyLength = keyStrength/8; @@ -150,12 +142,13 @@ QPair EncryptionHelper::gene qCInfo(lcCse()) << "Encryption key generated!"; QByteArray password((const char *)secretKey, keyLength); - return {password, salt}; + return password; } QByteArray EncryptionHelper::encryptPrivateKey( const QByteArray& key, - const QByteArray& privateKey + const QByteArray& privateKey, + const QByteArray& salt ) { QByteArray iv = generateRandom(12); @@ -225,6 +218,8 @@ QByteArray EncryptionHelper::encryptPrivateKey( QByteArray result = cipherTXT.toBase64(); result += "fA=="; result += iv.toBase64(); + result += "fA=="; + result += salt.toBase64(); return result; } @@ -958,9 +953,9 @@ void ClientSideEncryption::encryptPrivateKey() QString passPhrase = list.join(QString()).toLower(); qCInfo(lcCse()) << "Passphrase Generated:" << passPhrase; - /*TODO: C++17: auto [secretKey, salt]. */ - auto secretKey = EncryptionHelper::generatePassword(passPhrase); - auto cryptedText = EncryptionHelper::encryptPrivateKey(secretKey.first, _privateKey.toPem()); + auto salt = EncryptionHelper::generateRandom(40); + auto secretKey = EncryptionHelper::generatePassword(passPhrase, salt); + auto cryptedText = EncryptionHelper::encryptPrivateKey(secretKey, _privateKey.toPem(), salt); // Send private key to the server auto job = new StorePrivateKeyApiJob(_account, baseUrl() + "private-key", this); @@ -1009,10 +1004,17 @@ void ClientSideEncryption::decryptPrivateKey(const QByteArray &key) { _mnemonic = prev; QString mnemonic = prev.split(" ").join(QString()).toLower(); qCInfo(lcCse()) << "mnemonic:" << mnemonic; - auto pass = EncryptionHelper::generatePassword(mnemonic); - qCInfo(lcCse()) << "Generated key:" << pass.first; - QByteArray privateKey = EncryptionHelper::decryptPrivateKey(pass.first, key); + // split off salt + // Todo better place? + auto pos = key.lastIndexOf("fA=="); + QByteArray salt = QByteArray::fromBase64(key.mid(pos + 4)); + auto key2 = key.left(pos); + + auto pass = EncryptionHelper::generatePassword(mnemonic, salt); + qCInfo(lcCse()) << "Generated key:" << pass; + + QByteArray privateKey = EncryptionHelper::decryptPrivateKey(pass, key2); _privateKey = QSslKey(privateKey, QSsl::Rsa, QSsl::Pem, QSsl::PrivateKey); qCInfo(lcCse()) << "Private key: " << _privateKey.toPem(); From 0347cf4e9e1ca1db6a7ef994116978ffae67031f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 20 Dec 2017 19:37:04 +0100 Subject: [PATCH 166/262] [CSE] Store the sharing keys in memory --- src/libsync/clientsideencryption.cpp | 13 +++++++++++-- src/libsync/clientsideencryption.h | 1 + 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 3b9fe038c..32d489716 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1149,9 +1149,18 @@ void FolderMetadata::setupExistingMetadata() // Cool, We actually have the key, we can decrypt the rest of the metadata. qDebug() << "Sharing: " << sharing; - QByteArray sharingDecrypted = decryptJsonObject(sharing, _metadataKeys.last()); + auto sharingDecrypted = QByteArray::fromBase64(decryptJsonObject(sharing, _metadataKeys.last())); qDebug() << "Sharing Decrypted" << sharingDecrypted; - qDebug() << "Sharing B64 Decrypted" << QByteArray::fromBase64(sharingDecrypted); + + //Sharing is also a JSON object, so extract it and populate. + auto sharingDoc = QJsonDocument::fromJson(sharingDecrypted); + auto sharingObj = sharingDoc.object(); + for (auto it = sharingObj.constBegin(), end = sharingObj.constEnd(); it != end; it++) { + _sharing.push_back({it.key(), it.value().toString()}); + } + + // Retrieve the stored files, right now the metadata produced by this client doesn't have files, we should append one. + // TODO: Parse the "files" part. } // RSA/ECB/OAEPWithSHA-256AndMGF1Padding using private / public key. diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 10123166d..5246f9f92 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -118,6 +118,7 @@ private: QVector _metadataKeys; AccountPtr _account; QByteArray _metadata; + QVector> _sharing; }; class FileEncryptionJob : public QObject From 1e899f417228dbd215eb80f0eb1461f1cf99790f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 20 Dec 2017 20:00:59 +0100 Subject: [PATCH 167/262] [CSE] Do not create QPointer from raw pointers --- src/libsync/clientsideencryption.cpp | 3 ++- src/libsync/clientsideencryption.h | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 32d489716..73e5e9479 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1256,7 +1256,8 @@ bool ClientSideEncryption::isFolderEncrypted(const QString& path) { return (*it); } -FileEncryptionJob::FileEncryptionJob(QByteArray &key, QByteArray &iv, QFile *input, QFile *output, QObject *parent) +FileEncryptionJob::FileEncryptionJob(const QByteArray &key, const QByteArray &iv, + QPointer input, QPointer output, QObject *parent) : QObject(parent), _key(key), _iv(iv), diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 5246f9f92..e5753a009 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -125,7 +125,8 @@ class FileEncryptionJob : public QObject { Q_OBJECT public: - FileEncryptionJob(QByteArray &key, QByteArray &iv, QFile *input, QFile *output, QObject *parent = 0); + FileEncryptionJob(const QByteArray &key, const QByteArray &iv, + QPointer input, QPointer output, QObject *parent = 0); public slots: void start(); From c35b57cfbae6d539a7e0feb5197d5d2c1046dcc4 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Wed, 20 Dec 2017 20:49:16 +0100 Subject: [PATCH 168/262] Parse the files in the metadata Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 32d489716..d23e5f40d 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1134,6 +1134,7 @@ void FolderMetadata::setupExistingMetadata() QJsonObject metadataObj = metaDataDoc.object()["metadata"].toObject(); QJsonObject metadataKeys = metadataObj["metadataKeys"].toObject(); QByteArray sharing = metadataObj["sharing"].toString().toLocal8Bit(); + QJsonObject files = metaDataDoc.object()["files"].toObject(); QJsonDocument debugHelper; debugHelper.setObject(metadataKeys); @@ -1159,8 +1160,31 @@ void FolderMetadata::setupExistingMetadata() _sharing.push_back({it.key(), it.value().toString()}); } - // Retrieve the stored files, right now the metadata produced by this client doesn't have files, we should append one. - // TODO: Parse the "files" part. + for (auto it = files.constBegin(), end = files.constEnd(); it != end; it++) { + EncryptedFile file; + file.encryptedFilename = it.key(); + + auto fileObj = it.value().toObject(); + file.metadataKey = fileObj["metadataKey"].toInt(); + file.authenticationTag = QByteArray::fromBase64(fileObj["authenticationTag"].toString().toLocal8Bit()); + file.initializationVector = QByteArray::fromBase64(fileObj["initializationVector"].toString().toLocal8Bit()); + + //Decrypt encrypted part + //TODO actually get the right metadataKey; + QByteArray key = _metadataKeys.last(); + + auto encryptedFile = fileObj["encrypted"].toString().toLocal8Bit(); + auto decryptedFile = QByteArray::fromBase64(decryptJsonObject(encryptedFile, QByteArray::fromBase64(key))); + auto decryptedFileDoc = QJsonDocument::fromJson(decryptedFile); + auto decryptedFileObj = decryptedFileDoc.object(); + + file.originalFilename = decryptedFileObj["filename"].toString(); + file.encryptionKey = QByteArray::fromBase64(decryptedFileObj["key"].toString().toLocal8Bit()); + file.mimetype = decryptedFileObj["mimetype"].toString().toLocal8Bit(); + file.fileVersion = decryptedFileObj["version"].toInt(); + + _files.push_back(file); + } } // RSA/ECB/OAEPWithSHA-256AndMGF1Padding using private / public key. From b9f094cd943e0b49ae2a624f1b1f8121f4801ade Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Wed, 20 Dec 2017 21:17:41 +0100 Subject: [PATCH 169/262] Store metadata keys as keys Don't store them base64 encoded. But store them directly useable --- src/libsync/clientsideencryption.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index d23e5f40d..10a2ba905 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1144,7 +1144,7 @@ void FolderMetadata::setupExistingMetadata() // perhaps it's better to store a map instead of a vector, perhaps this just doesn't matter. for(auto it = metadataKeys.constBegin(), end = metadataKeys.constEnd(); it != end; it++) { QByteArray currB64Pass = it.value().toString().toLocal8Bit(); - QByteArray decryptedKey = decryptMetadataKey(currB64Pass); + QByteArray decryptedKey = QByteArray::fromBase64(decryptMetadataKey(currB64Pass)); _metadataKeys.push_back(decryptedKey); } @@ -1174,7 +1174,7 @@ void FolderMetadata::setupExistingMetadata() QByteArray key = _metadataKeys.last(); auto encryptedFile = fileObj["encrypted"].toString().toLocal8Bit(); - auto decryptedFile = QByteArray::fromBase64(decryptJsonObject(encryptedFile, QByteArray::fromBase64(key))); + auto decryptedFile = QByteArray::fromBase64(decryptJsonObject(encryptedFile, key)); auto decryptedFileDoc = QJsonDocument::fromJson(decryptedFile); auto decryptedFileObj = decryptedFileDoc.object(); From 5e23ca96584b54e142676289715aa2fca951dab5 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Wed, 20 Dec 2017 21:25:27 +0100 Subject: [PATCH 170/262] FolderMetaData: store metadataKeys in a QMap * This allows us to pick the right key for files Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 6 ++---- src/libsync/clientsideencryption.h | 4 +++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 10a2ba905..f7ec4a917 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1145,7 +1145,7 @@ void FolderMetadata::setupExistingMetadata() for(auto it = metadataKeys.constBegin(), end = metadataKeys.constEnd(); it != end; it++) { QByteArray currB64Pass = it.value().toString().toLocal8Bit(); QByteArray decryptedKey = QByteArray::fromBase64(decryptMetadataKey(currB64Pass)); - _metadataKeys.push_back(decryptedKey); + _metadataKeys.insert(it.key().toInt(), decryptedKey); } // Cool, We actually have the key, we can decrypt the rest of the metadata. @@ -1170,9 +1170,7 @@ void FolderMetadata::setupExistingMetadata() file.initializationVector = QByteArray::fromBase64(fileObj["initializationVector"].toString().toLocal8Bit()); //Decrypt encrypted part - //TODO actually get the right metadataKey; - QByteArray key = _metadataKeys.last(); - + QByteArray key = _metadataKeys[file.metadataKey]; auto encryptedFile = fileObj["encrypted"].toString().toLocal8Bit(); auto decryptedFile = QByteArray::fromBase64(decryptJsonObject(encryptedFile, key)); auto decryptedFileDoc = QJsonDocument::fromJson(decryptedFile); diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 5246f9f92..efec7ec0b 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include @@ -115,7 +117,7 @@ private: QByteArray decryptJsonObject(const QByteArray& encryptedJsonBlob, const QByteArray& pass) const; QVector _files; - QVector _metadataKeys; + QMap _metadataKeys; AccountPtr _account; QByteArray _metadata; QVector> _sharing; From 679bb1f18e0ff9492435cbd30b8b1aee962d1f7f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 20 Dec 2017 21:33:25 +0100 Subject: [PATCH 171/262] [CSE] Expose EncryptionHelper --- src/libsync/clientsideencryption.cpp | 35 ---------------------------- src/libsync/clientsideencryption.h | 35 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 3b56b0a80..29ab8080c 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -65,41 +65,6 @@ namespace { } } -class EncryptionHelper { -public: - static QByteArray generateRandom(int size); - static QByteArray generatePassword(const QString &wordlist, const QByteArray& salt); - static QByteArray encryptPrivateKey( - const QByteArray& key, - const QByteArray& privateKey, - const QByteArray &salt - ); - static QByteArray decryptPrivateKey( - const QByteArray& key, - const QByteArray& data - ); - static QByteArray encryptStringSymmetric( - const QByteArray& key, - const QByteArray& data - ); - static QByteArray decryptStringSymmetric( - const QByteArray& key, - const QByteArray& data - ); - - //TODO: change those two EVP_PKEY into QSslKey. - static QByteArray encryptStringAsymmetric( - EVP_PKEY *publicKey, - const QByteArray& data - ); - static QByteArray decryptStringAsymmetric( - EVP_PKEY *privateKey, - const QByteArray& data - ); - - static QByteArray BIO2ByteArray(BIO *b); -}; - QByteArray EncryptionHelper::generateRandom(int size) { unsigned char *tmp = (unsigned char *)malloc(sizeof(unsigned char) * size); diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index e5753a009..131238216 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -24,6 +24,41 @@ namespace OCC { QString baseUrl(); +class EncryptionHelper { +public: + static QByteArray generateRandom(int size); + static QByteArray generatePassword(const QString &wordlist, const QByteArray& salt); + static QByteArray encryptPrivateKey( + const QByteArray& key, + const QByteArray& privateKey, + const QByteArray &salt + ); + static QByteArray decryptPrivateKey( + const QByteArray& key, + const QByteArray& data + ); + static QByteArray encryptStringSymmetric( + const QByteArray& key, + const QByteArray& data + ); + static QByteArray decryptStringSymmetric( + const QByteArray& key, + const QByteArray& data + ); + + //TODO: change those two EVP_PKEY into QSslKey. + static QByteArray encryptStringAsymmetric( + EVP_PKEY *publicKey, + const QByteArray& data + ); + static QByteArray decryptStringAsymmetric( + EVP_PKEY *privateKey, + const QByteArray& data + ); + + static QByteArray BIO2ByteArray(BIO *b); +}; + class ClientSideEncryption : public QObject { Q_OBJECT public: From 74bc9213c560ec184fd79b5fc82e152ab12a4316 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Wed, 20 Dec 2017 22:04:26 +0100 Subject: [PATCH 172/262] Generate encrypted metadata on the fly Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 68 ++++++++++++++++++---------- src/libsync/clientsideencryption.h | 3 +- 2 files changed, 45 insertions(+), 26 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index f7ec4a917..f35a0e402 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1100,23 +1100,23 @@ void ClientSideEncryption::folderEncryptedStatusError(int error) qDebug() << "Failed to retrieve the status of the folders." << error; } -FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata) : _account(account), _metadata(metadata) +FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata) : _account(account) { if (metadata.isEmpty()) { qCInfo(lcCseMetadata()) << "Setupping Empty Metadata"; setupEmptyMetadata(); } else { qCInfo(lcCseMetadata()) << "Setting up existing metadata"; - setupExistingMetadata(); + setupExistingMetadata(metadata); } } -void FolderMetadata::setupExistingMetadata() +void FolderMetadata::setupExistingMetadata(const QByteArray& metadata) { /* This is the json response from the server, it contains two extra objects that we are *not* interested. * ocs and data. */ - QJsonDocument doc = QJsonDocument::fromJson(_metadata); + QJsonDocument doc = QJsonDocument::fromJson(metadata); qCInfo(lcCseMetadata()) << doc.toJson(QJsonDocument::Compact); // The metadata is being retrieved as a string stored in a json. @@ -1229,32 +1229,57 @@ QByteArray FolderMetadata::decryptJsonObject(const QByteArray& encryptedMetadata void FolderMetadata::setupEmptyMetadata() { qDebug() << "Settint up empty metadata"; QByteArray newMetadataPass = EncryptionHelper::generateRandom(16); - QByteArray encryptedMetadataPass = encryptMetadataKey(newMetadataPass); - QByteArray decryptedMetadataPass = decryptMetadataKey(encryptedMetadataPass); + _metadataKeys.insert(0, newMetadataPass); QString publicKey = _account->e2e()->_publicKey.toPem().toBase64(); QString displayName = _account->displayName(); - QJsonObject recepient = {{ - displayName, publicKey - }}; + _sharing.append({displayName, publicKey}); +} + +QByteArray FolderMetadata::encryptedMetadata() { + qDebug() << "Generating metadata"; + + QJsonObject metadataKeys; + for (auto it = _metadataKeys.constBegin(), end = _metadataKeys.constEnd(); it != end; it++) { + const QByteArray encryptedKey = encryptMetadataKey(it.value()); + metadataKeys.insert(QString::number(it.key()), QString(encryptedKey)); + } + + QJsonObject recepients; + for (auto it = _sharing.constBegin(), end = _sharing.constEnd(); it != end; it++) { + recepients.insert(it->first, it->second); + } QJsonDocument recepientDoc; - recepientDoc.setObject(recepient); + recepientDoc.setObject(recepients); + QString sharingEncrypted = encryptJsonObject(recepientDoc.toJson(QJsonDocument::Compact), _metadataKeys.last()); - QString sharingEncrypted = encryptJsonObject( - recepientDoc.toJson(QJsonDocument::Compact), newMetadataPass); - - QJsonObject metadataKeys = { - {"0", QString(encryptedMetadataPass)} - }; QJsonObject metadata = { {"metadataKeys", metadataKeys}, {"sharing", sharingEncrypted}, {"version", 1} }; - QJsonObject files = { - }; + QJsonObject files; + for (auto it = _files.constBegin(), end = _files.constEnd(); it != end; it++) { + QJsonObject encrypted; + encrypted.insert("key", QString(it->encryptionKey.toBase64())); + encrypted.insert("filename", it->originalFilename); + encrypted.insert("mimetype", QString(it->mimetype)); + encrypted.insert("version", it->fileVersion); + QJsonDocument encryptedDoc; + encryptedDoc.setObject(encrypted); + + QString encryptedEncrypted = encryptJsonObject(encryptedDoc.toJson(QJsonDocument::Compact), _metadataKeys.last()); + + QJsonObject file; + file.insert("encrypted", encryptedEncrypted); + file.insert("initializationVector", QString(it->initializationVector.toBase64())); + file.insert("authenticationTag", QString(it->authenticationTag.toBase64())); + file.insert("metadataKey", _metadataKeys.lastKey()); + + files.insert(it->encryptedFilename, file); + } QJsonObject metaObject = { {"metadata", metadata}, @@ -1263,12 +1288,7 @@ void FolderMetadata::setupEmptyMetadata() { QJsonDocument internalMetadata; internalMetadata.setObject(metaObject); - _metadata = internalMetadata.toJson(); - qDebug() << "Generated Json" << _metadata; -} - -QByteArray FolderMetadata::encryptedMetadata() { - return _metadata; + return internalMetadata.toJson(); } bool ClientSideEncryption::isFolderEncrypted(const QString& path) { diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index efec7ec0b..5508fb940 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -108,7 +108,7 @@ private: * to ease the port to Nlohmann Json API */ void setupEmptyMetadata(); - void setupExistingMetadata(); + void setupExistingMetadata(const QByteArray& metadata); QByteArray encryptMetadataKey(const QByteArray& metadataKey) const; QByteArray decryptMetadataKey(const QByteArray& encryptedKey) const; @@ -119,7 +119,6 @@ private: QVector _files; QMap _metadataKeys; AccountPtr _account; - QByteArray _metadata; QVector> _sharing; }; From 3c301a82824c8fb4b8780bd05fa90b6f6bcab9eb Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Wed, 20 Dec 2017 22:08:57 +0100 Subject: [PATCH 173/262] Add missing FolderMetadata functions Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index f35a0e402..48a267308 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1291,6 +1291,15 @@ QByteArray FolderMetadata::encryptedMetadata() { return internalMetadata.toJson(); } +void FolderMetadata::addEncryptedFile(const EncryptedFile &f) { + // TODO: check for duplicates + _files.append(f); +} + +QVector FolderMetadata::files() const { + return _files; +} + bool ClientSideEncryption::isFolderEncrypted(const QString& path) { auto it = _folder2encryptedStatus.find(path); if (it == _folder2encryptedStatus.end()) From 3760b86e07d5f8e45671d12a8916da49ff08a0ce Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 20 Dec 2017 22:28:01 +0100 Subject: [PATCH 174/262] [CSE] Mobe fileEncryption to a static function and I'll move all of those to a namespace latter. --- src/libsync/clientsideencryption.cpp | 36 ++++++++++------------------ src/libsync/clientsideencryption.h | 23 +++--------------- 2 files changed, 15 insertions(+), 44 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 540cab3ea..dc1b7b496 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1272,20 +1272,10 @@ bool ClientSideEncryption::isFolderEncrypted(const QString& path) { return (*it); } -FileEncryptionJob::FileEncryptionJob(const QByteArray &key, const QByteArray &iv, - QPointer input, QPointer output, QObject *parent) - : QObject(parent), - _key(key), - _iv(iv), - _input(input), - _output(output) +void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &iv, QFile *input, QFile *output) { -} - -void FileEncryptionJob::start() -{ - _input->open(QIODevice::ReadOnly); - _output->open(QIODevice::WriteOnly); + input->open(QIODevice::ReadOnly); + output->open(QIODevice::WriteOnly); // Init EVP_CIPHER_CTX *ctx; @@ -1305,13 +1295,13 @@ void FileEncryptionJob::start() EVP_CIPHER_CTX_set_padding(ctx, 0); /* Set IV length. */ - if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, _iv.size(), NULL)) { + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, iv.size(), NULL)) { qCInfo(lcCse()) << "Could not set iv length"; exit(-1); } /* Initialise key and IV */ - if(!EVP_EncryptInit_ex(ctx, NULL, NULL, (const unsigned char *)_key.constData(), (const unsigned char *)_iv.constData())) { + if(!EVP_EncryptInit_ex(ctx, NULL, NULL, (const unsigned char *)key.constData(), (const unsigned char *)iv.constData())) { qCInfo(lcCse()) << "Could not set key and iv"; exit(-1); } @@ -1320,8 +1310,8 @@ void FileEncryptionJob::start() int len = 0; int total_len = 0; - while(!_input->atEnd()) { - QByteArray data = _input->read(1024); + while(!input->atEnd()) { + QByteArray data = input->read(1024); if (data.size() == 0) { qCInfo(lcCse()) << "Could not read data from file"; @@ -1333,7 +1323,7 @@ void FileEncryptionJob::start() exit(-1); } - _output->write((char *)out, len); + output->write((char *)out, len); total_len += len; } @@ -1341,7 +1331,7 @@ void FileEncryptionJob::start() qCInfo(lcCse()) << "Could finalize encryption"; exit(-1); } - _output->write((char *)out, len); + output->write((char *)out, len); total_len += len; /* Get the tag */ @@ -1351,16 +1341,14 @@ void FileEncryptionJob::start() exit(-1); } - _output->write((char *)tag, 16); + output->write((char *)tag, 16); free(out); free(tag); EVP_CIPHER_CTX_free(ctx); - _input->close(); - _output->close(); - - emit finished(_output); + input->close(); + output->close(); } FileDecryptionJob::FileDecryptionJob(QByteArray &key, QByteArray &iv, QFile *input, QFile *output, QObject *parent) diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 9e0c0b4c7..b5a4e2a2d 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -59,6 +59,9 @@ public: ); static QByteArray BIO2ByteArray(BIO *b); + + static void fileEncryption(const QByteArray &key, const QByteArray &iv, + QFile *input, QFile *output); }; class ClientSideEncryption : public QObject { @@ -157,26 +160,6 @@ private: QVector> _sharing; }; -class FileEncryptionJob : public QObject -{ - Q_OBJECT -public: - FileEncryptionJob(const QByteArray &key, const QByteArray &iv, - QPointer input, QPointer output, QObject *parent = 0); - -public slots: - void start(); - -signals: - void finished(QFile *output); - -private: - QByteArray _key; - QByteArray _iv; - QPointer _input; - QPointer _output; -}; - class FileDecryptionJob : public QObject { Q_OBJECT From 7e83f0591b130175965a357bd78bd3d456a499d3 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 20 Dec 2017 22:32:38 +0100 Subject: [PATCH 175/262] [CSE] Create the Encrypted File on Disk --- src/libsync/propagateuploadencrypted.cpp | 39 +++++++++++++++++++++++- src/libsync/propagateuploadencrypted.h | 4 +++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 25d164420..cfa3da33b 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include namespace OCC { @@ -111,9 +113,44 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo { qDebug() << "Metadata Received, Preparing it for the new file." << json.toVariant(); + // Encrypt File! FolderMetadata metaData(_propagator->account(), json.toJson(QJsonDocument::Compact)); - qDebug() << "Unlockign folder because I didn't finished the metadata yet."; + QFileInfo info(_item->_file); + + //Todo: Move this to the MetadataHandler. + /* This should actually first verify if we don't have this file on the metadata already + * and construct this code if there isn't. + */ + + qDebug() << "Creating the encrypted file metadata helper."; + EncryptedFile encryptedFile; + encryptedFile.authenticationTag = "NOISE"; // TODO: Remove the noise. + encryptedFile.encryptedFilename = EncryptionHelper::generateRandom(20); + encryptedFile.encryptionKey = EncryptionHelper::generateRandom(16); + encryptedFile.fileVersion = 1; + encryptedFile.initializationVector = EncryptionHelper::generateRandom(16); + encryptedFile.metadataKey = 1; + encryptedFile.originalFilename = info.fileName(); + metaData.addEncryptedFile(encryptedFile); + + qDebug() << "Encrypting the file"; + QFile *input = new QFile(info.absoluteFilePath()); + + //TODO: Perhaps I should use a QTemporaryFile? + QFile *output = new QFile(QDir::tempPath() + encryptedFile.encryptedFilename); + + EncryptionHelper::fileEncryption(encryptedFile.encryptionKey, + encryptedFile.initializationVector, + input, output); + + + qDebug() << "Removing Temporary File Temporarely"; + output->remove(); + input->deleteLater(); + output->deleteLater(); + + qDebug() << "Unlockign folder because I didn't finished the metadata yet."; auto *unlockJob = new UnlockEncryptFolderApiJob(_propagator->account(), _folderId, _folderToken, this); connect(unlockJob, &UnlockEncryptFolderApiJob::success, this, &PropagateUploadEncrypted::slotUnlockEncryptedFolderSuccess); connect(unlockJob, &UnlockEncryptFolderApiJob::error, this, &PropagateUploadEncrypted::slotUnlockEncryptedFolderError); diff --git a/src/libsync/propagateuploadencrypted.h b/src/libsync/propagateuploadencrypted.h index 38d747e0b..979a514af 100644 --- a/src/libsync/propagateuploadencrypted.h +++ b/src/libsync/propagateuploadencrypted.h @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include "owncloudpropagator.h" @@ -57,6 +59,8 @@ private: bool _currentLockingInProgress; QByteArray _folderToken; QByteArray _folderId; + QByteArray _generatedKey; + QByteArray _generatedIv; }; From dd0528037d79098067bf794421c59f846be9608e Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 20 Dec 2017 23:09:28 +0100 Subject: [PATCH 176/262] [CSE] Generate a random name for the temporary file --- src/libsync/clientsideencryption.cpp | 16 +++++++++++++++- src/libsync/clientsideencryption.h | 1 + src/libsync/propagateuploadencrypted.cpp | 8 +++++--- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index dc1b7b496..3c36501b5 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -65,7 +65,21 @@ namespace { } } -QByteArray EncryptionHelper::generateRandom(int size) { +QByteArray EncryptionHelper::generateRandomString(int size) +{ + const QByteArray possibleCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + QByteArray randomString(size, '\0'); + for(int i=0; i < size; ++i) + { + int index = qrand() % possibleCharacters.length(); + randomString[i] = possibleCharacters.at(index); + } + return randomString; +} + +QByteArray EncryptionHelper::generateRandom(int size) +{ unsigned char *tmp = (unsigned char *)malloc(sizeof(unsigned char) * size); int ret = RAND_bytes(tmp, size); diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index b5a4e2a2d..7d5810a8d 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -28,6 +28,7 @@ QString baseUrl(); class EncryptionHelper { public: + static QByteArray generateRandomString(int size); static QByteArray generateRandom(int size); static QByteArray generatePassword(const QString &wordlist, const QByteArray& salt); static QByteArray encryptPrivateKey( diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index cfa3da33b..23d412601 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -126,7 +126,7 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo qDebug() << "Creating the encrypted file metadata helper."; EncryptedFile encryptedFile; encryptedFile.authenticationTag = "NOISE"; // TODO: Remove the noise. - encryptedFile.encryptedFilename = EncryptionHelper::generateRandom(20); + encryptedFile.encryptedFilename = EncryptionHelper::generateRandomString(20); encryptedFile.encryptionKey = EncryptionHelper::generateRandom(16); encryptedFile.fileVersion = 1; encryptedFile.initializationVector = EncryptionHelper::generateRandom(16); @@ -135,10 +135,12 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo metaData.addEncryptedFile(encryptedFile); qDebug() << "Encrypting the file"; - QFile *input = new QFile(info.absoluteFilePath()); + auto *input = new QFile(info.absoluteFilePath()); //TODO: Perhaps I should use a QTemporaryFile? - QFile *output = new QFile(QDir::tempPath() + encryptedFile.encryptedFilename); + qDebug() << "Creating " << QDir::tempPath() + QDir::separator() + encryptedFile.encryptedFilename; + + auto *output = new QFile(QDir::tempPath() + QDir::separator() + encryptedFile.encryptedFilename); EncryptionHelper::fileEncryption(encryptedFile.encryptionKey, encryptedFile.initializationVector, From 25d58ccd583aa4bf6598b5108e5046da9b328406 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 20 Dec 2017 23:16:42 +0100 Subject: [PATCH 177/262] Revert "Store metadata keys as keys" This commit broke decryption. This reverts commit b9f094cd943e0b49ae2a624f1b1f8121f4801ade. --- src/libsync/clientsideencryption.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 3c36501b5..5afaddd30 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1123,7 +1123,7 @@ void FolderMetadata::setupExistingMetadata(const QByteArray& metadata) // perhaps it's better to store a map instead of a vector, perhaps this just doesn't matter. for(auto it = metadataKeys.constBegin(), end = metadataKeys.constEnd(); it != end; it++) { QByteArray currB64Pass = it.value().toString().toLocal8Bit(); - QByteArray decryptedKey = QByteArray::fromBase64(decryptMetadataKey(currB64Pass)); + QByteArray decryptedKey = decryptMetadataKey(currB64Pass); _metadataKeys.insert(it.key().toInt(), decryptedKey); } @@ -1151,7 +1151,7 @@ void FolderMetadata::setupExistingMetadata(const QByteArray& metadata) //Decrypt encrypted part QByteArray key = _metadataKeys[file.metadataKey]; auto encryptedFile = fileObj["encrypted"].toString().toLocal8Bit(); - auto decryptedFile = QByteArray::fromBase64(decryptJsonObject(encryptedFile, key)); + auto decryptedFile = QByteArray::fromBase64(decryptJsonObject(encryptedFile, QByteArray::fromBase64(key))); auto decryptedFileDoc = QJsonDocument::fromJson(decryptedFile); auto decryptedFileObj = decryptedFileDoc.object(); From d87648c99a1594115bf8664bb7288b1bcda3dd20 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 20 Dec 2017 23:30:51 +0100 Subject: [PATCH 178/262] [CSE] Use the metadata as pointer. --- src/libsync/clientsideencryption.cpp | 1 + src/libsync/propagateuploadencrypted.cpp | 12 ++++-------- src/libsync/propagateuploadencrypted.h | 6 +++++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 5afaddd30..ac51fa251 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1363,6 +1363,7 @@ void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i input->close(); output->close(); + qDebug() << "File Encrypted Successfully"; } FileDecryptionJob::FileDecryptionJob(QByteArray &key, QByteArray &iv, QFile *input, QFile *output, QObject *parent) diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 23d412601..da0c5330c 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -14,7 +14,8 @@ namespace OCC { PropagateUploadEncrypted::PropagateUploadEncrypted(OwncloudPropagator *propagator, SyncFileItemPtr item) : _propagator(propagator), - _item(item) + _item(item), + _metadata(nullptr) { } @@ -114,7 +115,7 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo qDebug() << "Metadata Received, Preparing it for the new file." << json.toVariant(); // Encrypt File! - FolderMetadata metaData(_propagator->account(), json.toJson(QJsonDocument::Compact)); + _metadata = new FolderMetadata(_propagator->account(), json.toJson(QJsonDocument::Compact)); QFileInfo info(_item->_file); @@ -132,16 +133,11 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo encryptedFile.initializationVector = EncryptionHelper::generateRandom(16); encryptedFile.metadataKey = 1; encryptedFile.originalFilename = info.fileName(); - metaData.addEncryptedFile(encryptedFile); + _metadata->addEncryptedFile(encryptedFile); qDebug() << "Encrypting the file"; auto *input = new QFile(info.absoluteFilePath()); - - //TODO: Perhaps I should use a QTemporaryFile? - qDebug() << "Creating " << QDir::tempPath() + QDir::separator() + encryptedFile.encryptedFilename; - auto *output = new QFile(QDir::tempPath() + QDir::separator() + encryptedFile.encryptedFilename); - EncryptionHelper::fileEncryption(encryptedFile.encryptionKey, encryptedFile.initializationVector, input, output); diff --git a/src/libsync/propagateuploadencrypted.h b/src/libsync/propagateuploadencrypted.h index 979a514af..dd38a0dd7 100644 --- a/src/libsync/propagateuploadencrypted.h +++ b/src/libsync/propagateuploadencrypted.h @@ -13,7 +13,9 @@ #include "owncloudpropagator.h" namespace OCC { -/* This class is used if the server supports end to end encryption. +class FolderMetadata; + + /* This class is used if the server supports end to end encryption. * It will fire for *any* folder, encrypted or not, because when the * client starts the upload request we don't know if the folder is * encrypted on the server. @@ -24,6 +26,7 @@ namespace OCC { * folerNotEncrypted() if the file is within a folder that's not encrypted. * */ + class PropagateUploadEncrypted : public QObject { Q_OBJECT @@ -61,6 +64,7 @@ private: QByteArray _folderId; QByteArray _generatedKey; QByteArray _generatedIv; + FolderMetadata *_metadata; }; From 7be5f0a736f5729d415d7ff842c3520df4b0aabf Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Wed, 20 Dec 2017 23:36:49 +0100 Subject: [PATCH 179/262] [CSE] Update the finished signal --- src/libsync/propagateupload.cpp | 6 ++++-- src/libsync/propagateupload.h | 2 +- src/libsync/propagateuploadencrypted.cpp | 4 ++-- src/libsync/propagateuploadencrypted.h | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 383ae98c2..eee8bc205 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -185,9 +185,11 @@ void PropagateUploadFileCommon::start() } } -void PropagateUploadFileCommon::setupEncryptedFile() +void PropagateUploadFileCommon::setupEncryptedFile(const QString& path, const QString& filename, quint64 size) { - + _fileToUpload._path = path; + _fileToUpload._file = filename; + _fileToUpload._size = size; } void PropagateUploadFileCommon::setupUnencryptedFile() diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index adef62590..021bac4d5 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -252,7 +252,7 @@ public: /* start should setup the file, path and size that will be send to the server */ void start() Q_DECL_OVERRIDE; - void setupEncryptedFile(); + void setupEncryptedFile(const QString& path, const QString& filename, quint64 size); void setupUnencryptedFile(); void startUploadFile(); bool isLikelyFinishedQuickly() Q_DECL_OVERRIDE { return _item->_size < propagator()->smallFileSize(); } diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index da0c5330c..2d7a54adb 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -143,8 +143,8 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo input, output); - qDebug() << "Removing Temporary File Temporarely"; - output->remove(); + // File is Encrypted, Upload it. + emit finalized("","",0); input->deleteLater(); output->deleteLater(); diff --git a/src/libsync/propagateuploadencrypted.h b/src/libsync/propagateuploadencrypted.h index dd38a0dd7..08255c4cc 100644 --- a/src/libsync/propagateuploadencrypted.h +++ b/src/libsync/propagateuploadencrypted.h @@ -48,7 +48,7 @@ private slots: signals: // Emmited after the file is encrypted and everythign is setup. - void finalized(); + void finalized(const QString& path, const QString& filename, quint64 size); void error(); // Emited if the file is not in a encrypted folder. From df5fd3fbe65787f674171fdea53e0b5465d85c6e Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 21 Dec 2017 00:00:27 +0100 Subject: [PATCH 180/262] [CSE] More debug output, fix input file path --- src/libsync/clientsideencryption.cpp | 10 ++++++++-- src/libsync/propagateuploadencrypted.cpp | 13 +++++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index ac51fa251..8a626b48f 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1288,8 +1288,12 @@ bool ClientSideEncryption::isFolderEncrypted(const QString& path) { void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &iv, QFile *input, QFile *output) { - input->open(QIODevice::ReadOnly); - output->open(QIODevice::WriteOnly); + if (!input->open(QIODevice::ReadOnly)) { + qDebug() << "Could not open input file for reading" << input->errorString(); + } + if (!output->open(QIODevice::WriteOnly)) { + qDebug() << "Could not oppen output file for writting" << output->errorString(); + } // Init EVP_CIPHER_CTX *ctx; @@ -1324,6 +1328,7 @@ void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i int len = 0; int total_len = 0; + qDebug() << "Starting to encrypt the file" << input->fileName() << input->atEnd(); while(!input->atEnd()) { QByteArray data = input->read(1024); @@ -1332,6 +1337,7 @@ void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i exit(-1); } + qDebug() << "Encrypting " << data; if(!EVP_EncryptUpdate(ctx, out, &len, (unsigned char *)data.constData(), data.size())) { qCInfo(lcCse()) << "Could not encrypt"; exit(-1); diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 2d7a54adb..d9b4f31aa 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -117,14 +117,21 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo // Encrypt File! _metadata = new FolderMetadata(_propagator->account(), json.toJson(QJsonDocument::Compact)); - QFileInfo info(_item->_file); + QFileInfo info(_propagator->_localDir + QDir::separator() + _item->_file); //Todo: Move this to the MetadataHandler. /* This should actually first verify if we don't have this file on the metadata already * and construct this code if there isn't. */ + qDebug() << "Uploading to Remote Folder: " << _propagator->_remoteFolder; + qDebug() << "Uploading from Local Dir" << _propagator->_localDir; + qDebug() << "Local File" << _item->_file; + qDebug() << "Last Try" << QDir::cleanPath(_propagator->account()->url().path() + + QLatin1Char('/') + _propagator->account()->davPath() + + _propagator->_remoteFolder + _item->_file); qDebug() << "Creating the encrypted file metadata helper."; + EncryptedFile encryptedFile; encryptedFile.authenticationTag = "NOISE"; // TODO: Remove the noise. encryptedFile.encryptedFilename = EncryptionHelper::generateRandomString(20); @@ -144,7 +151,9 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo // File is Encrypted, Upload it. - emit finalized("","",0); + QFileInfo outputInfo(*output); + qDebug() << "Encrypted Info:" << outputInfo.path() <deleteLater(); output->deleteLater(); From b72113a53dfe68d8e31a6c8b0120807a1920ee0c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Thu, 21 Dec 2017 00:35:23 +0100 Subject: [PATCH 181/262] [CSE] Prepare the beginning of the upload. I still need to send the metadata. --- src/libsync/propagateupload.cpp | 24 +++++++++++++++++++----- src/libsync/propagateupload.h | 7 +++++++ src/libsync/propagateuploadencrypted.cpp | 12 +++--------- src/libsync/propagateuploadencrypted.h | 7 +++++-- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index eee8bc205..dbea904c5 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -172,14 +172,14 @@ void PropagateUploadFileCommon::setDeleteExisting(bool enabled) void PropagateUploadFileCommon::start() { if (propagator()->account()->capabilities().clientSideEncryptionAvaliable()) { - auto *encryptedJob = new PropagateUploadEncrypted(propagator(), _item); - connect(encryptedJob, &PropagateUploadEncrypted::folerNotEncrypted, + _uploadEncryptedHelper = new PropagateUploadEncrypted(propagator(), _item); + connect(_uploadEncryptedHelper, &PropagateUploadEncrypted::folerNotEncrypted, this, &PropagateUploadFileCommon::setupUnencryptedFile); - connect(encryptedJob, &PropagateUploadEncrypted::finalized, + connect(_uploadEncryptedHelper, &PropagateUploadEncrypted::finalized, this, &PropagateUploadFileCommon::setupEncryptedFile); - connect(encryptedJob, &PropagateUploadEncrypted::error, + connect(_uploadEncryptedHelper, &PropagateUploadEncrypted::error, []{ qDebug() << "Error setting up encryption."; }); - encryptedJob->start(); + _uploadEncryptedHelper->start(); } else { setupUnencryptedFile(); } @@ -187,13 +187,17 @@ void PropagateUploadFileCommon::start() void PropagateUploadFileCommon::setupEncryptedFile(const QString& path, const QString& filename, quint64 size) { + qDebug() << "Starting to upload encrypted file"; + _uploadingEncrypted = true; _fileToUpload._path = path; _fileToUpload._file = filename; _fileToUpload._size = size; + startUploadFile(); } void PropagateUploadFileCommon::setupUnencryptedFile() { + _uploadingEncrypted = false; _fileToUpload._file = _item->_file; _fileToUpload._size = _item->_size; _fileToUpload._path = propagator()->getFilePath(_fileToUpload._file); @@ -671,6 +675,16 @@ void PropagateUploadFileCommon::finalize() propagator()->_journal->commit("upload file start"); done(SyncFileItem::Success); + + if (_uploadingEncrypted) { + qDebug() << "Encrypted file upload Successfully"; + auto *unlockJob = new UnlockEncryptFolderApiJob(propagator()->account(), + _uploadEncryptedHelper->_folderId, _uploadEncryptedHelper->_folderToken, this); + + connect(unlockJob, &UnlockEncryptFolderApiJob::success, []{ qDebug() << "Successfully Unlocked"; }); + connect(unlockJob, &UnlockEncryptFolderApiJob::error, []{ qDebug() << "Unlock Error"; }); + unlockJob->start(); + } } void PropagateUploadFileCommon::prepareAbort(PropagatorJob::AbortType abortType) { diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index 021bac4d5..66e7de155 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -182,6 +182,8 @@ signals: void finishedSignal(); }; +class PropagateUploadEncrypted; + /** * @brief The PropagateUploadFileCommon class is the code common between all chunking algorithms * @ingroup libsync @@ -239,6 +241,8 @@ public: , _finished(false) , _deleteExisting(false) , _abortCount(0) + , _uploadEncryptedHelper(0) + , _uploadingEncrypted(false) { } @@ -300,6 +304,9 @@ protected: // Bases headers that need to be sent with every chunk QMap headers(); +private: + PropagateUploadEncrypted *_uploadEncryptedHelper; + bool _uploadingEncrypted; }; /** diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index d9b4f31aa..10262a791 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -149,19 +149,13 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo encryptedFile.initializationVector, input, output); - // File is Encrypted, Upload it. - QFileInfo outputInfo(*output); - qDebug() << "Encrypted Info:" << outputInfo.path() <fileName()); input->deleteLater(); output->deleteLater(); - qDebug() << "Unlockign folder because I didn't finished the metadata yet."; - auto *unlockJob = new UnlockEncryptFolderApiJob(_propagator->account(), _folderId, _folderToken, this); - connect(unlockJob, &UnlockEncryptFolderApiJob::success, this, &PropagateUploadEncrypted::slotUnlockEncryptedFolderSuccess); - connect(unlockJob, &UnlockEncryptFolderApiJob::error, this, &PropagateUploadEncrypted::slotUnlockEncryptedFolderError); - unlockJob->start(); + qDebug() << "Encrypted Info:" << outputInfo.path() << outputInfo.fileName() << outputInfo.size(); + emit finalized(outputInfo.path(),outputInfo.fileName(),outputInfo.size()); } void PropagateUploadEncrypted::slotUnlockEncryptedFolderSuccess(const QByteArray& fileId) diff --git a/src/libsync/propagateuploadencrypted.h b/src/libsync/propagateuploadencrypted.h index 08255c4cc..c6a6320ee 100644 --- a/src/libsync/propagateuploadencrypted.h +++ b/src/libsync/propagateuploadencrypted.h @@ -34,6 +34,10 @@ public: PropagateUploadEncrypted(OwncloudPropagator *propagator, SyncFileItemPtr item); void start(); + // Used by propagateupload + QByteArray _folderToken; + QByteArray _folderId; + private slots: void slotFolderEncryptedStatusFetched(const QMap& result); void slotFolderEncryptedStatusError(int error); @@ -60,8 +64,7 @@ private: QElapsedTimer _folderLockFirstTry; bool _currentLockingInProgress; - QByteArray _folderToken; - QByteArray _folderId; + QByteArray _generatedKey; QByteArray _generatedIv; FolderMetadata *_metadata; From 0dc7831336d42ca441ed39160cc9fe0708fd9245 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Thu, 21 Dec 2017 09:49:31 +0100 Subject: [PATCH 182/262] Key is already base64 decoded Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 8a626b48f..a1609b8c5 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1151,7 +1151,7 @@ void FolderMetadata::setupExistingMetadata(const QByteArray& metadata) //Decrypt encrypted part QByteArray key = _metadataKeys[file.metadataKey]; auto encryptedFile = fileObj["encrypted"].toString().toLocal8Bit(); - auto decryptedFile = QByteArray::fromBase64(decryptJsonObject(encryptedFile, QByteArray::fromBase64(key))); + auto decryptedFile = QByteArray::fromBase64(decryptJsonObject(encryptedFile, key)); auto decryptedFileDoc = QJsonDocument::fromJson(decryptedFile); auto decryptedFileObj = decryptedFileDoc.object(); From 65dfc47ac75d5039abcb0ea3d7b13b3acfc57b14 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Thu, 21 Dec 2017 10:20:03 +0100 Subject: [PATCH 183/262] Set public key once certificate is retrieved Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index a1609b8c5..9aed5af94 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -904,6 +904,7 @@ void ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) if (retCode == 200) { QString cert = json.object().value("ocs").toObject().value("data").toObject().value("public-key").toString(); _certificate = QSslCertificate(cert.toLocal8Bit(), QSsl::Pem); + _publicKey = _certificate.publicKey(); qCInfo(lcCse()) << "Certificate saved, Encrypting Private Key."; encryptPrivateKey(); } From 124a7253a4b20746378ff24a9e200d35e2581687 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 22 Dec 2017 22:02:16 +0100 Subject: [PATCH 184/262] [CSE] Create job to Update Metadata --- src/libsync/clientsideencryption.cpp | 1 - src/libsync/clientsideencryptionjobs.cpp | 49 ++++++++++++++++++++++++ src/libsync/clientsideencryptionjobs.h | 28 ++++++++++++++ 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 9aed5af94..980b5c04c 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1272,7 +1272,6 @@ QByteArray FolderMetadata::encryptedMetadata() { } void FolderMetadata::addEncryptedFile(const EncryptedFile &f) { - // TODO: check for duplicates _files.append(f); } diff --git a/src/libsync/clientsideencryptionjobs.cpp b/src/libsync/clientsideencryptionjobs.cpp index cdb25a900..05b49ade0 100644 --- a/src/libsync/clientsideencryptionjobs.cpp +++ b/src/libsync/clientsideencryptionjobs.cpp @@ -185,6 +185,55 @@ bool StoreMetaDataApiJob::finished() } + +UpdateMetadataApiJob::UpdateMetadataApiJob(const AccountPtr& account, + const QByteArray& fileId, + const QByteArray& b64Metadata, + cinst QByteArray& token, + QObject* parent) +: AbstractNetworkJob(account, baseUrl() + QStringLiteral("meta-data/") + fileId, parent), +_fileId(fileId), +_b64Metadata(b64Metadata), +_token(token) +{ +} + +void UpdateMetadataApiJob::start() +{ + QNetworkRequest req; + req.setRawHeader("OCS-APIREQUEST", "true"); + + QUrlQuery urlQuery; + urlQuery.addQueryItem(QStringLiteral("format"), QStringLiteral("json")); + urlQuery.addQueryItem( _token); + + QUrl url = Utility::concatUrlPath(account()->url(), path()); + url.setQuery(urlQuery); + + QByteArray data = QByteArray("metaData=") + QUrl::toPercentEncoding(_b64Metadata) + + QByteArray("token=") + _token; + + auto buffer = new QBuffer(this); + buffer->setData(data); + + qCInfo(lcCseJob()) << "updating the metadata for the fileId" << _fileId << "as encrypted"; + sendRequest("PUT", url, req, buffer); + AbstractNetworkJob::start(); +} + +bool UpdateMetadataApiJob::finished() +{ + int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (retCode != 200) { + qCInfo(lcCseJob()) << "error updating the metadata" << path() << errorString() << retCode; + emit error(_fileId, retCode); + } + + qCInfo(lcCseJob()) << "Metadata submited to the server successfully"; + emit success(_fileId); + return true; +} + UnlockEncryptFolderApiJob::UnlockEncryptFolderApiJob(const AccountPtr& account, const QByteArray& fileId, const QByteArray& token, diff --git a/src/libsync/clientsideencryptionjobs.h b/src/libsync/clientsideencryptionjobs.h index 021dcbc17..b3fdf5d6b 100644 --- a/src/libsync/clientsideencryptionjobs.h +++ b/src/libsync/clientsideencryptionjobs.h @@ -203,6 +203,34 @@ private: QByteArray _b64Metadata; }; +class OWNCLOUDSYNC_EXPORT UpdateMetadataApiJob : public AbstractNetworkJob +{ + Q_OBJECT +public: + explicit UpdateMetadataApiJob ( + const AccountPtr &account, + const QByteArray& fileId, + const QByteArray& b64Metadata, + const QByteArray& lockedToken, + QObject *parent = 0); + +public slots: + void start() override; + +protected: + bool finished() override; + +signals: + void success(const QByteArray& fileId); + void error(const QByteArray& fileId, int httpReturnCode); + +private: + QByteArray _fileId; + QByteArray _b64Metadata; + QByteArray _token; +}; + + class OWNCLOUDSYNC_EXPORT GetMetadataApiJob : public AbstractNetworkJob { Q_OBJECT From e70bf44aaf679e9abf7c104c52f346defdc07116 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 22 Dec 2017 22:39:45 +0100 Subject: [PATCH 185/262] [CSE] Fix the generation of the data for the update Metadata --- src/libsync/clientsideencryptionjobs.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libsync/clientsideencryptionjobs.cpp b/src/libsync/clientsideencryptionjobs.cpp index 05b49ade0..9cd1be092 100644 --- a/src/libsync/clientsideencryptionjobs.cpp +++ b/src/libsync/clientsideencryptionjobs.cpp @@ -184,12 +184,10 @@ bool StoreMetaDataApiJob::finished() return true; } - - UpdateMetadataApiJob::UpdateMetadataApiJob(const AccountPtr& account, const QByteArray& fileId, const QByteArray& b64Metadata, - cinst QByteArray& token, + const QByteArray& token, QObject* parent) : AbstractNetworkJob(account, baseUrl() + QStringLiteral("meta-data/") + fileId, parent), _fileId(fileId), @@ -205,14 +203,16 @@ void UpdateMetadataApiJob::start() QUrlQuery urlQuery; urlQuery.addQueryItem(QStringLiteral("format"), QStringLiteral("json")); - urlQuery.addQueryItem( _token); + urlQuery.addQueryItem(QStringLiteral("token"), _token); QUrl url = Utility::concatUrlPath(account()->url(), path()); url.setQuery(urlQuery); - QByteArray data = QByteArray("metaData=") + QUrl::toPercentEncoding(_b64Metadata) + - QByteArray("token=") + _token; + QUrlQuery params; + params.addQueryItem("metaData",QUrl::toPercentEncoding(_b64Metadata)); + params.addQueryItem("token",_token); + QByteArray data = params.query().toLocal8Bit(); auto buffer = new QBuffer(this); buffer->setData(data); From 60ef722b6057cb4d17c51d790bae4d5e730bce8a Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Fri, 22 Dec 2017 23:05:39 +0100 Subject: [PATCH 186/262] [CSE] Update the metadata when a file is being uploaded. --- src/libsync/propagateuploadencrypted.cpp | 35 ++++++++++++++++++++---- src/libsync/propagateuploadencrypted.h | 4 +++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 10262a791..a498b636b 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -141,12 +141,27 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo encryptedFile.metadataKey = 1; encryptedFile.originalFilename = info.fileName(); _metadata->addEncryptedFile(encryptedFile); + _encryptedFile = encryptedFile; - qDebug() << "Encrypting the file"; + qDebug() << "Metadata created, sending to the server."; + auto job = new UpdateMetadataApiJob(_propagator->account(), + _folderId, + _metadata->encryptedMetadata(), + _folderToken); + + connect(job, &UpdateMetadataApiJob::success, this, &PropagateUploadEncrypted::slotUpdateMetadataSuccess); + connect(job, &UpdateMetadataApiJob::error, this, &PropagateUploadEncrypted::slotUpdateMetadataError); + job->start(); +} + +void PropagateUploadEncrypted::slotUpdateMetadataSuccess(const QByteArray& fileId) +{ + qDebug() << "Uploading of the metadata success, Encrypting the file"; + QFileInfo info(_propagator->_localDir + QDir::separator() + _item->_file); auto *input = new QFile(info.absoluteFilePath()); - auto *output = new QFile(QDir::tempPath() + QDir::separator() + encryptedFile.encryptedFilename); - EncryptionHelper::fileEncryption(encryptedFile.encryptionKey, - encryptedFile.initializationVector, + auto *output = new QFile(QDir::tempPath() + QDir::separator() + _encryptedFile.encryptedFilename); + EncryptionHelper::fileEncryption(_encryptedFile.encryptionKey, + _encryptedFile.initializationVector, input, output); // File is Encrypted, Upload it. @@ -155,7 +170,17 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo output->deleteLater(); qDebug() << "Encrypted Info:" << outputInfo.path() << outputInfo.fileName() << outputInfo.size(); - emit finalized(outputInfo.path(),outputInfo.fileName(),outputInfo.size()); + // emit finalized(outputInfo.path(),outputInfo.fileName(),outputInfo.size()); + + auto *unlockJob = new UnlockEncryptFolderApiJob(_propagator->account(), _folderId, _folderToken, this); + connect(unlockJob, &UnlockEncryptFolderApiJob::success, []{ qDebug() << "Successfully Unlocked"; }); + connect(unlockJob, &UnlockEncryptFolderApiJob::error, []{ qDebug() << "Unlock Error"; }); + unlockJob->start(); +} + +void PropagateUploadEncrypted::slotUpdateMetadataError(const QByteArray& fileId, int httpErrorResponse) +{ + qDebug() << "Update metadata error for folder" << fileId << "with error" << httpErrorResponse; } void PropagateUploadEncrypted::slotUnlockEncryptedFolderSuccess(const QByteArray& fileId) diff --git a/src/libsync/propagateuploadencrypted.h b/src/libsync/propagateuploadencrypted.h index c6a6320ee..4b955e702 100644 --- a/src/libsync/propagateuploadencrypted.h +++ b/src/libsync/propagateuploadencrypted.h @@ -11,6 +11,7 @@ #include #include "owncloudpropagator.h" +#include "clientsideencryption.h" namespace OCC { class FolderMetadata; @@ -49,6 +50,8 @@ private slots: void slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode); void slotUnlockEncryptedFolderSuccess(const QByteArray& fileId); void slotUnlockEncryptedFolderError(const QByteArray& fileId, int httpReturnCode); + void slotUpdateMetadataSuccess(const QByteArray& fileId); + void slotUpdateMetadataError(const QByteArray& fileId, int httpReturnCode); signals: // Emmited after the file is encrypted and everythign is setup. @@ -68,6 +71,7 @@ private: QByteArray _generatedKey; QByteArray _generatedIv; FolderMetadata *_metadata; + EncryptedFile _encryptedFile; }; From 06c34ed61735b8fd1a6799eca9f8033d6428091c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 24 Dec 2017 16:30:39 +0100 Subject: [PATCH 187/262] [CSE] Assorted fixes - Unlock the folder even on error - Use the correct name of the file for upload --- src/libsync/propagateupload.cpp | 17 +++++++++++++---- src/libsync/propagateupload.h | 1 + src/libsync/propagateuploadencrypted.cpp | 8 ++------ 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index dbea904c5..e1b4c201a 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -329,10 +329,12 @@ void PropagateUploadFileCommon::slotStartUpload(const QByteArray &transmissionCh _item->_checksumHeader = _transmissionChecksumHeader; } - const QString fullFilePath = propagator()->getFilePath(_fileToUpload._file); + const QString fullFilePath = _fileToUpload._path; + const QString originalFilePath = propagator()->getFilePath(_item->_file); if (!FileSystem::fileExists(fullFilePath)) { - done(SyncFileItem::SoftError, tr("File Removed")); + callUnlockFolder(); + done(SyncFileItem::SoftError, tr("File Removed (start upload) %1").arg(fullFilePath)); return; } #ifdef WITH_TESTING @@ -343,9 +345,10 @@ void PropagateUploadFileCommon::slotStartUpload(const QByteArray &transmissionCh // but a potential checksum calculation could have taken some time during which the file could // have been changed again, so better check again here. - _item->_modtime = FileSystem::getModTime(fullFilePath); + _item->_modtime = FileSystem::getModTime(originalFilePath); if (prevModtime != _item->_modtime) { propagator()->_anotherSyncNeeded = true; + callUnlockFolder(); done(SyncFileItem::SoftError, tr("Local file changed during syncing. It will be resumed.")); return; } @@ -358,6 +361,7 @@ void PropagateUploadFileCommon::slotStartUpload(const QByteArray &transmissionCh // or not yet fully copied to the destination. if (fileIsStillChanging(*_item)) { propagator()->_anotherSyncNeeded = true; + callUnlockFolder(); done(SyncFileItem::SoftError, tr("Local file changed during sync.")); return; } @@ -657,6 +661,7 @@ QMap PropagateUploadFileCommon::headers() void PropagateUploadFileCommon::finalize() { + qDebug() << "Finalizing the upload. Check later if this is encrypted"; _finished = true; // Update the quota, if known @@ -674,10 +679,14 @@ void PropagateUploadFileCommon::finalize() propagator()->_journal->setUploadInfo(_item->_file, SyncJournalDb::UploadInfo()); propagator()->_journal->commit("upload file start"); + callUnlockFolder(); done(SyncFileItem::Success); +} +void PropagateUploadFileCommon::callUnlockFolder() +{ if (_uploadingEncrypted) { - qDebug() << "Encrypted file upload Successfully"; + qDebug() << "Calling Unlock"; auto *unlockJob = new UnlockEncryptFolderApiJob(propagator()->account(), _uploadEncryptedHelper->_folderId, _uploadEncryptedHelper->_folderToken, this); diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index 66e7de155..e483f2faf 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -259,6 +259,7 @@ public: void setupEncryptedFile(const QString& path, const QString& filename, quint64 size); void setupUnencryptedFile(); void startUploadFile(); + void callUnlockFolder(); bool isLikelyFinishedQuickly() Q_DECL_OVERRIDE { return _item->_size < propagator()->smallFileSize(); } private slots: diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index a498b636b..9c6ac8eac 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -170,12 +170,8 @@ void PropagateUploadEncrypted::slotUpdateMetadataSuccess(const QByteArray& fileI output->deleteLater(); qDebug() << "Encrypted Info:" << outputInfo.path() << outputInfo.fileName() << outputInfo.size(); - // emit finalized(outputInfo.path(),outputInfo.fileName(),outputInfo.size()); - - auto *unlockJob = new UnlockEncryptFolderApiJob(_propagator->account(), _folderId, _folderToken, this); - connect(unlockJob, &UnlockEncryptFolderApiJob::success, []{ qDebug() << "Successfully Unlocked"; }); - connect(unlockJob, &UnlockEncryptFolderApiJob::error, []{ qDebug() << "Unlock Error"; }); - unlockJob->start(); + qDebug() << "Finalizing the upload part, now the actuall uploader will take over"; + emit finalized(outputInfo.path(),outputInfo.fileName(),outputInfo.size()); } void PropagateUploadEncrypted::slotUpdateMetadataError(const QByteArray& fileId, int httpErrorResponse) From da9f1c412d1d4241dea893d0c3ae743154f17a8c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 24 Dec 2017 16:36:57 +0100 Subject: [PATCH 188/262] [CSE] Remove debug --- src/libsync/propagateuploadencrypted.cpp | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 9c6ac8eac..822b5dcc3 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -118,18 +118,6 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo _metadata = new FolderMetadata(_propagator->account(), json.toJson(QJsonDocument::Compact)); QFileInfo info(_propagator->_localDir + QDir::separator() + _item->_file); - - //Todo: Move this to the MetadataHandler. - /* This should actually first verify if we don't have this file on the metadata already - * and construct this code if there isn't. - */ - qDebug() << "Uploading to Remote Folder: " << _propagator->_remoteFolder; - qDebug() << "Uploading from Local Dir" << _propagator->_localDir; - qDebug() << "Local File" << _item->_file; - qDebug() << "Last Try" << QDir::cleanPath(_propagator->account()->url().path() - + QLatin1Char('/') + _propagator->account()->davPath() - + _propagator->_remoteFolder + _item->_file); - qDebug() << "Creating the encrypted file metadata helper."; EncryptedFile encryptedFile; From efc039863b096f0e39543001b6cb53937808bebf Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 24 Dec 2017 17:08:04 +0100 Subject: [PATCH 189/262] [CSE] Tons of debugs. --- src/libsync/propagateupload.cpp | 12 ++++++++---- src/libsync/propagateuploadv1.cpp | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index e1b4c201a..bf43472d2 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -189,7 +189,7 @@ void PropagateUploadFileCommon::setupEncryptedFile(const QString& path, const QS { qDebug() << "Starting to upload encrypted file"; _uploadingEncrypted = true; - _fileToUpload._path = path; + _fileToUpload._path = path + QDir::separator() + filename; _fileToUpload._file = filename; _fileToUpload._size = size; startUploadFile(); @@ -229,9 +229,11 @@ void PropagateUploadFileCommon::startUploadFile() { propagator()->_activeJobList.append(this); if (!_deleteExisting) { + qDebug() << "Running the compute checksum"; return slotComputeContentChecksum(); } + qDebug() << "Deleting the current"; auto job = new DeleteJob(propagator()->account(), propagator()->_remoteFolder + _fileToUpload._file, this); @@ -243,18 +245,20 @@ void PropagateUploadFileCommon::startUploadFile() { void PropagateUploadFileCommon::slotComputeContentChecksum() { + qDebug() << "Tryint to compute the checksum of the file"; + qDebug() << "Still trying to understand if this is the local file or the uploaded one"; if (propagator()->_abortRequested.fetchAndAddRelaxed(0)) { return; } - const QString filePath = propagator()->getFilePath(_fileToUpload._file); + const QString filePath = propagator()->getFilePath(_item->_file); // remember the modtime before checksumming to be able to detect a file // change during the checksum calculation - This goes inside of the _item->_file // and not the _fileToUpload because we are checking the original file, not there // probably temporary one. _item->_modtime = FileSystem::getModTime(filePath); - + qDebug() << "Mod time" << _item->_modtime; #ifdef WITH_TESTING _stopWatch.start(); #endif @@ -349,8 +353,8 @@ void PropagateUploadFileCommon::slotStartUpload(const QByteArray &transmissionCh if (prevModtime != _item->_modtime) { propagator()->_anotherSyncNeeded = true; callUnlockFolder(); + qDebug() << "prevModtime" << prevModtime << "Curr" << _item->_modtime; done(SyncFileItem::SoftError, tr("Local file changed during syncing. It will be resumed.")); - return; } quint64 fileSize = FileSystem::getSize(fullFilePath); diff --git a/src/libsync/propagateuploadv1.cpp b/src/libsync/propagateuploadv1.cpp index c8c212c06..63d42a9b6 100644 --- a/src/libsync/propagateuploadv1.cpp +++ b/src/libsync/propagateuploadv1.cpp @@ -109,6 +109,7 @@ void PropagateUploadFileV1::startNextChunk() } const QString fileName = _fileToUpload._path; + qDebug() << "Trying to upload" << fileName; if (!device->prepareAndOpen(fileName, chunkStart, currentChunkSize)) { qCWarning(lcPropagateUpload) << "Could not prepare upload device: " << device->errorString(); From c963259bfbd0dbfdc5723fb7cd870904ff94d054 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Thu, 28 Dec 2017 10:00:17 -0200 Subject: [PATCH 190/262] Fix URL in dialog that requests password An URL that had base like "http://localhost/nextcloud/" would get the last slash '/' removed and then appended with "index.php..." resulting in http://localhost/nextcloudindex.php --- src/gui/creds/httpcredentialsgui.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/gui/creds/httpcredentialsgui.cpp b/src/gui/creds/httpcredentialsgui.cpp index c4e0afdb0..08af9a8ee 100644 --- a/src/gui/creds/httpcredentialsgui.cpp +++ b/src/gui/creds/httpcredentialsgui.cpp @@ -139,22 +139,21 @@ void HttpCredentialsGui::showDialog() QString HttpCredentialsGui::requestAppPasswordText(const Account *account) { int version = account->serverVersionInt(); - QString path; + auto url = account->url().toString(); + if (url.endsWith('/')) + url.chop(1); if (version >= Account::makeServerVersion(13, 0, 0)) { - path = QLatin1String("index.php/settings/user/security"); + url += QLatin1String("/index.php/settings/user/security"); } else if (version >= Account::makeServerVersion(12, 0, 0)) { - path = QLatin1String("index.php/settings/personal#security"); + url += QLatin1String("/index.php/settings/personal#security"); } else if (version >= Account::makeServerVersion(11, 0, 0)) { - path = QLatin1String("index.php/settings/personal#apppasswords"); + url += QLatin1String("/index.php/settings/personal#apppasswords"); } else { return QString(); } - auto baseUrl = account->url().toString(); - if (baseUrl.endsWith('/')) - baseUrl.chop(1); return tr("Click here to request an app password from the web interface.") - .arg(baseUrl + path); + .arg(url); } } // namespace OCC From a63d34f870bbf65665c8dfe15479754211996689 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Thu, 28 Dec 2017 17:33:10 -0200 Subject: [PATCH 191/262] Prepend "nextcloud" for all logging categories Thus making easier to exclude logging from kio, qt and only enable "nextcloud.*" --- src/common/checksums.cpp | 2 +- src/common/filesystembase.cpp | 2 +- src/common/ownsql.cpp | 2 +- src/common/syncjournaldb.cpp | 2 +- src/common/utility.cpp | 2 +- src/csync/csync_reconcile.cpp | 2 +- src/csync/csync_update.cpp | 2 +- src/gui/accountmanager.cpp | 2 +- src/gui/accountsettings.cpp | 2 +- src/gui/accountstate.cpp | 2 +- src/gui/activitylistmodel.cpp | 2 +- src/gui/application.cpp | 2 +- src/gui/creds/credentialsfactory.cpp | 2 +- src/gui/creds/httpcredentialsgui.cpp | 2 +- src/gui/creds/oauth.cpp | 2 +- src/gui/creds/shibbolethcredentials.cpp | 2 +- src/gui/folder.cpp | 2 +- src/gui/folderman.cpp | 2 +- src/gui/folderstatusmodel.cpp | 2 +- src/gui/folderwatcher.cpp | 2 +- src/gui/guiutility.cpp | 2 +- src/gui/lockwatcher.cpp | 2 +- src/gui/navigationpanehelper.cpp | 2 +- src/gui/notificationwidget.cpp | 2 +- src/gui/ocsjob.cpp | 2 +- src/gui/proxyauthhandler.cpp | 2 +- src/gui/servernotificationhandler.cpp | 2 +- src/gui/sharee.cpp | 2 +- src/gui/socketapi.cpp | 2 +- src/gui/sslbutton.cpp | 2 +- src/gui/sslerrordialog.cpp | 2 +- src/gui/updater/updater.cpp | 2 +- src/gui/wizard/owncloudwizard.cpp | 2 +- src/libsync/abstractnetworkjob.cpp | 2 +- src/libsync/accessmanager.cpp | 2 +- src/libsync/account.cpp | 2 +- src/libsync/bandwidthmanager.cpp | 2 +- src/libsync/capabilities.cpp | 2 +- src/libsync/clientproxy.cpp | 2 +- src/libsync/clientsideencryption.cpp | 32 +++++++++++------------ src/libsync/clientsideencryptionjobs.cpp | 6 ++--- src/libsync/configfile.cpp | 2 +- src/libsync/connectionvalidator.cpp | 2 +- src/libsync/cookiejar.cpp | 2 +- src/libsync/creds/abstractcredentials.cpp | 2 +- src/libsync/creds/httpcredentials.cpp | 2 +- src/libsync/creds/tokencredentials.cpp | 2 +- src/libsync/discoveryphase.cpp | 2 +- src/libsync/logger.cpp | 2 +- src/libsync/networkjobs.cpp | 18 ++++++------- src/libsync/owncloudpropagator.cpp | 6 ++--- src/libsync/propagatedownload.cpp | 4 +-- src/libsync/propagateremotedelete.cpp | 4 +-- src/libsync/propagateremotemkdir.cpp | 2 +- src/libsync/propagateremotemove.cpp | 4 +-- src/libsync/propagateupload.cpp | 6 ++--- src/libsync/propagatorjobs.cpp | 6 ++--- src/libsync/syncengine.cpp | 2 +- src/libsync/syncfileitem.cpp | 2 +- src/libsync/syncfilestatustracker.cpp | 2 +- 60 files changed, 94 insertions(+), 94 deletions(-) diff --git a/src/common/checksums.cpp b/src/common/checksums.cpp index 2e9f8c95c..7cfb1353f 100644 --- a/src/common/checksums.cpp +++ b/src/common/checksums.cpp @@ -78,7 +78,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcChecksums, "sync.checksums", QtInfoMsg) +Q_LOGGING_CATEGORY(lcChecksums, "nextcloud.sync.checksums", QtInfoMsg) QByteArray makeChecksumHeader(const QByteArray &checksumType, const QByteArray &checksum) { diff --git a/src/common/filesystembase.cpp b/src/common/filesystembase.cpp index 1d95e3918..121632fd8 100644 --- a/src/common/filesystembase.cpp +++ b/src/common/filesystembase.cpp @@ -39,7 +39,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcFileSystem, "sync.filesystem", QtInfoMsg) +Q_LOGGING_CATEGORY(lcFileSystem, "nextcloud.sync.filesystem", QtInfoMsg) QString FileSystem::longWinPath(const QString &inpath) { diff --git a/src/common/ownsql.cpp b/src/common/ownsql.cpp index c4277a130..1e9ae97ba 100644 --- a/src/common/ownsql.cpp +++ b/src/common/ownsql.cpp @@ -40,7 +40,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcSql, "sync.database.sql", QtInfoMsg) +Q_LOGGING_CATEGORY(lcSql, "nextcloud.sync.database.sql", QtInfoMsg) SqlDatabase::SqlDatabase() : _db(0) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index 3710f36d3..41401ef1a 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -34,7 +34,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcDb, "sync.database", QtInfoMsg) +Q_LOGGING_CATEGORY(lcDb, "nextcloud.sync.database", QtInfoMsg) #define GET_FILE_RECORD_QUERY \ "SELECT path, inode, modtime, type, md5, fileid, remotePerm, filesize," \ diff --git a/src/common/utility.cpp b/src/common/utility.cpp index bdf603515..f5873af64 100644 --- a/src/common/utility.cpp +++ b/src/common/utility.cpp @@ -58,7 +58,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcUtility, "sync.utility", QtInfoMsg) +Q_LOGGING_CATEGORY(lcUtility, "nextcloud.sync.utility", QtInfoMsg) bool Utility::writeRandomFile(const QString &fname, int size) { diff --git a/src/csync/csync_reconcile.cpp b/src/csync/csync_reconcile.cpp index 97863fb8b..06ae9155a 100644 --- a/src/csync/csync_reconcile.cpp +++ b/src/csync/csync_reconcile.cpp @@ -29,7 +29,7 @@ #include "common/asserts.h" #include -Q_LOGGING_CATEGORY(lcReconcile, "sync.csync.reconciler", QtInfoMsg) +Q_LOGGING_CATEGORY(lcReconcile, "nextcloud.sync.csync.reconciler", QtInfoMsg) // Needed for PRIu64 on MinGW in C++ mode. #define __STDC_FORMAT_MACROS diff --git a/src/csync/csync_update.cpp b/src/csync/csync_update.cpp index ae7efba77..4f913fa99 100644 --- a/src/csync/csync_update.cpp +++ b/src/csync/csync_update.cpp @@ -52,7 +52,7 @@ #define __STDC_FORMAT_MACROS #include -Q_LOGGING_CATEGORY(lcUpdate, "sync.csync.updater", QtInfoMsg) +Q_LOGGING_CATEGORY(lcUpdate, "nextcloud.sync.csync.updater", QtInfoMsg) #ifdef NO_RENAME_EXTENSION /* Return true if the two path have the same extension. false otherwise. */ diff --git a/src/gui/accountmanager.cpp b/src/gui/accountmanager.cpp index 15758cc8a..9330987dc 100644 --- a/src/gui/accountmanager.cpp +++ b/src/gui/accountmanager.cpp @@ -38,7 +38,7 @@ static const char serverVersionC[] = "serverVersion"; namespace OCC { -Q_LOGGING_CATEGORY(lcAccountManager, "gui.account.manager", QtInfoMsg) +Q_LOGGING_CATEGORY(lcAccountManager, "nextcloud.gui.account.manager", QtInfoMsg) AccountManager *AccountManager::instance() { diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index cdcaa8f37..6046612f9 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -60,7 +60,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcAccountSettings, "gui.account.settings", QtInfoMsg) +Q_LOGGING_CATEGORY(lcAccountSettings, "nextcloud.gui.account.settings", QtInfoMsg) static const char progressBarStyleC[] = "QProgressBar {" diff --git a/src/gui/accountstate.cpp b/src/gui/accountstate.cpp index 0b19f3f88..f0bd9d62e 100644 --- a/src/gui/accountstate.cpp +++ b/src/gui/accountstate.cpp @@ -26,7 +26,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcAccountState, "gui.account.state", QtInfoMsg) +Q_LOGGING_CATEGORY(lcAccountState, "nextcloud.gui.account.state", QtInfoMsg) AccountState::AccountState(AccountPtr account) : QObject() diff --git a/src/gui/activitylistmodel.cpp b/src/gui/activitylistmodel.cpp index 33e57690d..39198fc9b 100644 --- a/src/gui/activitylistmodel.cpp +++ b/src/gui/activitylistmodel.cpp @@ -31,7 +31,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcActivity, "gui.activity", QtInfoMsg) +Q_LOGGING_CATEGORY(lcActivity, "nextcloud.gui.activity", QtInfoMsg) ActivityListModel::ActivityListModel(QWidget *parent) : QAbstractListModel(parent) diff --git a/src/gui/application.cpp b/src/gui/application.cpp index cbdb8514c..292da54c8 100644 --- a/src/gui/application.cpp +++ b/src/gui/application.cpp @@ -57,7 +57,7 @@ class QSocket; namespace OCC { -Q_LOGGING_CATEGORY(lcApplication, "gui.application", QtInfoMsg) +Q_LOGGING_CATEGORY(lcApplication, "nextcloud.gui.application", QtInfoMsg) namespace { diff --git a/src/gui/creds/credentialsfactory.cpp b/src/gui/creds/credentialsfactory.cpp index f9574f096..52e003b57 100644 --- a/src/gui/creds/credentialsfactory.cpp +++ b/src/gui/creds/credentialsfactory.cpp @@ -24,7 +24,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcGuiCredentials, "gui.credentials", QtInfoMsg) +Q_LOGGING_CATEGORY(lcGuiCredentials, "nextcloud.gui.credentials", QtInfoMsg) namespace CredentialsFactory { diff --git a/src/gui/creds/httpcredentialsgui.cpp b/src/gui/creds/httpcredentialsgui.cpp index 08af9a8ee..b896cb0cc 100644 --- a/src/gui/creds/httpcredentialsgui.cpp +++ b/src/gui/creds/httpcredentialsgui.cpp @@ -30,7 +30,7 @@ using namespace QKeychain; namespace OCC { -Q_LOGGING_CATEGORY(lcHttpCredentialsGui, "sync.credentials.http.gui", QtInfoMsg) +Q_LOGGING_CATEGORY(lcHttpCredentialsGui, "nextcloud.sync.credentials.http.gui", QtInfoMsg) void HttpCredentialsGui::askFromUser() { diff --git a/src/gui/creds/oauth.cpp b/src/gui/creds/oauth.cpp index 4980b8db9..b38ea10ce 100644 --- a/src/gui/creds/oauth.cpp +++ b/src/gui/creds/oauth.cpp @@ -25,7 +25,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcOauth, "sync.credentials.oauth", QtInfoMsg) +Q_LOGGING_CATEGORY(lcOauth, "nextcloud.sync.credentials.oauth", QtInfoMsg) OAuth::~OAuth() { diff --git a/src/gui/creds/shibbolethcredentials.cpp b/src/gui/creds/shibbolethcredentials.cpp index ae15699bb..fd5d096cc 100644 --- a/src/gui/creds/shibbolethcredentials.cpp +++ b/src/gui/creds/shibbolethcredentials.cpp @@ -39,7 +39,7 @@ using namespace QKeychain; namespace OCC { -Q_LOGGING_CATEGORY(lcShibboleth, "gui.credentials.shibboleth", QtInfoMsg) +Q_LOGGING_CATEGORY(lcShibboleth, "nextcloud.gui.credentials.shibboleth", QtInfoMsg) namespace { diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index d854f92ca..236cd2516 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -43,7 +43,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcFolder, "gui.folder", QtInfoMsg) +Q_LOGGING_CATEGORY(lcFolder, "nextcloud.gui.folder", QtInfoMsg) Folder::Folder(const FolderDefinition &definition, AccountState *accountState, diff --git a/src/gui/folderman.cpp b/src/gui/folderman.cpp index ca73ea0ec..89f8a1eb0 100644 --- a/src/gui/folderman.cpp +++ b/src/gui/folderman.cpp @@ -40,7 +40,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcFolderMan, "gui.folder.manager", QtInfoMsg) +Q_LOGGING_CATEGORY(lcFolderMan, "nextcloud.gui.folder.manager", QtInfoMsg) FolderMan *FolderMan::_instance = 0; diff --git a/src/gui/folderstatusmodel.cpp b/src/gui/folderstatusmodel.cpp index e6f36811b..587e18ae6 100644 --- a/src/gui/folderstatusmodel.cpp +++ b/src/gui/folderstatusmodel.cpp @@ -28,7 +28,7 @@ Q_DECLARE_METATYPE(QPersistentModelIndex) namespace OCC { -Q_LOGGING_CATEGORY(lcFolderStatus, "gui.folder.model", QtInfoMsg) +Q_LOGGING_CATEGORY(lcFolderStatus, "nextcloud.gui.folder.model", QtInfoMsg) static const char propertyParentIndexC[] = "oc_parentIndex"; static const char propertyPermissionMap[] = "oc_permissionMap"; diff --git a/src/gui/folderwatcher.cpp b/src/gui/folderwatcher.cpp index b39c15dcb..0b5799d34 100644 --- a/src/gui/folderwatcher.cpp +++ b/src/gui/folderwatcher.cpp @@ -36,7 +36,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcFolderWatcher, "gui.folderwatcher", QtInfoMsg) +Q_LOGGING_CATEGORY(lcFolderWatcher, "nextcloud.gui.folderwatcher", QtInfoMsg) FolderWatcher::FolderWatcher(const QString &root, Folder *folder) : QObject(folder) diff --git a/src/gui/guiutility.cpp b/src/gui/guiutility.cpp index 8f71b1cda..f354c39ca 100644 --- a/src/gui/guiutility.cpp +++ b/src/gui/guiutility.cpp @@ -23,7 +23,7 @@ using namespace OCC; -Q_LOGGING_CATEGORY(lcUtility, "gui.utility", QtInfoMsg) +Q_LOGGING_CATEGORY(lcUtility, "nextcloud.gui.utility", QtInfoMsg) bool Utility::openBrowser(const QUrl &url, QWidget *errorWidgetParent) { diff --git a/src/gui/lockwatcher.cpp b/src/gui/lockwatcher.cpp index cf76c097b..3f8030cde 100644 --- a/src/gui/lockwatcher.cpp +++ b/src/gui/lockwatcher.cpp @@ -20,7 +20,7 @@ using namespace OCC; -Q_LOGGING_CATEGORY(lcLockWatcher, "gui.lockwatcher", QtInfoMsg) +Q_LOGGING_CATEGORY(lcLockWatcher, "nextcloud.gui.lockwatcher", QtInfoMsg) static const int check_frequency = 20 * 1000; // ms diff --git a/src/gui/navigationpanehelper.cpp b/src/gui/navigationpanehelper.cpp index 47458bc4a..78a339cfc 100644 --- a/src/gui/navigationpanehelper.cpp +++ b/src/gui/navigationpanehelper.cpp @@ -22,7 +22,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcNavPane, "gui.folder.navigationpane", QtInfoMsg) +Q_LOGGING_CATEGORY(lcNavPane, "nextcloud.gui.folder.navigationpane", QtInfoMsg) NavigationPaneHelper::NavigationPaneHelper(FolderMan *folderMan) : _folderMan(folderMan) diff --git a/src/gui/notificationwidget.cpp b/src/gui/notificationwidget.cpp index cd879451e..1c48f4425 100644 --- a/src/gui/notificationwidget.cpp +++ b/src/gui/notificationwidget.cpp @@ -23,7 +23,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcNotifications, "gui.notifications", QtInfoMsg) +Q_LOGGING_CATEGORY(lcNotifications, "nextcloud.gui.notifications", QtInfoMsg) NotificationWidget::NotificationWidget(QWidget *parent) : QWidget(parent) diff --git a/src/gui/ocsjob.cpp b/src/gui/ocsjob.cpp index 6793bf09e..e06d7a43a 100644 --- a/src/gui/ocsjob.cpp +++ b/src/gui/ocsjob.cpp @@ -22,7 +22,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcOcs, "gui.sharing.ocs", QtInfoMsg) +Q_LOGGING_CATEGORY(lcOcs, "nextcloud.gui.sharing.ocs", QtInfoMsg) OcsJob::OcsJob(AccountPtr account) : AbstractNetworkJob(account, "") diff --git a/src/gui/proxyauthhandler.cpp b/src/gui/proxyauthhandler.cpp index f2ccd97ac..31dfda6e9 100644 --- a/src/gui/proxyauthhandler.cpp +++ b/src/gui/proxyauthhandler.cpp @@ -25,7 +25,7 @@ using namespace OCC; -Q_LOGGING_CATEGORY(lcProxy, "gui.credentials.proxy", QtInfoMsg) +Q_LOGGING_CATEGORY(lcProxy, "nextcloud.gui.credentials.proxy", QtInfoMsg) ProxyAuthHandler *ProxyAuthHandler::instance() { diff --git a/src/gui/servernotificationhandler.cpp b/src/gui/servernotificationhandler.cpp index 72b65b296..9a3eacfc3 100644 --- a/src/gui/servernotificationhandler.cpp +++ b/src/gui/servernotificationhandler.cpp @@ -22,7 +22,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcServerNotification, "gui.servernotification", QtInfoMsg) +Q_LOGGING_CATEGORY(lcServerNotification, "nextcloud.gui.servernotification", QtInfoMsg) const QString notificationsPath = QLatin1String("ocs/v2.php/apps/notifications/api/v1/notifications"); diff --git a/src/gui/sharee.cpp b/src/gui/sharee.cpp index e1cec2ff1..f35a818e6 100644 --- a/src/gui/sharee.cpp +++ b/src/gui/sharee.cpp @@ -21,7 +21,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcSharing, "gui.sharing", QtInfoMsg) +Q_LOGGING_CATEGORY(lcSharing, "nextcloud.gui.sharing", QtInfoMsg) Sharee::Sharee(const QString shareWith, const QString displayName, diff --git a/src/gui/socketapi.cpp b/src/gui/socketapi.cpp index 0cdc9f0de..276bfe456 100644 --- a/src/gui/socketapi.cpp +++ b/src/gui/socketapi.cpp @@ -83,7 +83,7 @@ static QString buildMessage(const QString &verb, const QString &path, const QStr namespace OCC { -Q_LOGGING_CATEGORY(lcSocketApi, "gui.socketapi", QtInfoMsg) +Q_LOGGING_CATEGORY(lcSocketApi, "nextcloud.gui.socketapi", QtInfoMsg) class BloomFilter { diff --git a/src/gui/sslbutton.cpp b/src/gui/sslbutton.cpp index e387f885e..71433aef4 100644 --- a/src/gui/sslbutton.cpp +++ b/src/gui/sslbutton.cpp @@ -26,7 +26,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcSsl, "gui.ssl", QtInfoMsg) +Q_LOGGING_CATEGORY(lcSsl, "nextcloud.gui.ssl", QtInfoMsg) SslButton::SslButton(QWidget *parent) : QToolButton(parent) diff --git a/src/gui/sslerrordialog.cpp b/src/gui/sslerrordialog.cpp index bd4c9b45b..b75fd425c 100644 --- a/src/gui/sslerrordialog.cpp +++ b/src/gui/sslerrordialog.cpp @@ -23,7 +23,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcSslErrorDialog, "gui.sslerrordialog", QtInfoMsg) +Q_LOGGING_CATEGORY(lcSslErrorDialog, "nextcloud.gui.sslerrordialog", QtInfoMsg) namespace Utility { // Used for QSSLCertificate::subjectInfo which returns a QStringList in Qt5, but a QString in Qt4 diff --git a/src/gui/updater/updater.cpp b/src/gui/updater/updater.cpp index 6249ebc3e..ad67b89a2 100644 --- a/src/gui/updater/updater.cpp +++ b/src/gui/updater/updater.cpp @@ -27,7 +27,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcUpdater, "gui.updater", QtInfoMsg) +Q_LOGGING_CATEGORY(lcUpdater, "nextcloud.gui.updater", QtInfoMsg) Updater *Updater::_instance = 0; diff --git a/src/gui/wizard/owncloudwizard.cpp b/src/gui/wizard/owncloudwizard.cpp index 991b5fc18..2397efbb8 100644 --- a/src/gui/wizard/owncloudwizard.cpp +++ b/src/gui/wizard/owncloudwizard.cpp @@ -36,7 +36,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcWizard, "gui.wizard", QtInfoMsg) +Q_LOGGING_CATEGORY(lcWizard, "nextcloud.gui.wizard", QtInfoMsg) OwncloudWizard::OwncloudWizard(QWidget *parent) : QWizard(parent) diff --git a/src/libsync/abstractnetworkjob.cpp b/src/libsync/abstractnetworkjob.cpp index f3fb71670..dd73074ad 100644 --- a/src/libsync/abstractnetworkjob.cpp +++ b/src/libsync/abstractnetworkjob.cpp @@ -38,7 +38,7 @@ Q_DECLARE_METATYPE(QTimer *) namespace OCC { -Q_LOGGING_CATEGORY(lcNetworkJob, "sync.networkjob", QtInfoMsg) +Q_LOGGING_CATEGORY(lcNetworkJob, "nextcloud.sync.networkjob", QtInfoMsg) // If not set, it is overwritten by the Application constructor with the value from the config int AbstractNetworkJob::httpTimeout = qEnvironmentVariableIntValue("OWNCLOUD_TIMEOUT"); diff --git a/src/libsync/accessmanager.cpp b/src/libsync/accessmanager.cpp index fdfa6758a..a386a88f1 100644 --- a/src/libsync/accessmanager.cpp +++ b/src/libsync/accessmanager.cpp @@ -29,7 +29,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcAccessManager, "sync.accessmanager", QtInfoMsg) +Q_LOGGING_CATEGORY(lcAccessManager, "nextcloud.sync.accessmanager", QtInfoMsg) AccessManager::AccessManager(QObject *parent) : QNetworkAccessManager(parent) diff --git a/src/libsync/account.cpp b/src/libsync/account.cpp index a83a414cc..3a52b6367 100644 --- a/src/libsync/account.cpp +++ b/src/libsync/account.cpp @@ -38,7 +38,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcAccount, "sync.account", QtInfoMsg) +Q_LOGGING_CATEGORY(lcAccount, "nextcloud.sync.account", QtInfoMsg) Account::Account(QObject *parent) : QObject(parent) diff --git a/src/libsync/bandwidthmanager.cpp b/src/libsync/bandwidthmanager.cpp index c1ed6adcd..00f833868 100644 --- a/src/libsync/bandwidthmanager.cpp +++ b/src/libsync/bandwidthmanager.cpp @@ -29,7 +29,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcBandwidthManager, "sync.bandwidthmanager", QtInfoMsg) +Q_LOGGING_CATEGORY(lcBandwidthManager, "nextcloud.sync.bandwidthmanager", QtInfoMsg) // Because of the many layers of buffering inside Qt (and probably the OS and the network) // we cannot lower this value much more. If we do, the estimated bw will be very high diff --git a/src/libsync/capabilities.cpp b/src/libsync/capabilities.cpp index 99020414f..0d2e891cf 100644 --- a/src/libsync/capabilities.cpp +++ b/src/libsync/capabilities.cpp @@ -20,7 +20,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcServerCapabilities, "sync.server.capabilities", QtInfoMsg) +Q_LOGGING_CATEGORY(lcServerCapabilities, "nextcloud.sync.server.capabilities", QtInfoMsg) Capabilities::Capabilities(const QVariantMap &capabilities) diff --git a/src/libsync/clientproxy.cpp b/src/libsync/clientproxy.cpp index e87f04b7c..11d456c25 100644 --- a/src/libsync/clientproxy.cpp +++ b/src/libsync/clientproxy.cpp @@ -21,7 +21,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcClientProxy, "sync.clientproxy", QtInfoMsg) +Q_LOGGING_CATEGORY(lcClientProxy, "nextcloud.sync.clientproxy", QtInfoMsg) ClientProxy::ClientProxy(QObject *parent) : QObject(parent) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 980b5c04c..118a6084c 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -43,9 +43,9 @@ using namespace QKeychain; namespace OCC { -Q_LOGGING_CATEGORY(lcCse, "sync.clientsideencryption", QtInfoMsg) -Q_LOGGING_CATEGORY(lcCseDecryption, "e2e", QtInfoMsg) -Q_LOGGING_CATEGORY(lcCseMetadata, "metadata", QtInfoMsg) +Q_LOGGING_CATEGORY(lcCse, "nextcloud.sync.clientsideencryption", QtInfoMsg) +Q_LOGGING_CATEGORY(lcCseDecryption, "nextcloud.e2e", QtInfoMsg) +Q_LOGGING_CATEGORY(lcCseMetadata, "nextcloud.metadata", QtInfoMsg) QString baseUrl(){ return QStringLiteral("ocs/v2.php/apps/end_to_end_encryption/api/v1/"); @@ -655,7 +655,7 @@ void ClientSideEncryption::publicKeyFetched(Job *incoming) { void ClientSideEncryption::setFolderEncryptedStatus(const QString& folder, bool status) { - qDebug() << "Setting folder" << folder << "as encrypted" << status; + qCDebug(lcCse) << "Setting folder" << folder << "as encrypted" << status; _folder2encryptedStatus[folder] = status; } @@ -1071,13 +1071,13 @@ void ClientSideEncryption::folderEncryptedStatusFetched(const QMapopen(QIODevice::ReadOnly)) { - qDebug() << "Could not open input file for reading" << input->errorString(); + qCDebug(lcCse) << "Could not open input file for reading" << input->errorString(); } if (!output->open(QIODevice::WriteOnly)) { - qDebug() << "Could not oppen output file for writting" << output->errorString(); + qCDebug(lcCse) << "Could not oppen output file for writting" << output->errorString(); } // Init @@ -1328,7 +1328,7 @@ void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i int len = 0; int total_len = 0; - qDebug() << "Starting to encrypt the file" << input->fileName() << input->atEnd(); + qCDebug(lcCse) << "Starting to encrypt the file" << input->fileName() << input->atEnd(); while(!input->atEnd()) { QByteArray data = input->read(1024); @@ -1337,7 +1337,7 @@ void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i exit(-1); } - qDebug() << "Encrypting " << data; + qCDebug(lcCse) << "Encrypting " << data; if(!EVP_EncryptUpdate(ctx, out, &len, (unsigned char *)data.constData(), data.size())) { qCInfo(lcCse()) << "Could not encrypt"; exit(-1); @@ -1369,7 +1369,7 @@ void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i input->close(); output->close(); - qDebug() << "File Encrypted Successfully"; + qCDebug(lcCse) << "File Encrypted Successfully"; } FileDecryptionJob::FileDecryptionJob(QByteArray &key, QByteArray &iv, QFile *input, QFile *output, QObject *parent) diff --git a/src/libsync/clientsideencryptionjobs.cpp b/src/libsync/clientsideencryptionjobs.cpp index 9cd1be092..eaa012365 100644 --- a/src/libsync/clientsideencryptionjobs.cpp +++ b/src/libsync/clientsideencryptionjobs.cpp @@ -18,9 +18,9 @@ #include "theme.h" #include "creds/abstractcredentials.h" -Q_LOGGING_CATEGORY(lcSignPublicKeyApiJob, "sync.networkjob.sendcsr", QtInfoMsg) -Q_LOGGING_CATEGORY(lcStorePrivateKeyApiJob, "sync.networkjob.storeprivatekey", QtInfoMsg) -Q_LOGGING_CATEGORY(lcCseJob, "sync.networkjob.clientsideencrypt", QtInfoMsg) +Q_LOGGING_CATEGORY(lcSignPublicKeyApiJob, "nextcloud.sync.networkjob.sendcsr", QtInfoMsg) +Q_LOGGING_CATEGORY(lcStorePrivateKeyApiJob, "nextcloud.sync.networkjob.storeprivatekey", QtInfoMsg) +Q_LOGGING_CATEGORY(lcCseJob, "nextcloud.sync.networkjob.clientsideencrypt", QtInfoMsg) namespace OCC { diff --git a/src/libsync/configfile.cpp b/src/libsync/configfile.cpp index 8aff35b3b..66616d75d 100644 --- a/src/libsync/configfile.cpp +++ b/src/libsync/configfile.cpp @@ -43,7 +43,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcConfigFile, "sync.configfile", QtInfoMsg) +Q_LOGGING_CATEGORY(lcConfigFile, "nextcloud.sync.configfile", QtInfoMsg) //static const char caCertsKeyC[] = "CaCertificates"; only used from account.cpp static const char remotePollIntervalC[] = "remotePollInterval"; diff --git a/src/libsync/connectionvalidator.cpp b/src/libsync/connectionvalidator.cpp index a55816baa..24592823f 100644 --- a/src/libsync/connectionvalidator.cpp +++ b/src/libsync/connectionvalidator.cpp @@ -28,7 +28,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcConnectionValidator, "sync.connectionvalidator", QtInfoMsg) +Q_LOGGING_CATEGORY(lcConnectionValidator, "nextcloud.sync.connectionvalidator", QtInfoMsg) // Make sure the timeout for this job is less than how often we get called // This makes sure we get tried often enough without "ConnectionValidator already running" diff --git a/src/libsync/cookiejar.cpp b/src/libsync/cookiejar.cpp index a15ac87d1..615220e8d 100644 --- a/src/libsync/cookiejar.cpp +++ b/src/libsync/cookiejar.cpp @@ -24,7 +24,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcCookieJar, "sync.cookiejar", QtInfoMsg) +Q_LOGGING_CATEGORY(lcCookieJar, "nextcloud.sync.cookiejar", QtInfoMsg) namespace { const unsigned int JAR_VERSION = 23; diff --git a/src/libsync/creds/abstractcredentials.cpp b/src/libsync/creds/abstractcredentials.cpp index 8b6ee3104..4febff813 100644 --- a/src/libsync/creds/abstractcredentials.cpp +++ b/src/libsync/creds/abstractcredentials.cpp @@ -21,7 +21,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcCredentials, "sync.credentials", QtInfoMsg) +Q_LOGGING_CATEGORY(lcCredentials, "nextcloud.sync.credentials", QtInfoMsg) AbstractCredentials::AbstractCredentials() : _account(0) diff --git a/src/libsync/creds/httpcredentials.cpp b/src/libsync/creds/httpcredentials.cpp index ca1fa7ba1..ca2a1982a 100644 --- a/src/libsync/creds/httpcredentials.cpp +++ b/src/libsync/creds/httpcredentials.cpp @@ -37,7 +37,7 @@ using namespace QKeychain; namespace OCC { -Q_LOGGING_CATEGORY(lcHttpCredentials, "sync.credentials.http", QtInfoMsg) +Q_LOGGING_CATEGORY(lcHttpCredentials, "nextcloud.sync.credentials.http", QtInfoMsg) namespace { const char userC[] = "user"; diff --git a/src/libsync/creds/tokencredentials.cpp b/src/libsync/creds/tokencredentials.cpp index 9f857ade5..2e443065e 100644 --- a/src/libsync/creds/tokencredentials.cpp +++ b/src/libsync/creds/tokencredentials.cpp @@ -29,7 +29,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcTokenCredentials, "sync.credentials.token", QtInfoMsg) +Q_LOGGING_CATEGORY(lcTokenCredentials, "nextcloud.sync.credentials.token", QtInfoMsg) namespace { diff --git a/src/libsync/discoveryphase.cpp b/src/libsync/discoveryphase.cpp index 4657e0488..7a6ca0456 100644 --- a/src/libsync/discoveryphase.cpp +++ b/src/libsync/discoveryphase.cpp @@ -31,7 +31,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcDiscovery, "sync.discovery", QtInfoMsg) +Q_LOGGING_CATEGORY(lcDiscovery, "nextcloud.sync.discovery", QtInfoMsg) /* Given a sorted list of paths ending with '/', return whether or not the given path is within one of the paths of the list*/ static bool findPathInList(const QStringList &list, const QString &path) diff --git a/src/libsync/logger.cpp b/src/libsync/logger.cpp index 31afbbe8c..96c80e55b 100644 --- a/src/libsync/logger.cpp +++ b/src/libsync/logger.cpp @@ -24,7 +24,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcCsync, "sync.csync", QtInfoMsg) +Q_LOGGING_CATEGORY(lcCsync, "nextcloud.sync.csync", QtInfoMsg) static void mirallLogCatcher(QtMsgType type, const QMessageLogContext &ctx, const QString &message) { diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index 01c1b0901..ce4e71daa 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -41,15 +41,15 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcEtagJob, "sync.networkjob.etag", QtInfoMsg) -Q_LOGGING_CATEGORY(lcLsColJob, "sync.networkjob.lscol", QtInfoMsg) -Q_LOGGING_CATEGORY(lcCheckServerJob, "sync.networkjob.checkserver", QtInfoMsg) -Q_LOGGING_CATEGORY(lcPropfindJob, "sync.networkjob.propfind", QtInfoMsg) -Q_LOGGING_CATEGORY(lcAvatarJob, "sync.networkjob.avatar", QtInfoMsg) -Q_LOGGING_CATEGORY(lcMkColJob, "sync.networkjob.mkcol", QtInfoMsg) -Q_LOGGING_CATEGORY(lcProppatchJob, "sync.networkjob.proppatch", QtInfoMsg) -Q_LOGGING_CATEGORY(lcJsonApiJob, "sync.networkjob.jsonapi", QtInfoMsg) -Q_LOGGING_CATEGORY(lcDetermineAuthTypeJob, "sync.networkjob.determineauthtype", QtInfoMsg) +Q_LOGGING_CATEGORY(lcEtagJob, "nextcloud.sync.networkjob.etag", QtInfoMsg) +Q_LOGGING_CATEGORY(lcLsColJob, "nextcloud.sync.networkjob.lscol", QtInfoMsg) +Q_LOGGING_CATEGORY(lcCheckServerJob, "nextcloud.sync.networkjob.checkserver", QtInfoMsg) +Q_LOGGING_CATEGORY(lcPropfindJob, "nextcloud.sync.networkjob.propfind", QtInfoMsg) +Q_LOGGING_CATEGORY(lcAvatarJob, "nextcloud.sync.networkjob.avatar", QtInfoMsg) +Q_LOGGING_CATEGORY(lcMkColJob, "nextcloud.sync.networkjob.mkcol", QtInfoMsg) +Q_LOGGING_CATEGORY(lcProppatchJob, "nextcloud.sync.networkjob.proppatch", QtInfoMsg) +Q_LOGGING_CATEGORY(lcJsonApiJob, "nextcloud.sync.networkjob.jsonapi", QtInfoMsg) +Q_LOGGING_CATEGORY(lcDetermineAuthTypeJob, "nextcloud.sync.networkjob.determineauthtype", QtInfoMsg) RequestEtagJob::RequestEtagJob(AccountPtr account, const QString &path, QObject *parent) : AbstractNetworkJob(account, path, parent) diff --git a/src/libsync/owncloudpropagator.cpp b/src/libsync/owncloudpropagator.cpp index e1d081141..85bc6594e 100644 --- a/src/libsync/owncloudpropagator.cpp +++ b/src/libsync/owncloudpropagator.cpp @@ -42,9 +42,9 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcPropagator, "sync.propagator", QtInfoMsg) -Q_LOGGING_CATEGORY(lcDirectory, "sync.propagator.directory", QtInfoMsg) -Q_LOGGING_CATEGORY(lcCleanupPolls, "sync.propagator.cleanuppolls", QtInfoMsg) +Q_LOGGING_CATEGORY(lcPropagator, "nextcloud.sync.propagator", QtInfoMsg) +Q_LOGGING_CATEGORY(lcDirectory, "nextcloud.sync.propagator.directory", QtInfoMsg) +Q_LOGGING_CATEGORY(lcCleanupPolls, "nextcloud.sync.propagator.cleanuppolls", QtInfoMsg) qint64 criticalFreeSpaceLimit() { diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index 2efc537ea..7813e4ea7 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -37,8 +37,8 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcGetJob, "sync.networkjob.get", QtInfoMsg) -Q_LOGGING_CATEGORY(lcPropagateDownload, "sync.propagator.download", QtInfoMsg) +Q_LOGGING_CATEGORY(lcGetJob, "nextcloud.sync.networkjob.get", QtInfoMsg) +Q_LOGGING_CATEGORY(lcPropagateDownload, "nextcloud.sync.propagator.download", QtInfoMsg) // Always coming in with forward slashes. // In csync_excluded_no_ctx we ignore all files with longer than 254 chars diff --git a/src/libsync/propagateremotedelete.cpp b/src/libsync/propagateremotedelete.cpp index f67b2b3af..cf230fc22 100644 --- a/src/libsync/propagateremotedelete.cpp +++ b/src/libsync/propagateremotedelete.cpp @@ -21,8 +21,8 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcDeleteJob, "sync.networkjob.delete", QtInfoMsg) -Q_LOGGING_CATEGORY(lcPropagateRemoteDelete, "sync.propagator.remotedelete", QtInfoMsg) +Q_LOGGING_CATEGORY(lcDeleteJob, "nextcloud.sync.networkjob.delete", QtInfoMsg) +Q_LOGGING_CATEGORY(lcPropagateRemoteDelete, "nextcloud.sync.propagator.remotedelete", QtInfoMsg) DeleteJob::DeleteJob(AccountPtr account, const QString &path, QObject *parent) : AbstractNetworkJob(account, path, parent) diff --git a/src/libsync/propagateremotemkdir.cpp b/src/libsync/propagateremotemkdir.cpp index 98a72d48a..4e5ae7ab4 100644 --- a/src/libsync/propagateremotemkdir.cpp +++ b/src/libsync/propagateremotemkdir.cpp @@ -24,7 +24,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcPropagateRemoteMkdir, "sync.propagator.remotemkdir", QtInfoMsg) +Q_LOGGING_CATEGORY(lcPropagateRemoteMkdir, "nextcloud.sync.propagator.remotemkdir", QtInfoMsg) void PropagateRemoteMkdir::start() { diff --git a/src/libsync/propagateremotemove.cpp b/src/libsync/propagateremotemove.cpp index 35a2486cf..7aa21a8a4 100644 --- a/src/libsync/propagateremotemove.cpp +++ b/src/libsync/propagateremotemove.cpp @@ -25,8 +25,8 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcMoveJob, "sync.networkjob.move", QtInfoMsg) -Q_LOGGING_CATEGORY(lcPropagateRemoteMove, "sync.propagator.remotemove", QtInfoMsg) +Q_LOGGING_CATEGORY(lcMoveJob, "nextcloud.sync.networkjob.move", QtInfoMsg) +Q_LOGGING_CATEGORY(lcPropagateRemoteMove, "nextcloud.sync.propagator.remotemove", QtInfoMsg) MoveJob::MoveJob(AccountPtr account, const QString &path, const QString &destination, QObject *parent) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index bf43472d2..49e38862e 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -43,9 +43,9 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcPutJob, "sync.networkjob.put", QtInfoMsg) -Q_LOGGING_CATEGORY(lcPollJob, "sync.networkjob.poll", QtInfoMsg) -Q_LOGGING_CATEGORY(lcPropagateUpload, "sync.propagator.upload", QtInfoMsg) +Q_LOGGING_CATEGORY(lcPutJob, "nextcloud.sync.networkjob.put", QtInfoMsg) +Q_LOGGING_CATEGORY(lcPollJob, "nextcloud.sync.networkjob.poll", QtInfoMsg) +Q_LOGGING_CATEGORY(lcPropagateUpload, "nextcloud.sync.propagator.upload", QtInfoMsg) /** * We do not want to upload files that are currently being modified. diff --git a/src/libsync/propagatorjobs.cpp b/src/libsync/propagatorjobs.cpp index 49ecfe156..2b24673cc 100644 --- a/src/libsync/propagatorjobs.cpp +++ b/src/libsync/propagatorjobs.cpp @@ -34,9 +34,9 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcPropagateLocalRemove, "sync.propagator.localremove", QtInfoMsg) -Q_LOGGING_CATEGORY(lcPropagateLocalMkdir, "sync.propagator.localmkdir", QtInfoMsg) -Q_LOGGING_CATEGORY(lcPropagateLocalRename, "sync.propagator.localrename", QtInfoMsg) +Q_LOGGING_CATEGORY(lcPropagateLocalRemove, "nextcloud.sync.propagator.localremove", QtInfoMsg) +Q_LOGGING_CATEGORY(lcPropagateLocalMkdir, "nextcloud.sync.propagator.localmkdir", QtInfoMsg) +Q_LOGGING_CATEGORY(lcPropagateLocalRename, "nextcloud.sync.propagator.localrename", QtInfoMsg) QByteArray localFileIdFromFullId(const QByteArray &id) { diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 6dc571c72..386e5d76b 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -53,7 +53,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcEngine, "sync.engine", QtInfoMsg) +Q_LOGGING_CATEGORY(lcEngine, "nextcloud.sync.engine", QtInfoMsg) static const int s_touchedFilesMaxAgeMs = 15 * 1000; bool SyncEngine::s_anySyncRunning = false; diff --git a/src/libsync/syncfileitem.cpp b/src/libsync/syncfileitem.cpp index bfea07163..3929ec1dc 100644 --- a/src/libsync/syncfileitem.cpp +++ b/src/libsync/syncfileitem.cpp @@ -21,7 +21,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcFileItem, "sync.fileitem", QtInfoMsg) +Q_LOGGING_CATEGORY(lcFileItem, "nextcloud.sync.fileitem", QtInfoMsg) SyncJournalFileRecord SyncFileItem::toSyncJournalFileRecordWithInode(const QString &localFileName) { diff --git a/src/libsync/syncfilestatustracker.cpp b/src/libsync/syncfilestatustracker.cpp index 1cae76986..6dc2514ff 100644 --- a/src/libsync/syncfilestatustracker.cpp +++ b/src/libsync/syncfilestatustracker.cpp @@ -23,7 +23,7 @@ namespace OCC { -Q_LOGGING_CATEGORY(lcStatusTracker, "sync.statustracker", QtInfoMsg) +Q_LOGGING_CATEGORY(lcStatusTracker, "nextcloud.sync.statustracker", QtInfoMsg) static int pathCompare( const QString& lhs, const QString& rhs ) { From e06e04ee933dcb739bc23d70ba5fffa4b9b9a021 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Fri, 29 Dec 2017 09:43:27 -0200 Subject: [PATCH 192/262] Fix parsing list of encrypted folders --- src/libsync/clientsideencryptionjobs.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libsync/clientsideencryptionjobs.cpp b/src/libsync/clientsideencryptionjobs.cpp index eaa012365..539ab9beb 100644 --- a/src/libsync/clientsideencryptionjobs.cpp +++ b/src/libsync/clientsideencryptionjobs.cpp @@ -81,10 +81,15 @@ bool GetFolderEncryptStatusJob::finished() auto type = reader.readNext(); if (type == QXmlStreamReader::StartElement) { if (reader.name() == QLatin1String("href")) { - // If the current file is not a folder, ignore it. - currFile = reader.readElementText(QXmlStreamReader::SkipChildElements).remove("/remote.php/webdav/"); - if (!currFile.endsWith('/')) - currFile.clear(); + // If the current file is not a folder, ignore it. + QString base = account()->url().path(); + if (base.endsWith(QLatin1Char('/'))) + base.chop(1); + + currFile = reader.readElementText(QXmlStreamReader::SkipChildElements); + currFile.remove(base + QLatin1String("/remote.php/webdav/")); + if (!currFile.endsWith('/')) + currFile.clear(); currEncryptedStatus = -1; } if (not currFile.isEmpty() && reader.name() == QLatin1String("is-encrypted")) { From fdc96d3a4f444f32436461b298cae26aaaa767b4 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Fri, 29 Dec 2017 09:45:55 -0200 Subject: [PATCH 193/262] Fix deprecated use of QUrl::setQueryItems() --- src/libsync/clientsideencryptionjobs.cpp | 35 ++++++++++-------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/src/libsync/clientsideencryptionjobs.cpp b/src/libsync/clientsideencryptionjobs.cpp index 539ab9beb..076bc990a 100644 --- a/src/libsync/clientsideencryptionjobs.cpp +++ b/src/libsync/clientsideencryptionjobs.cpp @@ -126,11 +126,10 @@ void GetMetadataApiJob::start() { QNetworkRequest req; req.setRawHeader("OCS-APIREQUEST", "true"); + QUrlQuery query; + query.addQueryItem(QLatin1String("format"), QLatin1String("json")); QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")), - }; - url.setQueryItems(params); + url.setQuery(query); qCInfo(lcCseJob()) << "Requesting the metadata for the fileId" << _fileId << "as encrypted"; sendRequest("GET", url, req); @@ -161,11 +160,10 @@ void StoreMetaDataApiJob::start() { QNetworkRequest req; req.setRawHeader("OCS-APIREQUEST", "true"); + QUrlQuery query; + query.addQueryItem(QLatin1String("format"), QLatin1String("json")); QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")), - }; - url.setQueryItems(params); + url.setQuery(query); QByteArray data = QByteArray("metaData=") + QUrl::toPercentEncoding(_b64Metadata); auto buffer = new QBuffer(this); @@ -316,11 +314,10 @@ void LockEncryptFolderApiJob::start() { QNetworkRequest req; req.setRawHeader("OCS-APIREQUEST", "true"); + QUrlQuery query; + query.addQueryItem(QLatin1String("format"), QLatin1String("json")); QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) - }; - url.setQueryItems(params); + url.setQuery(query); qCInfo(lcCseJob()) << "locking the folder with id" << _fileId << "as encrypted"; sendRequest("POST", url, req); @@ -392,11 +389,10 @@ void StorePrivateKeyApiJob::start() { QNetworkRequest req; req.setRawHeader("OCS-APIREQUEST", "true"); + QUrlQuery query; + query.addQueryItem(QLatin1String("format"), QLatin1String("json")); QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) - }; - url.setQueryItems(params); + url.setQuery(query); qCInfo(lcStorePrivateKeyApiJob) << "Sending the private key" << _privKey.data(); sendRequest("POST", url, req, &_privKey); @@ -431,11 +427,10 @@ void SignPublicKeyApiJob::start() { QNetworkRequest req; req.setRawHeader("OCS-APIREQUEST", "true"); + QUrlQuery query; + query.addQueryItem(QLatin1String("format"), QLatin1String("json")); QUrl url = Utility::concatUrlPath(account()->url(), path()); - QList> params = { - qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json")) - }; - url.setQueryItems(params); + url.setQuery(query); qCInfo(lcSignPublicKeyApiJob) << "Sending the CSR" << _csr.data(); sendRequest("POST", url, req, &_csr); From c6491d50bbd740f59ce8e627bb940a2a61873a0d Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Fri, 29 Dec 2017 13:21:13 -0200 Subject: [PATCH 194/262] Replace deprecated Qt API usage --- src/gui/application.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gui/application.cpp b/src/gui/application.cpp index 292da54c8..544ebe805 100644 --- a/src/gui/application.cpp +++ b/src/gui/application.cpp @@ -51,7 +51,7 @@ #include #include #include -#include +#include class QSocket; @@ -128,7 +128,7 @@ Application::Application(int &argc, char **argv) if (!QFileInfo(confDir).exists()) { // Migrate from version <= 2.4 setApplicationName(_theme->appNameGUI()); - QString oldDir = QDesktopServices::storageLocation(QDesktopServices::DataLocation); + QString oldDir = QStandardPaths::writableLocation(QStandardPaths::DataLocation); setApplicationName(_theme->appName()); if (QFileInfo(oldDir).isDir()) { qCInfo(lcApplication) << "Migrating old config from" << oldDir << "to" << confDir; From 50916bcda5ee1271db4796ef2b1cce8e2dc4f2f5 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Fri, 29 Dec 2017 13:22:23 -0200 Subject: [PATCH 195/262] Mark ClientSideEncryption::isFolderEncrypted() as const --- src/libsync/clientsideencryption.cpp | 6 +++--- src/libsync/clientsideencryption.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 118a6084c..0c07ec8b1 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1279,9 +1279,9 @@ QVector FolderMetadata::files() const { return _files; } -bool ClientSideEncryption::isFolderEncrypted(const QString& path) { - auto it = _folder2encryptedStatus.find(path); - if (it == _folder2encryptedStatus.end()) +bool ClientSideEncryption::isFolderEncrypted(const QString& path) const { + auto it = _folder2encryptedStatus.constFind(path); + if (it == _folder2encryptedStatus.constEnd()) return false; return (*it); } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 7d5810a8d..b5ffab5dc 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -81,7 +81,7 @@ public: void fetchFolderEncryptedStatus(); // to be used together with FolderStatusModel::FolderInfo::_path. - bool isFolderEncrypted(const QString& path); + bool isFolderEncrypted(const QString& path) const; void setFolderEncryptedStatus(const QString& path, bool status); void forgetSensitiveData(); From ca92c46970dc85da0d98f351db9d085f5adbd83e Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Fri, 29 Dec 2017 13:54:50 -0200 Subject: [PATCH 196/262] Add category logging to PropagateUploadEncrypted --- src/libsync/propagateuploadencrypted.cpp | 47 +++++++++++++----------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 822b5dcc3..eabf0eba1 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -9,9 +9,12 @@ #include #include #include +#include namespace OCC { +Q_DECLARE_LOGGING_CATEGORY(lcPropagateUpload) + PropagateUploadEncrypted::PropagateUploadEncrypted(OwncloudPropagator *propagator, SyncFileItemPtr item) : _propagator(propagator), _item(item), @@ -33,7 +36,7 @@ void PropagateUploadEncrypted::start() * * If the folder is unencrypted we just follow the old way. */ - qDebug() << "Starting to send an encrypted file!"; + qCDebug(lcPropagateUpload) << "Starting to send an encrypted file!"; QFileInfo info(_item->_file); auto getEncryptedStatus = new GetFolderEncryptStatusJob(_propagator->account(), info.path()); @@ -47,7 +50,7 @@ void PropagateUploadEncrypted::start() void PropagateUploadEncrypted::slotFolderEncryptedStatusFetched(const QMap& result) { - qDebug() << "Encrypted Status Fetched"; + qCDebug(lcPropagateUpload) << "Encrypted Status Fetched"; QFileInfo fileInfo(_item->_file); QString currFilePath = fileInfo.path(); if (!currFilePath.endsWith(QDir::separator())) @@ -55,7 +58,7 @@ void PropagateUploadEncrypted::slotFolderEncryptedStatusFetched(const QMap_file); LsColJob *job = new LsColJob(_propagator->account(), info.path(), this); job->setProperties({"resourcetype", "http://owncloud.org/ns:fileid"}); @@ -63,7 +66,7 @@ void PropagateUploadEncrypted::slotFolderEncryptedStatusFetched(const QMapstart(); } else { - qDebug() << "Folder is not encrypted, getting back to default."; + qCDebug(lcPropagateUpload) << "Folder is not encrypted, getting back to default."; emit folerNotEncrypted(); } } @@ -81,7 +84,7 @@ void PropagateUploadEncrypted::slotFolderEncryptedStatusFetched(const QMap(sender()); const auto& folderInfo = job->_folderInfos.value(list.first()); _folderLockFirstTry.start(); @@ -98,7 +101,7 @@ void PropagateUploadEncrypted::slotTryLock(const QByteArray& fileId) void PropagateUploadEncrypted::slotFolderLockedSuccessfully(const QByteArray& fileId, const QByteArray& token) { - qDebug() << "Folder" << fileId << "Locked Successfully for Upload, Fetching Metadata"; + qCDebug(lcPropagateUpload) << "Folder" << fileId << "Locked Successfully for Upload, Fetching Metadata"; // Should I use a mutex here? _currentLockingInProgress = true; _folderToken = token; @@ -112,13 +115,13 @@ void PropagateUploadEncrypted::slotFolderLockedSuccessfully(const QByteArray& fi void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode) { - qDebug() << "Metadata Received, Preparing it for the new file." << json.toVariant(); + qCDebug(lcPropagateUpload) << "Metadata Received, Preparing it for the new file." << json.toVariant(); // Encrypt File! _metadata = new FolderMetadata(_propagator->account(), json.toJson(QJsonDocument::Compact)); QFileInfo info(_propagator->_localDir + QDir::separator() + _item->_file); - qDebug() << "Creating the encrypted file metadata helper."; + qCDebug(lcPropagateUpload) << "Creating the encrypted file metadata helper."; EncryptedFile encryptedFile; encryptedFile.authenticationTag = "NOISE"; // TODO: Remove the noise. @@ -131,7 +134,7 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo _metadata->addEncryptedFile(encryptedFile); _encryptedFile = encryptedFile; - qDebug() << "Metadata created, sending to the server."; + qCDebug(lcPropagateUpload) << "Metadata created, sending to the server."; auto job = new UpdateMetadataApiJob(_propagator->account(), _folderId, _metadata->encryptedMetadata(), @@ -144,7 +147,7 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo void PropagateUploadEncrypted::slotUpdateMetadataSuccess(const QByteArray& fileId) { - qDebug() << "Uploading of the metadata success, Encrypting the file"; + qCDebug(lcPropagateUpload) << "Uploading of the metadata success, Encrypting the file"; QFileInfo info(_propagator->_localDir + QDir::separator() + _item->_file); auto *input = new QFile(info.absoluteFilePath()); auto *output = new QFile(QDir::tempPath() + QDir::separator() + _encryptedFile.encryptedFilename); @@ -157,24 +160,24 @@ void PropagateUploadEncrypted::slotUpdateMetadataSuccess(const QByteArray& fileI input->deleteLater(); output->deleteLater(); - qDebug() << "Encrypted Info:" << outputInfo.path() << outputInfo.fileName() << outputInfo.size(); - qDebug() << "Finalizing the upload part, now the actuall uploader will take over"; + qCDebug(lcPropagateUpload) << "Encrypted Info:" << outputInfo.path() << outputInfo.fileName() << outputInfo.size(); + qCDebug(lcPropagateUpload) << "Finalizing the upload part, now the actuall uploader will take over"; emit finalized(outputInfo.path(),outputInfo.fileName(),outputInfo.size()); } void PropagateUploadEncrypted::slotUpdateMetadataError(const QByteArray& fileId, int httpErrorResponse) { - qDebug() << "Update metadata error for folder" << fileId << "with error" << httpErrorResponse; + qCDebug(lcPropagateUpload) << "Update metadata error for folder" << fileId << "with error" << httpErrorResponse; } void PropagateUploadEncrypted::slotUnlockEncryptedFolderSuccess(const QByteArray& fileId) { - qDebug() << "Unlock Job worked for folder " << fileId; + qCDebug(lcPropagateUpload) << "Unlock Job worked for folder " << fileId; } void PropagateUploadEncrypted::slotUnlockEncryptedFolderError(const QByteArray& fileId, int httpStatusCode) { - qDebug() << "There was an error unlocking " << fileId << httpStatusCode; + qCDebug(lcPropagateUpload) << "There was an error unlocking " << fileId << httpStatusCode; } void PropagateUploadEncrypted::slotFolderLockedError(const QByteArray& fileId, int httpErrorCode) @@ -183,31 +186,31 @@ void PropagateUploadEncrypted::slotFolderLockedError(const QByteArray& fileId, i and fail if it's more than 5 minutes. */ QTimer::singleShot(5000, this, [this, fileId]{ if (!_currentLockingInProgress) { - qDebug() << "Error locking the folder while no other update is locking it up."; - qDebug() << "Perhaps another client locked it."; - qDebug() << "Abort"; + qCDebug(lcPropagateUpload) << "Error locking the folder while no other update is locking it up."; + qCDebug(lcPropagateUpload) << "Perhaps another client locked it."; + qCDebug(lcPropagateUpload) << "Abort"; return; } // Perhaps I should remove the elapsed timer if the lock is from this client? if (_folderLockFirstTry.elapsed() > /* five minutes */ 1000 * 60 * 5 ) { - qDebug() << "One minute passed, ignoring more attemps to lock the folder."; + qCDebug(lcPropagateUpload) << "One minute passed, ignoring more attemps to lock the folder."; return; } slotTryLock(fileId); }); - qDebug() << "Folder" << fileId << "Coundn't be locked."; + qCDebug(lcPropagateUpload) << "Folder" << fileId << "Coundn't be locked."; } void PropagateUploadEncrypted::slotFolderEncryptedIdError(QNetworkReply *r) { - qDebug() << "Error retrieving the Id of the encrypted folder."; + qCDebug(lcPropagateUpload) << "Error retrieving the Id of the encrypted folder."; } void PropagateUploadEncrypted::slotFolderEncryptedStatusError(int error) { - qDebug() << "Failed to retrieve the status of the folders." << error; + qCDebug(lcPropagateUpload) << "Failed to retrieve the status of the folders." << error; } } // namespace OCC From e2c895e61f4974907c8d109e11593bce2fe3ca2e Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Fri, 29 Dec 2017 14:23:28 -0200 Subject: [PATCH 197/262] Fix encrypted file location on the server --- src/libsync/propagateupload.cpp | 6 +++--- src/libsync/propagateuploadencrypted.cpp | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 49e38862e..281fbe525 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -178,7 +178,7 @@ void PropagateUploadFileCommon::start() connect(_uploadEncryptedHelper, &PropagateUploadEncrypted::finalized, this, &PropagateUploadFileCommon::setupEncryptedFile); connect(_uploadEncryptedHelper, &PropagateUploadEncrypted::error, - []{ qDebug() << "Error setting up encryption."; }); + []{ qCDebug(lcPropagateUpload) << "Error setting up encryption."; }); _uploadEncryptedHelper->start(); } else { setupUnencryptedFile(); @@ -187,9 +187,9 @@ void PropagateUploadFileCommon::start() void PropagateUploadFileCommon::setupEncryptedFile(const QString& path, const QString& filename, quint64 size) { - qDebug() << "Starting to upload encrypted file"; + qCDebug(lcPropagateUpload) << "Starting to upload encrypted file" << path << filename << size; _uploadingEncrypted = true; - _fileToUpload._path = path + QDir::separator() + filename; + _fileToUpload._path = path; _fileToUpload._file = filename; _fileToUpload._size = size; startUploadFile(); diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index eabf0eba1..819d360df 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -162,7 +162,9 @@ void PropagateUploadEncrypted::slotUpdateMetadataSuccess(const QByteArray& fileI qCDebug(lcPropagateUpload) << "Encrypted Info:" << outputInfo.path() << outputInfo.fileName() << outputInfo.size(); qCDebug(lcPropagateUpload) << "Finalizing the upload part, now the actuall uploader will take over"; - emit finalized(outputInfo.path(),outputInfo.fileName(),outputInfo.size()); + emit finalized(outputInfo.path() + QLatin1Char('/') + outputInfo.fileName(), + _item->_file.section(QLatin1Char('/'), 0, -2) + QLatin1Char('/') + outputInfo.fileName(), + outputInfo.size()); } void PropagateUploadEncrypted::slotUpdateMetadataError(const QByteArray& fileId, int httpErrorResponse) From 3abbbab6a0ab9db0f21bad758ce8bc0fa910e644 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Fri, 29 Dec 2017 17:45:48 -0200 Subject: [PATCH 198/262] Faster way to detecting CSE capability --- src/libsync/capabilities.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libsync/capabilities.cpp b/src/libsync/capabilities.cpp index 0d2e891cf..693d183d7 100644 --- a/src/libsync/capabilities.cpp +++ b/src/libsync/capabilities.cpp @@ -85,8 +85,10 @@ bool Capabilities::shareResharing() const bool Capabilities::clientSideEncryptionAvaliable() const { - return _capabilities.keys().indexOf("end-to-end-encryption") != -1 - ? _capabilities["end-to-end-encryption"].toMap()["enabled"].toBool() : false; + auto it = _capabilities.constFind(QStringLiteral("end-to-end-encryption")); + if (it != _capabilities.constEnd()) + return (*it).toMap().value(QStringLiteral("enabled"), false).toBool(); + return false; } bool Capabilities::notificationsAvailable() const From a7ee1a95a6ec0a6206b75615f6396ac472c9fa22 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Fri, 5 Jan 2018 18:37:31 -0200 Subject: [PATCH 199/262] Download and decrypt file if parent folder is marked as encrypted --- src/libsync/clientsideencryptionjobs.cpp | 5 +- src/libsync/clientsideencryptionjobs.h | 1 + src/libsync/propagatedownload.cpp | 95 +++++++++++++++++++++++- src/libsync/propagatedownload.h | 4 + src/libsync/propagateuploadencrypted.cpp | 15 ++-- src/libsync/propagateuploadencrypted.h | 2 +- 6 files changed, 109 insertions(+), 13 deletions(-) diff --git a/src/libsync/clientsideencryptionjobs.cpp b/src/libsync/clientsideencryptionjobs.cpp index 076bc990a..c603da064 100644 --- a/src/libsync/clientsideencryptionjobs.cpp +++ b/src/libsync/clientsideencryptionjobs.cpp @@ -34,7 +34,7 @@ void GetFolderEncryptStatusJob::start() QNetworkRequest req; req.setPriority(QNetworkRequest::HighPriority); req.setRawHeader("OCS-APIREQUEST", "true"); - req.setRawHeader("Content-Type", "application/xml"); + req.setHeader(QNetworkRequest::ContentTypeHeader, QByteArrayLiteral("application/xml")); QByteArray xml = " "; QBuffer *buf = new QBuffer(this); @@ -105,6 +105,7 @@ bool GetFolderEncryptStatusJob::finished() } emit encryptStatusReceived(folderStatus); + emit encryptStatusFolderReceived(_folder, folderStatus.value(_folder + QLatin1Char('/'))); } else { qCWarning(lcCseJob()) << "*not* successful, http result code is" << http_result_code << (http_result_code == 302 ? reply()->header(QNetworkRequest::LocationHeader).toString() : QLatin1String("")); @@ -160,6 +161,7 @@ void StoreMetaDataApiJob::start() { QNetworkRequest req; req.setRawHeader("OCS-APIREQUEST", "true"); + req.setHeader(QNetworkRequest::ContentTypeHeader, QByteArrayLiteral("application/x-www-form-urlencoded")); QUrlQuery query; query.addQueryItem(QLatin1String("format"), QLatin1String("json")); QUrl url = Utility::concatUrlPath(account()->url(), path()); @@ -203,6 +205,7 @@ void UpdateMetadataApiJob::start() { QNetworkRequest req; req.setRawHeader("OCS-APIREQUEST", "true"); + req.setHeader(QNetworkRequest::ContentTypeHeader, QByteArrayLiteral("application/x-www-form-urlencoded")); QUrlQuery urlQuery; urlQuery.addQueryItem(QStringLiteral("format"), QStringLiteral("json")); diff --git a/src/libsync/clientsideencryptionjobs.h b/src/libsync/clientsideencryptionjobs.h index b3fdf5d6b..c6d072ecb 100644 --- a/src/libsync/clientsideencryptionjobs.h +++ b/src/libsync/clientsideencryptionjobs.h @@ -293,6 +293,7 @@ protected: signals: void encryptStatusReceived(const QMap folderMetadata2EncryptionStatus); + void encryptStatusFolderReceived(const QString &folder, bool isEncrypted); void encryptStatusError(int statusCode); private: QString _folder; diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index 7813e4ea7..c3022bed6 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -24,6 +24,7 @@ #include "propagatorjobs.h" #include "common/checksums.h" #include "common/asserts.h" +#include "clientsideencryptionjobs.h" #include #include @@ -346,6 +347,66 @@ void PropagateDownloadFile::start() return; qCDebug(lcPropagateDownload) << _item->_file << propagator()->_activeJobList.count(); + if (propagator()->account()->capabilities().clientSideEncryptionAvaliable()) { + QFileInfo info(_item->_file); + auto getEncryptedStatus = new GetFolderEncryptStatusJob(propagator()->account(), info.path()); + connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusFolderReceived, + this, [this, info] (const QString &folder, bool isEncrypted) { + qCDebug(lcPropagateDownload) << "Get Folder is Encrypted Received" << folder << isEncrypted; + if (!isEncrypted) { + startAfterIsEncryptedIsChecked(); + return; + } + _isEncrypted = true; + + // Is encrypted Now we need the folder-id + auto job = new LsColJob(propagator()->account(), folder, this); + job->setProperties({"resourcetype", "http://owncloud.org/ns:fileid"}); + connect(job, &LsColJob::directoryListingSubfolders, this, [job, this, info] (const QStringList &list) { + const QString folderId = list.first(); + qCDebug(lcPropagateDownload) << "Received id of folder" << folderId; + const ExtraFolderInfo &folderInfo = job->_folderInfos.value(folderId); + + // Now that we have the folder-id we need it's JSON metadata + auto metadataJob = new GetMetadataApiJob(propagator()->account(), folderInfo.fileId); + connect(metadataJob, &GetMetadataApiJob::jsonReceived, + this, [this, info] (const QJsonDocument &json) { + qCDebug(lcPropagateDownload) << "Metadata Received reading" << json.toJson() << _item->_file; + const QString filename = info.fileName(); + auto meta = new FolderMetadata(propagator()->account(), json.toJson(QJsonDocument::Compact)); + const QVector files = meta->files(); + for (const EncryptedFile &file : files) { + qCDebug(lcPropagateDownload) << "file" << filename << file.encryptedFilename << file.originalFilename << file.encryptionKey; + if (filename == file.encryptedFilename) { + _encryptedInfo = file; + qCDebug(lcPropagateDownload) << "Found matching encrypted metadata for file, starting download"; + startAfterIsEncryptedIsChecked(); + return; + } + } + + qCDebug(lcPropagateDownload) << "Failed to find encrypted metadata information of remote file" << filename; + }); + metadataJob->start(); + }); + connect(job, &LsColJob::finishedWithError, this, []() { + qCDebug(lcPropagateDownload) << "Failed to get encrypted metadata of folder"; + }); + job->start(); + }); + connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusError, + this, [](int statusCode) { + qCDebug(lcPropagateDownload) << "Failed to get encrypted status of folder" << statusCode; + }); + + getEncryptedStatus->start(); + } else { + startAfterIsEncryptedIsChecked(); + } +} + +void PropagateDownloadFile::startAfterIsEncryptedIsChecked() +{ _stopwatch.start(); if (_deleteExisting) { @@ -785,7 +846,39 @@ void PropagateDownloadFile::contentChecksumComputed(const QByteArray &checksumTy { _item->_checksumHeader = makeChecksumHeader(checksumType, checksum); - downloadFinished(); + if (_isEncrypted) { + const QString tmpFileName = createDownloadTmpFileName(_item->_file + QLatin1String("_dec")); + qCDebug(lcPropagateDownload) << "Content Checksum Computed starting decryption" << tmpFileName; + + _tmpFile.close(); + auto _tmpOutput = new QFile(propagator()->getFilePath(tmpFileName), this); + auto job = new FileDecryptionJob(_encryptedInfo.encryptionKey, + _encryptedInfo.initializationVector, + &_tmpFile, + _tmpOutput); + connect(job, &FileDecryptionJob::finished, this, [this] (QFile *output) { + qCDebug(lcPropagateDownload) << "Decryption finished" << _tmpFile.fileName() << output->fileName(); + + _tmpFile.close(); + output->close(); + // we decripted the temporary into another temporary, so good bye old one + if (!_tmpFile.remove()) { + qCDebug(lcPropagateDownload) << "Failed to remove temporary file" << _tmpFile.errorString(); + done(SyncFileItem::NormalError, _tmpFile.errorString()); + return; + } + // Let's fool the rest of the logic into thinking this was the actual download + _tmpFile.setFileName(output->fileName()); + + // Let's fool the rest of the logic into thinking this is the right name of the DAV file + _item->_file = _item->_file.section(QLatin1Char('/'), 0, -2) + QLatin1Char('/') + _encryptedInfo.originalFilename; + + downloadFinished(); + }); + job->start(); + } else { + downloadFinished(); + } } void PropagateDownloadFile::downloadFinished() diff --git a/src/libsync/propagatedownload.h b/src/libsync/propagatedownload.h index 4b8988b65..0265d4506 100644 --- a/src/libsync/propagatedownload.h +++ b/src/libsync/propagatedownload.h @@ -15,6 +15,7 @@ #include "owncloudpropagator.h" #include "networkjobs.h" +#include "clientsideencryption.h" #include #include @@ -195,6 +196,7 @@ private slots: void slotChecksumFail(const QString &errMsg); private: + void startAfterIsEncryptedIsChecked(); void deleteExistingFolder(); quint64 _resumeStart; @@ -202,6 +204,8 @@ private: QPointer _job; QFile _tmpFile; bool _deleteExisting; + bool _isEncrypted = false; + EncryptedFile _encryptedInfo; QElapsedTimer _stopwatch; }; diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 819d360df..3356fc4a8 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -41,26 +41,21 @@ void PropagateUploadEncrypted::start() auto getEncryptedStatus = new GetFolderEncryptStatusJob(_propagator->account(), info.path()); - connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusReceived, + connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusFolderReceived, this, &PropagateUploadEncrypted::slotFolderEncryptedStatusFetched); connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusError, this, &PropagateUploadEncrypted::slotFolderEncryptedStatusError); getEncryptedStatus->start(); } -void PropagateUploadEncrypted::slotFolderEncryptedStatusFetched(const QMap& result) +void PropagateUploadEncrypted::slotFolderEncryptedStatusFetched(const QString &folder, bool isEncrypted) { - qCDebug(lcPropagateUpload) << "Encrypted Status Fetched"; - QFileInfo fileInfo(_item->_file); - QString currFilePath = fileInfo.path(); - if (!currFilePath.endsWith(QDir::separator())) - currFilePath += QDir::separator(); + qCDebug(lcPropagateUpload) << "Encrypted Status Fetched" << folder << isEncrypted; /* We are inside an encrypted folder, we need to find it's Id. */ - if (result[currFilePath] == true) { + if (isEncrypted) { qCDebug(lcPropagateUpload) << "Folder is encrypted, let's get the Id from it."; - QFileInfo info(_item->_file); - LsColJob *job = new LsColJob(_propagator->account(), info.path(), this); + auto job = new LsColJob(_propagator->account(), folder, this); job->setProperties({"resourcetype", "http://owncloud.org/ns:fileid"}); connect(job, &LsColJob::directoryListingSubfolders, this, &PropagateUploadEncrypted::slotFolderEncryptedIdReceived); connect(job, &LsColJob::finishedWithError, this, &PropagateUploadEncrypted::slotFolderEncryptedIdError); diff --git a/src/libsync/propagateuploadencrypted.h b/src/libsync/propagateuploadencrypted.h index 4b955e702..397c99aa3 100644 --- a/src/libsync/propagateuploadencrypted.h +++ b/src/libsync/propagateuploadencrypted.h @@ -40,7 +40,7 @@ public: QByteArray _folderId; private slots: - void slotFolderEncryptedStatusFetched(const QMap& result); + void slotFolderEncryptedStatusFetched(const QString &folder, bool isEncrypted); void slotFolderEncryptedStatusError(int error); void slotFolderEncryptedIdReceived(const QStringList &list); void slotFolderEncryptedIdError(QNetworkReply *r); From d1c84fd8593d0746bafcc45e0209ca14ba69a8fe Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Fri, 12 Jan 2018 15:21:30 +0100 Subject: [PATCH 200/262] Bump drone docer used for appimage Signed-off-by: Roeland Jago Douma --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 1da24e760..635f36b09 100644 --- a/.drone.yml +++ b/.drone.yml @@ -81,7 +81,7 @@ pipeline: TESTS: qt-5.9 AppImage-5.9: - image: nextcloudci/client-appimage-ci:client-appimage-ci-6 + image: nextcloudci/client-appimage-ci:client-appimage-ci-7 commands: - /bin/bash -c "./admin/linux/build-appimage.sh" when: From 99aebf292c47c2806a9dd0dcafc87468195cc629 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Fri, 12 Jan 2018 19:36:09 +0100 Subject: [PATCH 201/262] Yet another drone image Signed-off-by: Roeland Jago Douma --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 635f36b09..6e99b36d5 100644 --- a/.drone.yml +++ b/.drone.yml @@ -81,7 +81,7 @@ pipeline: TESTS: qt-5.9 AppImage-5.9: - image: nextcloudci/client-appimage-ci:client-appimage-ci-7 + image: nextcloudci/client-appimage-ci:client-appimage-ci-8 commands: - /bin/bash -c "./admin/linux/build-appimage.sh" when: From 5722d29e42a9e8ce5efe12d24275c880b196a5a1 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Mon, 15 Jan 2018 11:26:06 +0100 Subject: [PATCH 202/262] Double encode the metadatakeys This is required by a misunderstanding of the RFC. You need to resetup your test user. Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 30 +++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 0c07ec8b1..b81d51ef2 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1124,20 +1124,28 @@ void FolderMetadata::setupExistingMetadata(const QByteArray& metadata) // perhaps it's better to store a map instead of a vector, perhaps this just doesn't matter. for(auto it = metadataKeys.constBegin(), end = metadataKeys.constEnd(); it != end; it++) { QByteArray currB64Pass = it.value().toString().toLocal8Bit(); - QByteArray decryptedKey = decryptMetadataKey(currB64Pass); + /* + * We have to base64 decode the metadatakey here. This was a misunderstanding in the RFC + * Now we should be compatible with Android and IOS. Maybe we can fix it later. + */ + QByteArray decryptedKey = QByteArray::fromBase64(decryptMetadataKey(currB64Pass)); _metadataKeys.insert(it.key().toInt(), decryptedKey); } // Cool, We actually have the key, we can decrypt the rest of the metadata. qCDebug(lcCse) << "Sharing: " << sharing; - auto sharingDecrypted = QByteArray::fromBase64(decryptJsonObject(sharing, _metadataKeys.last())); - qCDebug(lcCse) << "Sharing Decrypted" << sharingDecrypted; + if (sharing.size()) { + auto sharingDecrypted = QByteArray::fromBase64(decryptJsonObject(sharing, _metadataKeys.last())); + qCDebug(lcCse) << "Sharing Decrypted" << sharingDecrypted; - //Sharing is also a JSON object, so extract it and populate. - auto sharingDoc = QJsonDocument::fromJson(sharingDecrypted); - auto sharingObj = sharingDoc.object(); - for (auto it = sharingObj.constBegin(), end = sharingObj.constEnd(); it != end; it++) { - _sharing.push_back({it.key(), it.value().toString()}); + //Sharing is also a JSON object, so extract it and populate. + auto sharingDoc = QJsonDocument::fromJson(sharingDecrypted); + auto sharingObj = sharingDoc.object(); + for (auto it = sharingObj.constBegin(), end = sharingObj.constEnd(); it != end; it++) { + _sharing.push_back({it.key(), it.value().toString()}); + } + } else { + qCDebug(lcCse) << "Skipping sharing section since it is empty"; } for (auto it = files.constBegin(), end = files.constEnd(); it != end; it++) { @@ -1222,7 +1230,11 @@ QByteArray FolderMetadata::encryptedMetadata() { QJsonObject metadataKeys; for (auto it = _metadataKeys.constBegin(), end = _metadataKeys.constEnd(); it != end; it++) { - const QByteArray encryptedKey = encryptMetadataKey(it.value()); + /* + * We have to already base64 encode the metadatakey here. This was a misunderstanding in the RFC + * Now we should be compatible with Android and IOS. Maybe we can fix it later. + */ + const QByteArray encryptedKey = encryptMetadataKey(it.value().toBase64()); metadataKeys.insert(QString::number(it.key()), QString(encryptedKey)); } From 6d613fb4d5300b637eb5a3332f393f9dded69c44 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Thu, 18 Jan 2018 22:10:55 +0100 Subject: [PATCH 203/262] Quick and dirty way of showing the mnemonic for now Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index b81d51ef2..7e5d42858 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -930,6 +931,14 @@ void ClientSideEncryption::encryptPrivateKey() _mnemonic = list.join(' '); qCInfo(lcCse()) << "mnemonic Generated:" << _mnemonic; + QMessageBox msgBox; + msgBox.setText(tr("Note your encryption passphrase")); + msgBox.setDetailedText(_mnemonic); + msgBox.setIcon(QMessageBox::Information); + msgBox.setStandardButtons(QMessageBox::Ok); + + msgBox.exec(); + QString passPhrase = list.join(QString()).toLower(); qCInfo(lcCse()) << "Passphrase Generated:" << passPhrase; From 4a2d0ab9e9657ed1873e8376c3990a10cee86943 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 21 Jan 2018 19:50:40 +0100 Subject: [PATCH 204/262] [CSE] Move code to display minemonic out of the libsync The libsync should not contain Qt Widget related code. --- src/gui/accountmanager.cpp | 15 +++++++++++++++ src/gui/accountmanager.h | 3 +++ src/libsync/clientsideencryption.cpp | 9 +-------- src/libsync/clientsideencryption.h | 1 + 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/gui/accountmanager.cpp b/src/gui/accountmanager.cpp index 9330987dc..e7f6517d9 100644 --- a/src/gui/accountmanager.cpp +++ b/src/gui/accountmanager.cpp @@ -23,6 +23,8 @@ #include #include #include +#include +#include "clientsideencryption.h" namespace { static const char urlC[] = "url"; @@ -324,10 +326,23 @@ AccountPtr AccountManager::createAccount() acc->setSslErrorHandler(new SslDialogErrorHandler); connect(acc.data(), &Account::proxyAuthenticationRequired, ProxyAuthHandler::instance(), &ProxyAuthHandler::handleProxyAuthenticationRequired); + + connect(acc.data()->e2e(), &ClientSideEncryption::mnemonicGenerated, + &AccountManager::displayMnemonic); + return acc; } +void AccountManager::displayMnemonic(const QString& mnemonic) +{ + QMessageBox msgBox; + msgBox.setText(tr("Note your encryption passphrase")); + msgBox.setDetailedText(mnemonic); + msgBox.setIcon(QMessageBox::Information); + msgBox.setStandardButtons(QMessageBox::Ok); + msgBox.exec(); +} void AccountManager::shutdown() { auto accountsCopy = _accounts; diff --git a/src/gui/accountmanager.h b/src/gui/accountmanager.h index a2c13bec2..35df3584a 100644 --- a/src/gui/accountmanager.h +++ b/src/gui/accountmanager.h @@ -97,6 +97,9 @@ public slots: /// Saves account state data, not including the account void saveAccountState(AccountState *a); + /// Display a Box with the mnemonic so the user can copy it to a safe place. + static void displayMnemonic(const QString& mnemonic); + Q_SIGNALS: void accountAdded(AccountState *account); diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 7e5d42858..4a31c403f 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -27,7 +27,6 @@ #include #include #include -#include #include @@ -931,13 +930,7 @@ void ClientSideEncryption::encryptPrivateKey() _mnemonic = list.join(' '); qCInfo(lcCse()) << "mnemonic Generated:" << _mnemonic; - QMessageBox msgBox; - msgBox.setText(tr("Note your encryption passphrase")); - msgBox.setDetailedText(_mnemonic); - msgBox.setIcon(QMessageBox::Information); - msgBox.setStandardButtons(QMessageBox::Ok); - - msgBox.exec(); + emit mnemonicGenerated(_mnemonic); QString passPhrase = list.join(QString()).toLower(); qCInfo(lcCse()) << "Passphrase Generated:" << passPhrase; diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index b5ffab5dc..4497dbcff 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -96,6 +96,7 @@ private slots: signals: void initializationFinished(); + void mnemonicGenerated(const QString& mnemonic); private: void getPrivateKeyFromServer(); From d5a76ea70dc8a9bf1bcaedf324e86bda30bc3771 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 21 Jan 2018 21:24:02 +0100 Subject: [PATCH 205/262] [CSE] Remove the DecryptionJob Transform it into an Static function - it was blocking anyway and this way it's easier to transform it into a thread in the future. --- src/libsync/clientsideencryption.cpp | 40 +++++++++++----------------- src/libsync/clientsideencryption.h | 23 +++------------- 2 files changed, 18 insertions(+), 45 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 4a31c403f..85478dad8 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1386,19 +1386,11 @@ void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i qCDebug(lcCse) << "File Encrypted Successfully"; } -FileDecryptionJob::FileDecryptionJob(QByteArray &key, QByteArray &iv, QFile *input, QFile *output, QObject *parent) - : QObject(parent), - _key(key), - _iv(iv), - _input(input), - _output(output) +void EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& iv, + QFile *input, QFile *output) { -} - -void FileDecryptionJob::start() -{ - _input->open(QIODevice::ReadOnly); - _output->open(QIODevice::WriteOnly); + input->open(QIODevice::ReadOnly); + output->open(QIODevice::WriteOnly); // Init EVP_CIPHER_CTX *ctx; @@ -1418,30 +1410,30 @@ void FileDecryptionJob::start() EVP_CIPHER_CTX_set_padding(ctx, 0); /* Set IV length. */ - if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, _iv.size(), NULL)) { + if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, iv.size(), NULL)) { qCInfo(lcCse()) << "Could not set iv length"; exit(-1); } /* Initialise key and IV */ - if(!EVP_DecryptInit_ex(ctx, NULL, NULL, (const unsigned char *)_key.constData(), (const unsigned char *)_iv.constData())) { + if(!EVP_DecryptInit_ex(ctx, NULL, NULL, (const unsigned char *) key.constData(), (const unsigned char *) iv.constData())) { qCInfo(lcCse()) << "Could not set key and iv"; exit(-1); } - qint64 size = _input->size() - 16; + qint64 size = input->size() - 16; unsigned char *out = (unsigned char *)malloc(sizeof(unsigned char) * (1024 + 16 -1)); int len = 0; - while(_input->pos() < size) { + while(input->pos() < size) { - int toRead = size - _input->pos(); + int toRead = size - input->pos(); if (toRead > 1024) { toRead = 1024; } - QByteArray data = _input->read(toRead); + QByteArray data = input->read(toRead); if (data.size() == 0) { qCInfo(lcCse()) << "Could not read data from file"; @@ -1453,10 +1445,10 @@ void FileDecryptionJob::start() exit(-1); } - _output->write((char *)out, len); + output->write((char *)out, len); } - QByteArray tag = _input->read(16); + QByteArray tag = input->read(16); /* Set expected tag value. Works in OpenSSL 1.0.1d and later */ if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, tag.size(), (unsigned char *)tag.constData())) { @@ -1468,15 +1460,13 @@ void FileDecryptionJob::start() qCInfo(lcCse()) << "Could finalize decryption"; exit(-1); } - _output->write((char *)out, len); + output->write((char *)out, len); free(out); EVP_CIPHER_CTX_free(ctx); - _input->close(); - _output->close(); - - emit finished(_output); + input->close(); + output->close(); } } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 4497dbcff..aa175e221 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -63,6 +63,9 @@ public: static void fileEncryption(const QByteArray &key, const QByteArray &iv, QFile *input, QFile *output); + + static void fileDecryption(const QByteArray &key, const QByteArray& iv, + QFile *input, QFile *output); }; class ClientSideEncryption : public QObject { @@ -162,25 +165,5 @@ private: QVector> _sharing; }; -class FileDecryptionJob : public QObject -{ - Q_OBJECT -public: - FileDecryptionJob(QByteArray &key, QByteArray &iv, QFile *input, QFile *output, QObject *parent = 0); - -public slots: - void start(); - -signals: - void finished(QFile *output); - -private: - QByteArray _key; - QByteArray _iv; - QPointer _input; - QPointer _output; -}; - - } // namespace OCC #endif From 009562da1e2ef491e56ffe470d3b05572a2c4d6f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 21 Jan 2018 21:25:06 +0100 Subject: [PATCH 206/262] [CSE] Remove lambas with lambdas It's hard to reason with lambdas with lambdas, so transform them into actuall signal / slot calls. --- src/libsync/CMakeLists.txt | 1 + src/libsync/propagatedownload.cpp | 92 +++++-------------------------- src/libsync/propagatedownload.h | 4 ++ 3 files changed, 19 insertions(+), 78 deletions(-) diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index ede6e7a6a..1243a7f34 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -54,6 +54,7 @@ set(libsync_SRCS propagateremotemove.cpp propagateremotemkdir.cpp propagateuploadencrypted.cpp + propagatedownloadencrypted.cpp syncengine.cpp syncfileitem.cpp syncfilestatus.cpp diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index c3022bed6..fe1b00cfc 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -25,6 +25,7 @@ #include "common/checksums.h" #include "common/asserts.h" #include "clientsideencryptionjobs.h" +#include "propagatedownloadencrypted.h" #include #include @@ -345,61 +346,19 @@ void PropagateDownloadFile::start() { if (propagator()->_abortRequested.fetchAndAddRelaxed(0)) return; + _isEncrypted = false; qCDebug(lcPropagateDownload) << _item->_file << propagator()->_activeJobList.count(); + if (propagator()->account()->capabilities().clientSideEncryptionAvaliable()) { - QFileInfo info(_item->_file); - auto getEncryptedStatus = new GetFolderEncryptStatusJob(propagator()->account(), info.path()); - connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusFolderReceived, - this, [this, info] (const QString &folder, bool isEncrypted) { - qCDebug(lcPropagateDownload) << "Get Folder is Encrypted Received" << folder << isEncrypted; - if (!isEncrypted) { - startAfterIsEncryptedIsChecked(); - return; - } - _isEncrypted = true; - - // Is encrypted Now we need the folder-id - auto job = new LsColJob(propagator()->account(), folder, this); - job->setProperties({"resourcetype", "http://owncloud.org/ns:fileid"}); - connect(job, &LsColJob::directoryListingSubfolders, this, [job, this, info] (const QStringList &list) { - const QString folderId = list.first(); - qCDebug(lcPropagateDownload) << "Received id of folder" << folderId; - const ExtraFolderInfo &folderInfo = job->_folderInfos.value(folderId); - - // Now that we have the folder-id we need it's JSON metadata - auto metadataJob = new GetMetadataApiJob(propagator()->account(), folderInfo.fileId); - connect(metadataJob, &GetMetadataApiJob::jsonReceived, - this, [this, info] (const QJsonDocument &json) { - qCDebug(lcPropagateDownload) << "Metadata Received reading" << json.toJson() << _item->_file; - const QString filename = info.fileName(); - auto meta = new FolderMetadata(propagator()->account(), json.toJson(QJsonDocument::Compact)); - const QVector files = meta->files(); - for (const EncryptedFile &file : files) { - qCDebug(lcPropagateDownload) << "file" << filename << file.encryptedFilename << file.originalFilename << file.encryptionKey; - if (filename == file.encryptedFilename) { - _encryptedInfo = file; - qCDebug(lcPropagateDownload) << "Found matching encrypted metadata for file, starting download"; - startAfterIsEncryptedIsChecked(); - return; - } - } - - qCDebug(lcPropagateDownload) << "Failed to find encrypted metadata information of remote file" << filename; - }); - metadataJob->start(); - }); - connect(job, &LsColJob::finishedWithError, this, []() { - qCDebug(lcPropagateDownload) << "Failed to get encrypted metadata of folder"; - }); - job->start(); + _downloadEncryptedHelper = new PropagateDownloadEncrypted(propagator(), _item); + connect(_downloadEncryptedHelper, &PropagateDownloadEncrypted::folderStatusNotEncrypted, [this] { + startAfterIsEncryptedIsChecked(); }); - connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusError, - this, [](int statusCode) { - qCDebug(lcPropagateDownload) << "Failed to get encrypted status of folder" << statusCode; + connect(_downloadEncryptedHelper, &PropagateDownloadEncrypted::folderStatusEncrypted, [this] { + _isEncrypted = true; + startAfterIsEncryptedIsChecked(); }); - - getEncryptedStatus->start(); } else { startAfterIsEncryptedIsChecked(); } @@ -847,35 +806,12 @@ void PropagateDownloadFile::contentChecksumComputed(const QByteArray &checksumTy _item->_checksumHeader = makeChecksumHeader(checksumType, checksum); if (_isEncrypted) { - const QString tmpFileName = createDownloadTmpFileName(_item->_file + QLatin1String("_dec")); - qCDebug(lcPropagateDownload) << "Content Checksum Computed starting decryption" << tmpFileName; + if (_downloadEncryptedHelper->decryptFile(_tmpFile)) { + downloadFinished(); + } else { + done(SyncFileItem::NormalError, _downloadEncryptedHelper->errorString()); + } - _tmpFile.close(); - auto _tmpOutput = new QFile(propagator()->getFilePath(tmpFileName), this); - auto job = new FileDecryptionJob(_encryptedInfo.encryptionKey, - _encryptedInfo.initializationVector, - &_tmpFile, - _tmpOutput); - connect(job, &FileDecryptionJob::finished, this, [this] (QFile *output) { - qCDebug(lcPropagateDownload) << "Decryption finished" << _tmpFile.fileName() << output->fileName(); - - _tmpFile.close(); - output->close(); - // we decripted the temporary into another temporary, so good bye old one - if (!_tmpFile.remove()) { - qCDebug(lcPropagateDownload) << "Failed to remove temporary file" << _tmpFile.errorString(); - done(SyncFileItem::NormalError, _tmpFile.errorString()); - return; - } - // Let's fool the rest of the logic into thinking this was the actual download - _tmpFile.setFileName(output->fileName()); - - // Let's fool the rest of the logic into thinking this is the right name of the DAV file - _item->_file = _item->_file.section(QLatin1Char('/'), 0, -2) + QLatin1Char('/') + _encryptedInfo.originalFilename; - - downloadFinished(); - }); - job->start(); } else { downloadFinished(); } diff --git a/src/libsync/propagatedownload.h b/src/libsync/propagatedownload.h index 0265d4506..2600ee560 100644 --- a/src/libsync/propagatedownload.h +++ b/src/libsync/propagatedownload.h @@ -21,6 +21,7 @@ #include namespace OCC { +class PropagateDownloadEncrypted; /** * @brief The GETFileJob class @@ -205,8 +206,11 @@ private: QFile _tmpFile; bool _deleteExisting; bool _isEncrypted = false; + EncryptedFile _encryptedInfo; QElapsedTimer _stopwatch; + + PropagateDownloadEncrypted *_downloadEncryptedHelper; }; } From 2d872f2a8dd4189ac164c8f0500423d471e5940d Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 21 Jan 2018 21:26:05 +0100 Subject: [PATCH 207/262] [CSE] Add missing files --- src/libsync/propagatedownloadencrypted.cpp | 131 +++++++++++++++++++++ src/libsync/propagatedownloadencrypted.h | 45 +++++++ 2 files changed, 176 insertions(+) create mode 100644 src/libsync/propagatedownloadencrypted.cpp create mode 100644 src/libsync/propagatedownloadencrypted.h diff --git a/src/libsync/propagatedownloadencrypted.cpp b/src/libsync/propagatedownloadencrypted.cpp new file mode 100644 index 000000000..378e434a3 --- /dev/null +++ b/src/libsync/propagatedownloadencrypted.cpp @@ -0,0 +1,131 @@ +#include "propagatedownloadencrypted.h" +#include "clientsideencryptionjobs.h" + +Q_LOGGING_CATEGORY(lcPropagateDownloadEncrypted, "nextcloud.sync.propagator.download.encrypted", QtInfoMsg) + + +namespace OCC { + +PropagateDownloadEncrypted::PropagateDownloadEncrypted(OwncloudPropagator *propagator, SyncFileItemPtr item) : + _propagator(propagator), _item(item), _info(_item->_file) + +{ +} + +void PropagateDownloadEncrypted::checkFolderEncryptedStatus() +{ + auto getEncryptedStatus = new GetFolderEncryptStatusJob(_propagator->account(), _info.path()); + connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusFolderReceived, + this, &PropagateDownloadEncrypted::folderStatusReceived); + + connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusError, + this, &PropagateDownloadEncrypted::folderStatusError); + + getEncryptedStatus->start(); +} + +void PropagateDownloadEncrypted::folderStatusError(int statusCode) +{ + qCDebug(lcPropagateDownloadEncrypted) << "Failed to get encrypted status of folder" << statusCode; +} + +void PropagateDownloadEncrypted::folderStatusReceived(const QString &folder, bool isEncrypted) +{ + qCDebug(lcPropagateDownloadEncrypted) << "Get Folder is Encrypted Received" << folder << isEncrypted; + if (!isEncrypted) { + emit folderStatusNotEncrypted(); + return; + } + + // Is encrypted Now we need the folder-id + auto job = new LsColJob(_propagator->account(), folder, this); + job->setProperties({"resourcetype", "http://owncloud.org/ns:fileid"}); + connect(job, &LsColJob::directoryListingSubfolders, + this, &PropagateDownloadEncrypted::checkFolderId); + connect(job, &LsColJob::finishedWithError, + this, &PropagateDownloadEncrypted::folderIdError); + job->start(); +} + +void PropagateDownloadEncrypted::folderIdError() +{ + qCDebug(lcPropagateDownloadEncrypted) << "Failed to get encrypted metadata of folder"; +} + +void PropagateDownloadEncrypted::checkFolderId(const QStringList &list) +{ + auto job = qobject_cast(sender()); + const QString folderId = list.first(); + qCDebug(lcPropagateDownloadEncrypted) << "Received id of folder" << folderId; + + const ExtraFolderInfo &folderInfo = job->_folderInfos.value(folderId); + + // Now that we have the folder-id we need it's JSON metadata + auto metadataJob = new GetMetadataApiJob(_propagator->account(), folderInfo.fileId); + connect(metadataJob, &GetMetadataApiJob::jsonReceived, + this, &PropagateDownloadEncrypted::checkFolderEncryptedMetadata); + + metadataJob->start(); +} + +void PropagateDownloadEncrypted::checkFolderEncryptedMetadata(const QJsonDocument &json) +{ + qCDebug(lcPropagateDownloadEncrypted) << "Metadata Received reading" << json.toJson() << _item->_file; + const QString filename = _info.fileName(); + auto meta = new FolderMetadata(_propagator->account(), json.toJson(QJsonDocument::Compact)); + const QVector files = meta->files(); + for (const EncryptedFile &file : files) { + qCDebug(lcPropagateDownloadEncrypted) << "file" << filename << file.encryptedFilename << file.originalFilename << file.encryptionKey; + if (filename == file.encryptedFilename) { + _encryptedInfo = file; + qCDebug(lcPropagateDownloadEncrypted) << "Found matching encrypted metadata for file, starting download"; + emit folderStatusEncrypted(); + return; + } + } + + qCDebug(lcPropagateDownloadEncrypted) << "Failed to find encrypted metadata information of remote file" << filename; +} + +// TODO: Fix this. Exported in the wrong place. +QString createDownloadTmpFileName(const QString &previous); + +bool PropagateDownloadEncrypted::decryptFile(QFile& tmpFile) +{ + const QString tmpFileName = createDownloadTmpFileName(_item->_file + QLatin1String("_dec")); + qCDebug(lcPropagateDownloadEncrypted) << "Content Checksum Computed starting decryption" << tmpFileName; + + tmpFile.close(); + auto _tmpOutput = new QFile(_propagator->getFilePath(tmpFileName), this); + EncryptionHelper::fileDecryption(_encryptedInfo.encryptionKey, + _encryptedInfo.initializationVector, + &tmpFile, + _tmpOutput); + + qCDebug(lcPropagateDownloadEncrypted) << "Decryption finished" << tmpFile.fileName() << _tmpOutput->fileName(); + + tmpFile.close(); + _tmpOutput->close(); + + // we decripted the temporary into another temporary, so good bye old one + if (!tmpFile.remove()) { + qCDebug(lcPropagateDownloadEncrypted) << "Failed to remove temporary file" << tmpFile.errorString(); + _errorString = tmpFile.errorString(); + return false; + } + + // Let's fool the rest of the logic into thinking this was the actual download + tmpFile.setFileName(_tmpOutput->fileName()); + + //TODO: This seems what's breaking the logic. + // Let's fool the rest of the logic into thinking this is the right name of the DAV file + _item->_file = _item->_file.section(QLatin1Char('/'), 0, -2) + QLatin1Char('/') + _encryptedInfo.originalFilename; + return true; +} + +QString PropagateDownloadEncrypted::errorString() const +{ + return _errorString; +} + +} diff --git a/src/libsync/propagatedownloadencrypted.h b/src/libsync/propagatedownloadencrypted.h new file mode 100644 index 000000000..379a9efc7 --- /dev/null +++ b/src/libsync/propagatedownloadencrypted.h @@ -0,0 +1,45 @@ +#ifndef PROPAGATEDOWNLOADENCRYPTED_H +#define PROPAGATEDOWNLOADENCRYPTED_H + +#include +#include + +#include "syncfileitem.h" +#include "owncloudpropagator.h" +#include "clientsideencryption.h" + +class QJsonDocument; + +namespace OCC { + +class PropagateDownloadEncrypted : public QObject { + Q_OBJECT +public: + PropagateDownloadEncrypted(OwncloudPropagator *propagator, SyncFileItemPtr item); + void checkFolderId(const QStringList &list); + bool decryptFile(QFile& tmpFile); + QString errorString() const; + +public slots: + void checkFolderEncryptedStatus(); + + void checkFolderEncryptedMetadata(const QJsonDocument &json); + void folderStatusReceived(const QString &folder, bool isEncrypted); + void folderStatusError(int httpErrorCode); + void folderIdError(); +signals: + void folderStatusEncrypted(); + void folderStatusNotEncrypted(); + + void decryptionFinished(); + +private: + OwncloudPropagator *_propagator; + SyncFileItemPtr _item; + QFileInfo _info; + EncryptedFile _encryptedInfo; + QString _errorString; +}; + +} +#endif From 261cedce3fac65b42af39f8d46a7aaeb56531186 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 21 Jan 2018 21:40:53 +0100 Subject: [PATCH 208/262] [CSE] Do not quit if encryption fails It's much more reasonable to return failure and handle it. --- src/libsync/clientsideencryption.cpp | 18 +++++++++--------- src/libsync/clientsideencryption.h | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 85478dad8..3923e457d 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1300,7 +1300,7 @@ bool ClientSideEncryption::isFolderEncrypted(const QString& path) const { return (*it); } -void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &iv, QFile *input, QFile *output) +bool EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &iv, QFile *input, QFile *output) { if (!input->open(QIODevice::ReadOnly)) { qCDebug(lcCse) << "Could not open input file for reading" << input->errorString(); @@ -1315,13 +1315,13 @@ void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i /* Create and initialise the context */ if(!(ctx = EVP_CIPHER_CTX_new())) { qCInfo(lcCse()) << "Could not create context"; - exit(-1); + return false; } /* Initialise the decryption operation. */ if(!EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { qCInfo(lcCse()) << "Could not init cipher"; - exit(-1); + return false; } EVP_CIPHER_CTX_set_padding(ctx, 0); @@ -1329,13 +1329,13 @@ void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i /* Set IV length. */ if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, iv.size(), NULL)) { qCInfo(lcCse()) << "Could not set iv length"; - exit(-1); + return false; } /* Initialise key and IV */ if(!EVP_EncryptInit_ex(ctx, NULL, NULL, (const unsigned char *)key.constData(), (const unsigned char *)iv.constData())) { qCInfo(lcCse()) << "Could not set key and iv"; - exit(-1); + return false; } unsigned char *out = (unsigned char *)malloc(sizeof(unsigned char) * (1024 + 16 -1)); @@ -1348,13 +1348,13 @@ void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i if (data.size() == 0) { qCInfo(lcCse()) << "Could not read data from file"; - exit(-1); + return false; } qCDebug(lcCse) << "Encrypting " << data; if(!EVP_EncryptUpdate(ctx, out, &len, (unsigned char *)data.constData(), data.size())) { qCInfo(lcCse()) << "Could not encrypt"; - exit(-1); + return false; } output->write((char *)out, len); @@ -1363,7 +1363,7 @@ void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i if(1 != EVP_EncryptFinal_ex(ctx, out, &len)) { qCInfo(lcCse()) << "Could finalize encryption"; - exit(-1); + return false; } output->write((char *)out, len); total_len += len; @@ -1372,7 +1372,7 @@ void EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i unsigned char *tag = (unsigned char *)malloc(sizeof(unsigned char) * 16); if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag)) { qCInfo(lcCse()) << "Could not get tag"; - exit(-1); + return false; } output->write((char *)tag, 16); diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index aa175e221..4b74aebcc 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -61,7 +61,7 @@ public: static QByteArray BIO2ByteArray(BIO *b); - static void fileEncryption(const QByteArray &key, const QByteArray &iv, + static bool fileEncryption(const QByteArray &key, const QByteArray &iv, QFile *input, QFile *output); static void fileDecryption(const QByteArray &key, const QByteArray& iv, From 5d6817e165fa57d310ca858ea3a86b8afee89274 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 21 Jan 2018 21:49:24 +0100 Subject: [PATCH 209/262] [CSE] Save the tag while encrypting. --- src/libsync/clientsideencryption.cpp | 3 ++- src/libsync/clientsideencryption.h | 2 +- src/libsync/propagateuploadencrypted.cpp | 5 ++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 3923e457d..6b2d8d2bb 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1300,7 +1300,7 @@ bool ClientSideEncryption::isFolderEncrypted(const QString& path) const { return (*it); } -bool EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &iv, QFile *input, QFile *output) +bool EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &iv, QFile *input, QFile *output, QByteArray& returnTag) { if (!input->open(QIODevice::ReadOnly)) { qCDebug(lcCse) << "Could not open input file for reading" << input->errorString(); @@ -1375,6 +1375,7 @@ bool EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i return false; } + returnTag = QByteArray((const char*) tag, 16); output->write((char *)tag, 16); free(out); diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 4b74aebcc..ebe87c540 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -62,7 +62,7 @@ public: static QByteArray BIO2ByteArray(BIO *b); static bool fileEncryption(const QByteArray &key, const QByteArray &iv, - QFile *input, QFile *output); + QFile *input, QFile *output, QByteArray& returnTag); static void fileDecryption(const QByteArray &key, const QByteArray& iv, QFile *input, QFile *output); diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 3356fc4a8..55a728454 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -146,9 +146,12 @@ void PropagateUploadEncrypted::slotUpdateMetadataSuccess(const QByteArray& fileI QFileInfo info(_propagator->_localDir + QDir::separator() + _item->_file); auto *input = new QFile(info.absoluteFilePath()); auto *output = new QFile(QDir::tempPath() + QDir::separator() + _encryptedFile.encryptedFilename); + + // TODO: Invert the operations. first enrypt, then generate the metadata. + QByteArray tag; EncryptionHelper::fileEncryption(_encryptedFile.encryptionKey, _encryptedFile.initializationVector, - input, output); + input, output, tag); // File is Encrypted, Upload it. QFileInfo outputInfo(output->fileName()); From cf2fa2ea355fe4a8e7d843d1c06991ab9194e79d Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 21 Jan 2018 21:59:19 +0100 Subject: [PATCH 210/262] [CSE] Get the tag of the encrypted file to feed the metadata Also, change the logging cattegory --- src/libsync/propagateuploadencrypted.cpp | 78 ++++++++++++------------ src/libsync/propagateuploadencrypted.h | 1 + 2 files changed, 41 insertions(+), 38 deletions(-) diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 55a728454..05ddbc0ce 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -13,7 +13,7 @@ namespace OCC { -Q_DECLARE_LOGGING_CATEGORY(lcPropagateUpload) +Q_LOGGING_CATEGORY(lcPropagateUploadEncrypted, "nextcloud.sync.propagator.upload.encrypted", QtInfoMsg) PropagateUploadEncrypted::PropagateUploadEncrypted(OwncloudPropagator *propagator, SyncFileItemPtr item) : _propagator(propagator), @@ -36,7 +36,7 @@ void PropagateUploadEncrypted::start() * * If the folder is unencrypted we just follow the old way. */ - qCDebug(lcPropagateUpload) << "Starting to send an encrypted file!"; + qCDebug(lcPropagateUploadEncrypted) << "Starting to send an encrypted file!"; QFileInfo info(_item->_file); auto getEncryptedStatus = new GetFolderEncryptStatusJob(_propagator->account(), info.path()); @@ -50,18 +50,18 @@ void PropagateUploadEncrypted::start() void PropagateUploadEncrypted::slotFolderEncryptedStatusFetched(const QString &folder, bool isEncrypted) { - qCDebug(lcPropagateUpload) << "Encrypted Status Fetched" << folder << isEncrypted; + qCDebug(lcPropagateUploadEncrypted) << "Encrypted Status Fetched" << folder << isEncrypted; /* We are inside an encrypted folder, we need to find it's Id. */ if (isEncrypted) { - qCDebug(lcPropagateUpload) << "Folder is encrypted, let's get the Id from it."; + qCDebug(lcPropagateUploadEncrypted) << "Folder is encrypted, let's get the Id from it."; auto job = new LsColJob(_propagator->account(), folder, this); job->setProperties({"resourcetype", "http://owncloud.org/ns:fileid"}); connect(job, &LsColJob::directoryListingSubfolders, this, &PropagateUploadEncrypted::slotFolderEncryptedIdReceived); connect(job, &LsColJob::finishedWithError, this, &PropagateUploadEncrypted::slotFolderEncryptedIdError); job->start(); } else { - qCDebug(lcPropagateUpload) << "Folder is not encrypted, getting back to default."; + qCDebug(lcPropagateUploadEncrypted) << "Folder is not encrypted, getting back to default."; emit folerNotEncrypted(); } } @@ -79,7 +79,7 @@ void PropagateUploadEncrypted::slotFolderEncryptedStatusFetched(const QString &f void PropagateUploadEncrypted::slotFolderEncryptedIdReceived(const QStringList &list) { - qCDebug(lcPropagateUpload) << "Received id of folder, trying to lock it so we can prepare the metadata"; + qCDebug(lcPropagateUploadEncrypted) << "Received id of folder, trying to lock it so we can prepare the metadata"; auto job = qobject_cast(sender()); const auto& folderInfo = job->_folderInfos.value(list.first()); _folderLockFirstTry.start(); @@ -96,7 +96,7 @@ void PropagateUploadEncrypted::slotTryLock(const QByteArray& fileId) void PropagateUploadEncrypted::slotFolderLockedSuccessfully(const QByteArray& fileId, const QByteArray& token) { - qCDebug(lcPropagateUpload) << "Folder" << fileId << "Locked Successfully for Upload, Fetching Metadata"; + qCDebug(lcPropagateUploadEncrypted) << "Folder" << fileId << "Locked Successfully for Upload, Fetching Metadata"; // Should I use a mutex here? _currentLockingInProgress = true; _folderToken = token; @@ -110,16 +110,31 @@ void PropagateUploadEncrypted::slotFolderLockedSuccessfully(const QByteArray& fi void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode) { - qCDebug(lcPropagateUpload) << "Metadata Received, Preparing it for the new file." << json.toVariant(); + qCDebug(lcPropagateUploadEncrypted) << "Metadata Received, Preparing it for the new file." << json.toVariant(); // Encrypt File! _metadata = new FolderMetadata(_propagator->account(), json.toJson(QJsonDocument::Compact)); QFileInfo info(_propagator->_localDir + QDir::separator() + _item->_file); - qCDebug(lcPropagateUpload) << "Creating the encrypted file metadata helper."; + qCDebug(lcPropagateUploadEncrypted) << "Creating the encrypted file."; + auto *input = new QFile(info.absoluteFilePath()); + auto *output = new QFile(QDir::tempPath() + QDir::separator() + _encryptedFile.encryptedFilename); + + // TODO: Invert the operations. first enrypt, then generate the metadata. + QByteArray tag; + EncryptionHelper::fileEncryption(_encryptedFile.encryptionKey, + _encryptedFile.initializationVector, + input, output, tag); + + input->deleteLater(); + output->deleteLater(); + + _completeFileName = output->fileName(); + + qCDebug(lcPropagateUploadEncrypted) << "Creating the metadata for the encrypted file."; EncryptedFile encryptedFile; - encryptedFile.authenticationTag = "NOISE"; // TODO: Remove the noise. + encryptedFile.authenticationTag = tag.toBase64(); // TODO: Check against Android to see if the metadata is correct.. encryptedFile.encryptedFilename = EncryptionHelper::generateRandomString(20); encryptedFile.encryptionKey = EncryptionHelper::generateRandom(16); encryptedFile.fileVersion = 1; @@ -129,7 +144,7 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo _metadata->addEncryptedFile(encryptedFile); _encryptedFile = encryptedFile; - qCDebug(lcPropagateUpload) << "Metadata created, sending to the server."; + qCDebug(lcPropagateUploadEncrypted) << "Metadata created, sending to the server."; auto job = new UpdateMetadataApiJob(_propagator->account(), _folderId, _metadata->encryptedMetadata(), @@ -142,24 +157,11 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo void PropagateUploadEncrypted::slotUpdateMetadataSuccess(const QByteArray& fileId) { - qCDebug(lcPropagateUpload) << "Uploading of the metadata success, Encrypting the file"; - QFileInfo info(_propagator->_localDir + QDir::separator() + _item->_file); - auto *input = new QFile(info.absoluteFilePath()); - auto *output = new QFile(QDir::tempPath() + QDir::separator() + _encryptedFile.encryptedFilename); + qCDebug(lcPropagateUploadEncrypted) << "Uploading of the metadata success, Encrypting the file"; + QFileInfo outputInfo(_completeFileName); - // TODO: Invert the operations. first enrypt, then generate the metadata. - QByteArray tag; - EncryptionHelper::fileEncryption(_encryptedFile.encryptionKey, - _encryptedFile.initializationVector, - input, output, tag); - - // File is Encrypted, Upload it. - QFileInfo outputInfo(output->fileName()); - input->deleteLater(); - output->deleteLater(); - - qCDebug(lcPropagateUpload) << "Encrypted Info:" << outputInfo.path() << outputInfo.fileName() << outputInfo.size(); - qCDebug(lcPropagateUpload) << "Finalizing the upload part, now the actuall uploader will take over"; + qCDebug(lcPropagateUploadEncrypted) << "Encrypted Info:" << outputInfo.path() << outputInfo.fileName() << outputInfo.size(); + qCDebug(lcPropagateUploadEncrypted) << "Finalizing the upload part, now the actuall uploader will take over"; emit finalized(outputInfo.path() + QLatin1Char('/') + outputInfo.fileName(), _item->_file.section(QLatin1Char('/'), 0, -2) + QLatin1Char('/') + outputInfo.fileName(), outputInfo.size()); @@ -167,17 +169,17 @@ void PropagateUploadEncrypted::slotUpdateMetadataSuccess(const QByteArray& fileI void PropagateUploadEncrypted::slotUpdateMetadataError(const QByteArray& fileId, int httpErrorResponse) { - qCDebug(lcPropagateUpload) << "Update metadata error for folder" << fileId << "with error" << httpErrorResponse; + qCDebug(lcPropagateUploadEncrypted) << "Update metadata error for folder" << fileId << "with error" << httpErrorResponse; } void PropagateUploadEncrypted::slotUnlockEncryptedFolderSuccess(const QByteArray& fileId) { - qCDebug(lcPropagateUpload) << "Unlock Job worked for folder " << fileId; + qCDebug(lcPropagateUploadEncrypted) << "Unlock Job worked for folder " << fileId; } void PropagateUploadEncrypted::slotUnlockEncryptedFolderError(const QByteArray& fileId, int httpStatusCode) { - qCDebug(lcPropagateUpload) << "There was an error unlocking " << fileId << httpStatusCode; + qCDebug(lcPropagateUploadEncrypted) << "There was an error unlocking " << fileId << httpStatusCode; } void PropagateUploadEncrypted::slotFolderLockedError(const QByteArray& fileId, int httpErrorCode) @@ -186,31 +188,31 @@ void PropagateUploadEncrypted::slotFolderLockedError(const QByteArray& fileId, i and fail if it's more than 5 minutes. */ QTimer::singleShot(5000, this, [this, fileId]{ if (!_currentLockingInProgress) { - qCDebug(lcPropagateUpload) << "Error locking the folder while no other update is locking it up."; - qCDebug(lcPropagateUpload) << "Perhaps another client locked it."; - qCDebug(lcPropagateUpload) << "Abort"; + qCDebug(lcPropagateUploadEncrypted) << "Error locking the folder while no other update is locking it up."; + qCDebug(lcPropagateUploadEncrypted) << "Perhaps another client locked it."; + qCDebug(lcPropagateUploadEncrypted) << "Abort"; return; } // Perhaps I should remove the elapsed timer if the lock is from this client? if (_folderLockFirstTry.elapsed() > /* five minutes */ 1000 * 60 * 5 ) { - qCDebug(lcPropagateUpload) << "One minute passed, ignoring more attemps to lock the folder."; + qCDebug(lcPropagateUploadEncrypted) << "One minute passed, ignoring more attemps to lock the folder."; return; } slotTryLock(fileId); }); - qCDebug(lcPropagateUpload) << "Folder" << fileId << "Coundn't be locked."; + qCDebug(lcPropagateUploadEncrypted) << "Folder" << fileId << "Coundn't be locked."; } void PropagateUploadEncrypted::slotFolderEncryptedIdError(QNetworkReply *r) { - qCDebug(lcPropagateUpload) << "Error retrieving the Id of the encrypted folder."; + qCDebug(lcPropagateUploadEncrypted) << "Error retrieving the Id of the encrypted folder."; } void PropagateUploadEncrypted::slotFolderEncryptedStatusError(int error) { - qCDebug(lcPropagateUpload) << "Failed to retrieve the status of the folders." << error; + qCDebug(lcPropagateUploadEncrypted) << "Failed to retrieve the status of the folders." << error; } } // namespace OCC diff --git a/src/libsync/propagateuploadencrypted.h b/src/libsync/propagateuploadencrypted.h index 397c99aa3..7ff52ff86 100644 --- a/src/libsync/propagateuploadencrypted.h +++ b/src/libsync/propagateuploadencrypted.h @@ -72,6 +72,7 @@ private: QByteArray _generatedIv; FolderMetadata *_metadata; EncryptedFile _encryptedFile; + QString _completeFileName; }; From cae6bbc853dafbeadd084975c4785cff0ed21f3c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 22 Jan 2018 14:08:32 +0100 Subject: [PATCH 211/262] [CSE] Start the download process --- src/gui/main.cpp | 1 - src/libsync/propagatedownload.cpp | 1 + src/libsync/propagatedownloadencrypted.cpp | 4 ++++ src/libsync/propagatedownloadencrypted.h | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/gui/main.cpp b/src/gui/main.cpp index d15c5d9d8..a4c6e8bff 100644 --- a/src/gui/main.cpp +++ b/src/gui/main.cpp @@ -52,7 +52,6 @@ int main(int argc, char **argv) Q_INIT_RESOURCE(client); /* Initialise the library */ - qDebug() << "LOADING OPENSSL STUFF"; ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); OPENSSL_config(NULL); diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index fe1b00cfc..25bb93576 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -359,6 +359,7 @@ void PropagateDownloadFile::start() _isEncrypted = true; startAfterIsEncryptedIsChecked(); }); + _downloadEncryptedHelper->start(); } else { startAfterIsEncryptedIsChecked(); } diff --git a/src/libsync/propagatedownloadencrypted.cpp b/src/libsync/propagatedownloadencrypted.cpp index 378e434a3..1deb4a8f2 100644 --- a/src/libsync/propagatedownloadencrypted.cpp +++ b/src/libsync/propagatedownloadencrypted.cpp @@ -12,6 +12,10 @@ PropagateDownloadEncrypted::PropagateDownloadEncrypted(OwncloudPropagator *propa { } +void PropagateDownloadEncrypted::start() { + checkFolderEncryptedStatus(); +} + void PropagateDownloadEncrypted::checkFolderEncryptedStatus() { auto getEncryptedStatus = new GetFolderEncryptStatusJob(_propagator->account(), _info.path()); diff --git a/src/libsync/propagatedownloadencrypted.h b/src/libsync/propagatedownloadencrypted.h index 379a9efc7..1f854753c 100644 --- a/src/libsync/propagatedownloadencrypted.h +++ b/src/libsync/propagatedownloadencrypted.h @@ -16,6 +16,7 @@ class PropagateDownloadEncrypted : public QObject { Q_OBJECT public: PropagateDownloadEncrypted(OwncloudPropagator *propagator, SyncFileItemPtr item); + void start(); void checkFolderId(const QStringList &list); bool decryptFile(QFile& tmpFile); QString errorString() const; From 33beb7aadeba2fe1d8a0f2c8fd035d5f764cfff0 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Mon, 22 Jan 2018 15:24:54 +0100 Subject: [PATCH 212/262] Don't generate values after we use them * Pregerenate IV * Pregenerate key * Pregenerate filename TODO: - Reuse existing file entries and update (we might need logic in the metadatahandling to do this properly). --- src/libsync/propagateuploadencrypted.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 05ddbc0ce..5313155d9 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -119,13 +119,15 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo qCDebug(lcPropagateUploadEncrypted) << "Creating the encrypted file."; auto *input = new QFile(info.absoluteFilePath()); - auto *output = new QFile(QDir::tempPath() + QDir::separator() + _encryptedFile.encryptedFilename); + // TODO: Get from metadata if it is a file update! + QByteArray encryptedFilename = EncryptionHelper::generateRandomString(20); + auto *output = new QFile(QDir::tempPath() + QDir::separator() + encryptedFilename); // TODO: Invert the operations. first enrypt, then generate the metadata. QByteArray tag; - EncryptionHelper::fileEncryption(_encryptedFile.encryptionKey, - _encryptedFile.initializationVector, - input, output, tag); + QByteArray key = EncryptionHelper::generateRandom(16); + QByteArray iv = EncryptionHelper::generateRandom(16); + EncryptionHelper::fileEncryption(key, iv, input, output, tag); input->deleteLater(); output->deleteLater(); @@ -133,12 +135,14 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo _completeFileName = output->fileName(); qCDebug(lcPropagateUploadEncrypted) << "Creating the metadata for the encrypted file."; + + // TODO: reuse existing and update it instead of always created a new file EncryptedFile encryptedFile; encryptedFile.authenticationTag = tag.toBase64(); // TODO: Check against Android to see if the metadata is correct.. - encryptedFile.encryptedFilename = EncryptionHelper::generateRandomString(20); - encryptedFile.encryptionKey = EncryptionHelper::generateRandom(16); + encryptedFile.encryptedFilename = encryptedFilename; + encryptedFile.encryptionKey = key; encryptedFile.fileVersion = 1; - encryptedFile.initializationVector = EncryptionHelper::generateRandom(16); + encryptedFile.initializationVector = iv; encryptedFile.metadataKey = 1; encryptedFile.originalFilename = info.fileName(); _metadata->addEncryptedFile(encryptedFile); From bb85e2ab2c2f97989af872a13a16e2fd0a5051cb Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Tue, 23 Jan 2018 11:36:15 +0100 Subject: [PATCH 213/262] Reuse existing EncryptedFile Signed-off-by: Roeland Jago Douma --- src/libsync/propagateuploadencrypted.cpp | 42 +++++++++++++++--------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 5313155d9..dc4a83c9b 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -116,18 +116,35 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo _metadata = new FolderMetadata(_propagator->account(), json.toJson(QJsonDocument::Compact)); QFileInfo info(_propagator->_localDir + QDir::separator() + _item->_file); + + // Find existing metadata for this file + EncryptedFile encryptedFile; + QVector files = _metadata->files(); + for(EncryptedFile &file : files) { + if (file.originalFilename == _item->File) { + encryptedFile = file; + } + } + + encryptedFile.encryptionKey = EncryptionHelper::generateRandom(16); + encryptedFile.initializationVector = EncryptionHelper::generateRandom(16); + + // New encrypted file so set it all up! + if (encryptedFile.encryptedFilename.isEmpty()) { + encryptedFile.encryptedFilename = EncryptionHelper::generateRandomString(20); + encryptedFile.fileVersion = 1; + encryptedFile.metadataKey = 1; + encryptedFile.originalFilename = info.fileName(); + } + + qCDebug(lcPropagateUploadEncrypted) << "Creating the encrypted file."; auto *input = new QFile(info.absoluteFilePath()); - // TODO: Get from metadata if it is a file update! - QByteArray encryptedFilename = EncryptionHelper::generateRandomString(20); - auto *output = new QFile(QDir::tempPath() + QDir::separator() + encryptedFilename); + auto *output = new QFile(QDir::tempPath() + QDir::separator() + encryptedFile.encryptedFilename); - // TODO: Invert the operations. first enrypt, then generate the metadata. QByteArray tag; - QByteArray key = EncryptionHelper::generateRandom(16); - QByteArray iv = EncryptionHelper::generateRandom(16); - EncryptionHelper::fileEncryption(key, iv, input, output, tag); + EncryptionHelper::fileEncryption(encryptedFile.encryptionKey, encryptedFile.initializationVector, input, output, tag); input->deleteLater(); output->deleteLater(); @@ -136,15 +153,8 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo qCDebug(lcPropagateUploadEncrypted) << "Creating the metadata for the encrypted file."; - // TODO: reuse existing and update it instead of always created a new file - EncryptedFile encryptedFile; - encryptedFile.authenticationTag = tag.toBase64(); // TODO: Check against Android to see if the metadata is correct.. - encryptedFile.encryptedFilename = encryptedFilename; - encryptedFile.encryptionKey = key; - encryptedFile.fileVersion = 1; - encryptedFile.initializationVector = iv; - encryptedFile.metadataKey = 1; - encryptedFile.originalFilename = info.fileName(); + encryptedFile.authenticationTag = tag.toBase64(); + _metadata->addEncryptedFile(encryptedFile); _encryptedFile = encryptedFile; From a2b8724adfe59743b707e4edb845df3de19677b2 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Tue, 23 Jan 2018 11:58:05 +0100 Subject: [PATCH 214/262] Hacky way to drop duplicates (we should really use the fileid!) Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 6b2d8d2bb..63c827392 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1286,6 +1286,14 @@ QByteArray FolderMetadata::encryptedMetadata() { } void FolderMetadata::addEncryptedFile(const EncryptedFile &f) { + + for (int i = 0; i < _files.size(); i++) { + if (_files.at(i).originalFilename == f.originalFilename) { + _files.removeAt(i); + break; + } + } + _files.append(f); } From 21d55c3321fc669a21718ba4d05e66fe46a634bc Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Tue, 23 Jan 2018 21:02:52 +0100 Subject: [PATCH 215/262] No sharing in metadata yet and PEM as PKCS#8 * Don't store the metadata yet this crashes android - Yes android should be fixed but for now this is quicker ;) * QSslKey exports PEM as PKCS#1 - This is not handled properly on android so use PKCS#8 helper Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 25 ++++++++++++++++++++++--- src/libsync/clientsideencryption.h | 2 ++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 63c827392..19f2d3bb3 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -381,6 +381,23 @@ QByteArray EncryptionHelper::decryptStringSymmetric(const QByteArray& key, const return result; } +QByteArray EncryptionHelper::privateKeyToPem(const QSslKey key) { + BIO *privateKeyBio = BIO_new(BIO_s_mem()); + QByteArray privateKeyPem = key.toPem(); + BIO_write(privateKeyBio, privateKeyPem.constData(), privateKeyPem.size()); + EVP_PKEY *pkey = PEM_read_bio_PrivateKey(privateKeyBio, NULL, NULL, NULL); + + BIO *pemBio = BIO_new(BIO_s_mem()); + PEM_write_bio_PKCS8PrivateKey(pemBio, pkey, NULL, NULL, 0, NULL, NULL); + QByteArray pem = BIO2ByteArray(pemBio); + + BIO_free_all(privateKeyBio); + BIO_free_all(pemBio); + EVP_PKEY_free(pkey); + + return pem; +} + QByteArray EncryptionHelper::encryptStringSymmetric(const QByteArray& key, const QByteArray& data) { QByteArray iv = generateRandom(16); @@ -706,7 +723,7 @@ void ClientSideEncryption::mnemonicKeyFetched(QKeychain::Job *incoming) { _mnemonic = readJob->textData(); - qCInfo(lcCse()) << "Mnemonic key fetched from keychain"; + qCInfo(lcCse()) << "Mnemonic key fetched from keychain: " << _mnemonic; emit initializationFinished(); } @@ -937,7 +954,7 @@ void ClientSideEncryption::encryptPrivateKey() auto salt = EncryptionHelper::generateRandom(40); auto secretKey = EncryptionHelper::generatePassword(passPhrase, salt); - auto cryptedText = EncryptionHelper::encryptPrivateKey(secretKey, _privateKey.toPem(), salt); + auto cryptedText = EncryptionHelper::encryptPrivateKey(secretKey, EncryptionHelper::privateKeyToPem(_privateKey), salt); // Send private key to the server auto job = new StorePrivateKeyApiJob(_account, baseUrl() + "private-key", this); @@ -1240,6 +1257,7 @@ QByteArray FolderMetadata::encryptedMetadata() { metadataKeys.insert(QString::number(it.key()), QString(encryptedKey)); } + /* NO SHARING IN V1 QJsonObject recepients; for (auto it = _sharing.constBegin(), end = _sharing.constEnd(); it != end; it++) { recepients.insert(it->first, it->second); @@ -1247,10 +1265,11 @@ QByteArray FolderMetadata::encryptedMetadata() { QJsonDocument recepientDoc; recepientDoc.setObject(recepients); QString sharingEncrypted = encryptJsonObject(recepientDoc.toJson(QJsonDocument::Compact), _metadataKeys.last()); + */ QJsonObject metadata = { {"metadataKeys", metadataKeys}, - {"sharing", sharingEncrypted}, + // {"sharing", sharingEncrypted}, {"version", 1} }; diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index ebe87c540..8ee5aa6a5 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -49,6 +49,8 @@ public: const QByteArray& data ); + static QByteArray privateKeyToPem(const QSslKey key); + //TODO: change those two EVP_PKEY into QSslKey. static QByteArray encryptStringAsymmetric( EVP_PKEY *publicKey, From 4892b0ec43c6a178ace4878535dd1af2b059ca45 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Tue, 23 Jan 2018 21:31:00 +0100 Subject: [PATCH 216/262] Properly fill EncryptedFile * Add a mimetype (mobile clients need this) * Add the tag unencoded as we encode it when we send the metadata already Signed-off-by: Roeland Jago Douma --- src/libsync/propagateuploadencrypted.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index dc4a83c9b..7b5d0b0d8 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace OCC { @@ -135,6 +136,8 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo encryptedFile.fileVersion = 1; encryptedFile.metadataKey = 1; encryptedFile.originalFilename = info.fileName(); + QMimeDatabase mdb; + encryptedFile.mimetype = mdb.mimeTypeForFile(info).name().toLocal8Bit(); } @@ -153,7 +156,7 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo qCDebug(lcPropagateUploadEncrypted) << "Creating the metadata for the encrypted file."; - encryptedFile.authenticationTag = tag.toBase64(); + encryptedFile.authenticationTag = tag; _metadata->addEncryptedFile(encryptedFile); _encryptedFile = encryptedFile; From b909bb6977b1e8c45e077f162af54dfcb871ca3c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 28 Jan 2018 16:41:07 +0100 Subject: [PATCH 217/262] [CSE] Change the Database to handle Encrypted Files Create a table to hold the information between the real file name and the fake file name --- src/common/syncjournaldb.cpp | 21 +++++++++++++++++++++ src/common/syncjournaldb.h | 4 ++++ 2 files changed, 25 insertions(+) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index 41401ef1a..b4b92daa2 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -455,6 +455,17 @@ bool SyncJournalDb::checkConnect() return sqlFail("Create table version", createQuery); } + /* maps the end to end filename in the server/metadata to the actuall filename in disk + */ + createQuery.prepare("CREATE TABLE IF NOT EXISTS e2efilemap(" + "mangledname TEXT UNIQUE," + "name TEXT UNIQUE" + ")"); + + if (!createQuery.exec()) { + return sqlFail("Create table e2efilemap", createQuery); + } + bool forceRemoteDiscovery = false; SqlQuery versionQuery("SELECT major, minor, patch FROM version;", _db); @@ -692,6 +703,16 @@ bool SyncJournalDb::checkConnect() return sqlFail("prepare _setDataFingerprintQuery2", *_setDataFingerprintQuery2); } + _getE2eFileMangledName.reset(new SqlQuery(_db)); + if (_getE2eFileMangledName->prepare("SELECT mangledname FROM e2efilemap WHERE mangledname =?1;")) { + return sqlFail("prepare _getE2eFileMangledName", *_getE2eFileMangledName); + } + + _setE2eFileRelationQuery.reset(new SqlQuery(_db)); + if (_setE2eFileRelationQuery->prepare("INSERT INTO e2efilemap (mangledname, name) VALUES (?1, ?2);")) { + return sqlFail("prepare _setE2eFileRelationQuery", *_setE2eFileRelationQuery); + } + // don't start a new transaction now commitInternal(QString("checkConnect End"), false); diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h index ad61a6ad2..46e1d9822 100644 --- a/src/common/syncjournaldb.h +++ b/src/common/syncjournaldb.h @@ -266,6 +266,10 @@ private: QScopedPointer _setDataFingerprintQuery1; QScopedPointer _setDataFingerprintQuery2; + // End to End Encryption Related Queries/ + QScopedPointer _setE2eFileRelationQuery; + QScopedPointer _getE2eFileMangledName; + /* This is the list of paths we called avoidReadFromDbOnNextSync on. * It means that they should not be written to the DB in any case since doing * that would write the etag and would void the purpose of avoidReadFromDbOnNextSync From be817748f8c160fc0d60beb41719000d89b86219 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 28 Jan 2018 17:46:20 +0100 Subject: [PATCH 218/262] [CSE] Reset the new Queries on close --- src/common/syncjournaldb.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index b4b92daa2..dfe69725e 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -761,6 +761,8 @@ void SyncJournalDb::close() _getDataFingerprintQuery.reset(0); _setDataFingerprintQuery1.reset(0); _setDataFingerprintQuery2.reset(0); + _setE2eFileRelationQuery.reset(0); + _getE2eFileMangledName.reset(0); _db.close(); _avoidReadFromDbOnNextSyncFilter.clear(); From 32eaaa0601e28945491aa99492b719ba63c2f26d Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 28 Jan 2018 17:50:10 +0100 Subject: [PATCH 219/262] [CSE] GetMangledName implementation --- src/common/syncjournaldb.cpp | 24 ++++++++++++++++++++++++ src/common/syncjournaldb.h | 3 +++ 2 files changed, 27 insertions(+) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index dfe69725e..15749a189 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -1420,6 +1420,30 @@ void SyncJournalDb::setDownloadInfo(const QString &file, const SyncJournalDb::Do } } +QString SyncJournalDb::getE2eMangledName(const QString& originalName) +{ + QMutexLocker locker(&_mutex); + if (checkConnect()) { + return QString{}; + } + + if (originalName.isEmpty()) { + return QString{}; + } + + _getE2eFileMangledName->reset_and_clear_bindings(); + _getE2eFileMangledName->bindValue(1, originalName); + if (!_getE2eFileMangledName->exec()) { + return QString{}; + } + + if (!_getE2eFileMangledName->next()) { + return QString{}; + } + + return _getE2eFileMangledName->stringValue(0); +} + QVector SyncJournalDb::getAndDeleteStaleDownloadInfos(const QSet &keep) { QVector empty_result; diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h index 46e1d9822..956ade884 100644 --- a/src/common/syncjournaldb.h +++ b/src/common/syncjournaldb.h @@ -137,6 +137,9 @@ public: void avoidRenamesOnNextSync(const QString &path) { avoidRenamesOnNextSync(path.toUtf8()); } void avoidRenamesOnNextSync(const QByteArray &path); void setPollInfo(const PollInfo &); + + QString getE2eMangledName(const QString& originalName); + QVector getPollInfos(); enum SelectiveSyncListType { From a003d216a59dce79d511df59f01c64914d7408a1 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 28 Jan 2018 18:05:28 +0100 Subject: [PATCH 220/262] [CSE] Implementation of the setE2eRelation This call creates a new entry on the database for e2e. --- src/common/syncjournaldb.cpp | 21 +++++++++++++++++++++ src/common/syncjournaldb.h | 1 + 2 files changed, 22 insertions(+) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index 15749a189..bbca1fb3f 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -1444,6 +1444,27 @@ QString SyncJournalDb::getE2eMangledName(const QString& originalName) return _getE2eFileMangledName->stringValue(0); } +bool SyncJournalDb::setE2eRelation(const QString& mangledName, const QString& originalName) +{ + Q_ASSERT(!mangledName.isEmpty()); + Q_ASSERT(!originalName.isEmpty()); + + if (mangledName.isEmpty()) { + qCDebug(lcDb) << "Cant create e2e relation on the database, mangled name is empty."; + return false; + } + + if (originalName.isEmpty()) { + qCDebug(lcDb) << "Cant create e2e relation on the database, original name is empty."; + return false; + } + + _setE2eFileRelationQuery->reset_and_clear_bindings(); + _setE2eFileRelationQuery->bindValue(1, mangledName); + _setE2eFileRelationQuery->bindValue(2, originalName); + return _setE2eFileRelationQuery->exec(); +} + QVector SyncJournalDb::getAndDeleteStaleDownloadInfos(const QSet &keep) { QVector empty_result; diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h index 956ade884..894220443 100644 --- a/src/common/syncjournaldb.h +++ b/src/common/syncjournaldb.h @@ -138,6 +138,7 @@ public: void avoidRenamesOnNextSync(const QByteArray &path); void setPollInfo(const PollInfo &); + bool setE2eRelation(const QString& mangledName, const QString& originalName); QString getE2eMangledName(const QString& originalName); QVector getPollInfos(); From e9a7a563597e1b5ffde6927fe80c50542b4c7441 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 28 Jan 2018 18:37:01 +0100 Subject: [PATCH 221/262] [CSE] Store the encrypted file name on the SyncFilePtr --- src/libsync/propagatedownloadencrypted.cpp | 7 ++++++- src/libsync/propagateuploadencrypted.cpp | 2 ++ src/libsync/syncfileitem.h | 6 ++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/libsync/propagatedownloadencrypted.cpp b/src/libsync/propagatedownloadencrypted.cpp index 1deb4a8f2..2cbee1ca5 100644 --- a/src/libsync/propagatedownloadencrypted.cpp +++ b/src/libsync/propagatedownloadencrypted.cpp @@ -123,7 +123,12 @@ bool PropagateDownloadEncrypted::decryptFile(QFile& tmpFile) //TODO: This seems what's breaking the logic. // Let's fool the rest of the logic into thinking this is the right name of the DAV file - _item->_file = _item->_file.section(QLatin1Char('/'), 0, -2) + QLatin1Char('/') + _encryptedInfo.originalFilename; + _item->_file = _item->_file.section(QLatin1Char('/'), 0, -2) + + QLatin1Char('/') + _encryptedInfo.originalFilename; + + _item->_isEndToEndEncrypted = true; + _item->_encryptedFileName = _encryptedInfo.encryptedFilename; + return true; } diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 7b5d0b0d8..657aa08d3 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -140,6 +140,8 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo encryptedFile.mimetype = mdb.mimeTypeForFile(info).name().toLocal8Bit(); } + _item->_isEndToEndEncrypted = true; + _item->_encryptedFileName = encryptedFile.encryptedFilename; qCDebug(lcPropagateUploadEncrypted) << "Creating the encrypted file."; diff --git a/src/libsync/syncfileitem.h b/src/libsync/syncfileitem.h index dd364ae43..08bcc66d7 100644 --- a/src/libsync/syncfileitem.h +++ b/src/libsync/syncfileitem.h @@ -195,6 +195,12 @@ public: // Variables useful for everybody QString _file; QString _renameTarget; + + /// Whether there's end to end encryption on this file. + /// If the file is encrypted, the _encryptedFilename is + /// the encrypted name on the server. + bool _isEndToEndEncrypted; + QString _encryptedFileName; Type _type BITFIELD(3); Direction _direction BITFIELD(3); bool _serverHasIgnoredFiles BITFIELD(1); From 3c084bb6eb22dc9572f26ba864c882aeee3ff6b4 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Sun, 28 Jan 2018 19:55:52 +0100 Subject: [PATCH 222/262] Bump drone image * Bump to 16.04 since 14.04 doesn't auto build here due to openssl fu * No more gcc7 since gcc5 is new enough Signed-off-by: Roeland Jago Douma --- .drone.yml | 2 +- admin/linux/build-appimage.sh | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.drone.yml b/.drone.yml index 6e99b36d5..e8674f9f2 100644 --- a/.drone.yml +++ b/.drone.yml @@ -81,7 +81,7 @@ pipeline: TESTS: qt-5.9 AppImage-5.9: - image: nextcloudci/client-appimage-ci:client-appimage-ci-8 + image: nextcloudci/client-appimage-ci:client-appimage-ci-9 commands: - /bin/bash -c "./admin/linux/build-appimage.sh" when: diff --git a/admin/linux/build-appimage.sh b/admin/linux/build-appimage.sh index 79bc3a477..d672034d1 100755 --- a/admin/linux/build-appimage.sh +++ b/admin/linux/build-appimage.sh @@ -5,9 +5,6 @@ set -xe mkdir /app mkdir /build -export CC=/usr/bin/gcc-7 -export CXX=/usr/bin/g++-7 - #Set Qt-5.9 export QT_BASE_DIR=/opt/qt59 export QTDIR=$QT_BASE_DIR From 11946d9ef8a88089e3dc29501d271c5afe5f4b83 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Sun, 28 Jan 2018 20:21:12 +0100 Subject: [PATCH 223/262] Bump Signed-off-by: Roeland Jago Douma --- .drone.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index e8674f9f2..8d24cd8e4 100644 --- a/.drone.yml +++ b/.drone.yml @@ -79,7 +79,6 @@ pipeline: when: matrix: TESTS: qt-5.9 - AppImage-5.9: image: nextcloudci/client-appimage-ci:client-appimage-ci-9 commands: From e8849a2cf911f94b352eafd02c24261dbeb59d36 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 28 Jan 2018 22:26:07 +0100 Subject: [PATCH 224/262] [CSE] Save the relation of the file / encrypted file names on the db --- src/libsync/propagateupload.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 281fbe525..c56b1767e 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -674,11 +674,21 @@ void PropagateUploadFileCommon::finalize() quotaIt.value() -= _fileToUpload._size; // Update the database entry - use the local file, not the temporary one. - if (!propagator()->_journal->setFileRecord(_item->toSyncJournalFileRecordWithInode(propagator()->getFilePath(_item->_file)))) { + const auto filePath = propagator()->getFilePath(_item->_file); + const auto fileRecord = _item->toSyncJournalFileRecordWithInode(filePath); + if (!propagator()->_journal->setFileRecord(fileRecord)) { done(SyncFileItem::FatalError, tr("Error writing metadata to the database")); return; } + if (_uploadingEncrypted) { + if (!propagator()->_journal->setE2eRelation(_item->_encryptedFileName, _item->_file)) { + qDebug() << "Error saving the encryption relation of the file."; + } else { + qDebug() << "Sabed the encryption relation of the file successfully"; + } + } + // Remove from the progress database: propagator()->_journal->setUploadInfo(_item->_file, SyncJournalDb::UploadInfo()); propagator()->_journal->commit("upload file start"); From 0c94142c977e58998763409bfd6e5eb3871637ac Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 28 Jan 2018 23:47:58 +0100 Subject: [PATCH 225/262] [CSE] More debug output --- src/common/syncjournaldb.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index bbca1fb3f..3f2ffac26 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -1423,25 +1423,34 @@ void SyncJournalDb::setDownloadInfo(const QString &file, const SyncJournalDb::Do QString SyncJournalDb::getE2eMangledName(const QString& originalName) { QMutexLocker locker(&_mutex); - if (checkConnect()) { + qCDebug(lcDb) << "######### Trying to find database entry for name " << originalName << ". #######"; + + if (! checkConnect()) { + qCDebug(lcDb) << "######### Connection to the database is wrong. #######"; return QString{}; } if (originalName.isEmpty()) { + qCDebug(lcDb) << "######### Original Name is Empty. #######"; return QString{}; } _getE2eFileMangledName->reset_and_clear_bindings(); _getE2eFileMangledName->bindValue(1, originalName); if (!_getE2eFileMangledName->exec()) { + qCDebug(lcDb) << "######### Error Executing query. #######"; return QString{}; } if (!_getE2eFileMangledName->next()) { + qCDebug(lcDb) << "######### Query returned empty. #######"; return QString{}; } - return _getE2eFileMangledName->stringValue(0); + const QString ret = _getE2eFileMangledName->stringValue(0); + + qCDebug(lcDb) << "The return of the mangled name is" << ret; + return ret; } bool SyncJournalDb::setE2eRelation(const QString& mangledName, const QString& originalName) @@ -1449,6 +1458,11 @@ bool SyncJournalDb::setE2eRelation(const QString& mangledName, const QString& or Q_ASSERT(!mangledName.isEmpty()); Q_ASSERT(!originalName.isEmpty()); + if (! checkConnect()) { + qCDebug(lcDb) << "######### Connection to the database is wrong. #######"; + return false; + } + if (mangledName.isEmpty()) { qCDebug(lcDb) << "Cant create e2e relation on the database, mangled name is empty."; return false; From 7b123650da200e9442f442c41b380e28676f8c8c Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 28 Jan 2018 23:48:25 +0100 Subject: [PATCH 226/262] [CSE] Search for name, not mangled name. --- src/common/syncjournaldb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index 3f2ffac26..bcdb6dffc 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -704,7 +704,7 @@ bool SyncJournalDb::checkConnect() } _getE2eFileMangledName.reset(new SqlQuery(_db)); - if (_getE2eFileMangledName->prepare("SELECT mangledname FROM e2efilemap WHERE mangledname =?1;")) { + if (_getE2eFileMangledName->prepare("SELECT mangledname FROM e2efilemap WHERE name =?1;")) { return sqlFail("prepare _getE2eFileMangledName", *_getE2eFileMangledName); } From d38e5e15295f931ae1c9c2930e9cbe3454b9a6b4 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 28 Jan 2018 23:48:51 +0100 Subject: [PATCH 227/262] [CSE] Bypass rename if it's e2e The client tracks the files in server and locally, but the names of the files differ in the server, but we do *not* wanna rename the files to the server one, we wanna keep it as is if it's an encrypted file. --- src/csync/csync_update.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/csync/csync_update.cpp b/src/csync/csync_update.cpp index 4f913fa99..66577ca40 100644 --- a/src/csync/csync_update.cpp +++ b/src/csync/csync_update.cpp @@ -343,8 +343,21 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr f csync_rename_record(ctx, base._path, fs->path); } - qCDebug(lcUpdate, "remote rename detected based on fileid %s --> %s", base._path.constData(), fs->path.constData()); - fs->instruction = CSYNC_INSTRUCTION_EVAL_RENAME; + /* A remote rename can also mean Encryption Mangled Name. + * if we find one of those in the database, we ignore it. + */ + qCDebug(lcUpdate) << "Tryig to get the mangled name!"; + QString remoteEncryptedName = ctx->statedb->getE2eMangledName(fs->path); + qCDebug(lcUpdate) << "Remote Encrypted Name stored for" << fs->path << "is" + << (remoteEncryptedName.isEmpty() ? "Empty" : remoteEncryptedName); + qCDebug(lcUpdate) << "And for the last part" << ctx->statedb->getE2eMangledName(fs->path.split('/')[1]); + + if (remoteEncryptedName.isEmpty()) { + qCDebug(lcUpdate, "remote rename detected based on fileid %s --> %s", base._path.constData(), fs->path.constData()); + fs->instruction = CSYNC_INSTRUCTION_EVAL_RENAME; + } else { + qCDebug(lcUpdate) << "Should *not* rename the file."; + } done = true; }; From 682bb2ad436e676f0ccddfa911d917396450ba70 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 28 Jan 2018 23:50:12 +0100 Subject: [PATCH 228/262] [CSE] Save the encrypted name relation on downloads --- src/libsync/propagatedownload.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index 25bb93576..9e14a22a9 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -928,6 +928,10 @@ void PropagateDownloadFile::updateMetadata(bool isConflict) return; } propagator()->_journal->setDownloadInfo(_item->_file, SyncJournalDb::DownloadInfo()); + if (_isEncrypted) { + propagator()->_journal->setE2eRelation(_item->_encryptedFileName, _item->_file); + } + propagator()->_journal->commit("download file start2"); done(isConflict ? SyncFileItem::Conflict : SyncFileItem::Success); From f4a50e88f5a387fb8b6b359ed8a7d0ae1569a813 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 28 Jan 2018 23:59:53 +0100 Subject: [PATCH 229/262] [CSE] Ignore renames for downloaded files. - TODO: Fix the incorrect "filename has been removed" popup. The file is not removed, just the popup shows. --- src/csync/csync_update.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/csync/csync_update.cpp b/src/csync/csync_update.cpp index 66577ca40..003bc8297 100644 --- a/src/csync/csync_update.cpp +++ b/src/csync/csync_update.cpp @@ -347,11 +347,7 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr f * if we find one of those in the database, we ignore it. */ qCDebug(lcUpdate) << "Tryig to get the mangled name!"; - QString remoteEncryptedName = ctx->statedb->getE2eMangledName(fs->path); - qCDebug(lcUpdate) << "Remote Encrypted Name stored for" << fs->path << "is" - << (remoteEncryptedName.isEmpty() ? "Empty" : remoteEncryptedName); - qCDebug(lcUpdate) << "And for the last part" << ctx->statedb->getE2eMangledName(fs->path.split('/')[1]); - + QString remoteEncryptedName = ctx->statedb->getE2eMangledName(base._path); if (remoteEncryptedName.isEmpty()) { qCDebug(lcUpdate, "remote rename detected based on fileid %s --> %s", base._path.constData(), fs->path.constData()); fs->instruction = CSYNC_INSTRUCTION_EVAL_RENAME; From dcc9ac7d2e02cc180caf56e580d4be39a008f3e7 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 29 Jan 2018 14:17:52 +0100 Subject: [PATCH 230/262] [Buildfix] Fix build from last merge --- src/gui/application.cpp | 2 +- src/libsync/propagateuploadencrypted.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gui/application.cpp b/src/gui/application.cpp index 1bf5be165..ada0df295 100644 --- a/src/gui/application.cpp +++ b/src/gui/application.cpp @@ -51,7 +51,7 @@ #include #include #include -#include +#include class QSocket; diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 657aa08d3..8be840d20 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -122,7 +122,7 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo EncryptedFile encryptedFile; QVector files = _metadata->files(); for(EncryptedFile &file : files) { - if (file.originalFilename == _item->File) { + if (file.originalFilename == _item->_file) { encryptedFile = file; } } From 296f46356eab10a369c0c5e0278a0f0165db11eb Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Mon, 29 Jan 2018 22:04:50 +0100 Subject: [PATCH 231/262] If the metadata is empty we should store (and not update). Android only creates the metadata file when the first encrypted file is added. We assumed it would be there. This hacky code makes us store the metadata if there wasn't any yet. Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 4 ++-- src/libsync/clientsideencryption.h | 2 +- src/libsync/propagateuploadencrypted.cpp | 20 +++++++++++++++----- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 19f2d3bb3..3d89eab76 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1099,9 +1099,9 @@ void ClientSideEncryption::folderEncryptedStatusError(int error) qCDebug(lcCse) << "Failed to retrieve the status of the folders." << error; } -FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata) : _account(account) +FolderMetadata::FolderMetadata(AccountPtr account, const QByteArray& metadata, int statusCode) : _account(account) { - if (metadata.isEmpty()) { + if (metadata.isEmpty() || statusCode == 404) { qCInfo(lcCseMetadata()) << "Setupping Empty Metadata"; setupEmptyMetadata(); } else { diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 8ee5aa6a5..c5759806f 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -142,7 +142,7 @@ struct EncryptedFile { class FolderMetadata { public: - FolderMetadata(AccountPtr account, const QByteArray& metadata = QByteArray()); + FolderMetadata(AccountPtr account, const QByteArray& metadata = QByteArray(), int statusCode = -1); QByteArray encryptedMetadata(); void addEncryptedFile(const EncryptedFile& f); QVector files() const; diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 8be840d20..e08864aec 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -114,7 +114,7 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo qCDebug(lcPropagateUploadEncrypted) << "Metadata Received, Preparing it for the new file." << json.toVariant(); // Encrypt File! - _metadata = new FolderMetadata(_propagator->account(), json.toJson(QJsonDocument::Compact)); + _metadata = new FolderMetadata(_propagator->account(), json.toJson(QJsonDocument::Compact), statusCode); QFileInfo info(_propagator->_localDir + QDir::separator() + _item->_file); @@ -164,14 +164,24 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo _encryptedFile = encryptedFile; qCDebug(lcPropagateUploadEncrypted) << "Metadata created, sending to the server."; - auto job = new UpdateMetadataApiJob(_propagator->account(), + + if (statusCode == 404) { + auto job = new StoreMetaDataApiJob(_propagator->account(), + _folderId, + _metadata->encryptedMetadata()); + connect(job, &StoreMetaDataApiJob::success, this, &PropagateUploadEncrypted::slotUpdateMetadataSuccess); + connect(job, &StoreMetaDataApiJob::error, this, &PropagateUploadEncrypted::slotUpdateMetadataError); + job->start(); + } else { + auto job = new UpdateMetadataApiJob(_propagator->account(), _folderId, _metadata->encryptedMetadata(), _folderToken); - connect(job, &UpdateMetadataApiJob::success, this, &PropagateUploadEncrypted::slotUpdateMetadataSuccess); - connect(job, &UpdateMetadataApiJob::error, this, &PropagateUploadEncrypted::slotUpdateMetadataError); - job->start(); + connect(job, &UpdateMetadataApiJob::success, this, &PropagateUploadEncrypted::slotUpdateMetadataSuccess); + connect(job, &UpdateMetadataApiJob::error, this, &PropagateUploadEncrypted::slotUpdateMetadataError); + job->start(); + } } void PropagateUploadEncrypted::slotUpdateMetadataSuccess(const QByteArray& fileId) From d85c2bf025793ca25fecbbae117f578f5315a592 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Tue, 30 Jan 2018 11:10:03 +0100 Subject: [PATCH 232/262] Fix building on windows Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index c5759806f..386f314ca 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -70,7 +70,7 @@ public: QFile *input, QFile *output); }; -class ClientSideEncryption : public QObject { +class OWNCLOUDSYNC_EXPORT ClientSideEncryption : public QObject { Q_OBJECT public: ClientSideEncryption(); @@ -140,7 +140,7 @@ struct EncryptedFile { int metadataKey; }; -class FolderMetadata { +class OWNCLOUDSYNC_EXPORT FolderMetadata { public: FolderMetadata(AccountPtr account, const QByteArray& metadata = QByteArray(), int statusCode = -1); QByteArray encryptedMetadata(); From f8d805d78be851d800d07fb331b450b8efb86ac8 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Tue, 30 Jan 2018 11:25:46 +0100 Subject: [PATCH 233/262] Use NC windows installer images Signed-off-by: Roeland Jago Douma --- admin/win/nsi/installer.ico | Bin 2998 -> 4286 bytes admin/win/nsi/page_header.bmp | Bin 34254 -> 9390 bytes admin/win/nsi/welcome.bmp | Bin 154542 -> 154542 bytes 3 files changed, 0 insertions(+), 0 deletions(-) diff --git a/admin/win/nsi/installer.ico b/admin/win/nsi/installer.ico index 94753ac296389ae5b90e56608a5fd1882692985f..d7d4a81a15759ca17ff6a4af91c94b9173155d56 100644 GIT binary patch literal 4286 zcmc&&TTqqP72f-#O`1$5Z7$Q!IC;oJJKFRuGkIuYl+;dN(lpK3rY2R&;bIUYDk{oN zKt!Xqi3(z(M$`ZSxe5VM@Nki9k$4M4jaEg_M1q#p@7w$ApHpY-YjGC)`tS9v z`$DBO{yp`S;%`;_6QzEtl$r#>mI?!L|9}0oWvPjWGId2qrXFg`(9#9!&eYP8q0ERg zIk;R&C-ylf@qFpaT4qhw5-~+fR2WHFnri3%DN0C@Q{AY=KUoj?C;LjG!_pEw*p?zN_AX;`MEa)FkjIN(dyQaslue|=sV4AC!Yi2-rM`#~(0 zKk@O*V0yufSBRl&g{~wo5_z?NNA)7`SctdZQu6U)tP_AGcuq`vur36DgBknvi9UJb zImiq-9d9$fNE}ud9H|Fl;25Nm(s(`Gn5>0*Z%(!H@!h&5N-7{{{+miR{Y^>j5{r*A z*I{0XlN!<$Zi%E@^kMM+hH zlES%4wni)27;gFUjCSY%-jCkWvN=M__82?P4!%uUjV*`)#C>Am=rEW&xCi--oCiZN z&<<_zJ<7EM@^6W&+d6vPsGT3S<|(o{?XckHYX0sOx_&l{5CkB zX3svHqh)uz@@!gw^#SOg7-)wg$iKtzg1ng19^_vOZ2Jt%W#AYi58S zct%dOykfzV{9~~fJ57P?J4JCyC`Z*oCHvstRq#Ffq0s~M(geSt&PhCPPgl}{y*AjN zzCaxo&4mvDdjYUeH``}ZXBNknNG*$BvU10P{~W`c{1t4kq&Y*c?8_2jj5-a^!O07K0pWo*D$q>eZHL6J)SF;ht{WMh0d43*Gb6>xB*E_MTN*?i~|@xfT( z`tNTb?q9GJfxI>Ecf4a`n=%FvhtxgaZB79$=!AHvn|#Q@?>G1#lGdWd^3CZ(GCFco zZh!x++&uq@^fx3*=gNl-9)81^=1AQ(EQ1`7jj`2^w`g0&5A~cAZslTr3xFr%oIXbx zXaoA-UW~<92i~3}S1vBiN9ASCd%O>A3#r^)?~XY^4;ZQGITb8wSDiKGoQ%q(Oc&HNYi$C zXQr0b(B*oJEB~yGtdzIAF4x?-b7xd8oH=af#B}<|0lCzF%&hqMdRDSu)3SncgR&rdRuDr@{L}85H}2h@AStgfC*CK0 zhxeKNYnRT-vX_;q-}F2B!q}Y=b9U}Pn#n2o$RVZa&v@^+NlxsOvI_j^ODl;3{E36K zCj3!%%^No_dItK@1_OP8KetCmz+X!;bg*HTr7Huy=4|NVF{w_SAQcNAkn=q|&0grQ zlI&^nF7*B`a=`joO4cGKSG{KSke~8==v6cSERZkRGaisN{t0q<;JDelc=n%$H*L_K z$vowg+V`ToZ^y>|Eq88PeQ>j{vp{YQUpD7MU$x2Rs2|9>*R`yjVRQW2*OjcE;gfTp zH<~-QZVt+^_F^;VFUrd|-A_eMgo)3TS9~__BIhz!479HPuY0`z_0c-n7WasBmL|K; z<;(68$%}nV)5T@qjIWimv?@+W`*R%JR+w)tMm3aA%@nnmBwE)U@!V?OB&)3 z#2m~X@5rX7%QFMin} zxsgATjj+weaOAPfFlj1WDvdi6I&+EIw5aIJ*ys; zE1ws7yzZWyI#eoKV}A^vvc5szT5tSId45LU&4$0wXBflPi`ZwNzXI(=y<@##zHUPQ z!~A>@c!-I87xOo%ck8oqp?#fPJ6R)xeILt(&Ud6M_t&x^9DWJj^iB3l#KGK2qAuLp z&6z(~3l5-Gu=it6!Jdcpj=dK#5l>?hMqp!oWF4^>Sua^v6#5m5wG}!fR>m|b&(Aoq zIB1Uzz>ouC?PPC|3ykbrF(0xIX%G5=vw!Gl>NV;meT2l?y&p0VGkuwJ z)|N8JQ308X(W~ypyd3qu9Q^?0q<(nU5$oYWt~mzr5Cd%<$~$yU-t5O5UN?5~yC8o4 z%|QLJ$74Uc3si<4gIJu*9Qlr;Ht_Gj`tBGA<=qM1UC3X=$nPCy*3-~;&D*%sfxGP8 zc%R>L#5cp_L%iJ}Vm7#g>rj68nmZd9nei@rcf&t8T3pnr)vL`lW_gd8-;_#4lKHXkxJ8BsAqS1;W~;kU9T{H<>X`C4-{DA` bdSmboeG@bcs^s7NWk literal 2998 zcmc)MKWH0g9LDjV6N8TqPFH90BGc8{op$J8YpEB%MkvWdDv$`envn4WpSL6%MXQ^LNCuEYnNE+KOLGOMmO~`pV9URzm zD%EP0qYjk$CrarvDx3aD?}B1w`)B58Wj2pq2A^bnPb9x(9~FZS@>?5eE9rkYEZPQ; zC#_80KKvv-OKNJ7oCkkZeQHca7_lbVapldmq0wze`;HdCN&+d1dR*7LU0LEHNN$lkEB*?#-p zbvB#l_ElQ4*|cpFnZguI+l;Jt8im}#+UGUd;-535w*frDt^rC!?<-R<8%0R{31SrAGbht)x;m;OZYsVl~$|V*yshS zD-Ykm*YP!c1<&G)S)e+91IpAKb)C9MjZnue(BF5ND}lV4tP1CG;ff08Tp zC#BofS2V@snhL}FdSQl&8J}0eb%R__XQ*`k3NEh<5Uw~>7``{28KGzU7guI_dORn% z1o>pB+~z%?^P4A6wkeyNo0FkD7+L4JXMHe~2lEg$`D_4&b8}m!e;E8pR^Ctr^ lCLe%txdGhf=zR40^d5?NRR+r&UY{Q5ep2}P`k()w@;93z)Y?6j6#gPXBSh7#XlRMCsA@~4iXw`rQd1*AloC^^p+OT8Lllu9WFRuVo2r^x zs#>b5UCkge5Shp%k&tu6u7pd6*IUAnHh8 zL?7*k=wtl}=7{keh?wJp2!j!GVhCbS3`MN>aKZ@O^BIYIzN2ve)EM0N`w(${V-e>+ z4i5sx6DHt6;6%g+O(IN2e6S_r6;qL*_!J4kG$e$~Kw^j$5`UbD#51#y^wS(9ot=y1 za|@7s-Wm@t*x=zsTf$;Iy0nC_6pwzfBP>J8<&{XevI?nJ?2-E08l+xbi?pljkoNm} zq+Q#H^glNfwjllbR%G08K*nF&@%W}A9{;@)nYVTkoRRsD3t=}hZ|_0Y?Y+qQcOStO zPyRnda6@*eJF>%$AUpghaw0tOG}4FQi)T@OcoywX2taO3AR!33vB3ld^6rHo@BSI& z#hpd|gP)Nff1Ypw`3VEGCqowyKm+hT7`q zgmTo?R1hjrS6fA>MqOPE>g#GzUtdS4=j&EJ$Q_V7Aa_9SfZPGO19AtNq5}rzpKWpV zJ^90TbBEH?baNSoAG-+3;X1i>GpWmLyRs;5>>Jc93VYvAvn)Xfn$Qdivk1~IOAzSo zrOAh>#VW`BE-S{0FNYWqAeH5{ap{r`$0R3slvtOs#n&$T9aoKQr|&XbST;y|jAvTy z-NZ_EzSoMmhQqdrzE>CKjz1#Rd2EV)FH@6sk`Vh?-V#{IeZ?%5?36$rS^*e+#Vzr$ zmfpDaHVikYDN$kQ>dym{)D=Hau$baa*lCczQI)RF)`@MUp(F_7X7a#N&E)fVaFK-; zdAcS`hh7J7E~Ri>y;IaY?8-iUUenZQbC^Xv8&=+GC*DB8>s3vf?!2i?PdGT3GLH1% zHa{%ZR`^^s=RSfahaBPoOKqO6{LL(VkU38`TxDCJb$e0plDv}HU#wn6dsq z3h?#3MoP9-EQ*djw9^|ar#4Gd>siE2o2M(E@8y2Ic)EV#_Q~|#*3RFTMfnI6f8*@U z8|a|T)0HpoR=ZxOe0}ybNDIQ3+WBXw=60LWWKU-vu+rw~%I_}j?k88yx93Yu@aAXM zqphYu%e|}&-xIu;)AsXMp5m5&H%*!jJ!>wYI#?~>_7|EGQ+UvBnu^D*g_Xm>d(Lt4 z7W{V1mR9j-ekQ3nXB(^O);k)!Poy&E?C=x39cQykxCw@ln#L@(lch9m)d=cjwXohz zOYzJpYa+wiK<}!J5O{~*&sB9D9n**}H3h|V-f>k`zKc3Eg=XV4Og>tEfGv4<`-v|< jT{JF*-dKhmJJAc0Jme0@9gsU9cR=od+yS`*njLrnHb0y6 literal 34254 zcmeI3_nVYO634lp?{By}FYDbYsED9qKyq9puD}unfhA`ImK+5MM>?W_fWUHOB(_VQdEeQYVFs6HtDa}J-?u}(-PPZ&uBz@WR$*_ye1-~T|9v+5 zHzxa6B>VT9e9vV6{h6-)HeWt|mWn_BJJFxYKrx^gPz)#r6a$I@#eiZ!F`yVw3@8Q^ z1BwB~fMP%~pcqgLCyt(^mpXulDK3!zhqD(n=`lNjQ)p^;ud7X^x*G-rQ z>b0p?RgN9pEBEf+l@o^#$mEfO!{&!E_i5i$KA$pH_U+grH@~}P>Mou$LxyH_l-i|V z3zK(={C|*dYgfqit5*cN_5DrRy>+AfXJ|i1hBe=PMb<7`AP0AEmmQnd%FzRR zWXAXrQu?Joh4F0Hpt@{YwM-7|+GfVnqh&)W_sZX7!-^$x=J*klKlR@qOQj@eSITD^>K-SD!>Gy}ZX_Z!49zA>*R_vBDGP`>DQdk{##{GNuWWuoi z&e*EI^OC`zyN>snzR}0=L;FhOs-;3?@^O#$roDUpf)(bs>RSa&-D+>YXykkR(0&=v zrIlH?tXy&w2wyj?kf??JO zYtxJ4o^2W%oO$^y%bXRqhQQ9QE$ii@=5<5*)F|a$v;Mu9Lq9NIpN<(8Qr|nCM7%dl zDJINyB$_j2oROPfogeSqmg>b`mL+pP^D7R_V@;=(E1DywECc$8y!^(DPDS_Fvay4E zN$++aI`8E7?a)llo<0$h2ew!Gfq<=1k|^)*}UfVBKvOpv&r(d(*dm#lp|YSLeTFy3ujMDo4Qrx;ll?uD`I-`=%KKE7uhln^u3igI=@z_ zBF;Pcysua|HzbdI^6FRo858l~ioLrB4?b3L0h2CSG`0=-S?)mP}sHXh)v&j+b)6R=4^e+3F<>{AVZ&_R3 zJvP6(*p3s24>0_?Hme(=yZ!1X8&~YLInniRubTa8`q)pxs$$1qyZTL7ek>V}9zBwd zAEe6Vix;A5w*HI7VeXVOsZp+K=aw>i@-KLZ7gyj0TN)e0`wiYl5A0!l zK3leV%@=}CVBdpMFRrMEuN!19vd-Llb}YDJjZq)JAgDjd#uaNOi17W`v#~FstgI{< z(Wk3fpX^2GD(v}0(AiVR<&$2WqGCmi!|r#;4POXdMGV2pFhu!8RX! zxZ_V?BM5`GFQk6B!mh^NZ&bOs({Jjsj=}bOMPG38W5E?X@DZ@x{bD+lCmUDj{2)5K zcbAOn-#x6Dee27U#|*BBPbGq|+gsL7iHa3-(X>WcW1k$_yVLkd?idtuTQGBqi3Rd( zf5ZmoP95*O+kEoU%Dxej-@aiD6C=a^#r|WQ_-Ed@5;~IgWqAdqez>A8-`&1pjp4t1 z{%nIkzG>qlmLGc2KKrHY-}oHvvc^?Pn!T1-7xG{n_V>-Di%GTHu?>P~=7dp#?HEM( zh{QY-Mfg-;HYZ|!(IxC5Zr$OQX`-HqArLQf$Hv^eiT~n@x%2#y(;uX-+t#o4oyyKlYRm0gKlqlBIW(g~R56M`2A+aw^cA>^ zMnA+m2eGyzUcerVE{xPRCtO8pn~bt#?}PN2nC6Y^p?Zh;V~>tR>>r%XB!bY(L%O%i zkrnzp8FGFcp2W1#>tHO1PhP|ovcRX=yn1=ayyCA$BF?%v?@I*XXXX^I=M2Vc-vrI$ zlg{tmwuy<$2E|A7Dz5B2a8`@Gz&+JzYlwzGY?C|CWt@d6D|6Dn&7|LdtICfaVKW=lKIlF zW4ehmU%GJ4Hg!_Y{qW5ChPr*tBYwwAA-~-@D4?!q~E?eKe(=Bhq+7l=;U^Vt{d#x%W&Q1!b+>=3zvJ@Z_HT8!Dc{|GdC&2V%a!i@wvq~+ zN`?A!eD)62)P+iyb$4Fg-*pAes*psUS08T#A4 z+5PAn<$;yS&u^CJ{NZICi9*s3zg}u`Vams6eXU!gW~<=VO0%Q#AGtv_TN}?U+>L6a zC9dVZ`Pps=Sje{Ng<>i&VV z2*df)U9kT0Z2#ri;l4GKJHI{L{Phcg?#Ebz_|5+U?%l0l?|CFr`Y^{k3^e%wG)B{% zTso7#nwsOYL6!@ZE{?TZ?u`2wYc&;Npn_VyJaVGKB%UV^z*#Z@BDbSU^HUi=SnJ-> z`H1wlBq|Nf9~rFt`L}&@KGr##YiV$LRYh|Ox=U!MAzL2ovLcO8uRTn#{_<@1$?3x4 z_l|dOAMRh-GrY8`fAi4r)}i5zL*U!JxU2u&hW?i;l^);J|9M~c-jV*ZQ~hUW`!AHb z4^DKyTH9~DX2QeY^7Y~7uk}9=?%mD*(t#}ag1C<=NrK%!{02|e>)|0av@(|q7ViI+ z7YF$h%>73BU^GS~0?@AVsUd~)PPL!y@^)3LUrVAEq!Y%-7APwOI^{p4l?Tm2?aKOg zwg6E@1!1_&^3bpjaHalSE>oyxtqSHr&ncKrcLin6(hiM(BCxtI&kP?O@83E&ym@eV z`|$AAfyqld`$sqQ&ur^o*ww$dyZhsw?z!#Vy(@-)yR-kv?ZaqyY*{&lF~(zM#=xU zt7~st-|{DzR#dqA%7WkpNEO{not$c|B{PYDg`ZnTK_{y+OvG zCP1rf1nVFrJ$wPyttJ^kRxeCBmrw$r^U67kW*cLzl5cg-?;74Y(tUKQfAc_p@2dVM zcMQK;)04pT%xzG3czS#H%HHmeyShg=cBgM0_TM<{xuHLId;jE?p2Ao6a_k@9+&wtc z|7vyr&^n37%e(q_4-X%lnEds~0Z|cEvQ>+hMrxB0q$^R?ZU>-ufWx&z-EFIoBDsfy}S4B{?x7Aj^+Kt(obxZ*M8 z*!0PdVzvE8Nh9fj_57qzwH@;ZMi@p|H-;8rpqgPS@Qr~Ufg5c<)%mSi%TKPI4iM;V zb>@`RB_A|7m0oQ>`70`&3*xDuBN~9S0VOid@?p5zN5 zxQ98GE($kX+izLgZ(k0qJwTJTTsLfA-fcJ5J8tMV8O^QNcAJ)|!sewN39L6<({EVX zw+vV-ErsC2!SD7zU)8<1a{%slj`p9P>KUMcHODW{;qVVXy|cgN8^aImq1A!dNI}H` zv&8}H1{u$9#37d`Jce&0Ob-_B_ z+j?HrDo;VPU7HF>HT9cUmO?+Uk!r{~W5^=;JdBK2A@ba6G?<%;Chfw@`a3ap)<~(a z={g`2;u|2WaRktvHvn0;?fQy7@^I}m4}Vpe3#IUt2W74TXrSM;6#dZc{ciV*RsA#D zHB!T2d;b{vyk~I7=q%&$dsgr#%UIiCWArVKOKVA0LZza>V~K)t1w}|TZni>70+1mV zaQtkrl2N~zlJ)XAh^tSqdI*VROoEj9DrrQvAr@I&a}0VYtPXd#=c(GTf}dQ{5_JkX z&ZM0NVonJ%%ZVPq4?jJ^U!-LLudG-HuFYQ>==F!#Ol(;u)V5rQjqXs$Tj1}t!#3rG zBY6@)kFnOz>(Jf>Yc9mfH>^Q?i}d-h|0Z0!$GMq=nQsq{QTsi zAOC0o?(fMtS7^oKFN-Cbt%Ndq23b66x^8>~#}$hi&W}%-n2wy}k}=lK7gNYp{9KUr zoSKKV7#FOO0bAtURE3qZVPy!FBMuS?MU5@ajdX2IE*IQM5*|C}WKxnwO8fN2H>PqD zfl)Q&${jVnZ_DdiTWZ?G(<@jTM=^3Db_}hoO^CyYa^eV@{jFhI6gCNIb0v!3& zAw%D~^P}*KFc^JK1!Wywb1W*grh2KgmNs=vzu9@S1k-U6BDqlt$zm z0~6H>(bAw`vIgK1msm5~W>H^0o6iPo?h;sK-|JOeSXhtz!m$d?^xCy`u=e0*8XRA_m6EHUft7wa8v<0 z;<4`?9bVouJionrVC}HyJN>G~z5c)&LE;Z-aHM1D6XR77%-MEq`07tn5*(ZrxK1n3 z16TGo>vGOH6%$Y1Kc}G6GwASuGCdZKP|Z9AM0%K-W{oeAMhcsnF5($PLkZ7(tX(Ll zI#$?!oh*p3MmgK&V#p}xMoP$%v4-Q4pW0w>R67+=XY}WnySuUi9hY^O=D6JL7_2!K z&@1s%GVeH52-jNYqxrAvXJg0l)v?{n(D2>v>xQ4)-oLb~d;8D;o!>v+y}GY|YIFD7 z_1!&pGHUA&eYfAWd|18MzJxte9yLJdnXAx}Ng+rXBj~A2Dz};`5ujJ}E3fQUUeV25 z*{?R%fYl@;e z=znR0upU4S@wB8?W3qsnx{9M)&>ex#_1jTouL^_=KXap8OUbSq`kjoeuJ2n0-+TM@ z0-CW{kR9vxl;qS@3CLtCZ8GBcY-Hl7ufL{ScTKl_S^tx}C(mr}U*6NdzQ23paQ`N8 zwgcVoHuooQ8IF9vJNAR&*!PD6-|c6vmJIWF>8^Bs>2P;DseA1+7;9jX0tsI3eCl8gLw0W39$<{MsNGb3#kw|L zh@YPj)=4)7)`qV$HcWmDh6&t}!hl<_wy3D2NQK5V%Il{qf3w?jgVNsR-M;Vi$>#_< z3o<=eb1Diya1%(eU)M7nlXr*PRz36CYdYXYpCA5Sf6uD!-|y=1z<<87|JjQE*K3D+ zR}RN+CKlHn{BC#T`~9)+_xo;~tX(2^A}BW$)T@;&oLX^3mvpy-rIh4sXp2RF&W)`R zjN0HFKx_L2?%GJX*W90_htrvmk&M`C_{h+{uomlei-M*qD5)vy2jquSR0!!(Ud|et zf}Pu^PW!0~>$P|KavmUOqtZ*>+8~y_T}uXYzjt|00%+jgeS;I3<>>aIVJ%pfDUzmi zJwxvf?sGPVKIk*7mL0c$4#6|lhT6jVYT)iko36#Lo5;aiN`+E+>$Uy1>rmhdyB)Zx zKYUZa=SIvonmM2t1>4;8JiUYmzWR!8)fL^!D+M|wImIyq$26~847GJFg;XG!<2r`& zDV&FO>etqUSIw)Dz*mpllxbY6bg^5Kz$t*H=c&wvQ#1uMXLZWTLCvZ%pkJwcz)H&v zpcHbXBu%NrrkO^BX4B!}Z&U&5h*ZTuD1Y7Qo1$||yMTB(QB$P`G@ug=YtZK`)ksal zSGeue*}fBeW30V?Nw;o^uqHnX;x!=vB;ZB^AQ)V};|4Xj$LeL@jos|h!oG9}s0Gss zkgf2Qh1w~k$J*HtnM(pUM_FwrNyr?H%~jJ2gIAj3)~!u4^HRH4%K)W_W}TVJp?uA& z|IPaOO#vKmpttA1S9zTt;K68z%Y!?LBHEsxO_LXSu;~fqIYmmGmJHnso$-W8gwA#n z9d&}srn5|~fRGVa%2`9B_&Et{E*&24-9S4Jc^x~gM{~Rn;=!5#PQzO20nK)N(NPoDZCSoM<>+l^0-4!i@s@$DYm-OfC+t7AzG z$0(U}`1n-HQ!QN`xKl?e_cA@R#KuV_79m$CccDatb%Zk*2|;sK;!z{yy8GosBwX1e z_W-S$ma!Z)ma%`xS&|4Y)=Lh<&~h||!{|y?g<0ToV{zjZXr|ZaB1#OCc;Es|di_`8K`yI;%)V^IWCL9ak@$KOnXvFi{~t!Lf8~v)zEGo=ZEqFfklOD$yOj43Q5QE{OvXw0RA_9l&StWLPFmUX^CCL3H*CzOZX z5OBY9d~a9_$|?IW*0$8fOQ0m7E->@}O=2Lndhr0YjG2F^K-jY-8IHk4&s?(olI=UKy3w<&*0d%UhMv}3p z$aZi?^6Rf2){vG=l`1_CIwS-DxEqLM06JaDF+UA`L75WJw9=kl1*6ki8f^KB`}A6{ zbDMe+WP`p{rmA|Al*-f*tiVU*f^QWGPRW#fd$_AkH;bFrv~kyk>17hR)r&jql=z6K zAU1Xab=LKBNrSr;k`cc1k{t;uGpycqy{se&92?NEIGEyW@3=wpp5bzifn$mhsA<+a zNe(uFwLt06IuSOFxD{D-utl@vcG}ZUF0|1v99PTN>5lmPIPG@TO2>koAqx#L!GiDZ z3nC=$^Kei{E~!bq-Eeek`YEPMlNWz4B9wbYy^pA^Cjmr3;7n~fHxkyC1pQRCQPPuq zo4bIueDVTWrKD?e6_$%ozn1h=G)t@uQcsD`Rh8N_l|eKmNg1k3T{xH{4RDzqGpTWL zxg{L$`DoU-O2fAzTdb`(%ivlF8u4Z5YC&UN$C0U6YaJdF%LD_HuB8i}-{d$>##FOiDHLIehrV%M3f+3=_Tx$)<;bWya_lh`i1T|3hOo6Sl=~-mJ zE_$w|X1x(jl^IXP&+W@uNOX&WbsXT)F=J}ESk+G^GY*Q97dI^lg3MDPfMPUJ;8PD= zUYWvZw$!G7{PW<>g>FYTV(v`GddC8H?jnt)y^h^M4>}0U_~C%!n|tj=YowOJP|6J;9^^I ziSH;O$*o~xxGc(BV+ZAWEM6b=Yl$*Z5jkdLojj*(uW^JF6APnlk&Dlk-fo?5tPA=S zqO`R*t#A|E%PgmX)~ci|T&Y-Vq8xlhYAn24{aObm2PHYtg~=0APPq(kj+qQiggtR(|llxgc&g^ z>!%mG6c^nkmV|l>1y@IIn=DJLzl4MbBhd&Un$=ek7MpS$>;j^Wn$Nm0iP7`w#0_)h z2#@Ag_(^|qoCjg}Zx#bI znwCPJ6|$7TcL{*SqbdOzcdEt2a~8!OwNLv{UPgaPd7aW7?F|ZdI|gd@IuO^WTw~|O z%$hACRKg${HycdTCmdU%uaJjOJFz%Ow{o#UcCz3a(@|M4jeC`RS|Eb}kX)i_DMxsG ziJ**}ddKW6vmV^vO{LE1HDQ-tS(yJk|1OadSYnO`PWDW5?JG2 z5TE%n#%}W(=R(SGqJTQjD{3O0-QUlOg5o#$}>gk~l7J zP2^=MqX=sjtHf|mK?iQaryh|7>sm*VH>KJ>4<^ZxdY@5eez`sOxc46>IButxc6>*$A%{KQA~2 z%`u}3zHYE#t#Kb>AT5D-FrlVNvFK-Is+z6v$=yN{Sj@ojPMr;o1=)&S`)SB3pb{ag z>%gi(nNJ1B+#+>S!clPhs#+qTRc@_r=mjWvHPn*VPq-fmjw(b?_N;?5mxO1V)#XhqeZz{wW)9sr{ zCs=jsDBnJwxhNdw7L+q-f|b9nS_GF^rPK$gs*5=aYXhoGOPJT&(31gICeRQ@AN9DJpd7 zz)xD>m<8mxPr}-h*L1$ImQiR)HLd@Jsb;jL0fe<7Gl;SdIUB@jhl$a0-*T1efa$oJ zTJu?7)vqD`=?F8N%3Np2l_Q-7rsP*wO+G%W{AhC@FpW}Cyy^T)9COLVD)_7rEHD+{ z+wqf;zYTVo;!Hb6(`&-3-Y(!;)l)})8Uj#PKvi1|o#zZ{3LNHXSznZBiOCpZMtfsh zL+8TN>*P3X0CzE3)0!w!NX^-3fI{0y4$`tzP=bq-w62Cxa=fQ*08J^CI&$GK$IN&N z+?wKy8D9%S0b0s|rg$Feuf3`xVZqm3r4?vOM8c&!<)J~+usc!laJC%n*R8r@SdF1K zp~76RVJk{t#O1=frZ>psY`7Mrz_)=*X6{ep>-_y?9~G^*Ga1r zC$XJPq%@=k#G=pZz0ky{HG4ANzB-E+lwjDPL!c-PKbCil%iE-@ov5(X` zmd>xBi<0X0LQG$-2&bOk$aS*su{#A`rdl^VEU)_p8Na)>O0?g2Zt0Iz-x*57caX87Bv*E&4q9=io8&&QR3 zj7SCtqpHpN;Mgk2eqnc#D-1^4KLSbXd_xYK5{*Un7Uc*DA8Jo2(F z2I!EEQ}TF?^_Y4M?jT|;EDxqC*IJ)zvQpQAugx?q2(L=aWC~s;X7aVdoSNJyk0cP6 z1bumBq*`CrJS91^j&xteCHf9#0F4+4rJsbAy{^%m{(sVe-Fq=b$~1yCZ(Kf z83hWXhIXeoVsaYU~&av86Gs|37 zj+f;q8kD(acIuheqIL$h>1si2qb$^{?QyE5vArz_!%c5flWIgnftzXuIxulca6~*R zM&*)H-da#Kv_w%9%>)3qOnH_v3T0m&^Yw!VZ|5#>FgEscqQ4Si!dC-TgpPG9*KS@Txk#<8? z)H0Bvr4>4>LAhjd2{lPHTYlO`-<5T(Ra#tMhQm#%GMGk(7oc&HOYCv#E#oXz7cSux zdDo;+3F3^gW?XEn1Eo(T;29Ph>oJ#WBSj35gzyf?n8V^2>>53%N~2og_5d`+ljxEu zqishd1C`*`Qm54%UzO?O zaMnYl5*x1^7p7A0hy-R@OZ{!8Jqiv?9Nm)w%O!=I3R}5{VYFpK4z^Z?SGZxTVG;pV zXr79gEGDn3i^V}ZZz5^IOBJZ^5*-6Bnbyc0tktrT+QA#Yx-4U+jvinsqNPTFdj?x< ze8H5;>x3-u*{ok>CE#bvm9OO|ojH#$1|{H5Ihz}C93hMKm)AWmsgI!x(=6?m z3ce{o*LCdNVz5Y9Hl*#5fdQL7d$omX?t`_51a46FPdNbVu9~0+_t%q^sZvm~I=Pn&fri<0#&`dL#7t(J@8 z1(75!Pa_3uYj`ts4vv&m=9cmW>y+oIjjX~ASfc0+xO;lksu=5RtZ@(U%EoYvv8Ftr zH-e9AwmeujZ#;#$ioE0DVssK%(_=6$UDq-yv@*n<7R@!IF~n*cTuPK0;u?di#9f`% zykC#P7A>bA6g{gRUD}Vz<(RGxD4{&;5D-a5O?9Rkd&>G;6&Wzx5|V~*%G;602uvKw z>t1eF?3O`qx0q^xb}g6qT0K~xuO(9vN@dDRg`%49>j=v$v5X4!4x{Hm(?A)Q3ut-c z#a_=t-(n!8UGY^W4x#v-6)bE#aRr)f-7+r|6c}MaTW}!>Yo;|N8)f1+SX(Dd<0@`P z@)N1h-MHGK4WK}3UVUt*3&MTBlSmTJCM zXR9oYP@YoWmFGAQ!8#DPBh`W|$JRdiBuX0eDZ!=3XKTP2vPQtd^qduK9renL)d*Ih z-x4^I&)Hzz+JxN=YxxqSxjHSJQcSy1UXGJS%43m`_SP8LLCC6~LwVWpsjqI7JyVI{ zMh3FtXfDT9C&O)~u^K&#$oalVNT>%!ZPY_gM!NVDoG&^`k}#rS8su1#mH)iIy1^85 z)-AT9mHzP&M_pmn3L=?Y5?w+4wO5pqL|@ZBb*RZw8I8)-j$2rwNSzU!r^btcMl&Zh zK*eqs8djZUMAqsA?np>914koXgo?^ztk*7XIx50kl%YFn4$77R8E(&0OC;!Ujv;LL ziw3|_k4spo0?^0+2+rfUfi7`!h{+vGn=vIL8B6&Xm)x2TCq)#gBTNNoQN^W>v33hO z71Samu3V5RGT^9M>7QWjm}--dT*#Aft@2k*s9YprZcR)!)|q%qTBFXot7XRt*HlUd zm#JbOW1Ttyn(~GKs zv=j8pwZ__VA_HNV3cmT7W9blM&7@cxc{yLYD;!^YWoZPsB`sG?r&OT!g>usPj^>nL zu=Y$rcr8gn0~?XRU1T6OH^2$&k!zjG964+LR+a9ROIe*5tQ+oApn-LSwL?ZzF`yI1 zHPv+>YdcPK1q!QUF_ghNuG;?(tZA*J@Y+~|8geqmI%tk%V2pK8&apLUsW2`TYZ@E_ zJ^f4nVXWy2u2MVy7}iCf7;8n^nM~0Dt*G2Uqht2w8%=(K^>`Rnah#ffQZfp>jD$yx z%O_C)0i{RWK!0_?bfmQ+jXGk&D$^~={sV>kG=u`v3Az?u!*0!rT6J46JUb>wPRiUR z!HbZZ26rCf@g#vYsw33q2uvi^pAM!}lArEvfv9AiV3doNXt;AZ&5|hL5?$rW>!-+e znG?|w&FsHdo#RYV(R5VG{UnLEP@=Qe-qMP-1A#K3bs8gWl+Mp&C{DE|D>rC{y+;x*$y1FXoAX~Nf9IWa2G1kBrI5;+-$5?Z(;A`|t zo@dr$#FSLE(4MMg##n=L;07iwU|7s95(iO@d%?QUU(DqeIr1lyv0Uly1wmu2My{F! zYcj-3qai7ogS9O?4T3eH73wf`LbHs>m^)j&Wa5>LdEpPb-C^@OOcUj+;|;h@Q|Ozw zh8RR1I7TG$N?5)Nz>CCrA=M)rhkNdt>{&ix0%+}$MQg6%874duZd;xi==1#f$rH97 zpvUOR^eS`~b0N_(tyPd0=QR)6$I*;68R)EaEF{0{QkTo17ikIB6uM=xIPLSZvD?Hf zj0OT#P)tCKCf}+>462Z*TEm(=7Y06MmSk)a%`Ew$P@Hujl)sn6lc;^CF)vC!l@||q z=0r{ax|MTnqmOPBZ9vPU#RK7Vum*AN$W9jWil}?0%n(;Q4vv@0p^`IS7nqHawJMmZ zcdq4Z!TMbP8SjleJMeDUFV9XsIW_PG_*eGy?;o9<`JqIeAi%}2Zk<3iIxRiCAl5(+ zf5$-2%L5QCT!?kr)TxEA<}4y%mZ|eqc_FNWG7m|{O%<$#xFc_z(w_jb;k7$FgS+4x zT;kg|mBJ+#V$H+QE%D6hx9D@KMOqrxD1TM6#z3D_ft?F$mQ0vbrJQYR#(h|?-3{Jk zsd>+s~{t+0C$ri8mk7v*X7@Lp!m zpF?W1x^yOkad3=(&Ujgx+~cn;*G=S&YdAJ_#f5lOy)zzvnUY?u#C%y?`T$gqh2Z=Q zh(LzUPd;@3ox+p=6I~jak;blLF_{Ke<7`rDrvF>Mot>8nb_MIoW&I3W<>BZB9XMbl z`iI@7XzbJ<9~=5it-2?Rdq6vBDm~wjMJFeyPLvf8E2iQ&v6O^Us(LTuYc9DE-*E}= ziT&Hz;rz+r_Z#~CH=@jKO%hV@n4iC;U7KkZiM4{i>afdSZ$BL2*=IA z+5%w8TlWfXGjy|Gjr)&a9egi_byy#)IRdvH@@grBV2$?`9I2U1{uX|!f`{t!fad2I z>+-kT&!J_VTC5Sb^sqHK6Q&Jo`fWk1S?k!ariGSb$AfHiwe||(*VXu^;TG0T&7hMh zSbrt2$GC&|>-#2;t{?W^IGMRxo1}!V9k%L$i1qwwJT1?bznu=DqhxlCo!l)nxq+Uu zMxsuq!toM}^?S>klh!h73V@`{3$@23HD9!JSPxjbM37Y_;0z^}5FKN?EdU}o8TvWk za#hjl!OK)6Ra2Coe$R11|A&EJ61G>b>wGGNs)xPSXwPpv8Zai==W5GF^?}^4p45R_(X>!-08XU*ShOHb^EkZRuR%GL1h15bV zl*K7&^~zyUtjj z2ku}^9syVoYk6OGq71aKrdqD`B~!vi4Cr})4mxA3DOW;(uhQFEf@ZN2vG-RNIVU;f*>^;*3_RRh-Z;d?0Shbyc+QsT-Mp6ExLC>tRZzqUJZJ} z=>Yw~iT>l``Zy+VW4r%P*lo}moxOj2(JxTx?4a?WgHaey)qHiSsrvWQUGzy5G98Yk zCIAww%K}tK&)7~94WLsaf(OllFR}ClVJdbFYXNGb?2@X+J%(x`VPt)(dgBooDkx*V z2mK%};0$-5Y+U9ie^Ho7KIIqwY}fko0-p%$kvJYWXwF<&Mq~}&A}92oOoQ%1aCJr9 z-rIQH^mZU_M`(_Q|MV1}W6%dQ-#yy%?Nl^5@ssvNt$)gnQ~AGs+|$A{TiHQ?N*A&a z9Ir!zGJHtpa1q_uDQhPYM@d`!O0cV<2Jt}|5UVDa|JL9hW1W=`DFK~kwb2;*S(#3r z%k@+4nZTt4rJ)Nw9#pxlb}|(D+8Apo|A31%r>lc3zy-~QV9QXRd^@VdDVJz9GPc_m z986=8vDe1hm^z%Ek+`*@=bofY^hj$J$@B=$Ivlt)vNqWp#4TeP(C3feI^4Z+pnLOR z&&L+uJ=(o{j1MArU+4=gJv1j&{!cNTkFoNhy=S*ic6|r`T=sqyv?>Qc0t zkpgee17rn0D^e?A^*QCKA52rHwot?);HrYJRUMM@ZowW}n43sx=sa`PgiecAEG3Rs zaZ}xwu0t@cq6PIK6BAjB9<44VZ<{`5`WuJzxlF#I_0sO{&wKkn@9F=%xBKg%9_iA~`P*@0?z(`n`Q5f9MvPOxA#`3{hWA;#6EEcJ0$6S^aNqEpF zv{U@Fcw*)5r9Y<+GXmX~I$nAa2E^55ZR6x+7H%3%DBT;GBVSmv($x>)gUGG z8=ZzXZcGVB+vh28E04CEU4kEr0@SuCVwGHu^#^4Jaexlf<)l@zzq%KQ^=Z@>cXof; z)iK}k${rn!c=z7%;S;{Hbh;;V+ao5vls-^Aegfa&0I^NPz!&JuPXS@u~8P*Y7Ib* zJCaX_$g%O@TII>u&k@|U9Lr9ZX4YE)UfR4$lFWT+0Um+4?6}z-Kkc0(d;u9=_b-7m zxbN>?-7mQRv}XYB*ADPm%Kxr?(G>zdNkGTCpRd2MQ>ac>%GTaT#zU3EDS@oO@4A>uQhFHFhHt8F2s0 z=JGQ@h^Kb;$O!AYg+EZL(DpiK=^T|8)nU7A^O0(tqWyw3n0m_(jnkPyIbiwm?5Bz8 z&Qvt>;q-M?Or_~tEf zq4dIz;SW3dKkVxMu(Nw^Tld^n9bedi+VA)Z_6NuN4^Ivs9PdaU9>)#nIlg~fpWFLf zI)4JkKRG$MXC=Ot!s!emT6CAc4!EV;6Ww+Am%oA#9PY=eEX=DKp!KtRQXL3nU<5!G zgHT%vKkQ~?*Sz%9q&$`B86h5h9+^=iX~(BpB;0DJ0Zk@(T5y*}5gG1c&8N1YF;^tz z!X=tAP>fXUjt`~_K?kjy`J;@cWxK5bIPz2+A%Y~qdM_N8mXM;5aHeuRRYFx6uW`p% zaE@;t?D+CEzS$cGm0mw6#9_3s-rfIk=kVf={)O!W>FI5J9JzaXYyb4N;pwfzQ(O9{ zwsm|l6ES#YcmLX6@tsd+LiX48cdsAllW_UoJuFH1^yK8TQxigM=!$z*@clImC1RazL{Le#kJA5)<@p;iRROMOexY9sH>SL7QW&u~ zSXZE>?W=|#nGY)sIlcCOT4$z98^pT1Eskg_UW;gyv_zpQ^PJ{BSQ9qkPj(Aatofrn zCKYf|1=0phT^MdO9^FMvl_M?k;gwup`iE1NRUPp|^bAYTlfz0|@VDUgA;w&oU_b#c zzLGYz-~j!H?cMV`^ig{Q-95WQspX%8^4q&V?$meBU)tTjEC;W9Z9m`M(&xASa-hRN zkkB6Q9Tmqv@kjSy`lly{w+{7>ZkQarNy{)~chVJJN)3qNxPcDb<;WjP%HN18kQ<<* z$XGORXl{L8HGK5E=?`sdv`vhbwr?es(H;VDZ9q(Bid|(8{V8d&2&oZM4+Mk0>l^** zfwc0V&#~~DKZiVjen`sE(vlJ(UfC^7YeXg86}X!bZ!I7;po}}pgRz$o-+)eg{}|u3 zWT@4nhe@CVQs5;Ejt;j12W!%^+Zjp0@s9NDw*L9;5(A|j=y|@FDXbwl*S>yWc>SQ* z4bA_0uz!o=VRZZO=^2tfun6L~c^~2Ioxq3;53QRV`JQ}c=^|+}hkT>Y*|ah@CF+Ys zBZ==`2H;`4tU28x!8T8W6j%fiWvV>T19sA+@B??P)8LjTpmxk|GE=N;&kJkci)VJV z0yodlDu?U{0F^1sDUOsVL>8ef$QGR-5GD3L8u!Rgu8IW*Q^wo7(jfJ3^>> zE|K4&ht#}k4C{MGdJm^qlk^&zf2J7(=h@t$8vEw_t7j}x;d{Ymf?R5>+ zySI)=UGRBa)-p+B9PK{DaO(s2J<&N3KYvPye`Lr0gJY9-jx2h9*W_m_CVOwhn8Nh7 zoI`eeoPu?ZVLfo82a>2+O=0P9P%yG{s2$wP(Qp^6FR<4ZiE?carvpJSXmV8$43tHU z&%qi=Q$&=jY8^iqa90NmwKu~vndUX*C-LYf%`aD_3s|*&!41Zf=an}#bn=&ikoea z@gjbxu##N&x`%y?b+$m_djNEXmS_ei93f#5ynUEkcm8~ zj#a%NE(TiKqXDg}e5!)NeGAP~M;EGu=??|Iz^gG)KRrCRTC?yJeO{`W=H*^R67&aZ zxkwU)wC{n8$~7dnKM=&x-b@g0U0xjK4~{8#^7hez7#i$`+JD}sAsB-%*bV6O5>C6) zUde+J+8a{u*8uIkWAej4If)-GSDXZ?iOwOQionIcF_J;%|jpL~eO9{0!oKgi<(k4}`D|qdWMs$XkJXwq zqWp;a@z~MQbV&Yo2@*l|a*1S&Nu^I@G(p6M+f2lE!){SqP8#-CDqRkn(`%3yLoAOD zuY33lF1M51Fw`~r!H}IlrIyjs47cI$=VylhJWHAbHxV|N{^?o44Z%M=p(6r7`QKU5 zdS+lUg~U~VJ3V>l@Z_m2i%#FF#SHjkt9V7fW^$$K@AP5}MGv%EL#d52@ZROzYQ$7{ zfkp*%aG}R4ZOgEW__^pQF5BX#j=7%_Bc*nzrV=O1hU!z2!c?fps(w=Gc}GrwJET)# zl(x8RB2z(0;Hu7u;}tt$tx>iwku8wYlZ!5u?iVvAimnQ7M@?OAyNMz_(SLBPYgQY# z9RDg1d*}qX{hJu#13k|3JJd7{(E#trf$uoJdqksYfQFvvWFU@R=7`dshw}I*aGyUZ zlwtSzGn#0-Py?j?7ij*olaEd;`qSS2*K7JC-=FY-A-a@d7k?TQCPj^pz4yB>XG*B? zE6ggLZ-A!O9C`+nz+@=xo>j0KYUl(+f>aqTRo?)ehq8F;RIn2z#p@L6syfORvW1D$ z*2HM)CmA@Z#XzV9@wX(fh6OTq)mJYrL2Z^EhER-#P`1JPoudOj)td(=tUw@8hE{(Y zG;MjIKWqh&;z%PQXlc~K3B^u7IN5)InMQ{n6Fp^@ZJ^=l7~GcESuIL8)+Tk9Ok50W zDu3}~$J@^h#NrT=&rT0$67>AP-Z9y6Eqw2Jl|h{X$QVr%f3O?#jn_&SHN41xHMnRb zZd=^2Zgm>yvC+E3WC!b#tiTv@+?e+v5HqZ7mP zn-`tDnJ_Goit>)=rp!_Gh&?@-ZX^RPBz{`=ikJb#oV7@3#Z>C>vp|_kA{8_ey}f}C z%~Vf21PtqGytZGs8GYMsori&mxk`L2!Qt|#+&7im5n+?SARBh)3UjdtmCI=g-v)i2 zL1_D9zc}47Z{0c@^-BWa9x`*``BQ3&%G9|Sm(vJ! z0<`Ys3pswGNtTXl;p`O0xh8(WaP?o!TzRbYva*44G#2+B!QDlzkt&sBoP>~qc9h?@$)Yw#YUeN+yq!ih{}TX|GC05!{_IQf4g0izwt|va9MD_ ztu71KSC_!@lr?}A_~_U$9lX-dqI}Jhfz*^xO_0@7)Z}uKFPU0%1jb0IskBxu)LoSL znNu7i1F0V<+gToul$ycLK@trL^wXN5aa`1n;dS?ti?6OWcS?{(f$FWuF!>*lZ>8CNCX=BLLb%p+TAKJ9eDe^Ylg| zBao>lg{e>rDEt)DYphIX4Z6;jyefc>E^&z(tXQ{9;8Qi%B$G78I>@R{#h1K+w*9r^ zA)SN3Tq`YP@`}PhtUTvoDK29#(Em7xUuazRk4no|*8Hz? zg!d=sZkw#VTFY^PSdMy~Py_VoP&BhA(^0cfFuvk%1b|FzG<}%7BCC!z>!ck_P^v=J z%;bSvm;0=+Gm(vKVOc|&On~M{o(sW_OZ>Ak)Tekz11)f)t*mY>nBCK#YW(87 z1_g}of*AdG6@m-5kfA?~|Ey8Dg0x!V*b&$v@B--PwUYTOeDatNDwAmK^97v$;csWM z_i)kHYZoEmF=`+V$ALRBRU;k}No@v&z|9fJoDwA~KLs>9$46($TO?Y2{cMfUY~ZLc zm7if>?pW?yrGok`md>eqVR_DAz2`gn%rUplV{=Iw{c}5+OC6ruuJp{7;n59DfA^1V>>r)&9^KGAzNve8m!@xcQ2^ug3kL-(gyz)4 zYyKBKgG<(IH&_!z$8&?-D0B688Fq!Xf+iX}aK{#tZ?rYL3&**ZQ_<}PNUiC@ku?uw zxrh@yYqK9X)1K$gsCqc9=r)v;`MxI4theDA`y`d}ThQbHtmtci|uw{PR;Cqy8_> zfOyZ`sMf#j)iT!?cl997L@Em>p5EL)zOj2`R&YNu+daIl|Lywz_nUg^5J6?S+(Gzi ziENU%#owX)nc=~i$@*(Va0Sx{s(YdwYB#t!MPC6B6ceU#!(>LmrS!SV>x5=-+<<}h z>L%A0>q~i62n|ijP5;PMxlt6t6iVxd(Isjv;z1qIR77P7-^9cl(VjmCN^ldaP zXAzA5CNCNS{_PB7ZB3RDo7IXqmNu~#&Wk9>fV&pFZR?)i(mk=Mdu&7Z$V~sxy8e+3 z-Ba88C$>;wc>563@ruvzw4dPsobEsJT%IHU!!K5Wi~SsbZ8I}vQ@GK4rp7$1g?=^A z5deNlDqP6tdcApJI%`(2jxDg9w9C^}{aHLWex|`7Zocb|s%Fj5rc(>tSFygm+kF#n z@}~l#IGqPCSO;+*xrAjQn$BR~2P+D&kKlV6+!f!zRuVyFKAMGO%z)70Zy%Ywb%dp5 z1Cy8}x}BIg^PW^-F}ak0W8h!O=zp-nmuKpm5$gIJ~hRnH*U4^!$$g znQh(gHg*rM?;l=2Ji4)ea*MF09EoLZ3`Sqk{!@}$U3-M(6B8zG2>3s+cCvLD?A9n7 z!yUL~wQHaWrnqmVfulclbOLC3+V1}fs9w9O_rmpgW z5YH?8n`SeKlPmVlB-$(cq*YkH$eLAltUSMScyd$!)D{-Cdf6MRr}k)BBUVNzp8#AL zDI7MG6(M)o2(_m1`1{8e9r}TG8pbUTyFnbX6LCtvH3-z<)qDpajPlej8>xU|G8TL( zaiI-vRro5jac{I$9j5@@Zc2sZIR?8NBT*T7ZPB;jdFhZ=Vy|ymWxZ36fI(;E4IPe?#=6$$S>@=JkPO5y zD=eI2#tlg(X)+xNJWHH>PQ1mT7?xg4b}fh9TJ{ij$4Se~2#la_)7#-`IhtGxja~dIm40%YX#3U9FLUy=QsI6@R1#!sr;;$W8wCy_H;jiUeZJh)DIc6@1(}%&xcokd#3#{ky zx&kD;A{FU#?6`kw>se51HA8K1m%*)wTfmTtT!r5ywIGg5Gw1=x2t=CzGPs3&e~01r z>{7kNn9JuO&Ls?Bp}=1c;(ODda$nzIBahbePg54Cw2};hfZNm-xnMUd44>N$vh;la z#3n)fQy6V;YU!7jLN*dhA1gvbHeG3+Z8$>T`I-+lQyN z_K$Dre!to7@oWdm53KDUoEaXN8SY=B5%CMV*bQ_ktTD}C^qJkfM#F0H-+lG_6>;Zdq;=Q(BU2(aGV=0pCbb_jv)PM&!V;0l%2wW zPSTQ|8eO4R3z{~ymEiQh`c~arDu{FesNicxS8cCqoRw*|uBca)QI*9&>^8X4$HKEqcJ5WLRq@l6xP z-O!mB8Ury!(kidj=LI0^)j~0Kc*QZoFjc$@VODX>ubUbr_ z#!bRU0_)#7zVX;>_sDue9K&zcb`P%Ye!Zr9aGhcutUt$V`utqSw4(_%hhkt#VN3Yl zz$9Oqfgw@TnG&9dIKUh1AR|DO*Zwd$&WQ4aJb!#|-%-mQ z`5Dhm+lo~_;S{lSwyLdLDnBiVW3S;(dR{10Tw+P-Qt%pM%}*Ev6G;_o*>uBNtZm8L z2Lm!0g%N*s`IO6|e_&0U?I`Vmy!qXp8~jC;4Gg#WDW9B6*e<69Y1L{=MZ9}-z>UGb zC9yl6EjeMalqAXPLS1eR-}HGh=Dx9(V;p<^G1WF zXSVNw$v4N)0`NYzn-^nBkuM`+036?QNh`(IPR&l)_0A3sG+S_uU z3S~;4i{10FmH>>fMiMr%e~|Yr`{fVTu5EZlgKD#3Z81+4cVBLVLGH!3SkST0v#8NWO4F4 z_OlPgT7{-`*7(LD)LSD&XZWk8d{bS^rqtcvVF80)0LwEsdrbu>+sn=lLexw78}O%B#iY}QPKBGt16Yv6uxwWtl;zb3P~du&$v zmJ#2jvF0q}aWu^P#|MsYYs)9xoUqjwUW?}10xbK&hz$8;lI{zpK|N;*$~Xpc0C>^j z;eiP1uU7Z>t?Ed>o*9U}Fqwi0U>C!C$A%A1OxSD)-=Epm|9pk;h1VpMISVo?)`7K} z-Jm(ex}Ghdh@M3TTr*ftb-raL!XWjlcLC`}RO3Z1yr(xJq@VuQJTX|oYUjUUojQS= zBLnTdH}>0?Gc5x25+NESfi-|;--m$q_8cPHPFgXpRo|XlR3{(RF+sogWC{=Fw0A6YNNvE3+e=nSdR=OE5RBv_y4Jv>A??0a8~cXjvP-Tl&7-?y3# z1>FN{cpmwU4`OT3KQnSI$ndmhey9aMiJF#Cwo#zd=u#QkYM0Ha@&yF)K;!-wWJ+cC zkk#@W*@$2`7Uws1^4b%rf_w`@HJhQA5n=2MzPXq0LP-Yr!jU`XcH5WryS~%!xL*6O zX==89Ar#7=_Zm(ncIN`kNS0ABvszvf#tJ~~15(0ABGQ6!$1GNM%Sxh4|AZ&JlMsin z&On=p79JY&%f@;DYq@GO>SVBfXgxwgq>hnMiPv2M>rYP&9}~7cQa3CdF~n>Y>pe+Jiu`+ zc1QE1AinRWF41y$EoJD%X2G%ad9YJg7p&><^nC050-0+H$|?zIM@QH#9Ic0`FvZ$% zS8Xqu@oSy91To`?5(vkm+Y8p$_q%UoLPWZKcBvf8ID1%)=uhMycNg3Rz4JNdY-p0R zU@cxB(s&$;i4PCE(V}4evPV7mNK>nAhrD2I#PQbbs`V)I`ylv?VlPBLcv;80NAq%( z0XGRsUWhw|OwdT%nA#zDIR4>D{UIzL%>$#rG0+$tVl2#g5nH=|b^ps?iYf6J5RoIp z*q@9%3M-rkbrxSj!im~JFzC15eir>SO^N>`yl*AXGfSxbUgFHsqx zpFDNIjVkUi5eQSgyFrNmC)z*|xmfe#3-WuyvM-sGxKX-AG~w z4Jr8?bGFnu3;Sf#71p+WWKi}RW1j=s);Go4F*L&C_RwUt=``_rgf)@02iA&Qzg*S- zdTo#2d*8~Q*U{r4e|lz-M`R0Q%~O4R+7pz9I}QdfUHt6K6lk87*I&PRWMDY(>$TiS z{3ZQnb%)RJn>B)%#20OV_|f%NI~%%(XQgnkFAeL5*J&Ja>=teP%f9ts9rKrty)@Q= zyWnC?{tse}$E6pqIS@JrYrl8>(pWPC!iBhPnC#8hXv?02$D%Su^U`_x*~1@UO?WNd zIOALc-4icOg4c|{z`9v&RtGWS2624yB-CR@VJOd^uk7w!*+IJBtk;kkMU5l<$!XbN zT&xQpiO~}xm%$)P=KSdi5B9~6rwixM-d~*Up4lQme?!9OhpK?9)M*U3wtHkswcY-smW>M0wS5$tK>yXWU&rZC_VaYeKMe&vIJO+jyiX zu3ccw@P*x*l3*Pjt~X;G>KWmJ4p_H&5F%Fur5o0<+C*2N*zY#U2$Fz1E)X6OR{IyL zg!M0044}{KGb;%{VXWlw5l;02np}>Y)|pUKF%YIZe@w|#D!Amm!?=YQ+(G0;MbGq* zbd*@4HvDcD^TE=!j!_u#6W|8=hi4~Lj(Egs|Ke_WM8O)AlL9)pkFgHo97SaF=#p3q zL4k ziy^yU`InACg;NYmiMbW5cWcribJ_G>@y)h&p7tuN(Q<-~8ih@Zjn7Nz+XdFVqXwPkvv-j(n&ND3iANNT7 z80$T3tT7xQ`1s~NPtuN0ZPwTj;pCNlyxNPnI(Z0=K+eM&ZGYb?0*{kb!dk=aBnTdq z;rKWce8D;yjt1H+26Xw@##)M6?$#x-mT!*^A7f1m7h=tDj6&pO4%Qx_@B1zv<>_`T z>-OI8Phd^|D>PkRfS?f>t`RFXd2a@xGrje0ha#l}C^W6PS$dlz-P=$`fyd)TT|321 zn&xq(2(@ANS^M)b0AtKWDGOntgWKe&yE^f;Ho>V68R1 z##%Gi0BwIP%2T=#MzaKQh++0< zfM2hc=ZubxvIP+|uejBt@T>kYA3j1x0bIsU~a5M#@K7t zBL(YdY=92RS;+m!dXdFLY|)u94uhbF)-v{yW6W8A%fRRUHFBSx+}uCALv!Mo30ATG zdA}mDtPQ3PiDlvBA2!O=M9fIMgbl?GvVJiaizvLR+%Y}Ph50>fFR(Ight{f@)EY`W zv%~W>n;qBncmd&Qi9dlqCyBoZnLP8Oe&2F9MIZM6e*5sVmHnO@wIBp2+4>z`Ta8a= zn)r#fPuxQ$8_;FS#d;s(PDgLs88btfP-8`F*qS3O#wc@D^VaOK_e4kIoJ$)l6Qgn0 zr7J{#Idbz%Gmg+WjnAUPNuOz_Hm}jps!_0JE^LamJ#fuz)_6Fpj~dqW8PE9ooG%Z^SA31=e(b&rnI+`2FPAMigXEO1DZuL2p%uu!xj_2 zCE_i+?6>oQHHhoQsw|Mx>zjsGSlqH(zL}uhD8zXfn&qh))(vPfJSP<%-=GoM{Vcg* zt?~w$9=yT(2^y>#PTxEsctVODPkc4FQ$hxqG*tJjnKB-=RWwauTVA1A6>>c*d4Rb_ zP*KyAi_1%qx+chxsi#PEuNN9M%J|GK@ zNG1y{Fy)wgTIOdgGkLPmkr&sTNSqfG8VkyAHKd%YudH$;t&x4#qAYJ z^?VI4`5}Oh(uUw*&Df9&0s0xH7#@6FfYM#zm2X2lg`xJK``(julCy#Aru{tJ6 z3hJkKXdfjDneyHnZOkd}TPrAMR1Ve=0Ej=u+D;cxLd6?tXF>n7%`^`(r#J+_s{uB#*-^ld1nt44~njd>FRL)%j!Z%Jjj0=k;f7AR|^ zRuh9YcCQe6RoI*rIJ`^Qq5H@NgDk;Xg{?{5g_%6knA#|L)DF_9ql4$1MI=bcw1@xs z-NU+T7_aG#J!Ko6)jP0SvA5Bgf*xWw;M#&WQ?B?7{59X3Hx7>W#00B?=eYMv2e2ju z;#APO1t-dxIU;2V3QM_gftbBz+(bWocx)PY9BXp&o)Oj@V$`Arx1|Zt93~&?=W=@#U4RAE>}$7;as- ze7G#I$)Xz8S7@0HZ&TDXjUqot0;|yo-Fwxm7fo5fxM|*!eId%6BH^MFN=@RXBu&wJ zIEHp&674}f4a?1ADI1QO#*9L z8rIqXxV3*a4{4(ltO0jE=v$-#QW!ZT%Bs<~Jqi>Ks^*~?g<)N&Kr8vAs3xBIfpwGn z*GztT=kW8D+ADhW7TIWR(9kYvKC#DOyfA%~sV>}Tkd6fpcGl9MEPU&cnd@Qa4H^~N zOixGU{CsFb|678?8yMg~I%b^+Cdg$Xe9aCEtU1ea=0TjFl>eQ*neyeBJ=Ee5+})S- zLRv2u2e_>YT(FAx8NFLx?9S2!^q%;*O>QuW`o2mleuX=mtgTg03$^FFC%5&$4Xv=J zPmBI~sWno8D&|_WH)|eg2sQVq2wwXZ%?QnC?)FlsQ{yqtcOw5FC{*zRN&+-2xZNA`9gtGab9!v0Nb@c+|T@ z!uFbh%x{P|>0u;|Bpwfe#6@8tH@hv`)`tJ883w0^lBx$JWreW;r@01>(^@rCU)6%!fHly!{NhtD(3Y3XEC@7( zV9gGF68aptVH4i3UN~RhRjVZ{yqh|c+AL#m*a|W-6v9oqXGPiatR4-;)$`~pACch$u1=B=&==RlOVEUZpUS&S+*pO`@z_yoA)Qb zyhnkuKk({%25a$8ylExT)UjhZ)h*1{rK3DHjGx94;27q!^rB%+kOvQ(#B`o!^t`Ug zQ|F#tosm;VCqr;KA@F^Z7DXKTo)E`NXMl!2_qT6k)3Mh~N&>NX?ZMg{tf4TUmiC;N z*{z=tUvR@^40k0p<7=2gLLZW%9ueTjH)=RTf>v)HV2zNb4lf3G5XUoT=MJ6B%m_Xn zJI8IalN=Ti<)@B&L=0eJc*sfc;H!Ffq0M*4734QV=}~U57`iG z?uGKf&0q_^SZrj+9AB{H5nOS{X~W`Z#j4v4tE_oOAMNNE5jK66czYRZ0R02&hI>~+ z^F=?oqd)fj{_ywMc0|yOG}&+s$*!6;xk#h*tb#0QJ+t3t z`~kPX<2oet1vJ8-k4>;vL3@qSNC=XY`3P&q*TNcOu^SV>E(Z3YNT~4V5o|cFwUM^Y zSR8>{-3CAp*7D2~K)?wha)Jq-#SIgMQQac!Y+;J*GB(=QoRC7I3Koen5rCC%{yd;R z?Dv7lpzA78p1o1$Zf9N+M{8IwQrmVHKdl&UlNa}F!iQ@~yOt|qwWHEGdZt@UXX}%U z9V^E|8QznyPYw?jmh{q|2}F2vfB)f8ZMfqF8q7T{7-MPZBMhua#1UddMjVw@i}bdf zWY10_73Z~(Np2EWl@YzS;V2x3(P#fAv?ZYdh+eWwi#R7=2-2tD9LFP?$I>zqjgWXA z$Q#X2X4Q33t#&iYN_mAR!>DcgS-s@8Pj!hrbkRXRcqW!(zfDECtiz z?9`U|-me#!6M)rJm+yiUS#?Zb)WcP%5f-TU-A&xlE5rR!$Y8B!4zD$`E?WtZUV9Pc z?GW>^#!oQEi|zJ^jJ(`4&x!_UX?s9N^Kb9be#B{gRyLhPUb5jm5nzqU)|%A|um*S3 zJCOo6@R!~z;y1>l9!ZR1cpVDH8otYkWoyg{89uZn{m)O7BsLn=Js)dF)gT~< zJKU-{P#P!B26;gZ0br#8=**j#ZtcKFo`;P^@1yfC5~^*LFOk8NScjY1N9Cu%eJKGp zG_ilg!BYG^rb6T!gx65p>{gJ6$&?x*2)^;zIP#MUT!q?i98V@FM;qX2U;@AkN4@^gsO{K>dW;G0w2ookV_k50vo$c5mqJJ0LXw8=|@ZhbU% zpJHw}1{rIE@^ew+_E&{|2wnkgF&4wE2qDn}Aa3S6+z06Rx)2}eA7i22dI8E@4>NQv zNtw$_FIc0@1-dw{1+JKK%?T$0v%ep&`!UNQ>StR5_mw<9gvZx}@F z5vj4-9-heR(KDbxrm)-k985{zZu8o`bCAh>xxVbOK|;7zuc!E`w_VZmpbV+gKGvL) zuf*&BxB{ZWy`)&$i?aQOb5u%7d{$1NmPa07u5&cjw!PNIw2uN`x4#A(>f>$rtr@D&jfVN%Bkb!X}lDX30ZtmvlvT_a%L1p*tYuTT2&Re9EH3*9vYY~F zG`Svj-E|`n+$4P^B~u1GZ*>j^QPF-QP4BndE`Pf zV)CLgS@4q+scy5IBW_z98(djU{qNR>_c!*@WBO_;VgqIHzAo(iRzfF3E~<%K%!(e? zfa_w0yIo}_O7pN zx%9LB_8au6G6mNBz9R^ZhX&9na3Dr$Gx+kfm!l}M-6c^WeTAdNJBvQEuEU9~`*ja3G zz~b4+$S7;V&Qo2U8%nSjm)ogKgaIXDAeByR%B%0tYv>B$rBOeL|vV zhIG^W@?4&JB}_^b6z=r zsjel|!>ca|CLp4EAEVyEI1lT1*DRbXW*4kI;9?w|A#q!F<_G>uspb`h*q) z8;HCS!RAqMFz#nqVSkdP*YM@VJp++#3==?;{$mA+O&^*G;w`}+A0OE2ZVNkI95sGI z@>qe-&rYBxBQlv|y|hYeE#_6b@F!UF*Ks-HI1Tyrw8$8-BiU<4UQ*rG;Y{+9g0(C! zU@`5&oSm1_fw)p#aswR9)waZJHE(|+sKN^5U>(~Hpx~=r>N|KJvZMDbq45LoItWx8 zbAlS9v2K~5ndj>bbX}u0at{b;p8b)&buLZLQdPg-jKwM&(C575AxqS}QOH7Qa8?Kvl(0dl-?5msveqyrg)6MtqUKwd|aExMZA^sCSCbDrER4@6ETQ+u-t zAF=D1rS-p91<=~slLWhexqA2|iw0-pDLFc~z=P%S*muwD24eHVZpGd)E*KgXjb8}v zf+c1*(H_0I8u`@RTppO+#K7$_HN)yyZA!HC)JNHLf8-t3 z?4qI(9xhST-P)U|K3IF+hgcn{i3LcY@uDm6p~h(>;Tp}YY}M8hC`>dSab;g~CIyI5Q2)j8@Kn(90}}72wSR(Q$Kucl@r4RW!at{# zB#sK)+#&`Dt-_xJIw7Nv3^z3ICdjB>;9_c!{x(y`i9qmVb8 zE9|dVyYL2APrdDtGGtPQJKB`Uf}mvN&vV==eq_iGPv(%?0gU5*@qRvpn)IB!~1Aho}jzfo^A zV%Y$=CkTlkp=LShN*t}2P zJ~{2^y*hAT9Z2N)lx9K|h&w(Y!;$3ok51k?L>CRL31LSSh77rl>yCfR+e`4=P~!kc zhMoA?j1S6`xKJhR+9pvs^{J`KXaF4hyZWEr#l%qmA9oV4(i{!8d%-^{wp$bKC%r}@ zTF8GC4kwD8U<-d8$wNquI6gcv;Y~7nkB7c8z*)V@iJeFEKM9EyD}}xiY5{>kD&9g9 z5|Z_F5}p_`gPZ^>?PK*A@8c+zIS4P0l+&n%I$DNd*<&YZkwAVTGg@zRR4SAycs}LB zh;-r^;T2!Q-IC6DqP8iK8PSNwwA(hB*;hXX8%KJj=YetI2<0-jnV zzyvp8N%>J4SyL@ESGe_ z0lFMHr_@XxTzXLc>77cf?zne_Lc15kI)4<1^PJl5?X@u==+6-7$q5bNlqv|!w=kHy zS9EBit^Lf%uqxv;FIH6~nc08%F4k+>RPwL4vH3MJC^{#IwhZ+CZcvzSGvS!o)A-(_l;9tLq_=>@yA*|BZMFM~J-TeKN{$N7>v9C@q%)grQG!9=ZZ z4ediy+OQE8!}kj!Y(>R5mp6n-Yn?O9zYKqHL5I6lT%-5V>&28JF~&1wvoUBw%wl)R zvWmf%In+YieCjJDFgSP3&H!zGFS#B8E-#$;dc=Ia-^7u9CusF9!Ox4Qx8X|7p?3q# zw?M8yM&e5)krworz8UM2sUL7d*s*q4?t0n(kos-J%m{_lYzzwctu{ginSg@B5lx=} zR;W2nP)hU2*ym2(K~648oo=H-2IFX3+?W83mJCIy31SiFyf$*?s;TWWwr#Pg*FFly@r8a;WN(Y-^lnhm)gu(8I==IdZ#*=9YlE_vJp zP+|>t)0Fsn;>bQqqx(#3?LDzIeKpcv zR8-`ER%9TEFi{xE;>zZ8*d3a+mIqF1?JL$EIcRbVvr5}Y2EG})7;s4=XQn0tJG1?Q zw;wFAMvNb1BfPTE(AprrU?Y6UBXc$wnX@6nO-q@Ohqum_Y7WDaZKYv7Z!Qg*lC+XS zOxx+y0I?e$KfKTfPL%f}!=0HOZpw*SJPrxEX=aNZaT!@)pL0N@RDUH6j855OIgiZo zRtAW4o4#DJdFWyNI|GG?{Ev4F-+ryl8t!g3O8^K@jvaY@BZN{Bv-1CF!X*OT@$Ql# z;rAn*@Xa<$%k3S~zgvxdcUv~J?9C>@*Rp02Wg& zI`Ft}>W~XXH0{C?GiG*h^~6Yj6wqVA9T+PGAelt>oZE?+4v# zyDE4&v^l-O?&~cA#rX-0NtOmkUqtW{19LSiGLHd@VMa)tvw2N?=xX^0cj(6d}Q86%nZWgxSqHw zK?buojI@CY!RV05^5x>B{q~kEIC|CwBP8~}f@V0igibqkyA~ugbKQ~Y>yA!cXJoH+ zM`o_awKZeC)=Vez>yA!aXLQ#3iagh-4O4Iod!WYbHJH?~^Ca7mtGLea=3!K$-?9C) z$9_C~KHlC92G7ZUeow@Q`!{x|V>h9l^@|Ku5Y(8IX`IE` zcGFs*jQf-kmWEoK!lSSj$$TUUwlRyNwVT--g)t!IxXbYnC~<=7fg7_8VGBlpg2trgTx(%tRvr5D55! z@R#Dt#Ze5w%UQ%+CnE)H`#p(5WI&-4jqwoj2z0_c4pT*%m)t?lG~8=g4yZYX73?L* zYzWh!uro^@fE#U_q%p>u>^CJueaLy)atCeP0(2uzKKoM4*V*tBYfqH2cEVIsnjqZg zgXRYH1b1hY=WR4HAIZPb$m|V3d1TJ|BOptfwVr6h2?mF+ov1^~h)-X8bo$!ueK*mp z2TZ{*U-keyF3OnE04;QV*gQA^V#|6JdJZDap}T2ri&?h)H*QOAtTQ@e9ez~)Vb)jT zKtsaQ0&Nlo7CM?Xm12r827p0eHtea7>c?q=zgk#Fd|d!4cYR7R!>ier?ONJcW+!%m z!*M4kA&w*HWYMlN<^dg#5ay!H2?Z#IyMTZ95-l;y1Myf^*xth>&dNS`pJ6y-kPCL# zHn14s`%j{Kk-8&$PY_cSkjcOfHV>eUwIGf{Ws;^2>7D3=$I;pq*2yxDe?ec&4&3uM z=t^0rVZb9cAHlZVYrWR=wMU$0uG5;cp7!Wq+W~g)!yeWE^b@#1&6I$&*tt>65tl+#M7f$*ox7E(C96?{WDU z#iT0($Hyqh}qgH}@Lm>;k z0Wk_%|B|SQD8QtLFeL4Bs#seLo(roCBr$!Rk!foyO&$h~4sEQ3`5W<>j3(Op zO#Qn2`N*HS>!amcY@~M3Chh$!8rmQu(`pnxng~)!IXee@CXSC2(h~)0(k%YuZ|))7DBT zyD4afc5Dnx6A6t@TUYeVC$kSW<|}S7&J;d%8x3qv+pYy(^4{QG!G+7>7jL$|hw==O zA}OOCq+?#jGd{2$+=_3ZW=NnZqhI}_9L&!jLaMbI&gON<9bATort zfegfAf-Ghj=dkz=dzDpP?fH{5K9V?<_QX<>yd~{Bv9<3c$Uu2>dPND0usWkHV_j9q z)Zz!F9e5lj&1j-E_#rfSG%GTjKcR)(u70J{0;)V?9ilsFZSa*zALu8n!&F1FW2nR6 zN`W;Cu1R3(v}Zucl|lixRuTmg2y2-T^TGA4X^_O)EjNo0z_>jS+}CZ* zT%RlEjfOlL(4op>mu7He_Koo}GZf_YZS3{wTApdn?f;cM8xx?9uHJl-2?ehb7i$_Tj>VVQOR$6G3sJwx7i1$$& z?IqSz*KSowqcem&V8PMAM>UIRGFkPLu_RCUhE#W+ryv_+J@qTC-M%7OL7KYe==8Pt zl`gz)U!YK7oHM|R@ys{~bsr@HTHd*sl}4*-0q-7?wDxa@C8=RGThQQ3M`&PQu~Y$% zH-3`XeN;gRoUou(s9Sj30d4okf(V9Sh>q)LX)qq{Q^gSxRE(PiTLRzOVVkq7a~ruN z6BnaMIx!8GtX|=xiuKZvszS;@m*}_ln}qo-J57`@5lx+0*l5asZqe;Vz#7GhT?B0) z+xbOl_6eia2tDAo49_AxXIx%f!5S6|vs~~YpMd~Cqg;IIWT24%GWCnc8Sdpe-5I34 zB|M~(G7=YPhp&uG`3E)%vkqR<*1jaU=3rZG){3+A zoS2Sh#!yk#Iygy=BSuoJOkq_T;ot1v$T~Hqq>Z&tYa+0atwktj-69THyfhqc%K=-# z-MEf5M$>*5go5nW!qv}YH!kRdH|DD~m~{efi$`NER)QHTQB0=5S{y1Z7UmgO%zL?6 zstN8U*6=UHkl_YX+6QaN`3h_4>yXuRHxeb7i_PO**8(Qj5vCv*JPTNSl$EdeeccX@ z(13VbTOPegYf1S^PXpM zro5sb750pW9D%I$N)axr^=O2J2N#1WgJo?e+kG4xpu_2icl0=|Ve(@Uk-qL|?UV=olt8?4&fvG{X5Pg0xN6gHY3Nt4 zW8#5|c-DE_16)fqosYqISXtdz-?4WaD-LrVhzV>Xs^moCTID>K4;dznq_s%iKLn0u z8z`?k25Y!Gvo=5*Yvj3tY}vqY8-=1_ilZGE$Kg&yFUV;>KW61-Gj*at_w=>e?h1eG zv~@>ktgj)76TM1CJRV7$SZLmE?zRk1$O5#*LbzEd92RTZH;?P5jjvy3Y^)JY{se(r zA&XS!-#}J&RF&#D8v6cNqs`%KV;!SBL^`00&6`-4C4d`~mGF2ATb&U)9EZHcreaSR zmWPwP=&V?4#tv^O8FU|n1l%N|hPV=?K)gZ7b47-;On@k8#&3aXMrh#$6WbVF78JfBy}665 z$b}5CYuvp6g)RvrX`fPob>_h`Bd=Om<`l|A4?P|b3tw;#drYnIfs{*Y(|FNrY9Su1 zZGFY&rN-8;^ycBT(SvXFqfUMwG z4EO56I)KL76xQoYk_&pJ#LK{qeC~ra&r0=MN`%DLV4Qy|1CNYlI9Fqh>4h0aEH@!i zH_urEW4PpN{B?|>qVqA^G!$Y$#B2+bk~mJ`M6&5nCX;F@aHdl?{V(dnSh!=d(tani8 z8eWvWx`ZcKheXxdYJ*Pvn?m4XRTn$wFzZ*t8pBLmLv(Dj1b4#PqPxNxG0sdhfd_1# zhzAvKs%DLe+SR$+a;?X#ijGXhyKcUQ6v3UKSquXXM&wL4VypJ0tC77iB4g&!9o`3P zkd^eF5a#q4SVIGJY>71us^z$rSeN|npT@gVI#>#T#>5)iJHZ_cE@^D}Y);qYTKD*? zu-pBhTSzuJDC39|&>gIOS|Wj|3QcE9l&Hek#HQqKcdvCfQ`O8`deI!A9n5#I;UOI^ zbsQ($al9Sp;f9W`c7Q1?aHO|5njr8hgiK*zo9gsZrmzWa#OHrDZvUOMHgM4{ZGzv} zZ~#jl^N%oc26s2k;!K&*MV!~eURbez4&lDn&uE{sW1Fqx_nRz@43uck#9RSQf+>u4 z=K*NGJT02I3~o(4%U%AJt0X2e;N^f%5+IRKaBJfILK<+_4~Cm1)#s@UbeVy#!LG)e zK;|UrEj=!I?!2EZ>szPeQj>fLsvtw55qCQ=_>%vPTX46waNHC2kw+Bv(OgN4t!h~F zFXcGd;3=)jXSWTaLnY4@ia;m}>8FJVlcCZfknpguzYD7&5>j1qs z)}AH4`Pu(ZE_JobU~;?U8$lwKa7R08?z5w#EbI=s^dqHBs(a0_eE8n2i;u z?}a;s<+rS>Z4pR7B$&-?kF7)i!Ok|qbDq#{swoPEATc6`iS zDdrF~@U5@4;Chw>5)3b|%N|-;Nr}Q^@X_I!i5)H2OEyk%)lklGcd1UZXW(U6v9uYE zDWa9h@lD$N;(`e+U}E5$wYJ&H(mRn#Ll?VFsu?_P#32A+GL+nPe&)KZT;KdFy&Tiu zs#9=uhbodtPnNk6c%f{bjMC=ca07S@^5@v0FMg>#i@ z4mdZ&vW~&+!n%N45+K6(#oe(oS8*naCy9A#1$c*Ndt+UVH7+k!XyQ~SnAtlLFpZ9c zKY3dgb+$Oai8yeEfxr!S<9Y>goOlR-pq#bt8o6W5p27>Tg1e(c<~N{VvjH*kpFEsP z0^j`X@K^&V0WBmNw@3$EdOL6w?)*y~6Jd!pa7Uev^v2c_`$w|PhOMql<$R;H8qky^ zpLwMeRA*g<$HSgYG(GScOP)9Dy95_2>(p?nz^%YX7?{KL!PNFPWI!fP$aPp?6;Kor zL@+P&xy(n0GgG*G=E5!u+?z*jq5TaAh9TzR1U3)Z44FWfilV_IvQ$UnvYwaY15VtE zouU;3$+m$DHqtqX3Eu+Zvc%#ym6!&CbIqHbn%(S-1tq(>D=7kP*1F-oBa{WKHt?FPm&ezRD|&>2oMxHDc0PX03sH&3u{6;dtxotW|Cs!=s{ma zAdGV0!VzB+CG1BKc+6}vfFeUeGO+7Dqd8&}84fV^SCs57^4QGVFxU!QDt7KdHe`DT z(~6xF)?H|J?BOQXHowKQF`39fY5s&ZixCdmgaE}bqShG+9v7^#n)nxQcJQW^FL_p9 z!t&P6g4OC+yqpNbB+&(N1~;&VLhL%rb@(mCe2T9X`_zQ*);aB)5$|l06Vd*#d9zN> z5+Vi(Cu3&}dY;deMvjD_W@QK3nQKlI=DTQZ{M9wS2G-2u-E-D*a#nXG@bPj-K$_M9 zVDcS0Sq>lEKF4jvVj=?bMsT|^Wj1SliBJa%saJuf;N@;Ut+A~tThdakiM5|i<*7}C z`N)A^6=lvLVogCxq81C}VDqGNGcIOZ1KQ2kK`ojoQ*yO<%A<0riI_wa6~V|gxEJoU z2B*qiIkgRgH&G+NmBj2eD;HR+(vu#VPf>zT1`=ZPa7MveEKm3!kzhmsBBimn3HX=L z7U+`OafKz7Se~=70!xe_(-S0QAjx^VEG*l7^0-KFSexIw`L_l-bzGVHzU6= zUpGqJpQz)`Qx)jB#3+|nnRzh^AuMHw_M@C#9!&~6)8o~9&8qNiDo&|0!3jFZKnii}$ zL_F)rCNH6(P~|eb6J)7GDp)0qpe3lq*O6q9J2EY7fXwferDu5#0oY7pHh~Ff{9r6k zmhIikG8c%IN5xQrGD~-fR3zq*K{_5Sf=TdTGbzT5WQN-r&i%P=uQvO<@mmK>l1O)Q zIB2lBhh4y(7U|TqcE!JG=AshjF(?afr)_CpabZ!xJkHgX{V(|(rX#Pnxjk z?{pUMYCk>g4n$CBY)L2WUz2YkXZWau?k6 zAuOZ8uO}a)Kgp|{xL-m_!uxHYT}VKYxgM5hG>yMPIAXa~oahcWOk7=IEUaB#-WBI& zGeA!D(Rmx_C0SE|VIK(E2djTc!MVn|B?1VE4LwA;BuR<2tuJuQ$vn=rB7lqyP4B?KED0f??5>Hs^0YW<8s~;$ z`_owo+KKDJoisTDgDaxAo56vb8B!w3*v|?DidAOIycFHdc<+j%t&yd>GgF&+J>SGN z{*}WMm$+jU@|2|!7y3+g#9eUr;}pW+W;&xR*_+%N{j{@*0az`rxw^#GRJt1y8@LzX zE|?Hl2SrK##1sj}xrXEg=Rypi2ELFVi;0YB=Ig-P(dJUtBr0&%vf8$HlG~Lh+|^hN zyTAOCp*_9YeX=$!`PLRJ*J49aHgVGSr?^Krd4$B-ZMdgcwyC9>I&Vvkti0w+h|6sy zul235Tt!%8%tz3vbbUD0Q4>U-Pz_TYGfCG%O@lQYE%d8GEO78uoUq~q0H}+v&Ai4m z*w%9&f=}`MF^#2sSJDzZ&sCY<_lYS^24$MwA{?SZjo#Lfxw6x`n72~a=6b7U<16VM z(%Apf!rTq2?N0_Iu#zVkeWl<6H!vD4fj6YjQ!HNN=#NC8i;mv|>dM%gUHc z!TZV@1jgxFp08y`tZ3H$IfRj)wNo2s3qP90wKHhy0~mZw*4WuLd7- zWH7KD1_pPlJmxYmhU1p_-CRrwd27<3A1F!$!1_2dH9$x;1j}=cb()ED&B%tb=e1_8 z&)I+(7QFxwETW%s2(Z3rlA&1*Yd9MJDpwiUX~l=ov_`H<#MdC3iNuZACif7Ym_ze5 zQJr|LCB46<>0reVXhPYG*>%cTg!pQHnT+z`jH=>}v;!wD=$t^Am3Imui*t*x_p~wN zc4B{)jj(S9t4;`OVL1=sOi%)oCTH(}zXekn$CpOX0@s!~ihC1ne?f^k-DFVlb#QmJ z_6Ob+rd49f1AI}AVrRNHz(Ox;aw=>j@oz)|i%x>y0bNnSTGzvap-hA6g0GW!7e5;P z$iTcclrx1yc^c4(fAy5G<^}ZAj(({ghqRDatOv)#vsWH2p|#QS6R^#+76A7ekg@nT z+1`O~pe*#kx>_$FTitLAecAKU*Jf=ZxK8FkidaWaz&2>NSS_*1 z8{FEp(%q8kA+e>SNgbd=Ho~*M8}KFG1h^QK1>?j1w$+2*{Ww$+Qx&Qoh;*YFJT8JE zW2EA@B#J31q(PzqmrHI3*4Tl8GFOtGQX`TfJd4(r=vdFzwZ_+AO*IHWI7Yysy)Gq3 zo1G1%jRXXaGL_Lz#!StXr~u*(*k=i2jEO4ZQn)f{7)zlQ6v{L-zGV=P7~m9zA_8z= zzfLz6tn7Ln8()WJ*AWIG2jVb6qR}J}bjE=M1+Hn9781Wij4@=!I$x%%pb&z<5!d@p z5{M*YZC_alw@WI-HL5bFBex?-ClZ8!(aS9-^6p-EDzKKI7VpE1FT*ZYNt}BM1;k=M_99~atW4Td1wqZfe;4k zl24m?p)Zv!nLq|P0C$H)VV_L=yKdUO%7fo>e58ez%zwqy6(MJ6*lTCB2?jfLTQ)Xn z>G+||$Y?8gCrSijBrp|8=&rDiTS*--bFIm15?k2C@-9|*5jzx-sEQTGU|`w;)miFg z9G-T6X@jg#=m1I`AOlAdLIBU$0h8NAIB-i^K&&mUM^JDTd|=H$J5VEU9>*w3TpW_w z_-3r>b|`_uVE?e<;&EY}9iB-P2c|hsMsi!p8Lto#bxF3gAiUfHG=P_zTl%e#0G=Na zQ)>z=qKsm!U9qhg)`-LPmIJ`e?3nX$%aYt|o*2BucbGgg)NG^xf&}?E6oD()nUSlj z{)Jo5-jP+?K^s+Qieph?Iy(h3c)btvk3cU?!9F>HJI!I7BWw5?mcwMH^JtoZp9*T^)pu(9Dp>l92-uu_4kGB4KllR zceqsv3O#2>Z8UVGrkoVU5P>V$W14@w?6o(D79a)xlj`R%%e7wI0MaFic=C zJ~RkI0I{Hbrx}%R0=0zD*s{fpHcvq(A{FAACXet#q9Q!-n*eAVK&`>S9n_5Y-ojcc z8wp^8+;(zMh2*wC#|%IqSOr9h+hHF7q>5xk6iU^W!>C|qNpH#Oq(Nv6Xy$In%ScMw z2n>9aoF~tWm?aerLI6e~$d6m%f>x-SqJ1Pm(U`FBFSsvIK0+_o&(EMefB_$NSW-C!)Y&0cf0DCW5(^HoG zK^4#sN(z0%%fQ`MyIkESS2*3ctNr9QM+GE63oZ^isRrYeLoixtD=$YZo&ul+c*2$F zbIn|PbHX{=NmVPdr)yI(4r9U%!ju*a&azDw?|*muHpf+5Hck5=dkx6NH4n2;pxf4y&T>oy6Lqh3Q^#bfwcHpG-38>kzF#ljTE2HlhlW<%1-D zh-G7uG(wPv%m9gzq(^8fxE|0unjK1Dtc}14i6OFy*{%RRVvK`@#mg4^T+P>+qp2@# z9#@&-l_!nDBei%|A(snxW=j?hrvooe$!Rme!ReWdhH!?EPU0+J4&vFSwQxt4476B8 z$xas_B|q2#ZRvv{$JT!t-&M@qI&TfO(?=KMzm><{``*NER|-drjcRLG%B@gBvm|Md zLRcfpk@~XwaI{WpqjN)&OJ9SvwK*k`NmvB*4~BSX=?Mke4$|xNhD%R zZ!vn}mZK-d!G9-i-C|opHYX%Jk8^4`m4gWjTw&UvGL9~RcxYJ;HLX;zKmr6Yia|{m97BpWDJK}Osl730X53h{&21y2m`lW-+5vS+|Sy!b>YsVH?k7H zF{@?hk>L#9G3VG9k;J}O+TD(cFpi&q3}Rd666^HTe4TLNIHfecW{!q1#n_lMB9}CE z;tF_KBdAH!C@IA`ASQ|Os)$F`N%L#uwIea6yHsQX+8hng=6C)z|H73pEd3v(QetiS z&?%t;&l*AUqM|SY;d*|Eh-k(U`f14-nf;IiSo13|HNe5n`&7HZFbXS~mpmwEzP42c z0jNAuR`Fr0TPCC{hAyHH4hma3ENU4`GJEil8296GbLl)ah1}!d(v*j{uhda-LLX1d zLiXs^=Ju^?rnatRv$7eQ;}$Q+VgYuc)Nv#+1$O#2%$c&)hGs?C`%WvicGGs{=KU-g z#Qt*$P!!w@xL^T#1`jzf0e9e~=rskfMPWief?i-==2Hx)!eFXAC9t+bl9Y+OPKOcn z3WsC$0g1<*v;$S}qikb{A-ENiFja6h4TOkDbm9_YZR4L3{smA>1rOB~eluzGkjb)X zc{lPn5JX5}t${nJ!PHXR@n8+>WT7R{rJ5vF16eey$ z+}sV~r0o4p8J*(F!Y*w_9b43D2Ao4lSb)Pm)r`l9H=`s^eU|fLB7m(&kqqqTkb%w_ z5Wipwb79)jCqfV-hU@5>X8K~jPc6r< za0L#+v}9=wVy5jvavJ%@QQ3?1hNYvVgYH- zLTtplqs>w4OpzS6X^RbM50qgeAP#3SQH8Z}Oc71u&EU$w7X*?2cvMIcq?*ED)V<}- zmO%-$BheA{LmG$%=p1^+T|^(4i=7&8dv5(dcT@T98gGTNFZU<{aAA8}7$Km1#cT5iR7?o2lf{B{uZmyYtMZ>}4 zHfGG`6?fMLvx(CgP3I(OFBiAAv?A$TZ_Rf^(!Y$v8`21>Fu3%%$`#5IW?naU zTLM&QYB0_Ubg^)SHI3BBqm);oUQJ%hvw`;sUY6s!$l>e8s}25P^UYkU)rOaZP?jA=6&*NNkW5zAwmzMR^24VYh>|^gbKT&t|e)D5FL> zZuPKO_jgF8i^aA|iDBVEr7(IW$q9Xl=oi9A58v`%AMuTUeaNQ2AZcImGUi!WX!50J zxXt4d3oyI*UGkO|JQ*kTjlV_XZm}M?09?$qiPO888qF)Nx8z4MXmTy4*U`~p_84IS zw^}QPU^Gh!sEJP_$(LeoEjU-Sp>P*zXH~;!moHsf8+Up}+|=9TZm+5IST#IUrIRYj z_KNF=`8PdoH>QxSh(xNP%2z)mF@mj#DTw8!{johF(-y|J3_yWnnlcmU9m_2VY6`0)v7kDY+y*R}ZB@Wb zV&o9r@GoG2pnwab%UByq=f%R^TueEY&EpwsJ_?GpfU=v~qcGT+-cDR8v%Ck^02h$? zt}0s&_=+P7=Vy0`^gbyAT{fF%OOJ}_@~9@k!nVn5&VT)Mo&FXvEyOjo7w+1YBOQlW zHP#m5$n%ioM{N0P*POJ&6HBlC=)?C5YzJjmta$FxyKgvW=F!{#n|(JP*_S1RtU$qz zqKmX&y_PJU$XOVigs#JCkxraZj>*wFXj5t1eKaB#^G53Mb;aYz@Q&TiFdSjda63@0 z;#I7y6U3q)l4jCFZWz!6ZpGck5sxc%pceSrLdNA(4#%22=afLPf;;TbFwCR}ZJxn3 z6(~kC|J%ugNY7Yn9%QWKHbJMrx6E5Rr1^2AN>l^uHcM`ggQz6iH7(AxDOvzy57Lr7 ziIH5{%w$fS@}HJofAR>0* z>A=(v#c|$y=AmmBE%0XPsyWJttNk-}G?U@DfJ$L7g8=!(c3-t!R516X+D#Ko1D zHQ8p6fpkITd=uOO<=&IlkVhSr&A)+lMjW6Gs}ylH!m5CVVdYhmW{>SO&HmM0Js8|A zzP9BFYs28l@(i~#7Miz5z^_8jYbG(-X&tO{IaDEOK%{X%OrB{+-7S)5u0hZ!bIX_t z!7i!Z!J1#B>x_Ou8=V*6-EagQy4e~xoVok^@BCz(NDfsU;(6e@BTw1>|HFe6t^+#| z(v_e}6M*PJn|Oe}mPK$DIj)kDi!)#2jW4m5b6z`mpanF^9s)tOH;`pp-5d?WrWl5b zVFOSxvhk%p5fM$S6Yd+Z;mqhE3^XFp3%JqUAP$4WEOs?>1w@b%w^l|tkO{jeH@+6p zXn{jwPKw!QSQv$lFCDPBt22v(B*gO36!Zgvk^_M|)hIDL8|4ycNW!;#4e}5ZojT>; zJ-zJ4pZxsuGP&T|DOV=YRMTqRbpeev!sACt*+^w0! zN)_5>^A)?71Ue+V=tnD_T(hF5Zmkop*^LUfHVxwJN&}@B8nX(FI0XESaJ?SXRYQ0~ zM#jc@iFM}U1hIZmPvw8FJbE&Nrg5OPI;!5aXaPhRbSTwR4Iz~W{bRRmcS__sPza?< zkV_2n8JO=!0F^`LVgqD)TOA%TSDv;8AUM=kdd!Dv=@|kc6g?T~eI@PTqndEH`4`05 zEhGgqmvSECe0|A^3XwJr$wj-flutjc6q8D3ZF8}^y2`ufby5jIWvsVVw8Mhf7pp+o zjeDMpB|i`YbPn$meU!1Tw06jcknJV^mih7pE-M{eH6x?r+~?gei6l-Z5c+i66v=vfkljEvxSF~0H&t*m1mC4@qW zmqVUI7^EiF2#2}hyfqx2QsjCCo{FtGs#x+Cj{!cSDrx@{ndr8%%$?)LulCQ1JHG=L;%7{WpmmN7_#fp!{4btj+ z;PwmVSR&c5v?4 zwT>7T%H`vRwY=VQ&~H6*-lXSmKlHU9Uh&rR_r3e-qwl=D?2RXHeRbJIPv0>2!888; zyj_2Zph4+w+wcWnyEEw=?i%iJ^^AOX?q99q@Od?0zGB7I$9&Ci$khnu7v|q)o|0p? z`sIrb`up!)IQOx87Cp24)>oc=_{Yyb`oiP)J$mouIPNYyY~o?x_>GLUM@E_J?Zyvb zu8B@OY%^>eJ%|>#f&c{K2hfUVYNG=kN1B&)W0fpSr_uoV>%YpT66_ zyKtYszTu>=KXl`Z{K_^zCo{E8ylm zxmpAnT)#Q_v-kSXPd&JJT<=Wfm!E#<{Dao3N%f4lx!W{=8~{gSZKsX7z672(nD_IdiY&ROr4!@Tm4$LF5fBf(z@P%m#YmATNGT6{QBqax$K}-u`8C}wdl|-f3rCxH;%>{9tUWxNR>zrJzmYX zG73{nYzP76H`1=cf;;yAuZ{o^?f^Y5tm)YI-Wa!Z&7GDr&n~~^aPhC6x&iQSC4nZW5#hFB>d=qt$2J#G=q*b$@HvH{!xrZ%% z)wd_U_x5Xjej?vO4xBmdOC>KG>*IF#ou`-Gl&h_D=Jl7KI%(Q}t(erJI9)5gRy>Tx zT+F1zOc}7|WFGgw>5OwYezYeEHFZ)Bnve z-%y+b?kP^x#aw*mroD&hu@B#Uec5UMr+oRGXHO6O0AT<~Yk*+e^Xf zj1KNO=r=!n?~VSCaF;qiIR3wuZ(hrv##V|Ej1k^`vAH4Lf`D+-grx(v0_&A~)@lpe~{1fCb|3z5e9DN1%ri*=C-lNi0rb1`4a*SjW_D7uINS>+uBr<|CIKw)Rg~tmyyi zXzder`T}s@dhU$=HCN^ClS{8+I*HfIzU43oH73;>C^qBWSSt}c(`~HL(K;FfzZqk+ ziAi7U)U!BIQFdCS8Vz8XOOIFb92#J-_QQ)dUzL|1_$;jY{iFWl!)N^cz?yG8yX5u- zYhX)wK!F-+8WjpmoyVDCOw+fBK=xc7xO+I}4lEsa*5iQw(9Nd|e1Tif-tCMjU;OZc zcL&y7l?qpW=c^g(n3WLFB)>c*xSNPTICN8vI?^sQ7$+Dc=qWN7Kb(uL6j&QyU-6lc z$`V=W)mRLy?_BUNR*T`?|Jz@E=z+`$(ldXw zXv<}%fBE4vzw+3n+koJT4~Lzi8ff98M-Vc+6-I@Xj;~EI9wjYK>SUWHMp&SExFV$$ zS3tBmUDph;FsCoM{NMo}i<}=M>fHjozx>YguQ_q+3l93ng$I23>JzqHcH3D4X(|8u zbtit)lH43!@-==5X3T?&li5}c5CZhgbT&#(i#af*N}Pd9GHaXnOCBfxlV|SEFVH9R z;yowad+_I*Zu9$(_~MV3E$Z{&@VAzKcRj!b=o;4is8b1gRj4IJjKbJP^~@bK?MQSJ z8#@r)gl!fgQG!uy9-MyiZeJV{%KoqY=-n5dI_3Ag>xd?IG(Aeh?8xyue*VFmPwxM$ z?>_(N_aHw2m9bH!cwq%=iGUSvnXoQDfkIr|?D?GG?qsI3+Bzj>rAe9P$=rFw9}V+j zk6pPlTHZPoU0L?IS?syn4jQJNK6O5L`;|Kv{6fjj&co?4Fd}(Mz47bmeOOS;#vnQX zn&Ta3x38JjW+OfnoE@_WeDlsT%WvuP8pGULa_IqVJYoe3A`X=-r$=u&bC+S-Ic4`h zQvr@28lLU8p z^XsR#&)9|;bH-?5qVx9t+kq&IRRJA+;;z5vKIbaKz=OL8sA`VKH`IQ2>1ltD&`?5p)KZkA zL7=xvO>2r4PWSrXnl{h;4n5p9A9~QDU}QVzakJ+1@>#!K`|08TedMl%;2TUWDIu)G zNDha*xEq*KezTGLul`p5mS1{&iC51Blos8>ROWXjD~C&f;{!_|N%=7j4)7>-yZi^N2t8kfFMYx~jaK z2BbKBtV0Qz@$EFQArbg$0A5P_TK=OC-s$rueQqs2W$Vn;fi*F)N#q)o@_Ol^YxQZ0 zTW`PdVrb{Ywl&1#Uv6dftw6X4jsj36qe(`%#}V@!Xiom3NbtDWJh1llqd_C4I}ZO7 zEBXv*nMoGlbOP06{@r)Vn*G17&t2T+9x`y{d{r9chjK}k(HWp54=9i}N)`fPy~FJ2 z^x$`I(dmsK79Y3 z!Dx5 zyeFEaggEEfQVDy2T~BQc+fAhY-TTxDD;E?UfS|+~$QpMvE!Jd4!PyN*~a( zOfWB-Y?|CniLt(Hcb!Yi@u@n7SVts}XbZAkIFs5b5HF<%IS0k?SIr zY|ai^>~@O{KqB+NdEe~+b$#yMddPqDz=1ZE!kpl@(!W(nzLhIn2OYBAu6EoN*3v0q zK;8fu0R7C7n<9A-VUysa_uuOCCVg%_x@Zez4*K*!V)j^2%t=yTRh%T=55lY9blKIO zj@fkNq5}p{KGQLv926oCfCksn&T3!K3w<+t`qwl7Nr11!vNbYWm+anhp0ge##slk$ zqjgL+d1U+R&o1rrY25nJ;{AkmF>=?Kj(@=&F-zzgV?D4AdGhQ%XZ3Hb&)vI@`(J6K zfS{19s0I!(l;|qSx2g=*W>0^D-Y^&+#LfN?!!7&#dY{+mf9u8jPveZU%cf|x$iZ4f ztZh;>FR6Xn+uA@NR@s0)qW;cH5A5ICOHbTqtTW<)b;sVzoK6K2Sbt{=0@n%A_fRX& z8ZIm`Tjm!AvH|hvg}aUt=xf}cx&OTW@AJ-UPhB)PpUwhLpS&>?J@>s=1; zW^ef~fAZ=xeVXFd@;lB65IuGd*KTlkcuu;^0L*fwxQk2e@);wSO_j-w8CF`|FgPXf z^#wa~MR8XtB>4ju@7Di)xV!AC*%$x})j8A&jxLk=-RM7a_eler>QmvFC1=PXP>F~# zkI(=|p;_If1WKR|XHs0gn_LRcQ%;GVjRYlR`H)}x;mh{u^BMzhJ$u)2oM@wQXjlad z1F;QI4VOh;fgv0h-O^IljS(`~|z{bm_$h{l&mnBCPJNQ#VBW3kV@D(8cL{YN#%t zr;aYWWY>Z1^sn&l8_!*`Co9NCFgZwMkW|idCC>T!StCdi;J$P>m0#*Q<#l|x`bM+ES3fXpy;~37}arC?$VPT1AINCsiI7T>dpSM#d zv30yn^fz7@9BIRwKXk?Hi}(D+i+8itrw)gSuHJLx=DmOI={vqXuz!ZBaN99|XNlBx zWeLgZFjZO#ZZNqNGgHWueh3>m}E@YTj=%nLfOZAacX?^mz~ zJP_~@0At{$dH?dU#dC)N`sJs85W8GzdK&n^8eQ*c*^1dq!(QmyGA9FnE#9lew$Gft z0j3KXT+1qxVRp$LYmkAXIa^%B5Qv|@Gb}F?{riiCm1%GP_^IXB?tj~NzI?-he|-C4 zfAalP*LwE$BR+V0fHx+0+Q9MMv-h8GKS0-2&Omq;5AEm;v!%zAy&>79^WY+dl@dO~ z%FiINz!IcO0&W{EAinI%RdR;sZz1#kn=d_a%i;H)vgRF!|LGlvf9bxH|L%z!_kZ{G zVMB25y>QA|>lKM~q#Ev1gC2afb#-+MQeT+M}0F8NF<`HV3Z3 z*CEMA&jnnz8{J+`IjEK{>i%r zgoVHk*Bca1pO>t&2PSI&1} zF7KM@Z90U}E}@%n?zuZ?UkJ|HV?7tAmY=>2r|Ib^c?wV8Zj^%*QQXY@K61sXYTE&y zxnjkA$NtsQ)7KqP?WaF|;o)m}?Pk(J=Q|>=&`=`~Ggtkq5?{5aVop3xO-S8|a!QbF z%dbCp;0ru(`OIq%_|w(9gGcj6mtDLQ{(%^`YB*EmIS)%_$yL*|A`%uB)@XC>xELe% zSdnLke@qatc&Ckhc>N5`;Ij$LY1<}_CUZL6ZNA=V^eT>D)C$~@yB2)zt>^A(zS8O{ zA^Hf9^j&-Z;xNw1yU*SV1c92#YH9?_<@5eMzrOs2!&le0 zH`{sb>ANqR*1oJ7R3=Xp<5mwjQEgZcp7HP8g3w(XbHrfLgJMF4zE#$!WbON$=nPaBlzN7wVn8@wsn-45``-Sh{ zz0be$?87s+>^HFfhws0A^#Olz&JI`?+N}s11NlQ@`#VJ3S(NOkbRNvRvy2c7l7%~u zVEvx8eM=QoQ_sfZ93`f6U@w~X`8QvAeBe(%_rRrYeBqRT@#>Rz4XnQs6+U?9^~L-B zfsD^QA;pM^iHu}Y;sn#~&5RvlUXrnoO3vR^Cm!03z~-NM(x;x8*Jdv7?Ct0oiR&Bp z0%g|Wb|(FS+rHul&t7XF<&XF4-hcCj?;rI?d7A0I!~X4x4+n>6zVrHXi{|_RiU!mG zk)5eHWsJAk3;Sz2*T~ocY2sU8Ik6KskM-)=7q=}*G!H3VEyc*&D;H{UIe&^3J(9XPY z=YUyVcinq$Jagxfe}3h3j`JHq++Dj@OZ(U>@Z2hc{rlUpotV*k(ucTV}WKI{c5{{mD>E&xUCqpzk!#Zx+XW#F7K&(;dZ{CDmM+`U?a zIjeh?9{ndZp!t^tCal#OrgM0WoQGmmGKJT-hzPAL1bRCHwu# zD?d0V*IbFrt54p2#ezS?@L<@B)pRC%V`4sSql^syv&J9q!Qvz1lL#u07r-V zM*6|pe)jf#fBEsNcYo)l!A?|!AZzptC#LO8K(rDKv(b~bj#Z>EJfoye$^hg z9{yh#_#l}%28D6KrMtBk$FV8a5e%ucDdJZ3<8@+VOgT1BH}Av#-H8ez4VZAzA!tUH z7`z;Y=X@@yD#Qd$vCiw3*&~d{K|G@$aXVV&$BHVbT?F4ZV6Vh+$ovym>^wXuKZ}|# zEIAe-mXjpmvSH zdC4I`9KPDqi)Z~TKpdvcmzJJ&TUryE;?T_3ogSztKchJ?GvtQ@`pq-t-bWyx7)PaK zxB(rkGuM}==j1&48XaBX*wNjLG)dOra5u9>6%AK%B$Gp#kQ?I7dUf;BMX0IeJ_5;MVfBjcMj? z{af;W@VM$HE*6~|Yjn4%Jzptj99kQjG(TrtB|kiG!b->c4UccCxnjjrH}4d;x5(o<$Qbbsrq5w0Ge5N|nY;J> zr5ElzY88$1KIp%)Y|)(u|ITW}cYIAHepe5}^;u5R&*Sa1@^>?WWGYSFvC8_JXHX*? zZG7Qq#LIZ!WEJZ_aNPg@#uGR7fovsiz5UX%ADp?S={a|lt!@Vg=E9#{CbXeEx3IHE zwE=f3$!wl8NdiQD3k~0*d&hdo68D@pYu7-}Eg*S}^~c81HS*vyA2{uQzxLR5t8TNs z`TYIM7EZopZZg9}{MAUr#tC1YyTzI42lXfa<~~^snQZh{worP3;PG21bf&6itf2?p z%0_aqp8tgT3>UrM_Fs6Tr4mi^y(<%i?tW#&?! zyZgBNj{ob7V(Byy<){2UGH3O?jEmk9_>#_W#ADM}O(@%eH^%zEjxIy5}?*OE2Gk z>7kb&Jn!*q_E>u2UvqweKQ>=8vOTfRca+*R2WnTkj&3x|_!b?KIy`z<`RK_$NY!r%{f;G^LGM8!70qbKlYh6!$VVsLt-FdN z3ZQ#qT|CA6WR}dCX8ZY84eMM^<`aVEFt6d)?o$(!hE6ccxSC54o^-Mqs?0ZgtPXVM z`fkpZN4i3`N2cTB>aqfCGZMCzy}fF3w>ge%K`Yk8^Z_;zr&1n~F5 zn9xanmRpo~x0WngJE}9(rR6}A0Bk3#T;7s2cCJ^8_FdSA9)zaXh>{Ui>wE^9Z{!w9h~w4tzp~`Jc-%c%2a)=f>K_YhnsxiS z(s*nt)ojNlV~Pc05~>YIX)vesw(KV83@d(6Xx4nu`jAfwzcN|`53;-B#~4@0D)d8 z3(0#hnE4S(Am0QU6Qd&@D)t0od9Lhj}mMQ)@5sciYt!g{xARY zch_+EyFBI^>l-t0isy--6l!WEUDe6XxDQ>Yc8h;YcW(JZm=cygElI@?y<$IDTkUlUWM-(zPtY4sR>uo zOOEs?Ik$}cu&mDdw+S~TtTWA>mPv?AuDq&PcWvt1!>P*;l~}JvxUD7g*L`|3V2!TN zSj(o3RQ4Nr6&AL4Mv!vnX(#u|V64NL)`fnf#N0}qn`SEp)f+h9Fq4(+Z0ATAd0lsB ztWBAIf5r2Q7ip{`7Ybt!6cUQ=woxD z{>9pBl#S;)%?yO6J{gX@Ymdv{5~&F#94aQA)pBZ9poCjJ%X6WymIUaBH4s3A&E$9E z>qMdjv-lIrlKy$>Q-RjtL^5;~c}CD>JFYWEJBdet!!Ma}(Ti0nzdwzSm3V>tqr3(^ z-zkZ9;!C?ew)RRUB%Y<*O$~p|;lw_PDPu&2?GtOMyh|RSsQ^9H%DewfBW|;^wLI4@ z?yiiATA>G|Spns{{!Ft>E*vX^_{zP3prcyuUdmHR1W7_v^+rG~0&#J;VIIGtDd9(FB@mCUxCvq7!{smV-`a4Z9}H-7ZWe*?1^lMp#6^ZTyhM zi?L4mqMSDMw(&TS^-PAqx2Q>dp2~AW+!e7!D#ZdN`Cs47B{T(s`Rj6~{PkQ|W_&Dy zu`3y~EOCC|6(T8HsO4GZok#BExyz&~-=0}MbA4G-<~@n#j%{9H1emc|KCTi0QB5?E zZ$4YTBzNgvz%nH?D^ntpi#qxBa*4|m*43EHg%&l~SGrr!A?2N?!m1G&1f6F=26R( zE{E}HDveo+*PuMt5h(_l(V!$ic=FSm*FK^8w*;$y-oW**!(DSXKj#>yUPe!ngg|#evfKFJKKm&JLFxa*HU!Zp{5LQX^ z)5^{)^IbKAUnp^f+EwMc)kAc+JR*6LAX2qTDCKe&syv!XKb>k8uifh>y56%JJKocG zo8M*L9~ag`^6Xe-0J5~uqy(3{aS=erw>DtN#v>&VZlNq0Z@%owKF`Ju5HH*EE?#QA&$M|uFW zJeN2_xo_!iX!&uXoHd`=p+8x&W_0IjYJKSawQn;T3tR=l2HCevQ+RafcX;QHKx z#f1kcrX`m&^XZXzNxgiW;|#3+3gcVmYbE&?$(R|NB`{;~qr_LUPFdgiIxlw_vdkh6)Mr4UwBX^H`t}ED&eL> zW6KtRH7DMr$g3Poxfb<9XjI~&G#iS1vOThV;8xK>+1kqId+>7Cd-iVGOtdN>4*9IH zPS-I$zxgsiKfaFY=G^?OMj}Vz(R-DIL>SC3WF(*mg#)}FOnrdBxQztF?#=#4r< zt(+uWo&E=K)uEs}spwuf< zvYWfb(qQN=$|%?Ss*A6CNDu1gXIE3kLcS37L;pl@Yb7*_r+SyK9545Ta$JSRcQs<| z28-_<4v6vfa9t9P`{yb;>C#yd_p2~(Qe~zFF^djMjaJcfRgSS5PgRxu$?=FJcJ`I! z`a|)Ndoo{#ffYXzX{?$wm2cJZ`B(a6y85c>9}0VQzP_|qdGt(2$_+ysmW0F998jmv zr}~K!SgMIitePHE}vNt(wZHD;LA?1ZNZL#Q&AXJyQK^w{ef!Nm zpse;=*3_llmHRno1_Jl9sDMcTJ99PA{DgHqJwVq{NXRkQdct(6QnSx<@6~R(&9bo6 z{>{5(%^~yI0Lbs~@n_tV8P=Yep9Pcy+k~UpkXgH+cbf#}ZmW`>Z<`BOt}~6@YR5X0 zgx`vIX@|%2Z%MAz-L66^P&ubu-_|%CQY$GfZ5SjTRGr+DbbjuooEeD7&#nS!qQpz{ zz;9*lcE0)RF4yV+=tQ5{m2#husZjzwMrD}?F(Lq(T7~>?My;8lPP4F9;&%L9iEazS zINdEunqTS?uKo&;?Nj0|>eEPmw%M8gj3xQ4a;Et*z9qxH*N`DU2&@@9XWU~=8mLvP z{afPbTV+YlyvfgRUgrI7F3IoNGn2oXyS4iHU+VOHmcsn_CqIL1mmV)9l0V9Co<7hk zG8%Qt&v&;}f-yB%6{bGqU&+bJ4d}Znk=T^X@#ALC8X4671$UP;-?r<`8ysD}bNOul zr?}gEu=%(f`NUoU8lI)wdGqpnhsifTzkI5Bml6zyJP>dNV-3o=i?Zae;amP&!`e;d z76xX@cawT*jC^y2+|*blxxyzu)70aB=lp3aouQqc&jPneE8hiZq&>?vWl>aubQtdD zsoV&e;d?tj#hV~!5N^$9T_=~<{>@*PD|~u0xi?li<8L4MxT_`cQdkAUU`^wz-osG4 zO}kZZ>gT=%quJ*(Gu*jOF3B0|?)-0Ltie0F-gSJnJck}~o1bMqh8{3L{eP%dhuw|3 zVeJydW#zG|W!>HXcR#!P3G9KfFm<^kd^K2bN#^lS;J1ACO8t9a0M@bE8hra#>;Ds< z-ThSbKxsbpf%{YOEuX#Q@I9~+So@LTf8w*6pUfU8p&#oFKf`*gp80f@msppWj+Os( zecfkoTlPTHVDsJ2ur7P)=Of?b>#<5o8Yt^D`=O+h=H1Uq{<(UfM6G|`Z`iEUeDlw+ LZuZG1q2&Jpc>|1$ literal 154542 zcmeI5cUV18_`MJM%)IBFcjkQFduHZLyVdHh_4~gf z`Jb}*R|5Yc@XzCSMT-2c$U6uXDKY@po5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* zAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8} z0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq z5+DH*a0-FY?rqjY%s(8r`{=>AnAMTPgSy%^8`Q4Jy2wSx686XMiVYt%P}T3DT)B59w5A5H$b;*oLxCOnMUedcwLm1q@V+#`Q*tBkPSV)^% z74&L)Nl^PH%VvKcziZp6q{GR_67jctYxL-l9(qkDN&=eKew3a4{Q2|C7tgO*Hh;x} znWv5>Ao}#_)8Ro~Y-P0d$5qg|aPCax-0;n7SLEg8J$?E_xDQcWx|n)4a^AE}t0QxA za`N(W1De-SO|`7??UN@@`gUlh3Jt^FrHW-|W)AP$&G4!aP8`xNC+9ITz5nywgL}5G zi&&^vPK@6Jg_Kjr#|;eBT-~&KIWFO|Z+lF{obc$VW%uvhMKm?#ltxn%J)3+2YWMHn ziC(pQ`P}fmu`$n|KZ67hi$+t8;+uc{>Sd_iy?rxgP2{3!6T}~qKYRA<^2JnD*@@u^ zgZm;=+$`XiEB41qAOE$|$8VMY;Li;_K3y_%GA_aO`s`3Pl&XK{mg=S0jcYP99?l#W zhE(5mYolI@UKQ~qFHe;qyJydy_UmZL*N?KZZ27uVL(i*e7m(w^@bMk$RaLW4!^`JR z%gfC*d{mPT?a$5086DEI+NXc7@Zn!@`xQU>D+0Y+H#mL#=)(v1J2voCO^I-u+7&Y% z+&_2vWS=&TG*@8?H+$@`aoytIxCYRn^6cE$%j5k&A+Jis-x7ZnK76x7{i>&rAGvk? zn#N|~qg|W$Am4@4##jC1?|N&VCI2zJZ&zR((z~O2wPA(NARN`-ptiwdZtK$dv!~;B zY>{@H;%ije$Sx@%UYa!w)GYHM)Oxh6tG^n*YwN@N_v)4V#Ixi-^;fHX`j6Bzr|=Bw zFWtJHZvNC~JiLE+{~k?3uP=(CB&hfBG-)o$X#H}Zph##OrNk)#v=N8ncHx_B%UZ>c z-o*3r_|YR(VQBA8sNM~m;D|X>QJ1#x`bO0kVYGk`?%$L0!u#6aw+6TwHoh_!5tuqM z6!bJgL|fOa01K#F;8XooNbm)y@kP-G_kPw?ZtIpWL}Ou?L<&Rv{a_r0loA9`f1;xB zDfPZ8)P$>)cqd}^_g5~S*T@SKZRzz53WINv;a6H(&TlTWGRKDYmXb+f{n6CQs2Z3z zd34U>$FQjiMR@$6Ks38jUU**v+JW7%(x6Eoa_+R7Mz!LmHIXM052EEVNeejB#|!}| zRVVY|1AKK=iDCF$@(DEk>ZRP=+`&CNRQ>ngRG}t}rlenorf9Y_^1?(HPoH${a+*)6 zVs?yt{@$^Kcrrbb z3C#grT8U`)X0;GC)YJ%D`IJ*NaSOL^UawsIZH<~H>Iz?PS-&#nalP8W_eRi z@inaBr5k0CW^D!9*Q*L*ZEIJQ3JGz$(5sYDy8&Ci&dkV|GHgKAPt00R7>9H5wj#%K z3he9Vo?qie@a+8ZbvW(V8uKYW+L^_gjrr&hqunm~|%-583t9>753JG2V5ANA<@uw73!CAhp_2RYHd@xp2gM)j$^n#C856`1o zB=fK>U!xtt13PEph-yZ|3$O`!ld7fhRUaALU1m`jUoV|G8DE>mmA;VH^Z^(78np@@ zncZ79)-3%2o^JK)je!)e&x>N*53Op~rZv?fIm_3uxnki=knGm1mTF&y(R|#vb_L^* zQX&F(Z{NZjoJMF8wdHFhiJU(jqnbVa>gjI(pZSiiTq-LwBb&V>1{%z4W z%75?|)an?}z;I%Ab{71iFyksae_cfA?4efIy{R%QgXB8u^ z=UG3d$X6CE!CP6!>BwlD1?4y)S@|2i}U|hEuOn-$sDw*n0a(! zM(%40W3YH4P?uV~1&hA6XzdqJ!!(U^$IX6$GP^DdwDRboS+wtVzIOT;sYOt)rkZ$N zTCre;mEzIYRw%v7I-^2*pe7G&=WDl+^tD|?uZTi3&#%yJGc3%=e%(>s;e`}aNTd1( zpsVla?V;DSrF{XuhtEw3=8>T-t<+A=Y)?QBxgpXLSX{0RUPhI zxfDjm@E(mPSN-gb-2~UD82x>{uU@*S>R^-)D|u8M407T%O-QgJ4hse_q@X4dVZ8QY z*%bzj)wG&0RFvhh%5WkL~ML(Ii0>nf|a~#-1%3vB)8?t^JDV_3<^<>1Aa+ z{2$Exw|;!Y>%R!&?S9O6`D-U84hhDvo_bZNVY%0&p}`vFm_WaK>!z@vCJ_>8A5-%e z&ZTr|;-g89=++;j&^^}ZYof?ney@-OYf5k5NEbTtf4n<~2DH`Nmy*}P*BJfA2p*V7 z@{*4Xh)bxkJ2lE*LSXoT4ptJ%RZ?y1RK`>tG$F^wFo_gOh0SYLNcj1aeaDUn3sTc0 zfK?e-;eg~AoOzMwOPPu+^otT@cCUl4v$8U+@Wdb+Mlw-Xn=8j|SOa=J{OXHgKjE4_w zFwAFr{bDKjODP?XMIGEEThro!+X@#NZw#mBb=yjgC;j0HDUpBKq6* zby#3$4Gohh5Vi0s3on@HU`_|UEGSrakfAjs7lZ%dBSQ@p%!Ml#&8q$N$5=qCSA*q5 z-J;^x!Pn-F(}2R(7_{tI#Hf+WD2BCtD}O0Avq)iK0;wsml-)nfu-G!JqbBG~!;?ykj0a|T6J=nq0eeW-eZH$U=$fk)ha={0M z#2N#KYI0!{Gk2aP^OpeWKM9V^*15ZvMftIRP0ztl0_KooksdZFV0A0PLL!t!7}M*1 zJ|2e23nRpgGiH`w?AUU^42pE^(nTx+!9sL3nJ6lj#QZiqX#FnEo;t2_uoVU$Olo&+ zQWG7sDZ}&EAb~%W1viljUGB2){vNw;Y#Aw-Tx5aO3!)F;kP$Q`4a%2{S~Ode%Hkxr zb&PJIDK_6}P-vop_OL*IY>>hBMhsbCO@7mAWw1sFn$q3aqt~QLDPWWmjh(_o@v@Bl zP-(i+Ji9iVR4I)Wvr-LZ!w?V}Ww-E-a|?OxN}A+g62P23*7sspg3!SujEN7tXrt$I zDk%}oG!}(oH)4PPmL^6Da|tU+uvJa0S#rouySkOJc~d6AEufIjW#}%}emZkfaxe_w zLBOyEwk>Q}xfBYB;v+Qo#l~P&vs@L4&8=&e%`-IO6k$+%U$HsVA!}H}iIo*->(?${ zfYB9q@rx_UiPucK=;lrug;o!P)zZsIBExB^=Z+rShw~6nypu-`*|k<+g_&c+@Q`8v z*182KV*t$++}zRT)R6;K#~EU*{q;v!bZ2-;3iEf+lz1D;gqN?Bgw~(xQ?4U zS{8%GvQE};`+h`d_vW?5^rhyK8qI&Mi@$v%{i-T-ZWx_`W|p68aXVNC?v5UDX_@cY z7zcy1aP+~zfOfXN3DTQ5JOLwSSl=!sP7ll+H>_QqO4i$JOoh7?nwuKOFa=Ibf_L;j z#m4*zgM#cj4zJZstCm^2Y=K?P0xR_G(6mq6MplxGyTA?(H+6@ri;XU>VWW)WKE_+{ zHSmIMNEbgVz9@jssA!ERdN=w-7U*ofq6wy&UlOgdv~L%@^|c)t#o>Tsl5 zI%^6ZZ467ww08YsT>(96?7-5mg40+KSsU&TdIf?1>P5ov6!ul2GbZ|DQ%8I!`et@L z2rpyf;pGc<7BXqPV=;>DKjJ_kyTzEu{+R8J-nPD7*ba=tD=_j21&kt_Ll#%Z4-CX1 z3>MvIT@9UloL__~HhmJcR5+(mJp#hkeNk>sAa45>J!g#s*fB8f_4#|-Hsl|nsd_=9 zX+0{8Oc)%5Npox^!}Ld3pg#@~#Z(AZ8R7IZ>^8#jp0mbsAbh~aE72F+D zOV08&?BM(3QS6V{ISbDo-oM-W5ljx<9hFrA*phG2gGhx%xfYe7X7~|LG#=g>U;GPa zazaO~D6qF$EDTml$gmR$+`e($&@ZZR!PIf+&sQ$~mN;@yb&#rNF^pXs&8n4o@+42S zBf{99;>25{DLnzQLV2&+_hzc^bWRcg!s4orsx|JhX&(-FI7`bEA>t<@+p$44JkDtDuccj7zW@lUANu!pcRRVtA z<f@j)EJH#?jbkLywzl%|(5xj8e#dqLoSuZk&#~SZrl6038vIf~uU7T3 z%nvIPA%~06ol8qSi+TXRVWB!#goPWEOB;%@i3 zR+DJL!mTLhkpuhWrOOvm@$)p!Dm?8tL;$lP(h+K4<-*wxt^HHes$rbLGPHN+@dJZ; zS~y_bp^{U%00i*EM(~cTU4^Hxy9Z0e_5EC8#! znNJ$hPiBf!+2LRL$XoHXP{vOYNh1un3jr}^ZpF*;Td{ZBCU?0zmQzDO43F4-x`hJQ za%(8|yC>djK}DigpDvg36O z+r9B)6Wcbd)}LefDi@w*BLNa10TLhq5+DH*AOR8}0TLhq5+DH*AOR8}0TLhq5+DH* jAOR8}0TLhq5+DH*AOR8}0TLhq5+DH*AORA14GH`Qke!Ck From c04d1f1f69f2a5d6e2fe4899edccf820b4534dc1 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Tue, 30 Jan 2018 11:53:52 +0100 Subject: [PATCH 234/262] Make different program so it doesn't mess with existing config Signed-off-by: Roeland Jago Douma --- NEXTCLOUD.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEXTCLOUD.cmake b/NEXTCLOUD.cmake index 180a9fa98..e2310e0a2 100644 --- a/NEXTCLOUD.cmake +++ b/NEXTCLOUD.cmake @@ -1,5 +1,5 @@ -set( APPLICATION_NAME "Nextcloud" ) -set( APPLICATION_SHORTNAME "Nextcloud" ) +set( APPLICATION_NAME "Nextcloud-e2e" ) +set( APPLICATION_SHORTNAME "Nextcloud-e2e" ) set( APPLICATION_EXECUTABLE "nextcloud" ) set( APPLICATION_DOMAIN "nextcloud.com" ) set( APPLICATION_VENDOR "Nextcloud GmbH" ) From 080c5ea67817eb0b4bb80e4ba28d7cfb7e516d82 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Wed, 31 Jan 2018 15:58:31 -0200 Subject: [PATCH 235/262] Better explanation for encryption passphrase FIXES: #149 --- src/gui/accountmanager.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/gui/accountmanager.cpp b/src/gui/accountmanager.cpp index e7f6517d9..45b75b3e3 100644 --- a/src/gui/accountmanager.cpp +++ b/src/gui/accountmanager.cpp @@ -336,8 +336,11 @@ AccountPtr AccountManager::createAccount() void AccountManager::displayMnemonic(const QString& mnemonic) { QMessageBox msgBox; - msgBox.setText(tr("Note your encryption passphrase")); + msgBox.setText(tr("All 12 words together make a very strong password, " + "letting only you view and make use of your encrypted files. " + "Please write it down and keep it somewhere safe.")); msgBox.setDetailedText(mnemonic); + msgBox.setWindowTitle(tr("Make note of your 12 word encryption password")); msgBox.setIcon(QMessageBox::Information); msgBox.setStandardButtons(QMessageBox::Ok); From 1eb7ba72f0ada54b92b2e1eb216b5d2d62cc30c6 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Fri, 9 Feb 2018 11:40:06 +0100 Subject: [PATCH 236/262] Use standardized filename See https://github.com/nextcloud/end_to_end_encryption_rfc/issues/13 Signed-off-by: Roeland Jago Douma --- src/libsync/clientsideencryption.cpp | 8 ++++---- src/libsync/clientsideencryption.h | 2 +- src/libsync/propagateuploadencrypted.cpp | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 3d89eab76..5fcdaf53b 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -65,12 +65,12 @@ namespace { } } -QByteArray EncryptionHelper::generateRandomString(int size) +QByteArray EncryptionHelper::generateRandomFilename() { - const QByteArray possibleCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + const QByteArray possibleCharacters = "0123456789abcdef"; - QByteArray randomString(size, '\0'); - for(int i=0; i < size; ++i) + QByteArray randomString(32, '\0'); + for(int i=0; i < 32; ++i) { int index = qrand() % possibleCharacters.length(); randomString[i] = possibleCharacters.at(index); diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 386f314ca..7a35a0606 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -28,7 +28,7 @@ QString baseUrl(); class EncryptionHelper { public: - static QByteArray generateRandomString(int size); + static QByteArray generateRandomFilename(); static QByteArray generateRandom(int size); static QByteArray generatePassword(const QString &wordlist, const QByteArray& salt); static QByteArray encryptPrivateKey( diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index e08864aec..08eb505f5 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -132,7 +132,7 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo // New encrypted file so set it all up! if (encryptedFile.encryptedFilename.isEmpty()) { - encryptedFile.encryptedFilename = EncryptionHelper::generateRandomString(20); + encryptedFile.encryptedFilename = EncryptionHelper::generateRandomFilename(); encryptedFile.fileVersion = 1; encryptedFile.metadataKey = 1; encryptedFile.originalFilename = info.fileName(); From 99117078f3b0a6e4673778e803ff48182f7fc2f7 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Mon, 12 Feb 2018 09:50:51 -0200 Subject: [PATCH 237/262] Remove table for e2e and add an e2eMangledName column (#169) If the code was not complex enough syncing two tables already started to give UNIQUE constrains errors on simple sync operations, this also adds initial support remote delete of an encrypted file --- src/common/syncjournaldb.cpp | 103 ++++----------------- src/common/syncjournaldb.h | 7 -- src/common/syncjournalfilerecord.h | 1 + src/csync/csync.cpp | 1 + src/csync/csync.h | 1 + src/csync/csync_update.cpp | 20 ++-- src/libsync/propagatedownload.cpp | 6 +- src/libsync/propagatedownloadencrypted.cpp | 8 +- src/libsync/propagateremotedelete.cpp | 11 ++- src/libsync/propagateupload.cpp | 8 -- src/libsync/propagateuploadencrypted.cpp | 4 +- src/libsync/syncfileitem.cpp | 4 +- 12 files changed, 56 insertions(+), 118 deletions(-) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index 57bb267e5..16571ceaf 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -39,7 +39,7 @@ Q_LOGGING_CATEGORY(lcDb, "nextcloud.sync.database", QtInfoMsg) #define GET_FILE_RECORD_QUERY \ "SELECT path, inode, modtime, type, md5, fileid, remotePerm, filesize," \ - " ignoredChildrenRemote, contentchecksumtype.name || ':' || contentChecksum" \ + " ignoredChildrenRemote, contentchecksumtype.name || ':' || contentChecksum, e2eMangledName " \ " FROM metadata" \ " LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id" @@ -55,6 +55,7 @@ static void fillFileRecordFromGetQuery(SyncJournalFileRecord &rec, SqlQuery &que rec._fileSize = query.int64Value(7); rec._serverHasIgnoredFiles = (query.intValue(8) > 0); rec._checksumHeader = query.baValue(9); + rec._e2eMangledName = query.baValue(10); } static QString defaultJournalMode(const QString &dbPath) @@ -468,17 +469,6 @@ bool SyncJournalDb::checkConnect() return sqlFail("Create table version", createQuery); } - /* maps the end to end filename in the server/metadata to the actuall filename in disk - */ - createQuery.prepare("CREATE TABLE IF NOT EXISTS e2efilemap(" - "mangledname TEXT UNIQUE," - "name TEXT UNIQUE" - ")"); - - if (!createQuery.exec()) { - return sqlFail("Create table e2efilemap", createQuery); - } - bool forceRemoteDiscovery = false; SqlQuery versionQuery("SELECT major, minor, patch FROM version;", _db); @@ -595,8 +585,8 @@ bool SyncJournalDb::checkConnect() _setFileRecordQuery.reset(new SqlQuery(_db)); if (_setFileRecordQuery->prepare("INSERT OR REPLACE INTO metadata " - "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId) " - "VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7, ?8 , ?9 , ?10, ?11, ?12, ?13, ?14, ?15, ?16);")) { + "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId, e2eMangledName) " + "VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7, ?8 , ?9 , ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17);")) { return sqlFail("prepare _setFileRecordQuery", *_setFileRecordQuery); } @@ -716,16 +706,6 @@ bool SyncJournalDb::checkConnect() return sqlFail("prepare _setDataFingerprintQuery2", *_setDataFingerprintQuery2); } - _getE2eFileMangledName.reset(new SqlQuery(_db)); - if (_getE2eFileMangledName->prepare("SELECT mangledname FROM e2efilemap WHERE name =?1;")) { - return sqlFail("prepare _getE2eFileMangledName", *_getE2eFileMangledName); - } - - _setE2eFileRelationQuery.reset(new SqlQuery(_db)); - if (_setE2eFileRelationQuery->prepare("INSERT INTO e2efilemap (mangledname, name) VALUES (?1, ?2);")) { - return sqlFail("prepare _setE2eFileRelationQuery", *_setE2eFileRelationQuery); - } - _getConflictRecordQuery.reset(new SqlQuery(_db)); if (_getConflictRecordQuery->prepare("SELECT baseFileId, baseModtime, baseEtag FROM conflicts WHERE path=?1;")) { return sqlFail("prepare _getConflictRecordQuery", *_getConflictRecordQuery); @@ -791,8 +771,6 @@ void SyncJournalDb::close() _getDataFingerprintQuery.reset(0); _setDataFingerprintQuery1.reset(0); _setDataFingerprintQuery2.reset(0); - _setE2eFileRelationQuery.reset(0); - _getE2eFileMangledName.reset(0); _getConflictRecordQuery.reset(0); _setConflictRecordQuery.reset(0); _deleteConflictRecordQuery.reset(0); @@ -814,7 +792,7 @@ bool SyncJournalDb::updateDatabaseStructure() bool SyncJournalDb::updateMetadataTableStructure() { - QStringList columns = tableColumns("metadata"); + const QStringList columns = tableColumns("metadata"); bool re = true; // check if the file_id column is there and create it if not @@ -905,6 +883,16 @@ bool SyncJournalDb::updateMetadataTableStructure() commitInternal("update database structure: add contentChecksumTypeId col"); } + if (!columns.contains(QLatin1String("e2eMangledName"))) { + SqlQuery query(_db); + query.prepare("ALTER TABLE metadata ADD COLUMN e2eMangledName TEXT;"); + if (!query.exec()) { + sqlFail("updateMetadataTableStructure: add e2eMangledName column", query); + re = false; + } + commitInternal("update database structure: add e2eMangledName col"); + } + if (!tableColumns("uploadinfo").contains("contentChecksum")) { SqlQuery query(_db); query.prepare("ALTER TABLE uploadinfo ADD COLUMN contentChecksum TEXT;"); @@ -1063,6 +1051,7 @@ bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record) _setFileRecordQuery->bindValue(14, record._serverHasIgnoredFiles ? 1 : 0); _setFileRecordQuery->bindValue(15, checksum); _setFileRecordQuery->bindValue(16, contentChecksumTypeId); + _setFileRecordQuery->bindValue(17, record._e2eMangledName); if (!_setFileRecordQuery->exec()) { return false; @@ -1382,6 +1371,7 @@ bool SyncJournalDb::setFileRecordMetadata(const SyncJournalFileRecord &record) existing._remotePerm = record._remotePerm; existing._fileSize = record._fileSize; existing._serverHasIgnoredFiles = record._serverHasIgnoredFiles; + existing._e2eMangledName = record._e2eMangledName; return setFileRecord(existing); } @@ -1463,65 +1453,6 @@ void SyncJournalDb::setDownloadInfo(const QString &file, const SyncJournalDb::Do } } -QString SyncJournalDb::getE2eMangledName(const QString& originalName) -{ - QMutexLocker locker(&_mutex); - qCDebug(lcDb) << "######### Trying to find database entry for name " << originalName << ". #######"; - - if (! checkConnect()) { - qCDebug(lcDb) << "######### Connection to the database is wrong. #######"; - return QString{}; - } - - if (originalName.isEmpty()) { - qCDebug(lcDb) << "######### Original Name is Empty. #######"; - return QString{}; - } - - _getE2eFileMangledName->reset_and_clear_bindings(); - _getE2eFileMangledName->bindValue(1, originalName); - if (!_getE2eFileMangledName->exec()) { - qCDebug(lcDb) << "######### Error Executing query. #######"; - return QString{}; - } - - if (!_getE2eFileMangledName->next()) { - qCDebug(lcDb) << "######### Query returned empty. #######"; - return QString{}; - } - - const QString ret = _getE2eFileMangledName->stringValue(0); - - qCDebug(lcDb) << "The return of the mangled name is" << ret; - return ret; -} - -bool SyncJournalDb::setE2eRelation(const QString& mangledName, const QString& originalName) -{ - Q_ASSERT(!mangledName.isEmpty()); - Q_ASSERT(!originalName.isEmpty()); - - if (! checkConnect()) { - qCDebug(lcDb) << "######### Connection to the database is wrong. #######"; - return false; - } - - if (mangledName.isEmpty()) { - qCDebug(lcDb) << "Cant create e2e relation on the database, mangled name is empty."; - return false; - } - - if (originalName.isEmpty()) { - qCDebug(lcDb) << "Cant create e2e relation on the database, original name is empty."; - return false; - } - - _setE2eFileRelationQuery->reset_and_clear_bindings(); - _setE2eFileRelationQuery->bindValue(1, mangledName); - _setE2eFileRelationQuery->bindValue(2, originalName); - return _setE2eFileRelationQuery->exec(); -} - QVector SyncJournalDb::getAndDeleteStaleDownloadInfos(const QSet &keep) { QVector empty_result; diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h index 7daf88132..a553ef4f1 100644 --- a/src/common/syncjournaldb.h +++ b/src/common/syncjournaldb.h @@ -139,9 +139,6 @@ public: void avoidRenamesOnNextSync(const QByteArray &path); void setPollInfo(const PollInfo &); - bool setE2eRelation(const QString& mangledName, const QString& originalName); - QString getE2eMangledName(const QString& originalName); - QVector getPollInfos(); enum SelectiveSyncListType { @@ -290,10 +287,6 @@ private: QScopedPointer _setConflictRecordQuery; QScopedPointer _deleteConflictRecordQuery; - // End to End Encryption Related Queries/ - QScopedPointer _setE2eFileRelationQuery; - QScopedPointer _getE2eFileMangledName; - /* This is the list of paths we called avoidReadFromDbOnNextSync on. * It means that they should not be written to the DB in any case since doing * that would write the etag and would void the purpose of avoidReadFromDbOnNextSync diff --git a/src/common/syncjournalfilerecord.h b/src/common/syncjournalfilerecord.h index b09006365..a29c2477d 100644 --- a/src/common/syncjournalfilerecord.h +++ b/src/common/syncjournalfilerecord.h @@ -64,6 +64,7 @@ public: RemotePermissions _remotePerm; bool _serverHasIgnoredFiles; QByteArray _checksumHeader; + QByteArray _e2eMangledName; }; bool OCSYNC_EXPORT diff --git a/src/csync/csync.cpp b/src/csync/csync.cpp index 992e60f21..25c82e291 100644 --- a/src/csync/csync.cpp +++ b/src/csync/csync.cpp @@ -409,5 +409,6 @@ std::unique_ptr csync_file_stat_s::fromSyncJournalFileRecord( st->size = rec._fileSize; st->has_ignored_files = rec._serverHasIgnoredFiles; st->checksumHeader = rec._checksumHeader; + st->e2eMangledName = rec._e2eMangledName; return st; } diff --git a/src/csync/csync.h b/src/csync/csync.h index e913a726a..23c57710c 100644 --- a/src/csync/csync.h +++ b/src/csync/csync.h @@ -194,6 +194,7 @@ struct OCSYNC_EXPORT csync_file_stat_s { // In the remote tree, this will have the server checksum, if available. // In both cases, the format is "SHA1:baff". QByteArray checksumHeader; + QByteArray e2eMangledName; CSYNC_STATUS error_status; diff --git a/src/csync/csync_update.cpp b/src/csync/csync_update.cpp index 7bc6936c9..cd6be8758 100644 --- a/src/csync/csync_update.cpp +++ b/src/csync/csync_update.cpp @@ -192,6 +192,12 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr f return -1; } + /* + * When file is encrypted it's phash (path hash) will not match the local file phash, + * we could match the e2eMangledName but that might be slow wihout index, and it's + * not UNIQUE at the moment. + */ + if(base.isValid()) { /* there is an entry in the database */ /* we have an update! */ qCInfo(lcUpdate, "Database entry found, compare: %" PRId64 " <-> %" PRId64 @@ -354,14 +360,14 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr f /* A remote rename can also mean Encryption Mangled Name. * if we find one of those in the database, we ignore it. */ - qCDebug(lcUpdate) << "Tryig to get the mangled name!"; - QString remoteEncryptedName = ctx->statedb->getE2eMangledName(base._path); - if (remoteEncryptedName.isEmpty()) { - qCDebug(lcUpdate, "remote rename detected based on fileid %s --> %s", base._path.constData(), fs->path.constData()); - fs->instruction = CSYNC_INSTRUCTION_EVAL_RENAME; - } else { - qCDebug(lcUpdate) << "Should *not* rename the file."; + if (!base._e2eMangledName.isEmpty()) { + qCWarning(lcUpdate, "Encrypted file can not rename"); + done = true; + return; } + + qCDebug(lcUpdate, "remote rename detected based on fileid %s --> %s", qPrintable(base._path), qPrintable(fs->path.constData())); + fs->instruction = CSYNC_INSTRUCTION_EVAL_RENAME; done = true; }; diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index 95afb8cca..f27663dab 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -934,9 +934,11 @@ void PropagateDownloadFile::updateMetadata(bool isConflict) done(SyncFileItem::FatalError, tr("Error writing metadata to the database")); return; } - propagator()->_journal->setDownloadInfo(_item->_file, SyncJournalDb::DownloadInfo()); + if (_isEncrypted) { - propagator()->_journal->setE2eRelation(_item->_encryptedFileName, _item->_file); + propagator()->_journal->setDownloadInfo(_item->_file, SyncJournalDb::DownloadInfo()); + } else { + propagator()->_journal->setDownloadInfo(_item->_encryptedFileName, SyncJournalDb::DownloadInfo()); } propagator()->_journal->commit("download file start2"); diff --git a/src/libsync/propagatedownloadencrypted.cpp b/src/libsync/propagatedownloadencrypted.cpp index 2cbee1ca5..92cab8460 100644 --- a/src/libsync/propagatedownloadencrypted.cpp +++ b/src/libsync/propagatedownloadencrypted.cpp @@ -123,11 +123,11 @@ bool PropagateDownloadEncrypted::decryptFile(QFile& tmpFile) //TODO: This seems what's breaking the logic. // Let's fool the rest of the logic into thinking this is the right name of the DAV file - _item->_file = _item->_file.section(QLatin1Char('/'), 0, -2) - + QLatin1Char('/') + _encryptedInfo.originalFilename; - _item->_isEndToEndEncrypted = true; - _item->_encryptedFileName = _encryptedInfo.encryptedFilename; + _item->_encryptedFileName = _item->_file; + _item->_file = _item->_file.section(QLatin1Char('/'), 0, -2) + + QLatin1Char('/') + _encryptedInfo.originalFilename; + return true; } diff --git a/src/libsync/propagateremotedelete.cpp b/src/libsync/propagateremotedelete.cpp index cf230fc22..d0bee9fbf 100644 --- a/src/libsync/propagateremotedelete.cpp +++ b/src/libsync/propagateremotedelete.cpp @@ -65,10 +65,17 @@ void PropagateRemoteDelete::start() if (propagator()->_abortRequested.fetchAndAddRelaxed(0)) return; - qCDebug(lcPropagateRemoteDelete) << _item->_file; + + QString remoteFile = _item->_file; + if (!_item->_encryptedFileName.isEmpty()) { + remoteFile = _item->_encryptedFileName; + // TODO update the JSON metadata + } + + qCDebug(lcPropagateRemoteDelete) << "local" << _item->_file << "remote" << remoteFile; _job = new DeleteJob(propagator()->account(), - propagator()->_remoteFolder + _item->_file, + propagator()->_remoteFolder + remoteFile, this); connect(_job.data(), &DeleteJob::finishedSignal, this, &PropagateRemoteDelete::slotDeleteJobFinished); propagator()->_activeJobList.append(this); diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 8b4f9f18e..4aac491bd 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -681,14 +681,6 @@ void PropagateUploadFileCommon::finalize() return; } - if (_uploadingEncrypted) { - if (!propagator()->_journal->setE2eRelation(_item->_encryptedFileName, _item->_file)) { - qDebug() << "Error saving the encryption relation of the file."; - } else { - qDebug() << "Sabed the encryption relation of the file successfully"; - } - } - // Remove from the progress database: propagator()->_journal->setUploadInfo(_item->_file, SyncJournalDb::UploadInfo()); propagator()->_journal->commit("upload file start"); diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 08eb505f5..afdcc5972 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -131,6 +131,7 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo encryptedFile.initializationVector = EncryptionHelper::generateRandom(16); // New encrypted file so set it all up! + if (encryptedFile.encryptedFilename.isEmpty()) { encryptedFile.encryptedFilename = EncryptionHelper::generateRandomFilename(); encryptedFile.fileVersion = 1; @@ -141,7 +142,8 @@ void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDo } _item->_isEndToEndEncrypted = true; - _item->_encryptedFileName = encryptedFile.encryptedFilename; + _item->_encryptedFileName = _item->_file.section(QLatin1Char('/'), 0, -2) + + QLatin1Char('/') + encryptedFile.encryptedFilename; qCDebug(lcPropagateUploadEncrypted) << "Creating the encrypted file."; diff --git a/src/libsync/syncfileitem.cpp b/src/libsync/syncfileitem.cpp index 563695d8c..04b60ad6e 100644 --- a/src/libsync/syncfileitem.cpp +++ b/src/libsync/syncfileitem.cpp @@ -35,6 +35,7 @@ SyncJournalFileRecord SyncFileItem::toSyncJournalFileRecordWithInode(const QStri rec._remotePerm = _remotePerm; rec._serverHasIgnoredFiles = _serverHasIgnoredFiles; rec._checksumHeader = _checksumHeader; + rec._e2eMangledName = _encryptedFileName.toUtf8(); // Go through csync vio just to get the inode. csync_file_stat_t fs; @@ -55,7 +56,7 @@ SyncJournalFileRecord SyncFileItem::toSyncJournalFileRecordWithInode(const QStri SyncFileItemPtr SyncFileItem::fromSyncJournalFileRecord(const SyncJournalFileRecord &rec) { SyncFileItemPtr item(new SyncFileItem); - item->_file = rec._path; + item->_file = QString::fromUtf8(rec._path); item->_inode = rec._inode; item->_modtime = rec._modtime; item->_type = rec._type; @@ -65,6 +66,7 @@ SyncFileItemPtr SyncFileItem::fromSyncJournalFileRecord(const SyncJournalFileRec item->_remotePerm = rec._remotePerm; item->_serverHasIgnoredFiles = rec._serverHasIgnoredFiles; item->_checksumHeader = rec._checksumHeader; + item->_encryptedFileName = QString::fromUtf8(rec._e2eMangledName); return item; } From 5faeca1b82b3b4a84a8049fcee7a9b9e77b0a7b7 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 18 Feb 2018 01:04:44 +0100 Subject: [PATCH 238/262] Move BIO2ByteArray to annonymous namespace This has no use outside of the clientsidenecryption.cpp --- src/libsync/clientsideencryption.cpp | 26 +++++++++++++------------- src/libsync/clientsideencryption.h | 2 -- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 5fcdaf53b..0118c9870 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -58,6 +58,17 @@ namespace { } // ns namespace { + QByteArray BIO2ByteArray(BIO *b) { + int pending = BIO_ctrl_pending(b); + char *tmp = (char *)calloc(pending+1, sizeof(char)); + BIO_read(b, tmp, pending); + + QByteArray res(tmp, pending); + free(tmp); + + return res; + } + void handleErrors(void) { ERR_print_errors_fp(stdout); // This line is not printing anything. @@ -592,17 +603,6 @@ QByteArray EncryptionHelper::encryptStringAsymmetric(EVP_PKEY *publicKey, const return raw.toBase64(); } -QByteArray EncryptionHelper::BIO2ByteArray(BIO *b) { - int pending = BIO_ctrl_pending(b); - char *tmp = (char *)calloc(pending+1, sizeof(char)); - BIO_read(b, tmp, pending); - - QByteArray res(tmp, pending); - free(tmp); - - return res; -} - ClientSideEncryption::ClientSideEncryption() { } @@ -847,7 +847,7 @@ void ClientSideEncryption::generateKeyPair() qCInfo(lcCse()) << "Could not read private key from bio."; return; } - QByteArray key = EncryptionHelper::BIO2ByteArray(privKey); + QByteArray key = BIO2ByteArray(privKey); _privateKey = QSslKey(key, QSsl::Rsa, QSsl::Pem, QSsl::PrivateKey); qCInfo(lcCse()) << "Keys generated correctly, sending to server."; @@ -907,7 +907,7 @@ void ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) BIO *out = BIO_new(BIO_s_mem()); ret = PEM_write_bio_X509_REQ(out, x509_req); - QByteArray output = EncryptionHelper::BIO2ByteArray(out); + QByteArray output = BIO2ByteArray(out); BIO_free(out); EVP_PKEY_free(keyPair); diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index 7a35a0606..b3dcdb17e 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -61,8 +61,6 @@ public: const QByteArray& data ); - static QByteArray BIO2ByteArray(BIO *b); - static bool fileEncryption(const QByteArray &key, const QByteArray &iv, QFile *input, QFile *output, QByteArray& returnTag); From 176a42a0621ebb451bf2414e952d81ca5a594d54 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 18 Feb 2018 01:13:23 +0100 Subject: [PATCH 239/262] Return the error reported by openssl --- src/libsync/clientsideencryption.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 0118c9870..3599d22fa 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -69,10 +69,13 @@ namespace { return res; } - void handleErrors(void) + QByteArray handleErrors(void) { - ERR_print_errors_fp(stdout); // This line is not printing anything. - fflush(stdout); + auto *bioErrors = BIO_new(BIO_s_mem()); + ERR_print_errors(bioErrors); // This line is not printing anything. + auto errors = BIO2ByteArray(bioErrors); + BIO_free_all(bioErrors); + return errors; } } From 7da0764b8d1238ea8e75c52fd140a7cd4cb71244 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 18 Feb 2018 01:23:59 +0100 Subject: [PATCH 240/262] Don't class what's not a class --- src/libsync/clientsideencryption.cpp | 22 +++++++++++---------- src/libsync/clientsideencryption.h | 29 ++++++++++++++-------------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 3599d22fa..07de977a8 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -79,7 +79,8 @@ namespace { } } -QByteArray EncryptionHelper::generateRandomFilename() +namespace EncryptionHelper { +QByteArray generateRandomFilename() { const QByteArray possibleCharacters = "0123456789abcdef"; @@ -92,7 +93,7 @@ QByteArray EncryptionHelper::generateRandomFilename() return randomString; } -QByteArray EncryptionHelper::generateRandom(int size) +QByteArray generateRandom(int size) { unsigned char *tmp = (unsigned char *)malloc(sizeof(unsigned char) * size); @@ -108,7 +109,7 @@ QByteArray EncryptionHelper::generateRandom(int size) return result; } -QByteArray EncryptionHelper::generatePassword(const QString& wordlist, const QByteArray& salt) { +QByteArray generatePassword(const QString& wordlist, const QByteArray& salt) { qCInfo(lcCse()) << "Start encryption key generation!"; const int iterationCount = 1024; @@ -138,7 +139,7 @@ QByteArray EncryptionHelper::generatePassword(const QString& wordlist, const QBy return password; } -QByteArray EncryptionHelper::encryptPrivateKey( +QByteArray encryptPrivateKey( const QByteArray& key, const QByteArray& privateKey, const QByteArray& salt @@ -217,7 +218,7 @@ QByteArray EncryptionHelper::encryptPrivateKey( return result; } -QByteArray EncryptionHelper::decryptPrivateKey(const QByteArray& key, const QByteArray& data) { +QByteArray decryptPrivateKey(const QByteArray& key, const QByteArray& data) { qCInfo(lcCse()) << "decryptStringSymmetric key: " << key; qCInfo(lcCse()) << "decryptStringSymmetric data: " << data; @@ -306,7 +307,7 @@ QByteArray EncryptionHelper::decryptPrivateKey(const QByteArray& key, const QByt return QByteArray::fromBase64(result); } -QByteArray EncryptionHelper::decryptStringSymmetric(const QByteArray& key, const QByteArray& data) { +QByteArray decryptStringSymmetric(const QByteArray& key, const QByteArray& data) { qCInfo(lcCse()) << "decryptStringSymmetric key: " << key; qCInfo(lcCse()) << "decryptStringSymmetric data: " << data; @@ -395,7 +396,7 @@ QByteArray EncryptionHelper::decryptStringSymmetric(const QByteArray& key, const return result; } -QByteArray EncryptionHelper::privateKeyToPem(const QSslKey key) { +QByteArray privateKeyToPem(const QSslKey key) { BIO *privateKeyBio = BIO_new(BIO_s_mem()); QByteArray privateKeyPem = key.toPem(); BIO_write(privateKeyBio, privateKeyPem.constData(), privateKeyPem.size()); @@ -412,7 +413,7 @@ QByteArray EncryptionHelper::privateKeyToPem(const QSslKey key) { return pem; } -QByteArray EncryptionHelper::encryptStringSymmetric(const QByteArray& key, const QByteArray& data) { +QByteArray encryptStringSymmetric(const QByteArray& key, const QByteArray& data) { QByteArray iv = generateRandom(16); EVP_CIPHER_CTX *ctx; @@ -484,7 +485,7 @@ QByteArray EncryptionHelper::encryptStringSymmetric(const QByteArray& key, const return result; } -QByteArray EncryptionHelper::decryptStringAsymmetric(EVP_PKEY *privateKey, const QByteArray& data) { +QByteArray decryptStringAsymmetric(EVP_PKEY *privateKey, const QByteArray& data) { int err = -1; qCInfo(lcCseDecryption()) << "Start to work the decryption."; @@ -552,7 +553,7 @@ QByteArray EncryptionHelper::decryptStringAsymmetric(EVP_PKEY *privateKey, const return raw; } -QByteArray EncryptionHelper::encryptStringAsymmetric(EVP_PKEY *publicKey, const QByteArray& data) { +QByteArray encryptStringAsymmetric(EVP_PKEY *publicKey, const QByteArray& data) { int err = -1; auto ctx = EVP_PKEY_CTX_new(publicKey, ENGINE_get_default_RSA()); @@ -606,6 +607,7 @@ QByteArray EncryptionHelper::encryptStringAsymmetric(EVP_PKEY *publicKey, const return raw.toBase64(); } +} ClientSideEncryption::ClientSideEncryption() { } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index b3dcdb17e..b2eb903f3 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -26,47 +26,46 @@ namespace OCC { QString baseUrl(); -class EncryptionHelper { -public: - static QByteArray generateRandomFilename(); - static QByteArray generateRandom(int size); - static QByteArray generatePassword(const QString &wordlist, const QByteArray& salt); - static QByteArray encryptPrivateKey( +namespace EncryptionHelper { + QByteArray generateRandomFilename(); + QByteArray generateRandom(int size); + QByteArray generatePassword(const QString &wordlist, const QByteArray& salt); + QByteArray encryptPrivateKey( const QByteArray& key, const QByteArray& privateKey, const QByteArray &salt ); - static QByteArray decryptPrivateKey( + QByteArray decryptPrivateKey( const QByteArray& key, const QByteArray& data ); - static QByteArray encryptStringSymmetric( + QByteArray encryptStringSymmetric( const QByteArray& key, const QByteArray& data ); - static QByteArray decryptStringSymmetric( + QByteArray decryptStringSymmetric( const QByteArray& key, const QByteArray& data ); - static QByteArray privateKeyToPem(const QSslKey key); + QByteArray privateKeyToPem(const QSslKey key); //TODO: change those two EVP_PKEY into QSslKey. - static QByteArray encryptStringAsymmetric( + QByteArray encryptStringAsymmetric( EVP_PKEY *publicKey, const QByteArray& data ); - static QByteArray decryptStringAsymmetric( + QByteArray decryptStringAsymmetric( EVP_PKEY *privateKey, const QByteArray& data ); - static bool fileEncryption(const QByteArray &key, const QByteArray &iv, + bool fileEncryption(const QByteArray &key, const QByteArray &iv, QFile *input, QFile *output, QByteArray& returnTag); - static void fileDecryption(const QByteArray &key, const QByteArray& iv, + void fileDecryption(const QByteArray &key, const QByteArray& iv, QFile *input, QFile *output); -}; +} class OWNCLOUDSYNC_EXPORT ClientSideEncryption : public QObject { Q_OBJECT From 96115be0c6c49c6a7ab9e1daaa74a038ee28e7d8 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 25 Mar 2018 21:54:08 +0200 Subject: [PATCH 241/262] Unlock folder in case of error --- src/libsync/clientsideencryptionjobs.cpp | 6 ++-- src/libsync/clientsideencryptionjobs.h | 1 + src/libsync/propagateupload.cpp | 29 ++++++++----------- src/libsync/propagateuploadencrypted.cpp | 36 ++++++++++++++++-------- src/libsync/propagateuploadencrypted.h | 7 +++-- 5 files changed, 45 insertions(+), 34 deletions(-) diff --git a/src/libsync/clientsideencryptionjobs.cpp b/src/libsync/clientsideencryptionjobs.cpp index c603da064..2a6eeaab8 100644 --- a/src/libsync/clientsideencryptionjobs.cpp +++ b/src/libsync/clientsideencryptionjobs.cpp @@ -140,9 +140,11 @@ void GetMetadataApiJob::start() bool GetMetadataApiJob::finished() { int retCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (retCode != 200) + if (retCode != 200) { qCInfo(lcCseJob()) << "error requesting the metadata" << path() << errorString() << retCode; - + emit error(_fileId, retCode); + return true; + } QJsonParseError error; auto json = QJsonDocument::fromJson(reply()->readAll(), &error); emit jsonReceived(json, reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt()); diff --git a/src/libsync/clientsideencryptionjobs.h b/src/libsync/clientsideencryptionjobs.h index c6d072ecb..ae363b87b 100644 --- a/src/libsync/clientsideencryptionjobs.h +++ b/src/libsync/clientsideencryptionjobs.h @@ -248,6 +248,7 @@ protected: signals: void jsonReceived(const QJsonDocument &json, int statusCode); + void error(const QByteArray& fileId, int httpReturnCode); private: QByteArray _fileId; diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 4aac491bd..062291ddc 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -328,8 +328,10 @@ void PropagateUploadFileCommon::slotStartUpload(const QByteArray &transmissionCh const QString originalFilePath = propagator()->getFilePath(_item->_file); if (!FileSystem::fileExists(fullFilePath)) { - callUnlockFolder(); - done(SyncFileItem::SoftError, tr("File Removed (start upload) %1").arg(fullFilePath)); + if (_uploadingEncrypted) { + _uploadEncryptedHelper->unlockFolder(); + } + done(SyncFileItem::SoftError, tr("File Removed (start upload) %1").arg(fullFilePath)); return; } time_t prevModtime = _item->_modtime; // the _item value was set in PropagateUploadFile::start() @@ -339,7 +341,9 @@ void PropagateUploadFileCommon::slotStartUpload(const QByteArray &transmissionCh _item->_modtime = FileSystem::getModTime(originalFilePath); if (prevModtime != _item->_modtime) { propagator()->_anotherSyncNeeded = true; - callUnlockFolder(); + if (_uploadingEncrypted) { + _uploadEncryptedHelper->unlockFolder(); + } qDebug() << "prevModtime" << prevModtime << "Curr" << _item->_modtime; done(SyncFileItem::SoftError, tr("Local file changed during syncing. It will be resumed.")); } @@ -352,7 +356,9 @@ void PropagateUploadFileCommon::slotStartUpload(const QByteArray &transmissionCh // or not yet fully copied to the destination. if (fileIsStillChanging(*_item)) { propagator()->_anotherSyncNeeded = true; - callUnlockFolder(); + if (_uploadingEncrypted) { + _uploadEncryptedHelper->unlockFolder(); + } done(SyncFileItem::SoftError, tr("Local file changed during sync.")); return; } @@ -685,21 +691,10 @@ void PropagateUploadFileCommon::finalize() propagator()->_journal->setUploadInfo(_item->_file, SyncJournalDb::UploadInfo()); propagator()->_journal->commit("upload file start"); - callUnlockFolder(); - done(SyncFileItem::Success); -} - -void PropagateUploadFileCommon::callUnlockFolder() -{ if (_uploadingEncrypted) { - qDebug() << "Calling Unlock"; - auto *unlockJob = new UnlockEncryptFolderApiJob(propagator()->account(), - _uploadEncryptedHelper->_folderId, _uploadEncryptedHelper->_folderToken, this); - - connect(unlockJob, &UnlockEncryptFolderApiJob::success, []{ qDebug() << "Successfully Unlocked"; }); - connect(unlockJob, &UnlockEncryptFolderApiJob::error, []{ qDebug() << "Unlock Error"; }); - unlockJob->start(); + _uploadEncryptedHelper->unlockFolder(); } + done(SyncFileItem::Success); } void PropagateUploadFileCommon::prepareAbort(PropagatorJob::AbortType abortType) { diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index afdcc5972..4d73ae2c8 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -105,11 +105,20 @@ void PropagateUploadEncrypted::slotFolderLockedSuccessfully(const QByteArray& fi auto job = new GetMetadataApiJob(_propagator->account(), _folderId); connect(job, &GetMetadataApiJob::jsonReceived, - this, &PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived); + this, &PropagateUploadEncrypted::slotFolderEncryptedMetadataReceived); + connect(job, &GetMetadataApiJob::error, + this, &PropagateUploadEncrypted::slotFolderEncryptedMetadataError); + job->start(); } -void PropagateUploadEncrypted::slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode) +void PropagateUploadEncrypted::slotFolderEncryptedMetadataError(const QByteArray& fileId, int httpReturnCode) +{ + qCDebug(lcPropagateUploadEncrypted()) << "Error Getting the encrypted metadata. unlock the folder."; + unlockFolder(); +} + +void PropagateUploadEncrypted::slotFolderEncryptedMetadataReceived(const QJsonDocument &json, int statusCode) { qCDebug(lcPropagateUploadEncrypted) << "Metadata Received, Preparing it for the new file." << json.toVariant(); @@ -201,16 +210,8 @@ void PropagateUploadEncrypted::slotUpdateMetadataSuccess(const QByteArray& fileI void PropagateUploadEncrypted::slotUpdateMetadataError(const QByteArray& fileId, int httpErrorResponse) { qCDebug(lcPropagateUploadEncrypted) << "Update metadata error for folder" << fileId << "with error" << httpErrorResponse; -} - -void PropagateUploadEncrypted::slotUnlockEncryptedFolderSuccess(const QByteArray& fileId) -{ - qCDebug(lcPropagateUploadEncrypted) << "Unlock Job worked for folder " << fileId; -} - -void PropagateUploadEncrypted::slotUnlockEncryptedFolderError(const QByteArray& fileId, int httpStatusCode) -{ - qCDebug(lcPropagateUploadEncrypted) << "There was an error unlocking " << fileId << httpStatusCode; + qCDebug(lcPropagateUploadEncrypted()) << "Unlocking the folder."; + unlockFolder(); } void PropagateUploadEncrypted::slotFolderLockedError(const QByteArray& fileId, int httpErrorCode) @@ -246,4 +247,15 @@ void PropagateUploadEncrypted::slotFolderEncryptedStatusError(int error) qCDebug(lcPropagateUploadEncrypted) << "Failed to retrieve the status of the folders." << error; } +void PropagateUploadEncrypted::unlockFolder() +{ + qDebug() << "Calling Unlock"; + auto *unlockJob = new UnlockEncryptFolderApiJob(_propagator->account(), + _folderId, _folderToken, this); + + connect(unlockJob, &UnlockEncryptFolderApiJob::success, []{ qDebug() << "Successfully Unlocked"; }); + connect(unlockJob, &UnlockEncryptFolderApiJob::error, []{ qDebug() << "Unlock Error"; }); + unlockJob->start(); +} + } // namespace OCC diff --git a/src/libsync/propagateuploadencrypted.h b/src/libsync/propagateuploadencrypted.h index 7ff52ff86..1c4d99b09 100644 --- a/src/libsync/propagateuploadencrypted.h +++ b/src/libsync/propagateuploadencrypted.h @@ -35,6 +35,8 @@ public: PropagateUploadEncrypted(OwncloudPropagator *propagator, SyncFileItemPtr item); void start(); + /* unlocks the current folder that holds this file */ + void unlockFolder(); // Used by propagateupload QByteArray _folderToken; QByteArray _folderId; @@ -47,9 +49,8 @@ private slots: void slotFolderLockedSuccessfully(const QByteArray& fileId, const QByteArray& token); void slotFolderLockedError(const QByteArray& fileId, int httpErrorCode); void slotTryLock(const QByteArray& fileId); - void slotFolderEncriptedMetadataReceived(const QJsonDocument &json, int statusCode); - void slotUnlockEncryptedFolderSuccess(const QByteArray& fileId); - void slotUnlockEncryptedFolderError(const QByteArray& fileId, int httpReturnCode); + void slotFolderEncryptedMetadataReceived(const QJsonDocument &json, int statusCode); + void slotFolderEncryptedMetadataError(const QByteArray& fileId, int httpReturnCode); void slotUpdateMetadataSuccess(const QByteArray& fileId); void slotUpdateMetadataError(const QByteArray& fileId, int httpReturnCode); From 11684682e6c51723384f319158cc7f328658bbe3 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Sun, 25 Mar 2018 22:31:49 +0200 Subject: [PATCH 242/262] Return empty metadata in case of error, Display error to the user. --- src/gui/accountsettings.cpp | 12 +++++++++++- src/libsync/clientsideencryption.cpp | 11 +++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 6046612f9..c47c01803 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -281,7 +281,17 @@ void AccountSettings::slotLockForEncryptionSuccess(const QByteArray& fileId, con accountsState()->account()->e2e()->setTokenForFolder(fileId, token); FolderMetadata emptyMetadata(accountsState()->account()); - auto storeMetadataJob = new StoreMetaDataApiJob(accountsState()->account(), fileId, emptyMetadata.encryptedMetadata()); + auto encryptedMetadata = emptyMetadata.encryptedMetadata(); + if (encryptedMetadata.isEmpty()) { + //TODO: Mark the folder as unencrypted as the metadata generation failed. + QMessageBox::warning(nullptr, "Warning", + "Could not generate the metadata for encryption, Unlocking the folder. \n" + "This can be an issue with your OpenSSL libraries, please note that OpenSSL 1.1 is \n" + "not compatible with Nextcloud yet." + ); + return; + } + auto storeMetadataJob = new StoreMetaDataApiJob(accountsState()->account(), fileId, emptyMetadata.encryptedMetadata()); connect(storeMetadataJob, &StoreMetaDataApiJob::success, this, &AccountSettings::slotUploadMetadataSuccess); connect(storeMetadataJob, &StoreMetaDataApiJob::error, diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 07de977a8..901fd9cef 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -421,12 +421,14 @@ QByteArray encryptStringSymmetric(const QByteArray& key, const QByteArray& data) if(!(ctx = EVP_CIPHER_CTX_new())) { qCInfo(lcCse()) << "Error creating cipher"; handleErrors(); + return {}; } /* Initialise the decryption operation. */ if(!EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { qCInfo(lcCse()) << "Error initializing context with aes_128"; handleErrors(); + return {}; } // No padding @@ -436,12 +438,14 @@ QByteArray encryptStringSymmetric(const QByteArray& key, const QByteArray& data) if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, iv.size(), NULL)) { qCInfo(lcCse()) << "Error setting iv length"; handleErrors(); + return {}; } /* Initialise key and IV */ if(!EVP_EncryptInit_ex(ctx, NULL, NULL, (unsigned char *)key.constData(), (unsigned char *)iv.constData())) { qCInfo(lcCse()) << "Error initialising key and iv"; handleErrors(); + return {}; } // We write the data base64 encoded @@ -455,6 +459,7 @@ QByteArray encryptStringSymmetric(const QByteArray& key, const QByteArray& data) if(!EVP_EncryptUpdate(ctx, ctext, &len, (unsigned char *)dataB64.constData(), dataB64.size())) { qCInfo(lcCse()) << "Error encrypting"; handleErrors(); + return {}; } int clen = len; @@ -465,6 +470,7 @@ QByteArray encryptStringSymmetric(const QByteArray& key, const QByteArray& data) if(1 != EVP_EncryptFinal_ex(ctx, ctext + len, &len)) { qCInfo(lcCse()) << "Error finalizing encryption"; handleErrors(); + return {}; } clen += len; @@ -473,6 +479,7 @@ QByteArray encryptStringSymmetric(const QByteArray& key, const QByteArray& data) if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag)) { qCInfo(lcCse()) << "Error getting the tag"; handleErrors(); + return {}; } QByteArray cipherTXT((char *)ctext, clen); @@ -1289,6 +1296,9 @@ QByteArray FolderMetadata::encryptedMetadata() { encryptedDoc.setObject(encrypted); QString encryptedEncrypted = encryptJsonObject(encryptedDoc.toJson(QJsonDocument::Compact), _metadataKeys.last()); + if (encryptedEncrypted.isEmpty()) { + qCDebug(lcCse) << "Metadata generation failed!"; + } QJsonObject file; file.insert("encrypted", encryptedEncrypted); @@ -1417,6 +1427,7 @@ bool EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i input->close(); output->close(); qCDebug(lcCse) << "File Encrypted Successfully"; + return true; } void EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& iv, From ec28465e01561747f5e12116517fb6cda32d5576 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 26 Mar 2018 20:51:14 +0200 Subject: [PATCH 243/262] Correctly handle systems without client side encryption --- src/libsync/clientsideencryption.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 901fd9cef..f12fa93b2 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -630,6 +630,7 @@ void ClientSideEncryption::initialize() if (!_account->capabilities().clientSideEncryptionAvaliable()) { qCInfo(lcCse()) << "No Client side encryption avaliable on server."; emit initializationFinished(); + return; } fetchFromKeyChain(); From 5fe9717598f05a0089dd6f7e6c1b999492174011 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 26 Mar 2018 21:37:13 +0200 Subject: [PATCH 244/262] Potentially fix crash acessing a deleted pointer. --- src/libsync/propagateuploadencrypted.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 4d73ae2c8..96de63036 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -156,16 +156,13 @@ void PropagateUploadEncrypted::slotFolderEncryptedMetadataReceived(const QJsonDo qCDebug(lcPropagateUploadEncrypted) << "Creating the encrypted file."; - auto *input = new QFile(info.absoluteFilePath()); - auto *output = new QFile(QDir::tempPath() + QDir::separator() + encryptedFile.encryptedFilename); + QFile input(info.absoluteFilePath()); + QFile output(QDir::tempPath() + QDir::separator() + encryptedFile.encryptedFilename); QByteArray tag; - EncryptionHelper::fileEncryption(encryptedFile.encryptionKey, encryptedFile.initializationVector, input, output, tag); + EncryptionHelper::fileEncryption(encryptedFile.encryptionKey, encryptedFile.initializationVector, &input, &output, tag); - input->deleteLater(); - output->deleteLater(); - - _completeFileName = output->fileName(); + _completeFileName = output.fileName(); qCDebug(lcPropagateUploadEncrypted) << "Creating the metadata for the encrypted file."; From 6c88367ac6d7ec0c56141eab8366db0239f3718a Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 26 Mar 2018 21:44:46 +0200 Subject: [PATCH 245/262] Do not crash if there was an error encrypting the file. --- src/libsync/logger.cpp | 4 ++-- src/libsync/propagateuploadencrypted.cpp | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/libsync/logger.cpp b/src/libsync/logger.cpp index cfba000ed..40c7a2da5 100644 --- a/src/libsync/logger.cpp +++ b/src/libsync/logger.cpp @@ -44,9 +44,9 @@ Logger::Logger(QObject *parent) , _logExpire(0) , _logDebug(false) { - qSetMessagePattern("%{time MM-dd hh:mm:ss:zzz} [ %{type} %{category} ]%{if-debug}\t[ %{function} ]%{endif}:\t%{message}"); + qSetMessagePattern("[%{function} \t%{message}"); #ifndef NO_MSG_HANDLER - qInstallMessageHandler(mirallLogCatcher); + // qInstallMessageHandler(mirallLogCatcher); #else Q_UNUSED(mirallLogCatcher) #endif diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 96de63036..966420722 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -160,7 +160,16 @@ void PropagateUploadEncrypted::slotFolderEncryptedMetadataReceived(const QJsonDo QFile output(QDir::tempPath() + QDir::separator() + encryptedFile.encryptedFilename); QByteArray tag; - EncryptionHelper::fileEncryption(encryptedFile.encryptionKey, encryptedFile.initializationVector, &input, &output, tag); + bool encryptionResult = EncryptionHelper::fileEncryption( + encryptedFile.encryptionKey, + encryptedFile.initializationVector, + &input, &output, tag); + + if (!encryptionResult) { + qCDebug(lcPropagateUploadEncrypted()) << "There was an error encrypting the file, aborting upload."; + unlockFolder(); + return; + } _completeFileName = output.fileName(); From 461aeca20078c5ccd300ec3b71646bc75c87a916 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 26 Mar 2018 22:02:46 +0200 Subject: [PATCH 246/262] Do not crash when failing to decrypt the metadata --- src/libsync/clientsideencryption.cpp | 39 +++++++++++++++++----------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index f12fa93b2..0f9b7e678 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -500,32 +500,32 @@ QByteArray decryptStringAsymmetric(EVP_PKEY *privateKey, const QByteArray& data) if (!ctx) { qCInfo(lcCseDecryption()) << "Could not create the PKEY context."; handleErrors(); - exit(1); + return {}; } err = EVP_PKEY_decrypt_init(ctx); if (err <= 0) { qCInfo(lcCseDecryption()) << "Could not init the decryption of the metadata"; handleErrors(); - exit(1); + return {}; } if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) <= 0) { qCInfo(lcCseDecryption()) << "Error setting the encryption padding."; handleErrors(); - exit(1); + return {}; } if (EVP_PKEY_CTX_set_rsa_oaep_md(ctx, EVP_sha256()) <= 0) { qCInfo(lcCseDecryption()) << "Error setting OAEP SHA 256"; handleErrors(); - exit(1); + return {}; } if (EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, EVP_sha256()) <= 0) { qCInfo(lcCseDecryption()) << "Error setting MGF1 padding"; handleErrors(); - exit(1); + return {}; } size_t outlen = 0; @@ -533,7 +533,7 @@ QByteArray decryptStringAsymmetric(EVP_PKEY *privateKey, const QByteArray& data) if (err <= 0) { qCInfo(lcCseDecryption()) << "Could not determine the buffer length"; handleErrors(); - exit(1); + return {}; } else { qCInfo(lcCseDecryption()) << "Size of output is: " << outlen; qCInfo(lcCseDecryption()) << "Size of data is: " << data.size(); @@ -543,13 +543,13 @@ QByteArray decryptStringAsymmetric(EVP_PKEY *privateKey, const QByteArray& data) if (!out) { qCInfo(lcCseDecryption()) << "Could not alloc space for the decrypted metadata"; handleErrors(); - exit(1); + return {}; } if (EVP_PKEY_decrypt(ctx, out, &outlen, (unsigned char *)data.constData(), data.size()) <= 0) { qCInfo(lcCseDecryption()) << "Could not decrypt the data."; ERR_print_errors_fp(stdout); // This line is not printing anything. - exit(1); + return {}; } else { qCInfo(lcCseDecryption()) << "data decrypted successfully"; } @@ -1160,7 +1160,13 @@ void FolderMetadata::setupExistingMetadata(const QByteArray& metadata) * We have to base64 decode the metadatakey here. This was a misunderstanding in the RFC * Now we should be compatible with Android and IOS. Maybe we can fix it later. */ - QByteArray decryptedKey = QByteArray::fromBase64(decryptMetadataKey(currB64Pass)); + QByteArray b64DecryptedKey = decryptMetadataKey(currB64Pass); + if (b64DecryptedKey.isEmpty()) { + qCDebug(lcCse()) << "Could not decrypt metadata for key" << it.key(); + continue; + } + + QByteArray decryptedKey = QByteArray::fromBase64(b64DecryptedKey); _metadataKeys.insert(it.key().toInt(), decryptedKey); } @@ -1227,12 +1233,15 @@ QByteArray FolderMetadata::decryptMetadataKey(const QByteArray& encryptedMetadat EVP_PKEY *key = PEM_read_bio_PrivateKey(privateKeyBio, NULL, NULL, NULL); // Also base64 decode the result - return QByteArray::fromBase64( - EncryptionHelper::decryptStringAsymmetric( - key, - QByteArray::fromBase64(encryptedMetadata) - ) - ); + QByteArray decryptResult = EncryptionHelper::decryptStringAsymmetric( + key, QByteArray::fromBase64(encryptedMetadata)); + + if (decryptResult.isEmpty()) + { + qCDebug(lcCse()) << "ERROR. Could not decrypt the metadata key"; + return {}; + } + return QByteArray::fromBase64(decryptResult); } // AES/GCM/NoPadding (128 bit key size) From ed6f56257e98e78b1d31131f3284613ef6296564 Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Mon, 26 Mar 2018 22:18:07 +0200 Subject: [PATCH 247/262] Dont use new with QFiles. --- src/libsync/propagatedownloadencrypted.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libsync/propagatedownloadencrypted.cpp b/src/libsync/propagatedownloadencrypted.cpp index 92cab8460..815b56cef 100644 --- a/src/libsync/propagatedownloadencrypted.cpp +++ b/src/libsync/propagatedownloadencrypted.cpp @@ -100,16 +100,16 @@ bool PropagateDownloadEncrypted::decryptFile(QFile& tmpFile) qCDebug(lcPropagateDownloadEncrypted) << "Content Checksum Computed starting decryption" << tmpFileName; tmpFile.close(); - auto _tmpOutput = new QFile(_propagator->getFilePath(tmpFileName), this); + QFile _tmpOutput(_propagator->getFilePath(tmpFileName), this); EncryptionHelper::fileDecryption(_encryptedInfo.encryptionKey, _encryptedInfo.initializationVector, &tmpFile, - _tmpOutput); + &_tmpOutput); - qCDebug(lcPropagateDownloadEncrypted) << "Decryption finished" << tmpFile.fileName() << _tmpOutput->fileName(); + qCDebug(lcPropagateDownloadEncrypted) << "Decryption finished" << tmpFile.fileName() << _tmpOutput.fileName(); tmpFile.close(); - _tmpOutput->close(); + _tmpOutput.close(); // we decripted the temporary into another temporary, so good bye old one if (!tmpFile.remove()) { @@ -119,7 +119,7 @@ bool PropagateDownloadEncrypted::decryptFile(QFile& tmpFile) } // Let's fool the rest of the logic into thinking this was the actual download - tmpFile.setFileName(_tmpOutput->fileName()); + tmpFile.setFileName(_tmpOutput.fileName()); //TODO: This seems what's breaking the logic. // Let's fool the rest of the logic into thinking this is the right name of the DAV file From b4b3e422de0f7367edfdc4b3fe9074225e3a175f Mon Sep 17 00:00:00 2001 From: Tomaz Canabrava Date: Tue, 27 Mar 2018 09:18:54 +0200 Subject: [PATCH 248/262] Handle gracefully failures in file decryption --- src/libsync/clientsideencryption.cpp | 19 ++++++++++--------- src/libsync/clientsideencryption.h | 2 +- src/libsync/propagatedownloadencrypted.cpp | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 0f9b7e678..904fe7f5b 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1440,7 +1440,7 @@ bool EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i return true; } -void EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& iv, +bool EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& iv, QFile *input, QFile *output) { input->open(QIODevice::ReadOnly); @@ -1452,13 +1452,13 @@ void EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& i /* Create and initialise the context */ if(!(ctx = EVP_CIPHER_CTX_new())) { qCInfo(lcCse()) << "Could not create context"; - exit(-1); + return false; } /* Initialise the decryption operation. */ if(!EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), NULL, NULL, NULL)) { qCInfo(lcCse()) << "Could not init cipher"; - exit(-1); + return false; } EVP_CIPHER_CTX_set_padding(ctx, 0); @@ -1466,13 +1466,13 @@ void EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& i /* Set IV length. */ if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, iv.size(), NULL)) { qCInfo(lcCse()) << "Could not set iv length"; - exit(-1); + return false; } /* Initialise key and IV */ if(!EVP_DecryptInit_ex(ctx, NULL, NULL, (const unsigned char *) key.constData(), (const unsigned char *) iv.constData())) { qCInfo(lcCse()) << "Could not set key and iv"; - exit(-1); + return false; } qint64 size = input->size() - 16; @@ -1491,12 +1491,12 @@ void EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& i if (data.size() == 0) { qCInfo(lcCse()) << "Could not read data from file"; - exit(-1); + return false; } if(!EVP_DecryptUpdate(ctx, out, &len, (unsigned char *)data.constData(), data.size())) { qCInfo(lcCse()) << "Could not decrypt"; - exit(-1); + return false; } output->write((char *)out, len); @@ -1507,12 +1507,12 @@ void EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& i /* Set expected tag value. Works in OpenSSL 1.0.1d and later */ if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, tag.size(), (unsigned char *)tag.constData())) { qCInfo(lcCse()) << "Could not set expected tag"; - exit(-1); + return false; } if(1 != EVP_DecryptFinal_ex(ctx, out, &len)) { qCInfo(lcCse()) << "Could finalize decryption"; - exit(-1); + return false; } output->write((char *)out, len); @@ -1521,6 +1521,7 @@ void EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& i input->close(); output->close(); + return true; } } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index b2eb903f3..b5087d5d5 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -63,7 +63,7 @@ namespace EncryptionHelper { bool fileEncryption(const QByteArray &key, const QByteArray &iv, QFile *input, QFile *output, QByteArray& returnTag); - void fileDecryption(const QByteArray &key, const QByteArray& iv, + bool fileDecryption(const QByteArray &key, const QByteArray& iv, QFile *input, QFile *output); } diff --git a/src/libsync/propagatedownloadencrypted.cpp b/src/libsync/propagatedownloadencrypted.cpp index 815b56cef..b07dec6fa 100644 --- a/src/libsync/propagatedownloadencrypted.cpp +++ b/src/libsync/propagatedownloadencrypted.cpp @@ -101,7 +101,7 @@ bool PropagateDownloadEncrypted::decryptFile(QFile& tmpFile) tmpFile.close(); QFile _tmpOutput(_propagator->getFilePath(tmpFileName), this); - EncryptionHelper::fileDecryption(_encryptedInfo.encryptionKey, + bool fileDecrypted = EncryptionHelper::fileDecryption(_encryptedInfo.encryptionKey, _encryptedInfo.initializationVector, &tmpFile, &_tmpOutput); From d5ab642e46f55a80f90f4f1aa65e0f8d422a8e3e Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Wed, 28 Mar 2018 18:12:48 -0300 Subject: [PATCH 249/262] Fix sync status running when encryption data is missing When a file on the server of an encrypted folder do not have a matching entry on the JSON metadata, we need to report an error and be done with the job so that Sync status is not "running". This eventually should cause the file to be removed from the server as it can not be recovered anymore. --- src/libsync/propagatedownload.cpp | 4 ++++ src/libsync/propagatedownloadencrypted.cpp | 4 +++- src/libsync/propagatedownloadencrypted.h | 1 + src/libsync/wordlist.cpp | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index f27663dab..e0907f7f3 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -359,6 +359,10 @@ void PropagateDownloadFile::start() _isEncrypted = true; startAfterIsEncryptedIsChecked(); }); + connect(_downloadEncryptedHelper, &PropagateDownloadEncrypted::failed, [this] { + done(SyncFileItem::NormalError, + tr("File %1 can not be downloaded because encryption information is missing.").arg(QDir::toNativeSeparators(_item->_file))); + }); _downloadEncryptedHelper->start(); } else { startAfterIsEncryptedIsChecked(); diff --git a/src/libsync/propagatedownloadencrypted.cpp b/src/libsync/propagatedownloadencrypted.cpp index b07dec6fa..69f31cb73 100644 --- a/src/libsync/propagatedownloadencrypted.cpp +++ b/src/libsync/propagatedownloadencrypted.cpp @@ -78,6 +78,7 @@ void PropagateDownloadEncrypted::checkFolderEncryptedMetadata(const QJsonDocumen const QString filename = _info.fileName(); auto meta = new FolderMetadata(_propagator->account(), json.toJson(QJsonDocument::Compact)); const QVector files = meta->files(); + for (const EncryptedFile &file : files) { qCDebug(lcPropagateDownloadEncrypted) << "file" << filename << file.encryptedFilename << file.originalFilename << file.encryptionKey; if (filename == file.encryptedFilename) { @@ -88,7 +89,8 @@ void PropagateDownloadEncrypted::checkFolderEncryptedMetadata(const QJsonDocumen } } - qCDebug(lcPropagateDownloadEncrypted) << "Failed to find encrypted metadata information of remote file" << filename; + emit failed(); + qCCritical(lcPropagateDownloadEncrypted) << "Failed to find encrypted metadata information of remote file" << filename; } // TODO: Fix this. Exported in the wrong place. diff --git a/src/libsync/propagatedownloadencrypted.h b/src/libsync/propagatedownloadencrypted.h index 1f854753c..98dc788af 100644 --- a/src/libsync/propagatedownloadencrypted.h +++ b/src/libsync/propagatedownloadencrypted.h @@ -31,6 +31,7 @@ public slots: signals: void folderStatusEncrypted(); void folderStatusNotEncrypted(); + void failed(); void decryptionFinished(); diff --git a/src/libsync/wordlist.cpp b/src/libsync/wordlist.cpp index 032210d88..69134ac3e 100644 --- a/src/libsync/wordlist.cpp +++ b/src/libsync/wordlist.cpp @@ -1,3 +1,4 @@ + #include "wordlist.h" #include From 60fc5d6dd4304448ff3808df65fec6bcee6673b0 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Thu, 29 Mar 2018 11:09:04 -0300 Subject: [PATCH 250/262] Fix upload of existing encrypted files by matching only filename --- src/libsync/propagateuploadencrypted.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 966420722..f1ad8584f 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -126,27 +126,31 @@ void PropagateUploadEncrypted::slotFolderEncryptedMetadataReceived(const QJsonDo _metadata = new FolderMetadata(_propagator->account(), json.toJson(QJsonDocument::Compact), statusCode); QFileInfo info(_propagator->_localDir + QDir::separator() + _item->_file); + const QString fileName = info.fileName(); // Find existing metadata for this file + bool found = false; EncryptedFile encryptedFile; QVector files = _metadata->files(); for(EncryptedFile &file : files) { - if (file.originalFilename == _item->_file) { + if (file.originalFilename == fileName) { encryptedFile = file; + found = true; } } - encryptedFile.encryptionKey = EncryptionHelper::generateRandom(16); - encryptedFile.initializationVector = EncryptionHelper::generateRandom(16); + // New encrypted file so set it all up! - - if (encryptedFile.encryptedFilename.isEmpty()) { + if (!found) { + encryptedFile.encryptionKey = EncryptionHelper::generateRandom(16); encryptedFile.encryptedFilename = EncryptionHelper::generateRandomFilename(); + encryptedFile.initializationVector = EncryptionHelper::generateRandom(16); encryptedFile.fileVersion = 1; encryptedFile.metadataKey = 1; - encryptedFile.originalFilename = info.fileName(); - QMimeDatabase mdb; + encryptedFile.originalFilename = fileName; + + static thread_local QMimeDatabase mdb; encryptedFile.mimetype = mdb.mimeTypeForFile(info).name().toLocal8Bit(); } From c0ef36b8fa8bb02529965184ed013d6ece459949 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Fri, 30 Mar 2018 14:28:00 -0300 Subject: [PATCH 251/262] Use QUuid to generate random names --- src/libsync/clientsideencryption.cpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 904fe7f5b..5c9a058d4 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -82,15 +83,7 @@ namespace { namespace EncryptionHelper { QByteArray generateRandomFilename() { - const QByteArray possibleCharacters = "0123456789abcdef"; - - QByteArray randomString(32, '\0'); - for(int i=0; i < 32; ++i) - { - int index = qrand() % possibleCharacters.length(); - randomString[i] = possibleCharacters.at(index); - } - return randomString; + return QUuid::createUuid().toRfc4122().toHex(); } QByteArray generateRandom(int size) From d3b6aacf3f0db7a87b0ce3c361635e87efac0a64 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Fri, 30 Mar 2018 16:23:18 -0300 Subject: [PATCH 252/262] Fix merging remote and local trees when e2e files are involved When populating the tree from the filesystem we need to fill the e2eMangledName from DB and we when trying to find another match on the oposite tree we need to take in account that names don't match and search with the mangled name information --- src/common/syncjournaldb.cpp | 2 +- src/csync/csync_private.h | 11 ++++++ src/csync/csync_reconcile.cpp | 16 +++++++-- src/csync/csync_update.cpp | 5 +++ src/libsync/clientsideencryption.cpp | 50 ++++++++++++++-------------- 5 files changed, 56 insertions(+), 28 deletions(-) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index 16571ceaf..06c4fcd13 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -1018,7 +1018,7 @@ bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record) qCInfo(lcDb) << "Updating file record for path:" << record._path << "inode:" << record._inode << "modtime:" << record._modtime << "type:" << record._type << "etag:" << record._etag << "fileId:" << record._fileId << "remotePerm:" << record._remotePerm.toString() - << "fileSize:" << record._fileSize << "checksum:" << record._checksumHeader; + << "fileSize:" << record._fileSize << "checksum:" << record._checksumHeader << "e2eMangledName:" << record._e2eMangledName; qlonglong phash = getPHash(record._path); if (checkConnect()) { diff --git a/src/csync/csync_private.h b/src/csync/csync_private.h index f73a1b2d5..2af7f8af9 100644 --- a/src/csync/csync_private.h +++ b/src/csync/csync_private.h @@ -120,6 +120,17 @@ struct OCSYNC_EXPORT csync_s { auto it = find(key); return it != end() ? it->second.get() : nullptr; } + csync_file_stat_t *findFileMangledName(const ByteArrayRef &key) const { + auto it = begin(); + while (it != end()) { + csync_file_stat_t *fs = it->second.get(); + if (fs->e2eMangledName == key) { + return fs; + } + ++it; + } + return nullptr; + } }; struct { diff --git a/src/csync/csync_reconcile.cpp b/src/csync/csync_reconcile.cpp index 55c5d1f12..f3e253c14 100644 --- a/src/csync/csync_reconcile.cpp +++ b/src/csync/csync_reconcile.cpp @@ -110,7 +110,20 @@ static int _csync_merge_algorithm_visitor(csync_file_stat_t *cur, CSYNC * ctx) { break; } - csync_file_stat_t *other = other_tree->findFile(cur->path);; + csync_file_stat_t *other; + if (!cur->e2eMangledName.isEmpty()) { + // When the file has the e2e mangled name data the other tree + // must be the remote one, so search with the mangled name that will match there + other = other_tree->findFile(cur->e2eMangledName); + } else { + other = other_tree->findFile(cur->path); + + if (!other && ctx->current == REMOTE_REPLICA) { + // The file was not found and the other is the local tree + // so check if the path doesn't match a mangled file name + other = other_tree->findFileMangledName(cur->path); + } + } if (!other) { /* Check the renamed path as well. */ @@ -318,7 +331,6 @@ static int _csync_merge_algorithm_visitor(csync_file_stat_t *cur, CSYNC * ctx) { auto localNode = ctx->current == REMOTE_REPLICA ? other : cur; remoteNode->instruction = CSYNC_INSTRUCTION_NONE; localNode->instruction = up._modtime == localNode->modtime ? CSYNC_INSTRUCTION_UPDATE_METADATA : CSYNC_INSTRUCTION_SYNC; - // Update the etag and other server metadata in the journal already // (We can't use a typical CSYNC_INSTRUCTION_UPDATE_METADATA because // we must not store the size/modtime from the file system) diff --git a/src/csync/csync_update.cpp b/src/csync/csync_update.cpp index cd6be8758..d807e13d4 100644 --- a/src/csync/csync_update.cpp +++ b/src/csync/csync_update.cpp @@ -192,6 +192,7 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr f return -1; } + /* * When file is encrypted it's phash (path hash) will not match the local file phash, * we could match the e2eMangledName but that might be slow wihout index, and it's @@ -199,6 +200,10 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr f */ if(base.isValid()) { /* there is an entry in the database */ + // When the file is loaded from the file system it misses + // the e2e mangled name + fs->e2eMangledName = base._e2eMangledName; + /* we have an update! */ qCInfo(lcUpdate, "Database entry found, compare: %" PRId64 " <-> %" PRId64 ", etag: %s <-> %s, inode: %" PRId64 " <-> %" PRId64 diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 5c9a058d4..8e39c3210 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -938,13 +938,13 @@ void ClientSideEncryption::generateCSR(EVP_PKEY *keyPair) void ClientSideEncryption::setTokenForFolder(const QByteArray& folderId, const QByteArray& token) { - _folder2token[folderId] = token; + _folder2token[folderId] = token; } QByteArray ClientSideEncryption::tokenForFolder(const QByteArray& folderId) const { - Q_ASSERT(_folder2token.contains(folderId)); - return _folder2token[folderId]; + Q_ASSERT(_folder2token.contains(folderId)); + return _folder2token[folderId]; } void ClientSideEncryption::encryptPrivateKey() @@ -963,23 +963,23 @@ void ClientSideEncryption::encryptPrivateKey() auto cryptedText = EncryptionHelper::encryptPrivateKey(secretKey, EncryptionHelper::privateKeyToPem(_privateKey), salt); // Send private key to the server - auto job = new StorePrivateKeyApiJob(_account, baseUrl() + "private-key", this); + auto job = new StorePrivateKeyApiJob(_account, baseUrl() + "private-key", this); job->setPrivateKey(cryptedText); - connect(job, &StorePrivateKeyApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { - Q_UNUSED(doc); - switch(retCode) { - case 200: + connect(job, &StorePrivateKeyApiJob::jsonReceived, [this](const QJsonDocument& doc, int retCode) { + Q_UNUSED(doc); + switch(retCode) { + case 200: qCInfo(lcCse()) << "Private key stored encrypted on server."; writePrivateKey(); writeCertificate(); writeMnemonic(); - emit initializationFinished(); - break; - default: - qCInfo(lcCse()) << "Store private key failed, return code:" << retCode; - } - }); - job->start(); + emit initializationFinished(); + break; + default: + qCInfo(lcCse()) << "Store private key failed, return code:" << retCode; + } + }); + job->start(); } void ClientSideEncryption::decryptPrivateKey(const QByteArray &key) { @@ -1083,25 +1083,25 @@ void ClientSideEncryption::getPublicKeyFromServer() } void ClientSideEncryption::fetchFolderEncryptedStatus() { - _refreshingEncryptionStatus = true; - auto getEncryptedStatus = new GetFolderEncryptStatusJob(_account, QString()); - connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusReceived, - this, &ClientSideEncryption::folderEncryptedStatusFetched); - connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusError, - this, &ClientSideEncryption::folderEncryptedStatusError); - getEncryptedStatus->start(); + _refreshingEncryptionStatus = true; + auto getEncryptedStatus = new GetFolderEncryptStatusJob(_account, QString()); + connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusReceived, + this, &ClientSideEncryption::folderEncryptedStatusFetched); + connect(getEncryptedStatus, &GetFolderEncryptStatusJob::encryptStatusError, + this, &ClientSideEncryption::folderEncryptedStatusError); + getEncryptedStatus->start(); } void ClientSideEncryption::folderEncryptedStatusFetched(const QMap& result) { - _refreshingEncryptionStatus = false; - _folder2encryptedStatus = result; + _refreshingEncryptionStatus = false; + _folder2encryptedStatus = result; qCDebug(lcCse) << "Retrieved correctly the encrypted status of the folders." << result; } void ClientSideEncryption::folderEncryptedStatusError(int error) { - _refreshingEncryptionStatus = false; + _refreshingEncryptionStatus = false; qCDebug(lcCse) << "Failed to retrieve the status of the folders." << error; } From d93f7269b89c159327a9d106d7a26745ec9865a5 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Tue, 3 Apr 2018 11:24:38 -0300 Subject: [PATCH 253/262] Properly mark encrypted files for removal --- src/common/syncjournaldb.cpp | 46 +++++++++++++++++++++++++++++++++++ src/common/syncjournaldb.h | 2 ++ src/csync/csync_reconcile.cpp | 18 ++++++-------- src/csync/csync_update.cpp | 16 +++++++++--- 4 files changed, 68 insertions(+), 14 deletions(-) diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index 06c4fcd13..becb8ceee 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -547,6 +547,13 @@ bool SyncJournalDb::checkConnect() return sqlFail("prepare _getFileRecordQuery", *_getFileRecordQuery); } + _getFileRecordQueryByMangledName.reset(new SqlQuery(_db)); + if (_getFileRecordQueryByMangledName->prepare( + GET_FILE_RECORD_QUERY + " WHERE e2eMangledName=?1")) { + return sqlFail("prepare _getFileRecordQueryByMangledName", *_getFileRecordQueryByMangledName); + } + _getFileRecordQueryByInode.reset(new SqlQuery(_db)); if (_getFileRecordQueryByInode->prepare( GET_FILE_RECORD_QUERY @@ -747,6 +754,7 @@ void SyncJournalDb::close() commitTransaction(); _getFileRecordQuery.reset(0); + _getFileRecordQueryByMangledName.reset(0); _getFileRecordQueryByInode.reset(0); _getFileRecordQueryByFileId.reset(0); _getFilesBelowPathQuery.reset(0); @@ -1136,6 +1144,44 @@ bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRec return true; } +bool SyncJournalDb::getFileRecordByE2eMangledName(const QString &mangledName, SyncJournalFileRecord *rec) +{ + QMutexLocker locker(&_mutex); + + // Reset the output var in case the caller is reusing it. + Q_ASSERT(rec); + rec->_path.clear(); + Q_ASSERT(!rec->isValid()); + + if (_metadataTableIsEmpty) + return true; // no error, yet nothing found (rec->isValid() == false) + + if (!checkConnect()) + return false; + + if (!mangledName.isEmpty()) { + _getFileRecordQueryByMangledName->reset_and_clear_bindings(); + _getFileRecordQueryByMangledName->bindValue(1, mangledName); + + if (!_getFileRecordQueryByMangledName->exec()) { + close(); + return false; + } + + if (_getFileRecordQueryByMangledName->next()) { + fillFileRecordFromGetQuery(*rec, *_getFileRecordQueryByMangledName); + } else { + int errId = _getFileRecordQueryByMangledName->errorId(); + if (errId != SQLITE_DONE) { // only do this if the problem is different from SQLITE_DONE + QString err = _getFileRecordQueryByMangledName->error(); + qCWarning(lcDb) << "No journal entry found for mangled name" << mangledName << "Error: " << err; + close(); + } + } + } + return true; +} + bool SyncJournalDb::getFileRecordByInode(quint64 inode, SyncJournalFileRecord *rec) { QMutexLocker locker(&_mutex); diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h index a553ef4f1..e561ada64 100644 --- a/src/common/syncjournaldb.h +++ b/src/common/syncjournaldb.h @@ -57,6 +57,7 @@ public: // To verify that the record could be found check with SyncJournalFileRecord::isValid() bool getFileRecord(const QString &filename, SyncJournalFileRecord *rec) { return getFileRecord(filename.toUtf8(), rec); } bool getFileRecord(const QByteArray &filename, SyncJournalFileRecord *rec); + bool getFileRecordByE2eMangledName(const QString &mangledName, SyncJournalFileRecord *rec); bool getFileRecordByInode(quint64 inode, SyncJournalFileRecord *rec); bool getFileRecordsByFileId(const QByteArray &fileId, const std::function &rowCallback); bool getFilesBelowPath(const QByteArray &path, const std::function &rowCallback); @@ -259,6 +260,7 @@ private: // NOTE! when adding a query, don't forget to reset it in SyncJournalDb::close QScopedPointer _getFileRecordQuery; + QScopedPointer _getFileRecordQueryByMangledName; QScopedPointer _getFileRecordQueryByInode; QScopedPointer _getFileRecordQueryByFileId; QScopedPointer _getFilesBelowPathQuery; diff --git a/src/csync/csync_reconcile.cpp b/src/csync/csync_reconcile.cpp index f3e253c14..ed35b9d43 100644 --- a/src/csync/csync_reconcile.cpp +++ b/src/csync/csync_reconcile.cpp @@ -110,18 +110,14 @@ static int _csync_merge_algorithm_visitor(csync_file_stat_t *cur, CSYNC * ctx) { break; } - csync_file_stat_t *other; - if (!cur->e2eMangledName.isEmpty()) { - // When the file has the e2e mangled name data the other tree - // must be the remote one, so search with the mangled name that will match there - other = other_tree->findFile(cur->e2eMangledName); - } else { - other = other_tree->findFile(cur->path); - - if (!other && ctx->current == REMOTE_REPLICA) { - // The file was not found and the other is the local tree - // so check if the path doesn't match a mangled file name + csync_file_stat_t *other = other_tree->findFile(cur->path); + if (!other) { + if (ctx->current == REMOTE_REPLICA) { + // The file was not found and the other tree is the local one + // check if the path doesn't match a mangled file name other = other_tree->findFileMangledName(cur->path); + } else { + other = other_tree->findFile(cur->e2eMangledName); } } diff --git a/src/csync/csync_update.cpp b/src/csync/csync_update.cpp index d807e13d4..10eda33ef 100644 --- a/src/csync/csync_update.cpp +++ b/src/csync/csync_update.cpp @@ -198,6 +198,12 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr f * we could match the e2eMangledName but that might be slow wihout index, and it's * not UNIQUE at the moment. */ + if (!base.isValid()) { + if(!ctx->statedb->getFileRecordByE2eMangledName(fs->path, &base)) { + ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL; + return -1; + } + } if(base.isValid()) { /* there is an entry in the database */ // When the file is loaded from the file system it misses @@ -205,12 +211,16 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr f fs->e2eMangledName = base._e2eMangledName; /* we have an update! */ - qCInfo(lcUpdate, "Database entry found, compare: %" PRId64 " <-> %" PRId64 + qCInfo(lcUpdate, "Database entry found for %s, compare: %" PRId64 " <-> %" PRId64 ", etag: %s <-> %s, inode: %" PRId64 " <-> %" PRId64 - ", size: %" PRId64 " <-> %" PRId64 ", perms: %x <-> %x, ignore: %d", + ", size: %" PRId64 " <-> %" PRId64 ", perms: %x <-> %x, ignore: %d, e2e: %s", + base._path.constData(), ((int64_t) fs->modtime), ((int64_t) base._modtime), fs->etag.constData(), base._etag.constData(), (uint64_t) fs->inode, (uint64_t) base._inode, - (uint64_t) fs->size, (uint64_t) base._fileSize, *reinterpret_cast(&fs->remotePerm), *reinterpret_cast(&base._remotePerm), base._serverHasIgnoredFiles ); + (uint64_t) fs->size, (uint64_t) base._fileSize, + *reinterpret_cast(&fs->remotePerm), *reinterpret_cast(&base._remotePerm), + base._serverHasIgnoredFiles, + base._e2eMangledName.constData()); if (ctx->current == REMOTE_REPLICA && fs->etag != base._etag) { fs->instruction = CSYNC_INSTRUCTION_EVAL; From a7a8de8787f51f550a9728613f29bbdac670510f Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Tue, 3 Apr 2018 14:52:44 -0300 Subject: [PATCH 254/262] Fix matching encrypted file for download --- src/libsync/propagatedownloadencrypted.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsync/propagatedownloadencrypted.cpp b/src/libsync/propagatedownloadencrypted.cpp index 69f31cb73..08cf6f986 100644 --- a/src/libsync/propagatedownloadencrypted.cpp +++ b/src/libsync/propagatedownloadencrypted.cpp @@ -81,7 +81,7 @@ void PropagateDownloadEncrypted::checkFolderEncryptedMetadata(const QJsonDocumen for (const EncryptedFile &file : files) { qCDebug(lcPropagateDownloadEncrypted) << "file" << filename << file.encryptedFilename << file.originalFilename << file.encryptionKey; - if (filename == file.encryptedFilename) { + if (filename == file.originalFilename) { _encryptedInfo = file; qCDebug(lcPropagateDownloadEncrypted) << "Found matching encrypted metadata for file, starting download"; emit folderStatusEncrypted(); From 6e63a538b7766ad198d61a53c03a9636bb9ca56d Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Tue, 3 Apr 2018 16:38:10 -0300 Subject: [PATCH 255/262] Make sure bool _currentLockingInProgress is first defined to false --- src/libsync/propagateuploadencrypted.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsync/propagateuploadencrypted.h b/src/libsync/propagateuploadencrypted.h index 1c4d99b09..b9a85633c 100644 --- a/src/libsync/propagateuploadencrypted.h +++ b/src/libsync/propagateuploadencrypted.h @@ -67,7 +67,7 @@ private: SyncFileItemPtr _item; QElapsedTimer _folderLockFirstTry; - bool _currentLockingInProgress; + bool _currentLockingInProgress = false; QByteArray _generatedKey; QByteArray _generatedIv; From a9208f6581a0e97e35abfc2d7495666115f49fac Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Tue, 3 Apr 2018 19:08:34 -0300 Subject: [PATCH 256/262] Remove unused encrypted property --- src/libsync/propagatedownloadencrypted.cpp | 1 - src/libsync/propagateuploadencrypted.cpp | 2 +- src/libsync/syncfileitem.h | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libsync/propagatedownloadencrypted.cpp b/src/libsync/propagatedownloadencrypted.cpp index 08cf6f986..9b247cdff 100644 --- a/src/libsync/propagatedownloadencrypted.cpp +++ b/src/libsync/propagatedownloadencrypted.cpp @@ -125,7 +125,6 @@ bool PropagateDownloadEncrypted::decryptFile(QFile& tmpFile) //TODO: This seems what's breaking the logic. // Let's fool the rest of the logic into thinking this is the right name of the DAV file - _item->_isEndToEndEncrypted = true; _item->_encryptedFileName = _item->_file; _item->_file = _item->_file.section(QLatin1Char('/'), 0, -2) + QLatin1Char('/') + _encryptedInfo.originalFilename; diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index f1ad8584f..3946858bd 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -132,6 +132,7 @@ void PropagateUploadEncrypted::slotFolderEncryptedMetadataReceived(const QJsonDo bool found = false; EncryptedFile encryptedFile; QVector files = _metadata->files(); + for(EncryptedFile &file : files) { if (file.originalFilename == fileName) { encryptedFile = file; @@ -154,7 +155,6 @@ void PropagateUploadEncrypted::slotFolderEncryptedMetadataReceived(const QJsonDo encryptedFile.mimetype = mdb.mimeTypeForFile(info).name().toLocal8Bit(); } - _item->_isEndToEndEncrypted = true; _item->_encryptedFileName = _item->_file.section(QLatin1Char('/'), 0, -2) + QLatin1Char('/') + encryptedFile.encryptedFilename; diff --git a/src/libsync/syncfileitem.h b/src/libsync/syncfileitem.h index 184568ec8..244f77a6d 100644 --- a/src/libsync/syncfileitem.h +++ b/src/libsync/syncfileitem.h @@ -205,7 +205,6 @@ public: /// Whether there's end to end encryption on this file. /// If the file is encrypted, the _encryptedFilename is /// the encrypted name on the server. - bool _isEndToEndEncrypted; QString _encryptedFileName; ItemType _type BITFIELD(3); Direction _direction BITFIELD(3); From b12066f1aab519e3d9fd0026926ea8988dc75e44 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Tue, 3 Apr 2018 19:41:11 -0300 Subject: [PATCH 257/262] Properly remove encryption data from JSON of deleted file --- src/libsync/CMakeLists.txt | 1 + src/libsync/clientsideencryption.cpp | 10 ++ src/libsync/clientsideencryption.h | 1 + src/libsync/propagatedownloadencrypted.cpp | 2 +- src/libsync/propagateremotedelete.cpp | 22 ++- src/libsync/propagateremotedelete.h | 1 + .../propagateremotedeleteencrypted.cpp | 130 ++++++++++++++++++ src/libsync/propagateremotedeleteencrypted.h | 40 ++++++ src/libsync/syncengine.cpp | 1 + 9 files changed, 200 insertions(+), 8 deletions(-) create mode 100644 src/libsync/propagateremotedeleteencrypted.cpp create mode 100644 src/libsync/propagateremotedeleteencrypted.h diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index 7fb1168e8..e03921cb6 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -43,6 +43,7 @@ set(libsync_SRCS propagateuploadv1.cpp propagateuploadng.cpp propagateremotedelete.cpp + propagateremotedeleteencrypted.cpp propagateremotemove.cpp propagateremotemkdir.cpp propagateuploadencrypted.cpp diff --git a/src/libsync/clientsideencryption.cpp b/src/libsync/clientsideencryption.cpp index 8e39c3210..f84127f40 100644 --- a/src/libsync/clientsideencryption.cpp +++ b/src/libsync/clientsideencryption.cpp @@ -1334,6 +1334,16 @@ void FolderMetadata::addEncryptedFile(const EncryptedFile &f) { _files.append(f); } +void FolderMetadata::removeEncryptedFile(const EncryptedFile &f) +{ + for (int i = 0; i < _files.size(); i++) { + if (_files.at(i).originalFilename == f.originalFilename) { + _files.removeAt(i); + break; + } + } +} + QVector FolderMetadata::files() const { return _files; } diff --git a/src/libsync/clientsideencryption.h b/src/libsync/clientsideencryption.h index b5087d5d5..103e606a2 100644 --- a/src/libsync/clientsideencryption.h +++ b/src/libsync/clientsideencryption.h @@ -142,6 +142,7 @@ public: FolderMetadata(AccountPtr account, const QByteArray& metadata = QByteArray(), int statusCode = -1); QByteArray encryptedMetadata(); void addEncryptedFile(const EncryptedFile& f); + void removeEncryptedFile(const EncryptedFile& f); QVector files() const; diff --git a/src/libsync/propagatedownloadencrypted.cpp b/src/libsync/propagatedownloadencrypted.cpp index 9b247cdff..773732888 100644 --- a/src/libsync/propagatedownloadencrypted.cpp +++ b/src/libsync/propagatedownloadencrypted.cpp @@ -81,7 +81,7 @@ void PropagateDownloadEncrypted::checkFolderEncryptedMetadata(const QJsonDocumen for (const EncryptedFile &file : files) { qCDebug(lcPropagateDownloadEncrypted) << "file" << filename << file.encryptedFilename << file.originalFilename << file.encryptionKey; - if (filename == file.originalFilename) { + if (filename == file.encryptedFilename) { _encryptedInfo = file; qCDebug(lcPropagateDownloadEncrypted) << "Found matching encrypted metadata for file, starting download"; emit folderStatusEncrypted(); diff --git a/src/libsync/propagateremotedelete.cpp b/src/libsync/propagateremotedelete.cpp index d0bee9fbf..4ff177466 100644 --- a/src/libsync/propagateremotedelete.cpp +++ b/src/libsync/propagateremotedelete.cpp @@ -13,6 +13,7 @@ */ #include "propagateremotedelete.h" +#include "propagateremotedeleteencrypted.h" #include "owncloudpropagator_p.h" #include "account.h" #include "common/asserts.h" @@ -65,18 +66,25 @@ void PropagateRemoteDelete::start() if (propagator()->_abortRequested.fetchAndAddRelaxed(0)) return; - - QString remoteFile = _item->_file; if (!_item->_encryptedFileName.isEmpty()) { - remoteFile = _item->_encryptedFileName; - // TODO update the JSON metadata + auto job = new PropagateRemoteDeleteEncrypted(propagator(), _item, this); + connect(job, &PropagateRemoteDeleteEncrypted::finished, this, [this] (bool success) { + Q_UNUSED(success) // Should we skip file deletion in case of failure? + createDeleteJob(_item->_encryptedFileName); + }); + job->start(); + } else { + createDeleteJob(_item->_file); } +} - qCDebug(lcPropagateRemoteDelete) << "local" << _item->_file << "remote" << remoteFile; +void PropagateRemoteDelete::createDeleteJob(const QString &filename) +{ + qCInfo(lcPropagateRemoteDelete) << "Deleting file, local" << _item->_file << "remote" << filename; _job = new DeleteJob(propagator()->account(), - propagator()->_remoteFolder + remoteFile, - this); + propagator()->_remoteFolder + filename, + this); connect(_job.data(), &DeleteJob::finishedSignal, this, &PropagateRemoteDelete::slotDeleteJobFinished); propagator()->_activeJobList.append(this); _job->start(); diff --git a/src/libsync/propagateremotedelete.h b/src/libsync/propagateremotedelete.h index 4d07843e4..559d64743 100644 --- a/src/libsync/propagateremotedelete.h +++ b/src/libsync/propagateremotedelete.h @@ -52,6 +52,7 @@ public: { } void start() Q_DECL_OVERRIDE; + void createDeleteJob(const QString &filename); void abort(PropagatorJob::AbortType abortType) Q_DECL_OVERRIDE; bool isLikelyFinishedQuickly() Q_DECL_OVERRIDE { return !_item->isDirectory(); } diff --git a/src/libsync/propagateremotedeleteencrypted.cpp b/src/libsync/propagateremotedeleteencrypted.cpp new file mode 100644 index 000000000..40644d3ba --- /dev/null +++ b/src/libsync/propagateremotedeleteencrypted.cpp @@ -0,0 +1,130 @@ +#include "propagateremotedeleteencrypted.h" +#include "clientsideencryptionjobs.h" +#include "clientsideencryption.h" +#include "owncloudpropagator.h" + +#include +#include +#include +#include + +using namespace OCC; + +Q_LOGGING_CATEGORY(PROPAGATE_REMOVE_ENCRYPTED, "nextcloud.sync.propagator.remove.encrypted") + +PropagateRemoteDeleteEncrypted::PropagateRemoteDeleteEncrypted(OwncloudPropagator *propagator, SyncFileItemPtr item, QObject *parent) + : QObject(parent) + , _propagator(propagator) + , _item(item) +{ + +} + +void PropagateRemoteDeleteEncrypted::start() +{ + QFileInfo info(_item->_file); + qCDebug(PROPAGATE_REMOVE_ENCRYPTED) << "Folder is encrypted, let's get the Id from it."; + auto job = new LsColJob(_propagator->account(), info.path(), this); + job->setProperties({"resourcetype", "http://owncloud.org/ns:fileid"}); + connect(job, &LsColJob::directoryListingSubfolders, this, &PropagateRemoteDeleteEncrypted::slotFolderEncryptedIdReceived); + connect(job, &LsColJob::finishedWithError, this, &PropagateRemoteDeleteEncrypted::taskFailed); + job->start(); +} + +void PropagateRemoteDeleteEncrypted::slotFolderEncryptedIdReceived(const QStringList &list) +{ + qCDebug(PROPAGATE_REMOVE_ENCRYPTED) << "Received id of folder, trying to lock it so we can prepare the metadata"; + auto job = qobject_cast(sender()); + const ExtraFolderInfo folderInfo = job->_folderInfos.value(list.first()); + slotTryLock(folderInfo.fileId); +} + +void PropagateRemoteDeleteEncrypted::slotTryLock(const QByteArray &folderId) +{ + auto lockJob = new LockEncryptFolderApiJob(_propagator->account(), folderId, this); + connect(lockJob, &LockEncryptFolderApiJob::success, this, &PropagateRemoteDeleteEncrypted::slotFolderLockedSuccessfully); + connect(lockJob, &LockEncryptFolderApiJob::error, this, &PropagateRemoteDeleteEncrypted::taskFailed); + lockJob->start(); +} + +void PropagateRemoteDeleteEncrypted::slotFolderLockedSuccessfully(const QByteArray &folderId, const QByteArray &token) +{ + qCDebug(PROPAGATE_REMOVE_ENCRYPTED) << "Folder id" << folderId << "Locked Successfully for Upload, Fetching Metadata"; + _folderLocked = true; + _folderToken = token; + _folderId = folderId; + + auto job = new GetMetadataApiJob(_propagator->account(), _folderId); + connect(job, &GetMetadataApiJob::jsonReceived, this, &PropagateRemoteDeleteEncrypted::slotFolderEncryptedMetadataReceived); + connect(job, &GetMetadataApiJob::error, this, &PropagateRemoteDeleteEncrypted::taskFailed); + job->start(); +} + +void PropagateRemoteDeleteEncrypted::slotFolderEncryptedMetadataReceived(const QJsonDocument &json, int statusCode) +{ + if (statusCode == 404) { + qCDebug(PROPAGATE_REMOVE_ENCRYPTED) << "Metadata not found, ignoring."; + unlockFolder(); + return; + } + + qCDebug(PROPAGATE_REMOVE_ENCRYPTED) << "Metadata Received, Preparing it for the new file."; + + // Encrypt File! + FolderMetadata metadata(_propagator->account(), json.toJson(QJsonDocument::Compact), statusCode); + + QFileInfo info(_propagator->_localDir + QDir::separator() + _item->_file); + const QString fileName = info.fileName(); + + // Find existing metadata for this file + bool found = false; + const QVector files = metadata.files(); + for (const EncryptedFile &file : files) { + if (file.encryptedFilename == fileName) { + metadata.removeEncryptedFile(file); + found = true; + break; + } + } + + if (!found) { + // The removed file was not in the JSON so nothing else to do + unlockFolder(); + } + + qCDebug(PROPAGATE_REMOVE_ENCRYPTED) << "Metadata updated, sending to the server."; + + auto job = new UpdateMetadataApiJob(_propagator->account(), + _folderId, + metadata.encryptedMetadata(), + _folderToken); + + connect(job, &UpdateMetadataApiJob::success, this, &PropagateRemoteDeleteEncrypted::unlockFolder); + connect(job, &UpdateMetadataApiJob::error, this, &PropagateRemoteDeleteEncrypted::taskFailed); + job->start(); +} + +void PropagateRemoteDeleteEncrypted::unlockFolder() +{ + qCDebug(PROPAGATE_REMOVE_ENCRYPTED) << "Unlocking folder" << _folderId; + auto unlockJob = new UnlockEncryptFolderApiJob(_propagator->account(), + _folderId, _folderToken, this); + + connect(unlockJob, &UnlockEncryptFolderApiJob::success, [this] { + qCDebug(PROPAGATE_REMOVE_ENCRYPTED) << "Folder successfully unlocked" << _folderId; + _folderLocked = false; + emit finished(true); + }); + connect(unlockJob, &UnlockEncryptFolderApiJob::error, this, &PropagateRemoteDeleteEncrypted::taskFailed); + unlockJob->start(); +} + +void PropagateRemoteDeleteEncrypted::taskFailed() +{ + qCDebug(PROPAGATE_REMOVE_ENCRYPTED) << "Task failed of job" << sender(); + if (_folderLocked) { + unlockFolder(); + } else { + emit finished(false); + } +} diff --git a/src/libsync/propagateremotedeleteencrypted.h b/src/libsync/propagateremotedeleteencrypted.h new file mode 100644 index 000000000..8e0d44f90 --- /dev/null +++ b/src/libsync/propagateremotedeleteencrypted.h @@ -0,0 +1,40 @@ +#ifndef PROPAGATEREMOTEDELETEENCRYPTED_H +#define PROPAGATEREMOTEDELETEENCRYPTED_H + +#include +#include + +#include "syncfileitem.h" + +namespace OCC { + +class OwncloudPropagator; +class PropagateRemoteDeleteEncrypted : public QObject +{ + Q_OBJECT +public: + PropagateRemoteDeleteEncrypted(OwncloudPropagator *_propagator, SyncFileItemPtr item, QObject *parent); + + void start(); + +signals: + void finished(bool success); + +private: + void slotFolderEncryptedIdReceived(const QStringList &list); + void slotTryLock(const QByteArray &folderId); + void slotFolderLockedSuccessfully(const QByteArray &fileId, const QByteArray &token); + void slotFolderEncryptedMetadataReceived(const QJsonDocument &json, int statusCode); + void unlockFolder(); + void taskFailed(); + + OwncloudPropagator *_propagator; + SyncFileItemPtr _item; + QByteArray _folderToken; + QByteArray _folderId; + bool _folderLocked = false; +}; + +} + +#endif // PROPAGATEREMOTEDELETEENCRYPTED_H diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 66cc72aa8..beecbc299 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -424,6 +424,7 @@ int SyncEngine::treewalkFile(csync_file_stat_t *file, csync_file_stat_t *other, item->_file = fileUtf8; } item->_originalFile = item->_file; + item->_encryptedFileName = file->e2eMangledName; if (item->_instruction == CSYNC_INSTRUCTION_NONE || (item->_instruction == CSYNC_INSTRUCTION_IGNORE && instruction != CSYNC_INSTRUCTION_NONE)) { From defe2d7631bb1cf5cf20e8687b66d44a28ef042a Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Tue, 3 Apr 2018 19:56:05 -0300 Subject: [PATCH 258/262] Fix matching encrypted data for downloading --- src/libsync/propagatedownloadencrypted.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsync/propagatedownloadencrypted.cpp b/src/libsync/propagatedownloadencrypted.cpp index 773732888..753478e40 100644 --- a/src/libsync/propagatedownloadencrypted.cpp +++ b/src/libsync/propagatedownloadencrypted.cpp @@ -79,9 +79,9 @@ void PropagateDownloadEncrypted::checkFolderEncryptedMetadata(const QJsonDocumen auto meta = new FolderMetadata(_propagator->account(), json.toJson(QJsonDocument::Compact)); const QVector files = meta->files(); + const QString encryptedFilename = _item->_encryptedFileName.section(QLatin1Char('/'), -1); for (const EncryptedFile &file : files) { - qCDebug(lcPropagateDownloadEncrypted) << "file" << filename << file.encryptedFilename << file.originalFilename << file.encryptionKey; - if (filename == file.encryptedFilename) { + if (encryptedFilename == file.encryptedFilename) { _encryptedInfo = file; qCDebug(lcPropagateDownloadEncrypted) << "Found matching encrypted metadata for file, starting download"; emit folderStatusEncrypted(); From 3eb3535d0ff905e8f14ef58edae28be459866ee8 Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Wed, 4 Apr 2018 10:08:35 -0300 Subject: [PATCH 259/262] Fix downloading NEW encrypted files --- src/libsync/propagatedownloadencrypted.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libsync/propagatedownloadencrypted.cpp b/src/libsync/propagatedownloadencrypted.cpp index 753478e40..f8de072e2 100644 --- a/src/libsync/propagatedownloadencrypted.cpp +++ b/src/libsync/propagatedownloadencrypted.cpp @@ -74,12 +74,15 @@ void PropagateDownloadEncrypted::checkFolderId(const QStringList &list) void PropagateDownloadEncrypted::checkFolderEncryptedMetadata(const QJsonDocument &json) { - qCDebug(lcPropagateDownloadEncrypted) << "Metadata Received reading" << json.toJson() << _item->_file; + qCDebug(lcPropagateDownloadEncrypted) << "Metadata Received reading" << + csync_instruction_str(_item->_instruction) << _item->_file << _item->_encryptedFileName; const QString filename = _info.fileName(); auto meta = new FolderMetadata(_propagator->account(), json.toJson(QJsonDocument::Compact)); const QVector files = meta->files(); - const QString encryptedFilename = _item->_encryptedFileName.section(QLatin1Char('/'), -1); + const QString encryptedFilename = _item->_instruction == CSYNC_INSTRUCTION_NEW ? + _item->_file.section(QLatin1Char('/'), -1) : + _item->_encryptedFileName.section(QLatin1Char('/'), -1); for (const EncryptedFile &file : files) { if (encryptedFilename == file.encryptedFilename) { _encryptedInfo = file; From 3e05bd1da09a5b1be8dbb934e209ae6f42fde0ab Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Wed, 4 Apr 2018 10:13:46 -0300 Subject: [PATCH 260/262] Do not detach encrypted files vector --- src/libsync/propagateuploadencrypted.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsync/propagateuploadencrypted.cpp b/src/libsync/propagateuploadencrypted.cpp index 3946858bd..b7021023f 100644 --- a/src/libsync/propagateuploadencrypted.cpp +++ b/src/libsync/propagateuploadencrypted.cpp @@ -131,9 +131,9 @@ void PropagateUploadEncrypted::slotFolderEncryptedMetadataReceived(const QJsonDo // Find existing metadata for this file bool found = false; EncryptedFile encryptedFile; - QVector files = _metadata->files(); + const QVector files = _metadata->files(); - for(EncryptedFile &file : files) { + for(const EncryptedFile &file : files) { if (file.originalFilename == fileName) { encryptedFile = file; found = true; From 16aa54ed647abdc2b1ac3f6029ae52891739a1ac Mon Sep 17 00:00:00 2001 From: Daniel Nicoletti Date: Wed, 4 Apr 2018 10:29:14 -0300 Subject: [PATCH 261/262] Do not create new db entries when the remote tree does not have proper values --- src/csync/csync_update.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/csync/csync_update.cpp b/src/csync/csync_update.cpp index 10eda33ef..2058157e9 100644 --- a/src/csync/csync_update.cpp +++ b/src/csync/csync_update.cpp @@ -208,7 +208,10 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr f if(base.isValid()) { /* there is an entry in the database */ // When the file is loaded from the file system it misses // the e2e mangled name - fs->e2eMangledName = base._e2eMangledName; + if (fs->e2eMangledName.isEmpty() && !base._e2eMangledName.isEmpty()) { + fs->e2eMangledName = base._e2eMangledName; + fs->path = base._path; + } /* we have an update! */ qCInfo(lcUpdate, "Database entry found for %s, compare: %" PRId64 " <-> %" PRId64 From 81ae682e872fe20cfa67cc9903934599ecf891a8 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Sun, 22 Apr 2018 21:04:37 +0200 Subject: [PATCH 262/262] Revert "Make different program so it doesn't mess with existing config" This reverts commit c04d1f1f69f2a5d6e2fe4899edccf820b4534dc1. --- NEXTCLOUD.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEXTCLOUD.cmake b/NEXTCLOUD.cmake index e2310e0a2..180a9fa98 100644 --- a/NEXTCLOUD.cmake +++ b/NEXTCLOUD.cmake @@ -1,5 +1,5 @@ -set( APPLICATION_NAME "Nextcloud-e2e" ) -set( APPLICATION_SHORTNAME "Nextcloud-e2e" ) +set( APPLICATION_NAME "Nextcloud" ) +set( APPLICATION_SHORTNAME "Nextcloud" ) set( APPLICATION_EXECUTABLE "nextcloud" ) set( APPLICATION_DOMAIN "nextcloud.com" ) set( APPLICATION_VENDOR "Nextcloud GmbH" )