Merge pull request #3348 from nextcloud/feature/vfs-hydration-with-decrypt-file

Allow hydration of VFS placeholders that are E2E encrypted.

src/common/constants.h

@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) by Oleksandr Zolotov <alex@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#pragma once
+
+#include <QtGlobal>
+
+namespace OCC {
+namespace Constants {
+    constexpr qint32 e2EeTagSize = 16;
+};
+};

src/libsync/clientsideencryption.cpp

@@ -36,7 +36,8 @@
 #include <QRandomGenerator>
 
 #include <qt5keychain/keychain.h>
-#include "common/utility.h"
+#include <common/utility.h>
+#include <common/constants.h>
 
 #include "wordlist.h"
 
@@ -66,6 +67,8 @@ namespace {
     const char e2e_private[] = "_e2e-private";
     const char e2e_mnemonic[] = "_e2e-mnemonic";
 
+    constexpr qint64 blockSize = 1024;
+
     QList<QByteArray> oldCipherFormatSplit(const QByteArray &cipher)
     {
         const auto separator = QByteArrayLiteral("fA=="); // BASE64 encoded '|'
@@ -423,17 +426,17 @@ QByteArray encryptPrivateKey(
     }
     clen += len;
 
-    /* Get the tag */
+    /* Get the e2EeTag */
-    QByteArray tag(16, '\0');
+    QByteArray e2EeTag(OCC::Constants::e2EeTagSize, '\0');
-    if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, unsignedData(tag))) {
+    if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, OCC::Constants::e2EeTagSize, unsignedData(e2EeTag))) {
-        qCInfo(lcCse()) << "Error getting the tag";
+        qCInfo(lcCse()) << "Error getting the e2EeTag";
         handleErrors();
     }
 
     QByteArray cipherTXT;
-    cipherTXT.reserve(clen + 16);
+    cipherTXT.reserve(clen + OCC::Constants::e2EeTagSize);
     cipherTXT.append(ctext, clen);
-    cipherTXT.append(tag);
+    cipherTXT.append(e2EeTag);
 
     QByteArray result = cipherTXT.toBase64();
     result += '|';
@@ -463,8 +466,8 @@ QByteArray decryptPrivateKey(const QByteArray& key, const QByteArray& data) {
     QByteArray cipherTXT = QByteArray::fromBase64(cipherTXT64);
     QByteArray iv = QByteArray::fromBase64(ivB64);
 
-    QByteArray tag = cipherTXT.right(16);
+    const QByteArray e2EeTag = cipherTXT.right(OCC::Constants::e2EeTagSize);
-    cipherTXT.chop(16);
+    cipherTXT.chop(OCC::Constants::e2EeTagSize);
 
     // Init
     CipherCtx ctx;
@@ -493,7 +496,7 @@ QByteArray decryptPrivateKey(const QByteArray& key, const QByteArray& data) {
         return QByteArray();
     }
 
-    QByteArray ptext(cipherTXT.size() + 16, '\0');
+    QByteArray ptext(cipherTXT.size() + OCC::Constants::e2EeTagSize, '\0');
     int plen = 0;
 
     /* Provide the message to be decrypted, and obtain the plaintext output.
@@ -504,9 +507,9 @@ QByteArray decryptPrivateKey(const QByteArray& key, const QByteArray& data) {
         return QByteArray();
     }
 
-    /* Set expected tag value. Works in OpenSSL 1.0.1d and later */
+    /* Set expected e2EeTag value. Works in OpenSSL 1.0.1d and later */
-    if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, tag.size(), (unsigned char *)tag.constData())) {
+    if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, e2EeTag.size(), (unsigned char *)e2EeTag.constData())) {
-        qCInfo(lcCse()) << "Could not set tag";
+        qCInfo(lcCse()) << "Could not set e2EeTag";
         return QByteArray();
     }
 
@@ -553,8 +556,8 @@ QByteArray decryptStringSymmetric(const QByteArray& key, const QByteArray& data)
     QByteArray cipherTXT = QByteArray::fromBase64(cipherTXT64);
     QByteArray iv = QByteArray::fromBase64(ivB64);
 
-    QByteArray tag = cipherTXT.right(16);
+    const QByteArray e2EeTag = cipherTXT.right(OCC::Constants::e2EeTagSize);
-    cipherTXT.chop(16);
+    cipherTXT.chop(OCC::Constants::e2EeTagSize);
 
     // Init
     CipherCtx ctx;
@@ -583,7 +586,7 @@ QByteArray decryptStringSymmetric(const QByteArray& key, const QByteArray& data)
         return QByteArray();
     }
 
-    QByteArray ptext(cipherTXT.size() + 16, '\0');
+    QByteArray ptext(cipherTXT.size() + OCC::Constants::e2EeTagSize, '\0');
     int plen = 0;
 
     /* Provide the message to be decrypted, and obtain the plaintext output.
@@ -594,9 +597,9 @@ QByteArray decryptStringSymmetric(const QByteArray& key, const QByteArray& data)
         return QByteArray();
     }
 
-    /* Set expected tag value. Works in OpenSSL 1.0.1d and later */
+    /* Set expected e2EeTag value. Works in OpenSSL 1.0.1d and later */
-    if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, tag.size(), (unsigned char *)tag.constData())) {
+    if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, e2EeTag.size(), (unsigned char *)e2EeTag.constData())) {
-        qCInfo(lcCse()) << "Could not set tag";
+        qCInfo(lcCse()) << "Could not set e2EeTag";
         return QByteArray();
     }
 
@@ -686,18 +689,18 @@ QByteArray encryptStringSymmetric(const QByteArray& key, const QByteArray& data)
     }
     clen += len;
 
-    /* Get the tag */
+    /* Get the e2EeTag */
-    QByteArray tag(16, '\0');
+    QByteArray e2EeTag(OCC::Constants::e2EeTagSize, '\0');
-    if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, unsignedData(tag))) {
+    if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, OCC::Constants::e2EeTagSize, unsignedData(e2EeTag))) {
-        qCInfo(lcCse()) << "Error getting the tag";
+        qCInfo(lcCse()) << "Error getting the e2EeTag";
         handleErrors();
         return {};
     }
 
     QByteArray cipherTXT;
-    cipherTXT.reserve(clen + 16);
+    cipherTXT.reserve(clen + OCC::Constants::e2EeTagSize);
     cipherTXT.append(ctext, clen);
-    cipherTXT.append(tag);
+    cipherTXT.append(e2EeTag);
 
     QByteArray result = cipherTXT.toBase64();
     result += '|';
@@ -753,7 +756,7 @@ QByteArray decryptStringAsymmetric(EVP_PKEY *privateKey, const QByteArray& data)
         qCInfo(lcCseDecryption()) << "Size of data is: " << data.size();
     }
 
-    QByteArray out(outlen, '\0');
+    QByteArray out(static_cast<int>(outlen), '\0');
 
     if (EVP_PKEY_decrypt(ctx, unsignedData(out), &outlen, (unsigned char *)data.constData(), data.size()) <= 0) {
         const auto error = handleErrors();
@@ -804,7 +807,7 @@ QByteArray encryptStringAsymmetric(EVP_PKEY *publicKey, const QByteArray& data)
         qCInfo(lcCse()) << "Encryption Length:" << outLen;
     }
 
-    QByteArray out(outLen, '\0');
+    QByteArray out(static_cast<int>(outLen), '\0');
     if (EVP_PKEY_encrypt(ctx, unsignedData(out), &outLen, (unsigned char *)data.constData(), data.size()) != 1) {
         qCInfo(lcCse()) << "Could not encrypt key." << err;
         exit(1);
@@ -1650,13 +1653,13 @@ bool EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i
         return false;
     }
 
-    QByteArray out(1024 + 16 - 1, '\0');
+    QByteArray out(blockSize + OCC::Constants::e2EeTagSize - 1, '\0');
     int len = 0;
     int total_len = 0;
 
     qCDebug(lcCse) << "Starting to encrypt the file" << input->fileName() << input->atEnd();
     while(!input->atEnd()) {
-        QByteArray data = input->read(1024);
+        const auto data = input->read(blockSize);
 
         if (data.size() == 0) {
             qCInfo(lcCse()) << "Could not read data from file";
@@ -1680,15 +1683,15 @@ bool EncryptionHelper::fileEncryption(const QByteArray &key, const QByteArray &i
     output->write(out, len);
     total_len += len;
 
-    /* Get the tag */
+    /* Get the e2EeTag */
-    QByteArray tag(16, '\0');
+    QByteArray e2EeTag(OCC::Constants::e2EeTagSize, '\0');
-    if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, unsignedData(tag))) {
+    if(1 != EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, OCC::Constants::e2EeTagSize, unsignedData(e2EeTag))) {
-        qCInfo(lcCse()) << "Could not get tag";
+        qCInfo(lcCse()) << "Could not get e2EeTag";
         return false;
     }
 
-    returnTag = tag;
+    returnTag = e2EeTag;
-    output->write(tag, 16);
+    output->write(e2EeTag, OCC::Constants::e2EeTagSize);
 
     input->close();
     output->close();
@@ -1731,16 +1734,16 @@ bool EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& i
         return false;
     }
 
-    qint64 size = input->size() - 16;
+    qint64 size = input->size() - OCC::Constants::e2EeTagSize;
 
-    QByteArray out(1024 + 16 - 1, '\0');
+    QByteArray out(blockSize + OCC::Constants::e2EeTagSize - 1, '\0');
     int len = 0;
 
     while(input->pos() < size) {
 
         auto toRead = size - input->pos();
-        if (toRead > 1024) {
+        if (toRead > blockSize) {
-            toRead = 1024;
+            toRead = blockSize;
         }
 
         QByteArray data = input->read(toRead);
@@ -1758,11 +1761,11 @@ bool EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& i
         output->write(out, len);
     }
 
-    QByteArray tag = input->read(16);
+    const QByteArray e2EeTag = input->read(OCC::Constants::e2EeTagSize);
 
-    /* Set expected tag value. Works in OpenSSL 1.0.1d and later */
+    /* Set expected e2EeTag value. Works in OpenSSL 1.0.1d and later */
-    if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, tag.size(), (unsigned char *)tag.constData())) {
+    if(!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, e2EeTag.size(), (unsigned char *)e2EeTag.constData())) {
-        qCInfo(lcCse()) << "Could not set expected tag";
+        qCInfo(lcCse()) << "Could not set expected e2EeTag";
         return false;
     }
 
@@ -1777,4 +1780,181 @@ bool EncryptionHelper::fileDecryption(const QByteArray &key, const QByteArray& i
     return true;
 }
 
+EncryptionHelper::StreamingDecryptor::StreamingDecryptor(const QByteArray &key, const QByteArray &iv, quint64 totalSize) : _totalSize(totalSize)
+{
+    if (_ctx && !key.isEmpty() && !iv.isEmpty() && totalSize > 0) {
+        _isInitialized = true;
+
+        /* Initialize the decryption operation. */
+        if(!EVP_DecryptInit_ex(_ctx, EVP_aes_128_gcm(), nullptr, nullptr, nullptr)) {
+            qCritical(lcCse()) << "Could not init cipher";
+            _isInitialized = false;
+        }
+
+        EVP_CIPHER_CTX_set_padding(_ctx, 0);
+
+        /* Set IV length. */
+        if(!EVP_CIPHER_CTX_ctrl(_ctx, EVP_CTRL_GCM_SET_IVLEN, iv.size(), nullptr)) {
+            qCritical(lcCse()) << "Could not set iv length";
+            _isInitialized = false;
+        }
+
+        /* Initialize key and IV */
+        if(!EVP_DecryptInit_ex(_ctx, nullptr, nullptr, reinterpret_cast<const unsigned char*>(key.constData()), reinterpret_cast<const unsigned char*>(iv.constData()))) {
+            qCritical(lcCse()) << "Could not set key and iv";
+            _isInitialized = false;
+        }
+    }
 }
+
+QByteArray EncryptionHelper::StreamingDecryptor::chunkDecryption(const char *input, quint64 chunkSize)
+{
+    QByteArray byteArray;
+    QBuffer buffer(&byteArray);
+    buffer.open(QIODevice::WriteOnly);
+
+    Q_ASSERT(isInitialized());
+    if (!isInitialized()) {
+        qCritical(lcCse()) << "Decryption failed. Decryptor is not initialized!";
+        return QByteArray();
+    }
+
+    Q_ASSERT(buffer.isOpen() && buffer.isWritable());
+    if (!buffer.isOpen() || !buffer.isWritable()) {
+        qCritical(lcCse()) << "Decryption failed. Incorrect output device!";
+        return QByteArray();
+    }
+
+    Q_ASSERT(input);
+    if (!input) {
+        qCritical(lcCse()) << "Decryption failed. Incorrect input!";
+        return QByteArray();
+    }
+
+    Q_ASSERT(chunkSize > 0);
+    if (chunkSize <= 0) {
+        qCritical(lcCse()) << "Decryption failed. Incorrect chunkSize!";
+        return QByteArray();
+    }
+
+    if (_decryptedSoFar == 0) {
+        qCDebug(lcCse()) << "Decryption started";
+    }
+
+    Q_ASSERT(_decryptedSoFar + chunkSize <= _totalSize);
+    if (_decryptedSoFar + chunkSize > _totalSize) {
+        qCritical(lcCse()) << "Decryption failed. Chunk is out of range!";
+        return QByteArray();
+    }
+
+    Q_ASSERT(_decryptedSoFar + chunkSize < OCC::Constants::e2EeTagSize || _totalSize - OCC::Constants::e2EeTagSize >= _decryptedSoFar + chunkSize - OCC::Constants::e2EeTagSize);
+    if (_decryptedSoFar + chunkSize > OCC::Constants::e2EeTagSize && _totalSize - OCC::Constants::e2EeTagSize < _decryptedSoFar + chunkSize - OCC::Constants::e2EeTagSize) {
+        qCritical(lcCse()) << "Decryption failed. Incorrect chunk!";
+        return QByteArray();
+    }
+
+    const bool isLastChunk = _decryptedSoFar + chunkSize == _totalSize;
+
+    // last OCC::Constants::e2EeTagSize bytes is ALWAYS a e2EeTag!!!
+    const qint64 size = isLastChunk ? chunkSize - OCC::Constants::e2EeTagSize : chunkSize;
+
+    // either the size is more than 0 and an e2EeTag is at the end of chunk, or, chunk is the e2EeTag itself
+    Q_ASSERT(size > 0 || chunkSize == OCC::Constants::e2EeTagSize);
+    if (size <= 0 && chunkSize != OCC::Constants::e2EeTagSize) {
+        qCritical(lcCse()) << "Decryption failed. Invalid input size: " << size << " !";
+        return QByteArray();
+    }
+
+    qint64 bytesWritten = 0;
+    qint64 inputPos = 0;
+
+    QByteArray decryptedBlock(blockSize + OCC::Constants::e2EeTagSize - 1, '\0');
+
+    while(inputPos < size) {
+        // read blockSize or less bytes
+        const QByteArray encryptedBlock(input + inputPos, qMin(size - inputPos, blockSize));
+
+        if (encryptedBlock.size() == 0) {
+            qCritical(lcCse()) << "Could not read data from the input buffer.";
+            return QByteArray();
+        }
+
+        int outLen = 0;
+
+        if(!EVP_DecryptUpdate(_ctx, unsignedData(decryptedBlock), &outLen, reinterpret_cast<const unsigned char*>(encryptedBlock.data()), encryptedBlock.size())) {
+            qCritical(lcCse()) << "Could not decrypt";
+            return QByteArray();
+        }
+
+        const auto writtenToOutput = buffer.write(decryptedBlock, outLen);
+
+        Q_ASSERT(writtenToOutput == outLen);
+        if (writtenToOutput != outLen) {
+            qCritical(lcCse()) << "Failed to write decrypted data to device.";
+            return QByteArray();
+        }
+
+        bytesWritten += writtenToOutput;
+
+        // advance input position for further read
+        inputPos += encryptedBlock.size();
+
+        _decryptedSoFar += encryptedBlock.size();
+    }
+
+    if (isLastChunk) {
+        // if it's a last chunk, we'd need to read a e2EeTag at the end and finalize the decryption
+
+        Q_ASSERT(chunkSize - inputPos == OCC::Constants::e2EeTagSize);
+        if (chunkSize - inputPos != OCC::Constants::e2EeTagSize) {
+            qCritical(lcCse()) << "Decryption failed. e2EeTag is missing!";
+            return QByteArray();
+        }
+
+        int outLen = 0;
+
+        QByteArray e2EeTag = QByteArray(input + inputPos, OCC::Constants::e2EeTagSize);
+
+        /* Set expected e2EeTag value. Works in OpenSSL 1.0.1d and later */
+        if(!EVP_CIPHER_CTX_ctrl(_ctx, EVP_CTRL_GCM_SET_TAG, e2EeTag.size(), reinterpret_cast<unsigned char*>(e2EeTag.data()))) {
+            qCritical(lcCse()) << "Could not set expected e2EeTag";
+            return QByteArray();
+        }
+
+        if(1 != EVP_DecryptFinal_ex(_ctx, unsignedData(decryptedBlock), &outLen)) {
+            qCritical(lcCse()) << "Could finalize decryption";
+            return QByteArray();
+        }
+
+        const auto writtenToOutput = buffer.write(decryptedBlock, outLen);
+
+        Q_ASSERT(writtenToOutput == outLen);
+        if (writtenToOutput != outLen) {
+            qCritical(lcCse()) << "Failed to write decrypted data to device.";
+            return QByteArray();
+        }
+
+        bytesWritten += writtenToOutput;
+
+        _decryptedSoFar += OCC::Constants::e2EeTagSize;
+
+        _isFinished = true;
+    }
+
+    if (isFinished()) {
+        qCDebug(lcCse()) << "Decryption complete";
+    }
+
+    return byteArray;
+}
+
+bool EncryptionHelper::StreamingDecryptor::isInitialized() const
+{
+    return _isInitialized;
+}
+
+bool EncryptionHelper::StreamingDecryptor::isFinished() const
+{
+    return _isFinished;
+}
+};

src/libsync/clientsideencryption.h

@@ -27,7 +27,7 @@ QString baseUrl();
 
 namespace EncryptionHelper {
     QByteArray generateRandomFilename();
-    QByteArray generateRandom(int size);
+    OWNCLOUDSYNC_EXPORT QByteArray generateRandom(int size);
     QByteArray generatePassword(const QString &wordlist, const QByteArray& salt);
     OWNCLOUDSYNC_EXPORT QByteArray encryptPrivateKey(
             const QByteArray& key,
@@ -60,11 +60,57 @@ namespace EncryptionHelper {
             const QByteArray& data
     );
 
-    bool fileEncryption(const QByteArray &key, const QByteArray &iv,
+    OWNCLOUDSYNC_EXPORT bool fileEncryption(const QByteArray &key, const QByteArray &iv,
                       QFile *input, QFile *output, QByteArray& returnTag);
 
-    bool fileDecryption(const QByteArray &key, const QByteArray& iv,
+    OWNCLOUDSYNC_EXPORT bool fileDecryption(const QByteArray &key, const QByteArray &iv,
                                QFile *input, QFile *output);
+
+//
+// Simple classes for safe (RAII) handling of OpenSSL
+// data structures
+//
+class CipherCtx {
+public:
+    CipherCtx() : _ctx(EVP_CIPHER_CTX_new())
+    {
+    }
+
+    ~CipherCtx()
+    {
+        EVP_CIPHER_CTX_free(_ctx);
+    }
+
+    operator EVP_CIPHER_CTX*()
+    {
+        return _ctx;
+    }
+
+private:
+    Q_DISABLE_COPY(CipherCtx)
+    EVP_CIPHER_CTX *_ctx;
+};
+
+class OWNCLOUDSYNC_EXPORT StreamingDecryptor
+{
+public:
+    StreamingDecryptor(const QByteArray &key, const QByteArray &iv, quint64 totalSize);
+    ~StreamingDecryptor() = default;
+
+    QByteArray chunkDecryption(const char *input, quint64 chunkSize);
+
+    bool isInitialized() const;
+    bool isFinished() const;
+
+private:
+    Q_DISABLE_COPY(StreamingDecryptor)
+
+    CipherCtx _ctx;
+    bool _isInitialized = false;
+    bool _isFinished = false;
+    quint64 _decryptedSoFar = 0;
+    quint64 _totalSize = 0;
+};
 }
 
 class OWNCLOUDSYNC_EXPORT ClientSideEncryption : public QObject {

src/libsync/discovery.cpp

@@ -24,7 +24,8 @@
 #include <QFileInfo>
 #include <QFile>
 #include <QThreadPool>
-#include "common/checksums.h"
+#include <common/checksums.h>
+#include <common/constants.h>
 #include "csync_exclude.h"
 #include "csync.h"
 
@@ -459,13 +460,19 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(
 
     // The file is known in the db already
     if (dbEntry.isValid()) {
+        const bool isDbEntryAnE2EePlaceholder = dbEntry.isVirtualFile() && !dbEntry.e2eMangledName().isEmpty();
+        Q_ASSERT(!isDbEntryAnE2EePlaceholder || serverEntry.size >= Constants::e2EeTagSize);
+        const bool isVirtualE2EePlaceholder = isDbEntryAnE2EePlaceholder && serverEntry.size >= Constants::e2EeTagSize;
+        const qint64 sizeOnServer = isVirtualE2EePlaceholder ? serverEntry.size - Constants::e2EeTagSize : serverEntry.size;
+        const bool metaDataSizeNeedsUpdateForE2EeFilePlaceholder = isVirtualE2EePlaceholder && dbEntry._fileSize == serverEntry.size;
+
         if (serverEntry.isDirectory != dbEntry.isDirectory()) {
             // If the type of the entity changed, it's like NEW, but
             // needs to delete the other entity first.
             item->_instruction = CSYNC_INSTRUCTION_TYPE_CHANGE;
             item->_direction = SyncFileItem::Down;
             item->_modtime = serverEntry.modtime;
-            item->_size = serverEntry.size;
+            item->_size = sizeOnServer;
         } else if ((dbEntry._type == ItemTypeVirtualFileDownload || localEntry.type == ItemTypeVirtualFileDownload)
             && (localEntry.isValid() || _queryLocal == ParentNotChanged)) {
             // The above check for the localEntry existing is important. Otherwise it breaks
@@ -476,7 +483,7 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(
         } else if (dbEntry._etag != serverEntry.etag) {
             item->_direction = SyncFileItem::Down;
             item->_modtime = serverEntry.modtime;
-            item->_size = serverEntry.size;
+            item->_size = sizeOnServer;
             if (serverEntry.isDirectory) {
                 ENFORCE(dbEntry.isDirectory());
                 item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
@@ -486,11 +493,43 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(
             } else {
                 item->_instruction = CSYNC_INSTRUCTION_SYNC;
             }
-        } else if (dbEntry._remotePerm != serverEntry.remotePerm || dbEntry._fileId != serverEntry.fileId) {
+        } else if (dbEntry._remotePerm != serverEntry.remotePerm || dbEntry._fileId != serverEntry.fileId || metaDataSizeNeedsUpdateForE2EeFilePlaceholder) {
+            if (metaDataSizeNeedsUpdateForE2EeFilePlaceholder) {
+                // we are updating placeholder sizes after migrating from older versions with VFS + E2EE implicit hydration not supported
+                qCDebug(lcDisco) << "Migrating the E2EE VFS placeholder " << dbEntry.path() << " from older version. The old size is " << item->_size << ". The new size is " << sizeOnServer;
+                item->_size = sizeOnServer;
+            }
             item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA;
             item->_direction = SyncFileItem::Down;
         } else {
-            processFileAnalyzeLocalInfo(item, path, localEntry, serverEntry, dbEntry, ParentNotChanged);
+            // if (is virtual mode enabled and folder is encrypted - check if the size is the same as on the server and then - trigger server query
+            // to update a placeholder with corrected size (-16 Bytes)
+            // or, maybe, add a flag to the database - vfsE2eeSizeCorrected? if it is not set - subtract it from the placeholder's size and re-create/update a placeholder?
+            const QueryMode serverQueryMode = [this, &dbEntry, &serverEntry]() {
+                const bool isVfsModeOn = _discoveryData && _discoveryData->_syncOptions._vfs && _discoveryData->_syncOptions._vfs->mode() != Vfs::Off;
+                if (isVfsModeOn && dbEntry.isDirectory() && dbEntry._isE2eEncrypted) {
+                    qint64 localFolderSize = 0;
+                    const auto listFilesCallback = [this, &localFolderSize](const OCC::SyncJournalFileRecord &record) {
+                        if (record.isFile()) {
+                            // add Constants::e2EeTagSize so we will know the size of E2EE file on the server
+                            localFolderSize += record._fileSize + Constants::e2EeTagSize;
+                        } else if (record.isVirtualFile()) {
+                            // just a virtual file, so, the size must contain Constants::e2EeTagSize if it was not corrected already
+                            localFolderSize += record._fileSize;
+                        }
+                    };
+
+                    const bool listFilesSucceeded = _discoveryData->_statedb->listFilesInPath(dbEntry.path().toUtf8(), listFilesCallback);
+
+                    if (listFilesSucceeded && localFolderSize != 0 && localFolderSize == serverEntry.sizeOfFolder) {
+                        qCInfo(lcDisco) << "Migration of E2EE folder " << dbEntry.path() << " from older version to the one, supporting the implicit VFS hydration.";
+                        return NormalQuery;
+                    }
+                }
+                return ParentNotChanged;
+            }();
+            
+            processFileAnalyzeLocalInfo(item, path, localEntry, serverEntry, dbEntry, serverQueryMode);
             return;
         }
 
@@ -531,6 +570,15 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(
             if (isVfsWithSuffix())
                 addVirtualFileSuffix(tmp_path._original);
         }
+
+        if (opts._vfs->mode() != Vfs::Off && !item->_encryptedFileName.isEmpty()) {
+            // We are syncing a file for the first time (local entry is invalid) and it is encrypted file that will be virtual once synced
+            // to avoid having error of "file has changed during sync" when trying to hydrate it excplicitly - we must remove Constants::e2EeTagSize bytes from the end
+            // as explicit hydration does not care if these bytes are present in the placeholder or not, but, the size must not change in the middle of the sync
+            // this way it works for both implicit and explicit hydration by making a placeholder size that does not includes encryption tag Constants::e2EeTagSize bytes
+            // another scenario - we are syncing a file which is on disk but not in the database (database was removed or file was not written there yet)
+            item->_size = serverEntry.size - Constants::e2EeTagSize;
+        }
         processFileAnalyzeLocalInfo(item, tmp_path, localEntry, serverEntry, dbEntry, _queryServer);
     };
 

src/libsync/discoveryphase.cpp

@@ -349,6 +349,7 @@ void DiscoverySingleDirectoryJob::start()
           << "getlastmodified"
           << "getcontentlength"
           << "getetag"
+          << "http://owncloud.org/ns:size"
           << "http://owncloud.org/ns:id"
           << "http://owncloud.org/ns:downloadURL"
           << "http://owncloud.org/ns:dDC"
@@ -429,6 +430,10 @@ static void propertyMapToRemoteInfo(const QMap<QString, QString> &map, RemoteInf
             result.isE2eEncrypted = true;
         }
     }
+
+    if (result.isDirectory && map.contains("size")) {
+        result.sizeOfFolder = map.value("size").toInt();
+    }
 }
 
 void DiscoverySingleDirectoryJob::directoryListingIteratedSlot(const QString &file, const QMap<QString, QString> &map)
@@ -455,6 +460,9 @@ void DiscoverySingleDirectoryJob::directoryListingIteratedSlot(const QString &fi
             _isE2eEncrypted = true;
             Q_ASSERT(!_fileId.isEmpty());
         }
+        if (map.contains("size")) {
+            _size = map.value("size").toInt();
+        }
     } else {
 
         RemoteInfo result;

src/libsync/discoveryphase.h

@@ -55,6 +55,7 @@ struct RemoteInfo
     OCC::RemotePermissions remotePerm;
     time_t modtime = 0;
     int64_t size = 0;
+    int64_t sizeOfFolder = 0;
     bool isDirectory = false;
     bool isE2eEncrypted = false;
     QString e2eMangledName;
@@ -153,6 +154,7 @@ private:
     // If this directory is e2ee
     bool _isE2eEncrypted;
     // If set, the discovery will finish with an error
+    int64_t _size = 0;
     QString _error;
     QPointer<LsColJob> _lsColJob;
 

src/libsync/propagatedownload.cpp

@@ -22,8 +22,9 @@
 #include "common/utility.h"
 #include "filesystem.h"
 #include "propagatorjobs.h"
-#include "common/checksums.h"
+#include <common/checksums.h>
-#include "common/asserts.h"
+#include <common/asserts.h>
+#include <common/constants.h>
 #include "clientsideencryptionjobs.h"
 #include "propagatedownloadencrypted.h"
 #include "common/vfs.h"
@@ -93,7 +94,6 @@ GETFileJob::GETFileJob(AccountPtr account, const QUrl &url, QIODevice *device,
     const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume,
     qint64 resumeStart, QObject *parent)
     : AbstractNetworkJob(account, url.toEncoded(), parent)
-    , _device(device)
     , _headers(headers)
     , _expectedEtagForResume(expectedEtagForResume)
     , _expectedContentLength(-1)
@@ -107,6 +107,7 @@ GETFileJob::GETFileJob(AccountPtr account, const QUrl &url, QIODevice *device,
     , _bandwidthManager(nullptr)
     , _hasEmittedFinishedSignal(false)
     , _lastModified()
+    , _device(device)
 {
 }
 
@@ -284,6 +285,11 @@ qint64 GETFileJob::currentDownloadPosition()
     return _resumeStart;
 }
 
+qint64 GETFileJob::writeToDevice(const QByteArray &data)
+{
+    return _device->write(data);
+}
+
 void GETFileJob::slotReadyRead()
 {
     if (!reply())
@@ -306,8 +312,8 @@ void GETFileJob::slotReadyRead()
             _bandwidthQuota -= toRead;
         }
 
-        qint64 r = reply()->read(buffer.data(), toRead);
+        const qint64 readBytes = reply()->read(buffer.data(), toRead);
-        if (r < 0) {
+        if (readBytes < 0) {
             _errorString = networkReplyErrorString(*reply());
             _errorStatus = SyncFileItem::NormalError;
             qCWarning(lcGetJob) << "Error while reading from device: " << _errorString;
@@ -315,11 +321,11 @@ void GETFileJob::slotReadyRead()
             return;
         }
 
-        qint64 w = _device->write(buffer.constData(), r);
+        const qint64 writtenBytes = writeToDevice(QByteArray::fromRawData(buffer.constData(), readBytes));
-        if (w != r) {
+        if (writtenBytes != readBytes) {
             _errorString = _device->errorString();
             _errorStatus = SyncFileItem::NormalError;
-            qCWarning(lcGetJob) << "Error while writing to file" << w << r << _errorString;
+            qCWarning(lcGetJob) << "Error while writing to file" << writtenBytes << readBytes << _errorString;
             reply()->abort();
             return;
         }
@@ -371,6 +377,75 @@ QString GETFileJob::errorString() const
     return AbstractNetworkJob::errorString();
 }
 
+GETEncryptedFileJob::GETEncryptedFileJob(AccountPtr account, const QString &path, QIODevice *device,
+    const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume,
+    qint64 resumeStart, EncryptedFile encryptedInfo, QObject *parent)
+    : GETFileJob(account, path, device, headers, expectedEtagForResume, resumeStart, parent)
+    , _encryptedFileInfo(encryptedInfo)
+{
+}
+
+GETEncryptedFileJob::GETEncryptedFileJob(AccountPtr account, const QUrl &url, QIODevice *device,
+    const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume,
+    qint64 resumeStart, EncryptedFile encryptedInfo, QObject *parent)
+    : GETFileJob(account, url, device, headers, expectedEtagForResume, resumeStart, parent)
+    , _encryptedFileInfo(encryptedInfo)
+{
+}
+
+qint64 GETEncryptedFileJob::writeToDevice(const QByteArray &data)
+{
+    if (!_decryptor) {
+        // only initialize the decryptor once, because, according to Qt documentation, metadata might get changed during the processing of the data sometimes
+        // https://doc.qt.io/qt-5/qnetworkreply.html#metaDataChanged
+        _decryptor.reset(new EncryptionHelper::StreamingDecryptor(_encryptedFileInfo.encryptionKey, _encryptedFileInfo.initializationVector, _contentLength));
+    }
+
+    if (!_decryptor->isInitialized()) {
+        return -1;
+    }
+
+    const auto bytesRemaining = _contentLength - _processedSoFar - data.length();
+
+    if (bytesRemaining != 0 && bytesRemaining < OCC::Constants::e2EeTagSize) {
+        // decryption is going to fail if last chunk does not include or does not equal to OCC::Constants::e2EeTagSize bytes tag
+        // we may end up receiving packets beyond OCC::Constants::e2EeTagSize bytes tag at the end
+        // in that case, we don't want to try and decrypt less than OCC::Constants::e2EeTagSize ending bytes of tag, we will accumulate all the incoming data till the end
+        // and then, we are going to decrypt the entire chunk containing OCC::Constants::e2EeTagSize bytes at the end
+        _pendingBytes += QByteArray(data.constData(), data.length());
+        _processedSoFar += data.length();
+        if (_processedSoFar != _contentLength) {
+            return data.length();
+        }
+    }
+
+    if (!_pendingBytes.isEmpty()) {
+        const auto decryptedChunk = _decryptor->chunkDecryption(_pendingBytes.constData(), _pendingBytes.size());
+
+        if (decryptedChunk.isEmpty()) {
+            qCCritical(lcPropagateDownload) << "Decryption failed!";
+            return -1;
+        }
+
+        GETFileJob::writeToDevice(decryptedChunk);
+
+        return data.length();
+    }
+
+    const auto decryptedChunk = _decryptor->chunkDecryption(data.constData(), data.length());
+
+    if (decryptedChunk.isEmpty()) {
+        qCCritical(lcPropagateDownload) << "Decryption failed!";
+        return -1;
+    }
+
+    GETFileJob::writeToDevice(decryptedChunk);
+
+    _processedSoFar += data.length();
+
+    return data.length();
+}
+
 void PropagateDownloadFile::start()
 {
     if (propagator()->_abortRequested)

src/libsync/propagatedownload.h

@@ -36,7 +36,6 @@ class OWNCLOUDSYNC_EXPORT GETFileJob : public AbstractNetworkJob
     QString _errorString;
     QByteArray _expectedEtagForResume;
     qint64 _expectedContentLength;
-    qint64 _contentLength;
     qint64 _resumeStart;
     SyncFileItem::Status _errorStatus;
     QUrl _directDownloadUrl;
@@ -51,6 +50,9 @@ class OWNCLOUDSYNC_EXPORT GETFileJob : public AbstractNetworkJob
     /// Will be set to true once we've seen a 2xx response header
     bool _saveBodyToFile = false;
 
+protected:
+    qint64 _contentLength;
+
 public:
     // DOES NOT take ownership of the device.
     explicit GETFileJob(AccountPtr account, const QString &path, QIODevice *device,
@@ -110,6 +112,9 @@ public:
     qint64 expectedContentLength() const { return _expectedContentLength; }
     void setExpectedContentLength(qint64 size) { _expectedContentLength = size; }
 
+protected:
+    virtual qint64 writeToDevice(const QByteArray &data);
+
 signals:
     void finishedSignal();
     void downloadProgress(qint64, qint64);
@@ -118,6 +123,34 @@ private slots:
     void slotMetaDataChanged();
 };
 
+/**
+ * @brief The GETEncryptedFileJob class that provides file decryption on the fly while the download is running
+ * @ingroup libsync
+ */
+class OWNCLOUDSYNC_EXPORT GETEncryptedFileJob : public GETFileJob
+{
+    Q_OBJECT
+
+public:
+    // DOES NOT take ownership of the device.
+    explicit GETEncryptedFileJob(AccountPtr account, const QString &path, QIODevice *device,
+        const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume,
+        qint64 resumeStart, EncryptedFile encryptedInfo, QObject *parent = nullptr);
+    explicit GETEncryptedFileJob(AccountPtr account, const QUrl &url, QIODevice *device,
+        const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume,
+        qint64 resumeStart, EncryptedFile encryptedInfo, QObject *parent = nullptr);
+    virtual ~GETEncryptedFileJob() = default;
+
+protected:
+    virtual qint64 writeToDevice(const QByteArray &data) override;
+
+private:
+    QSharedPointer<EncryptionHelper::StreamingDecryptor> _decryptor;
+    EncryptedFile _encryptedFileInfo = {};
+    QByteArray _pendingBytes;
+    qint64 _processedSoFar = 0;
+};
+
 /**
  * @brief The PropagateDownloadFile class
  * @ingroup libsync
@@ -219,6 +252,6 @@ private:
 
     QElapsedTimer _stopwatch;
 
-    PropagateDownloadEncrypted *_downloadEncryptedHelper;
+    PropagateDownloadEncrypted *_downloadEncryptedHelper = nullptr;
 };
 }

src/libsync/vfs/cfapi/hydrationjob.cpp

@@ -17,6 +17,9 @@
 #include "common/syncjournaldb.h"
 #include "propagatedownload.h"
 #include "vfs/cfapi/vfs_cfapi.h"
+#include <clientsideencryptionjobs.h>
+
+#include "filesystem.h"
 
 #include <QLocalServer>
 #include <QLocalSocket>
@@ -88,6 +91,26 @@ void OCC::HydrationJob::setFolderPath(const QString &folderPath)
     _folderPath = folderPath;
 }
 
+bool OCC::HydrationJob::isEncryptedFile() const
+{
+    return _isEncryptedFile;
+}
+
+void OCC::HydrationJob::setIsEncryptedFile(bool isEncrypted)
+{
+    _isEncryptedFile = isEncrypted;
+}
+
+QString OCC::HydrationJob::e2eMangledName() const
+{
+    return _e2eMangledName;
+}
+
+void OCC::HydrationJob::setE2eMangledName(const QString &e2eMangledName)
+{
+    _e2eMangledName = e2eMangledName;
+}
+
 OCC::HydrationJob::Status OCC::HydrationJob::status() const
 {
     return _status;
@@ -137,6 +160,70 @@ void OCC::HydrationJob::start()
     connect(_transferDataServer, &QLocalServer::newConnection, this, &HydrationJob::onNewConnection);
 }
 
+void OCC::HydrationJob::slotFolderIdError()
+{
+    // TODO: the following code is borrowed from PropagateDownloadEncrypted (see HydrationJob::onNewConnection() for explanation of next steps)
+    qCCritical(lcHydration) << "Failed to get encrypted metadata of folder" << _requestId << _localPath << _folderPath;
+    emitFinished(Error);
+}
+
+void OCC::HydrationJob::slotCheckFolderId(const QStringList &list)
+{
+    // TODO: the following code is borrowed from PropagateDownloadEncrypted (see HydrationJob::onNewConnection() for explanation of next steps)
+    auto job = qobject_cast<LsColJob *>(sender());
+    const QString folderId = list.first();
+    qCDebug(lcHydration) << "Received id of folder" << folderId;
+
+    const ExtraFolderInfo &folderInfo = job->_folderInfos.value(folderId);
+
+    // Now that we have the folder-id we need it's JSON metadata
+    auto metadataJob = new GetMetadataApiJob(_account, folderInfo.fileId);
+    connect(metadataJob, &GetMetadataApiJob::jsonReceived,
+        this, &HydrationJob::slotCheckFolderEncryptedMetadata);
+    connect(metadataJob, &GetMetadataApiJob::error,
+        this, &HydrationJob::slotFolderEncryptedMetadataError);
+
+    metadataJob->start();
+}
+
+void OCC::HydrationJob::slotFolderEncryptedMetadataError(const QByteArray & /*fileId*/, int /*httpReturnCode*/)
+{
+    // TODO: the following code is borrowed from PropagateDownloadEncrypted (see HydrationJob::onNewConnection() for explanation of next steps)
+    qCCritical(lcHydration) << "Failed to find encrypted metadata information of remote file" << e2eMangledName();
+    emitFinished(Error);
+    return;
+}
+
+void OCC::HydrationJob::slotCheckFolderEncryptedMetadata(const QJsonDocument &json)
+{
+    // TODO: the following code is borrowed from PropagateDownloadEncrypted (see HydrationJob::onNewConnection() for explanation of next steps)
+    qCDebug(lcHydration) << "Metadata Received reading" << e2eMangledName();
+    const QString filename = e2eMangledName();
+    auto meta = new FolderMetadata(_account, json.toJson(QJsonDocument::Compact));
+    const QVector<EncryptedFile> files = meta->files();
+
+    EncryptedFile encryptedInfo = {};
+
+    const QString encryptedFileExactName = e2eMangledName().section(QLatin1Char('/'), -1);
+    for (const EncryptedFile &file : files) {
+        if (encryptedFileExactName == file.encryptedFilename) {
+            EncryptedFile encryptedInfo = file;
+            encryptedInfo = file;
+
+            qCDebug(lcHydration) << "Found matching encrypted metadata for file, starting download" << _requestId << _folderPath;
+            _transferDataSocket = _transferDataServer->nextPendingConnection();
+            _job = new GETEncryptedFileJob(_account, _remotePath + e2eMangledName(), _transferDataSocket, {}, {}, 0, encryptedInfo, this);
+
+            connect(qobject_cast<GETEncryptedFileJob *>(_job), &GETEncryptedFileJob::finishedSignal, this, &HydrationJob::onGetFinished);
+            _job->start();
+            return;
+        }
+    }
+
+    qCCritical(lcHydration) << "Failed to find encrypted metadata information of a remote file" << filename;
+    emitFinished(Error);
+}
+
 void OCC::HydrationJob::cancel()
 {
     Q_ASSERT(_signalSocket);
@@ -184,11 +271,11 @@ void OCC::HydrationJob::onNewConnection()
     Q_ASSERT(!_transferDataSocket);
     Q_ASSERT(!_job);
 
-    qCInfo(lcHydration) << "Got new connection starting GETFileJob" << _requestId << _folderPath;
+    if (isEncryptedFile()) {
-    _transferDataSocket = _transferDataServer->nextPendingConnection();
+        handleNewConnectionForEncryptedFile();
-    _job = new GETFileJob(_account, _remotePath + _folderPath, _transferDataSocket, {}, {}, 0, this);
+    } else {
-    connect(_job, &GETFileJob::finishedSignal, this, &HydrationJob::onGetFinished);
+        handleNewConnection();
-    _job->start();
+    }
 }
 
 void OCC::HydrationJob::finalize(OCC::VfsCfApi *vfs)
@@ -214,6 +301,9 @@ void OCC::HydrationJob::finalize(OCC::VfsCfApi *vfs)
     }
 
     record._type = ItemTypeFile;
+    // store the actual size of a file that has been decrypted as we will need its actual size when dehydrating it if requested
+    record._fileSize = FileSystem::getSize(localPath() + folderPath());
+
     _journal->setFileRecord(record);
 }
 
@@ -234,3 +324,38 @@ void OCC::HydrationJob::onGetFinished()
     }
     emitFinished(Success);
 }
+
+void OCC::HydrationJob::handleNewConnection()
+{
+    qCInfo(lcHydration) << "Got new connection starting GETFileJob" << _requestId << _folderPath;
+    _transferDataSocket = _transferDataServer->nextPendingConnection();
+    _job = new GETFileJob(_account, _remotePath + _folderPath, _transferDataSocket, {}, {}, 0, this);
+    connect(_job, &GETFileJob::finishedSignal, this, &HydrationJob::onGetFinished);
+    _job->start();
+}
+
+void OCC::HydrationJob::handleNewConnectionForEncryptedFile()
+{
+    // TODO: the following code is borrowed from PropagateDownloadEncrypted (should we factor it out and reuse? YES! Should we do it now? Probably not, as, this would imply modifying PropagateDownloadEncrypted, so we need a separate PR)
+    qCInfo(lcHydration) << "Got new connection for encrypted file. Getting required info for decryption...";
+    const auto rootPath = [=]() {
+        const auto result = _remotePath;
+        if (result.startsWith('/')) {
+            return result.mid(1);
+        } else {
+            return result;
+        }
+    }();
+
+    const auto remoteFilename = e2eMangledName();
+    const auto remotePath = QString(rootPath + remoteFilename);
+    const auto remoteParentPath = remotePath.left(remotePath.lastIndexOf('/'));
+
+    auto job = new LsColJob(_account, remoteParentPath, this);
+    job->setProperties({ "resourcetype", "http://owncloud.org/ns:fileid" });
+    connect(job, &LsColJob::directoryListingSubfolders,
+        this, &HydrationJob::slotCheckFolderId);
+    connect(job, &LsColJob::finishedWithError,
+        this, &HydrationJob::slotFolderIdError);
+    job->start();
+}

src/libsync/vfs/cfapi/hydrationjob.h

@@ -25,6 +25,10 @@ class GETFileJob;
 class SyncJournalDb;
 class VfsCfApi;
 
+namespace EncryptionHelper {
+    class StreamingDecryptor;
+};
+
 class HydrationJob : public QObject
 {
     Q_OBJECT
@@ -56,12 +60,27 @@ public:
     QString folderPath() const;
     void setFolderPath(const QString &folderPath);
 
+    bool isEncryptedFile() const;
+    void setIsEncryptedFile(bool isEncrypted);
+
+    QString e2eMangledName() const;
+    void setE2eMangledName(const QString &e2eMangledName);
+
+    qint64 fileTotalSize() const;
+    void setFileTotalSize(qint64 totalSize);
+
     Status status() const;
 
     void start();
     void cancel();
     void finalize(OCC::VfsCfApi *vfs);
 
+public slots:
+    void slotCheckFolderId(const QStringList &list);
+    void slotFolderIdError();
+    void slotCheckFolderEncryptedMetadata(const QJsonDocument &json);
+    void slotFolderEncryptedMetadataError(const QByteArray &fileId, int httpReturnCode);
+
 signals:
     void finished(HydrationJob *job);
 
@@ -72,6 +91,11 @@ private:
     void onCancellationServerNewConnection();
     void onGetFinished();
 
+    void handleNewConnection();
+    void handleNewConnectionForEncryptedFile();
+
+    void startServerAndWaitForConnections();
+
     AccountPtr _account;
     QString _remotePath;
     QString _localPath;
@@ -81,6 +105,9 @@ private:
     QString _requestId;
     QString _folderPath;
 
+    bool _isEncryptedFile = false;
+    QString _e2eMangledName;
+
     QLocalServer *_transferDataServer = nullptr;
     QLocalServer *_signalServer = nullptr;
     QLocalSocket *_transferDataSocket = nullptr;

src/libsync/vfs/cfapi/vfs_cfapi.cpp

@@ -16,7 +16,6 @@
 
 #include <QDir>
 #include <QFile>
-#include <QMessageBox>
 
 #include "cfapiwrapper.h"
 #include "hydrationjob.h"
@@ -323,28 +322,8 @@ void VfsCfApi::requestHydration(const QString &requestId, const QString &path)
         return;
     }
 
-    // This is impossible to handle with CfAPI since the file size is generally different
-    // between the encrypted and the decrypted file which would make CfAPI reject the hydration
-    // of the placeholder with decrypted data
-    if (record._isE2eEncrypted || !record._e2eMangledName.isEmpty()) {
-        qCInfo(lcCfApi) << "Couldn't hydrate, the file is E2EE this is not supported";
-
-        QMessageBox e2eeFileDownloadRequestWarningMsgBox;
-        e2eeFileDownloadRequestWarningMsgBox.setText(tr("Download of end-to-end encrypted file failed"));
-        e2eeFileDownloadRequestWarningMsgBox.setInformativeText(tr("It seems that you are trying to download a virtual file that"
-                                                                   " is end-to-end encrypted. Implicitly downloading such files is not"
-                                                                   " supported at the moment. To workaround this issue, go to the"
-                                                                   " settings and mark the encrypted folder with \"Make always available"
-                                                                   " locally\"."));
-        e2eeFileDownloadRequestWarningMsgBox.setIcon(QMessageBox::Warning);
-        e2eeFileDownloadRequestWarningMsgBox.exec();
-
-        emit hydrationRequestFailed(requestId);
-        return;
-    }
-
     // All good, let's hydrate now
-    scheduleHydrationJob(requestId, relativePath);
+    scheduleHydrationJob(requestId, relativePath, record);
 }
 
 void VfsCfApi::fileStatusChanged(const QString &systemFileName, SyncFileStatus fileStatus)
@@ -353,7 +332,7 @@ void VfsCfApi::fileStatusChanged(const QString &systemFileName, SyncFileStatus f
     Q_UNUSED(fileStatus);
 }
 
-void VfsCfApi::scheduleHydrationJob(const QString &requestId, const QString &folderPath)
+void VfsCfApi::scheduleHydrationJob(const QString &requestId, const QString &folderPath, const SyncJournalFileRecord &record)
 {
     const auto jobAlreadyScheduled = std::any_of(std::cbegin(d->hydrationJobs), std::cend(d->hydrationJobs), [=](HydrationJob *job) {
         return job->requestId() == requestId || job->folderPath() == folderPath;
@@ -376,6 +355,8 @@ void VfsCfApi::scheduleHydrationJob(const QString &requestId, const QString &fol
     job->setJournal(params().journal);
     job->setRequestId(requestId);
     job->setFolderPath(folderPath);
+    job->setIsEncryptedFile(record._isE2eEncrypted);
+    job->setE2eMangledName(record._e2eMangledName);
     connect(job, &HydrationJob::finished, this, &VfsCfApi::onHydrationJobFinished);
     d->hydrationJobs << job;
     job->start();

src/libsync/vfs/cfapi/vfs_cfapi.h

@@ -22,6 +22,7 @@
 namespace OCC {
 class HydrationJob;
 class VfsCfApiPrivate;
+class SyncJournalFileRecord;
 
 class VfsCfApi : public Vfs
 {
@@ -71,7 +72,7 @@ protected:
     void startImpl(const VfsSetupParams &params) override;
 
 private:
-    void scheduleHydrationJob(const QString &requestId, const QString &folderPath);
+    void scheduleHydrationJob(const QString &requestId, const QString &folderPath, const SyncJournalFileRecord &record);
     void onHydrationJobFinished(HydrationJob *job);
     HydrationJob *findHydrationJob(const QString &requestId) const;
 

test/testclientsideencryption.cpp

@@ -6,6 +6,11 @@
 
 #include <QtTest>
 
+#include <QTemporaryFile>
+#include <QRandomGenerator>
+
+#include <common/constants.h>
+
 #include "clientsideencryption.h"
 
 using namespace OCC;
@@ -132,6 +137,116 @@ private slots:
         // THEN
         QCOMPARE(data, originalData);
     }
+
+    void testStreamingDecryptor_data()
+    {
+        QTest::addColumn<int>("totalBytes");
+        QTest::addColumn<int>("bytesToRead");
+
+        QTest::newRow("data1") << 64  << 2;
+        QTest::newRow("data2") << 32  << 8;
+        QTest::newRow("data3") << 76  << 64;
+        QTest::newRow("data4") << 272 << 256;
+    }
+
+    void testStreamingDecryptor()
+    {
+        QFETCH(int, totalBytes);
+
+        QTemporaryFile dummyInputFile;
+
+        QVERIFY(dummyInputFile.open());
+
+        const auto dummyFileRandomContents = EncryptionHelper::generateRandom(totalBytes);
+
+        QCOMPARE(dummyInputFile.write(dummyFileRandomContents), dummyFileRandomContents.size());
+
+        const auto generateHash = [](const QByteArray &data) {
+            QCryptographicHash hash(QCryptographicHash::Sha1);
+            hash.addData(data);
+            return hash.result();
+        };
+
+        const QByteArray originalFileHash = generateHash(dummyFileRandomContents);
+
+        QVERIFY(!originalFileHash.isEmpty());
+
+        dummyInputFile.close();
+        QVERIFY(!dummyInputFile.isOpen());
+
+        const auto encryptionKey = EncryptionHelper::generateRandom(16);
+        const auto initializationVector = EncryptionHelper::generateRandom(16);
+
+        // test normal file encryption/decryption
+        QTemporaryFile dummyEncryptionOutputFile;
+
+        QByteArray tag;
+
+        QVERIFY(EncryptionHelper::fileEncryption(encryptionKey, initializationVector, &dummyInputFile, &dummyEncryptionOutputFile, tag));
+        dummyInputFile.close();
+        QVERIFY(!dummyInputFile.isOpen());
+
+        dummyEncryptionOutputFile.close();
+        QVERIFY(!dummyEncryptionOutputFile.isOpen());
+
+        QTemporaryFile dummyDecryptionOutputFile;
+
+        QVERIFY(EncryptionHelper::fileDecryption(encryptionKey, initializationVector, &dummyEncryptionOutputFile, &dummyDecryptionOutputFile));
+        QVERIFY(dummyDecryptionOutputFile.open());
+        const auto dummyDecryptionOutputFileHash = generateHash(dummyDecryptionOutputFile.readAll());
+        QCOMPARE(dummyDecryptionOutputFileHash, originalFileHash);
+
+        // test streaming decryptor
+        EncryptionHelper::StreamingDecryptor streamingDecryptor(encryptionKey, initializationVector, dummyEncryptionOutputFile.size());
+        QVERIFY(streamingDecryptor.isInitialized());
+
+        QBuffer chunkedOutputDecrypted;
+        QVERIFY(chunkedOutputDecrypted.open(QBuffer::WriteOnly));
+
+        QVERIFY(dummyEncryptionOutputFile.open());
+
+        QByteArray pendingBytes;
+
+        QFETCH(int, bytesToRead);
+
+        while (dummyEncryptionOutputFile.pos() < dummyEncryptionOutputFile.size()) {
+            const auto bytesRemaining = dummyEncryptionOutputFile.size() - dummyEncryptionOutputFile.pos();
+            auto toRead = bytesRemaining > bytesToRead ? bytesToRead : bytesRemaining;
+
+            if (dummyEncryptionOutputFile.pos() + toRead > dummyEncryptionOutputFile.size()) {
+                toRead = dummyEncryptionOutputFile.size() - dummyEncryptionOutputFile.pos();
+            }
+
+            if (bytesRemaining - toRead != 0 && bytesRemaining - toRead < OCC::Constants::e2EeTagSize) {
+                // decryption is going to fail if last chunk does not include or does not equal to OCC::Constants::e2EeTagSize bytes tag
+                // since we are emulating random size of network packets, we may end up reading beyond OCC::Constants::e2EeTagSize bytes tag at the end
+                // in that case, we don't want to try and decrypt less than OCC::Constants::e2EeTagSize ending bytes of tag, we will accumulate all the incoming data till the end
+                // and then, we are going to decrypt the entire chunk containing OCC::Constants::e2EeTagSize bytes at the end
+                pendingBytes += dummyEncryptionOutputFile.read(bytesRemaining);
+                continue;
+            }
+
+            const auto decryptedChunk = streamingDecryptor.chunkDecryption(dummyEncryptionOutputFile.read(toRead).constData(), toRead);
+
+            QVERIFY(decryptedChunk.size() == toRead || streamingDecryptor.isFinished() || !pendingBytes.isEmpty());
+
+            chunkedOutputDecrypted.write(decryptedChunk);
+        }
+
+        if (!pendingBytes.isEmpty()) {
+            const auto decryptedChunk = streamingDecryptor.chunkDecryption(pendingBytes.constData(), pendingBytes.size());
+
+            QVERIFY(decryptedChunk.size() == pendingBytes.size() || streamingDecryptor.isFinished());
+
+            chunkedOutputDecrypted.write(decryptedChunk);
+        }
+
+        chunkedOutputDecrypted.close();
+
+        QVERIFY(chunkedOutputDecrypted.open(QBuffer::ReadOnly));
+        QCOMPARE(generateHash(chunkedOutputDecrypted.readAll()), originalFileHash);
+        chunkedOutputDecrypted.close();
+    }
 };
 
 QTEST_APPLESS_MAIN(TestClientSideEncryption)