Merge pull request #2793 from nextcloud/xattr_backend_for_vfs

XAttr backend for VFS

src/common/vfs.cpp

@@ -53,6 +53,8 @@ QString Vfs::modeToString(Mode mode)
         return QStringLiteral("suffix");
     case WindowsCfApi:
         return QStringLiteral("wincfapi");
+    case XAttr:
+        return QStringLiteral("xattr");
     }
     return QStringLiteral("off");
 }
@@ -145,6 +147,8 @@ static QString modeToPluginName(Vfs::Mode mode)
         return QStringLiteral("suffix");
     if (mode == Vfs::WindowsCfApi)
         return QStringLiteral("win");
+    if (mode == Vfs::XAttr)
+        return QStringLiteral("xattr");
     return QString();
 }
 
@@ -171,9 +175,32 @@ Vfs::Mode OCC::bestAvailableVfsMode()
 {
     if (isVfsPluginAvailable(Vfs::WindowsCfApi)) {
         return Vfs::WindowsCfApi;
-    } else if (isVfsPluginAvailable(Vfs::WithSuffix)) {
+    }
+
+    if (isVfsPluginAvailable(Vfs::WithSuffix)) {
         return Vfs::WithSuffix;
     }
+
+    // For now the "suffix" backend has still precedence over the "xattr" backend.
+    // Ultimately the order of those ifs will change when xattr will be more mature.
+    // But what does "more mature" means here?
+    //
+    //  * On Mac when it properly reads and writes com.apple.LaunchServices.OpenWith
+    // This will require reverse engineering to see what they stuff in there. Maybe a good
+    // starting point:
+    // https://eclecticlight.co/2017/12/20/xattr-com-apple-launchservices-openwith-sets-a-custom-app-to-open-a-file/
+    //
+    //  * On Linux when our user.nextcloud.hydrate_exec is adopted by at least KDE and Gnome
+    // the "user.nextcloud" prefix might turn into "user.xdg" in the process since it would
+    // be best to have a freedesktop.org spec for it.
+    // When that time comes, it might still require detecting at runtime if that's indeed
+    // supported in the user session or even per sync folder (in case user would pick a folder
+    // which wouldn't support xattr for some reason)
+
+    if (isVfsPluginAvailable(Vfs::XAttr)) {
+        return Vfs::XAttr;
+    }
+
     return Vfs::Off;
 }
 

src/common/vfs.h

@@ -95,6 +95,7 @@ public:
         Off,
         WithSuffix,
         WindowsCfApi,
+        XAttr,
     };
     Q_ENUM(Mode)
     static QString modeToString(Mode mode);

src/csync/vio/csync_vio_local_unix.cpp

@@ -124,7 +124,7 @@ std::unique_ptr<csync_file_stat_t> csync_vio_local_readdir(csync_vio_handle_t *h
   if (vfs) {
       // Directly modifies file_stat->type.
       // We can ignore the return value since we're done here anyway.
-      vfs->statTypeVirtualFile(file_stat.get(), nullptr);
+      vfs->statTypeVirtualFile(file_stat.get(), &handle->path);
   }
 
   return file_stat;

src/libsync/CMakeLists.txt

@@ -1,4 +1,6 @@
 project(libsync)
+include(DefinePlatformDefaults)
+
 set(CMAKE_AUTOMOC TRUE)
 
 if ( APPLE )
@@ -72,6 +74,13 @@ if (WIN32)
     )
     add_definitions(-D_WIN32_WINNT=_WIN32_WINNT_WIN10)
     list(APPEND OS_SPECIFIC_LINK_LIBRARIES cldapi)
+elseif(LINUX) # elseif(LINUX OR APPLE)
+    set(libsync_SRCS ${libsync_SRCS} vfs/xattr/vfs_xattr.cpp)
+    if (APPLE)
+        set(libsync_SRCS ${libsync_SRCS} vfs/xattr/xattrwrapper_mac.cpp)
+    else()
+        set(libsync_SRCS ${libsync_SRCS} vfs/xattr/xattrwrapper_linux.cpp)
+    endif()
 endif()
 
 if(TOKEN_AUTH_ONLY)

src/libsync/propagatedownload.cpp

@@ -985,7 +985,13 @@ void PropagateDownloadFile::downloadFinished()
         previousFileExists = false;
     }
 
-    if (previousFileExists) {
+    const auto vfs = propagator()->syncOptions()._vfs;
+
+    // In the case of an hydration, this size is likely to change for placeholders
+    // (except with the cfapi backend)
+    const auto isVirtualDownload = _item->_type == ItemTypeVirtualFileDownload;
+    const auto isCfApiVfs = vfs && vfs->mode() == Vfs::WindowsCfApi;
+    if (previousFileExists && (isCfApiVfs || !isVirtualDownload)) {
         // Check whether the existing file has changed since the discovery
         // phase by comparing size and mtime to the previous values. This
         // is necessary to avoid overwriting user changes that happened between
@@ -1027,7 +1033,6 @@ void PropagateDownloadFile::downloadFinished()
     if (_conflictRecord.isValid())
         propagator()->_journal->setConflictRecord(_conflictRecord);
 
-    auto vfs = propagator()->syncOptions()._vfs;
     if (vfs && vfs->mode() == Vfs::WithSuffix) {
         // If the virtual file used to have a different name and db
         // entry, remove it transfer its old pin state.

src/libsync/vfs/xattr/vfs_xattr.cpp

@@ -0,0 +1,186 @@
+/*
+ * Copyright (C) by Kevin Ottens <kevin.ottens@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#include "vfs_xattr.h"
+
+#include <QFile>
+
+#include "syncfileitem.h"
+#include "filesystem.h"
+#include "common/syncjournaldb.h"
+
+#include "xattrwrapper.h"
+
+namespace xattr {
+using namespace OCC::XAttrWrapper;
+}
+
+namespace OCC {
+
+VfsXAttr::VfsXAttr(QObject *parent)
+    : Vfs(parent)
+{
+}
+
+VfsXAttr::~VfsXAttr() = default;
+
+Vfs::Mode VfsXAttr::mode() const
+{
+    return XAttr;
+}
+
+QString VfsXAttr::fileSuffix() const
+{
+    return QString();
+}
+
+void VfsXAttr::startImpl(const VfsSetupParams &)
+{
+}
+
+void VfsXAttr::stop()
+{
+}
+
+void VfsXAttr::unregisterFolder()
+{
+}
+
+bool VfsXAttr::socketApiPinStateActionsShown() const
+{
+    return true;
+}
+
+bool VfsXAttr::isHydrating() const
+{
+    return false;
+}
+
+Result<void, QString> VfsXAttr::updateMetadata(const QString &filePath, time_t modtime, qint64, const QByteArray &)
+{
+    FileSystem::setModTime(filePath, modtime);
+    return {};
+}
+
+Result<void, QString> VfsXAttr::createPlaceholder(const SyncFileItem &item)
+{
+    const auto path = QString(_setupParams.filesystemPath + item._file);
+    QFile file(path);
+    if (file.exists() && file.size() > 1
+        && !FileSystem::verifyFileUnchanged(path, item._size, item._modtime)) {
+        return QStringLiteral("Cannot create a placeholder because a file with the placeholder name already exist");
+    }
+
+    if (!file.open(QFile::ReadWrite | QFile::Truncate)) {
+        return file.errorString();
+    }
+
+    file.write(" ");
+    file.close();
+    FileSystem::setModTime(path, item._modtime);
+    return xattr::addNextcloudPlaceholderAttributes(path);
+}
+
+Result<void, QString> VfsXAttr::dehydratePlaceholder(const SyncFileItem &item)
+{
+    const auto path = QString(_setupParams.filesystemPath + item._file);
+    QFile file(path);
+    if (!file.remove()) {
+        return QStringLiteral("Couldn't remove the original file to dehydrate");
+    }
+    auto r = createPlaceholder(item);
+    if (!r) {
+        return r;
+    }
+
+    // Ensure the pin state isn't contradictory
+    const auto pin = pinState(item._file);
+    if (pin && *pin == PinState::AlwaysLocal) {
+        setPinState(item._renameTarget, PinState::Unspecified);
+    }
+    return {};
+}
+
+Result<void, QString> VfsXAttr::convertToPlaceholder(const QString &, const SyncFileItem &, const QString &)
+{
+    // Nothing necessary
+    return {};
+}
+
+bool VfsXAttr::needsMetadataUpdate(const SyncFileItem &)
+{
+    return false;
+}
+
+bool VfsXAttr::isDehydratedPlaceholder(const QString &filePath)
+{
+    const auto fi = QFileInfo(filePath);
+    return fi.exists() &&
+            xattr::hasNextcloudPlaceholderAttributes(filePath);
+}
+
+bool VfsXAttr::statTypeVirtualFile(csync_file_stat_t *stat, void *statData)
+{
+    if (stat->type == ItemTypeDirectory) {
+        return false;
+    }
+
+    const auto parentPath = static_cast<QByteArray *>(statData);
+    Q_ASSERT(!parentPath->endsWith('/'));
+    Q_ASSERT(!stat->path.startsWith('/'));
+
+    const auto path = QByteArray(*parentPath + '/' + stat->path);
+    const auto pin = [=] {
+        const auto absolutePath = QString::fromUtf8(path);
+        Q_ASSERT(absolutePath.startsWith(params().filesystemPath.toUtf8()));
+        const auto folderPath = absolutePath.mid(params().filesystemPath.length());
+        return pinState(folderPath);
+    }();
+
+    if (xattr::hasNextcloudPlaceholderAttributes(path)) {
+        const auto shouldDownload = pin && (*pin == PinState::AlwaysLocal);
+        stat->type = shouldDownload ? ItemTypeVirtualFileDownload : ItemTypeVirtualFile;
+        return true;
+    } else {
+        const auto shouldDehydrate = pin && (*pin == PinState::OnlineOnly);
+        if (shouldDehydrate) {
+            stat->type = ItemTypeVirtualFileDehydration;
+            return true;
+        }
+    }
+    return false;
+}
+
+bool VfsXAttr::setPinState(const QString &folderPath, PinState state)
+{
+    return setPinStateInDb(folderPath, state);
+}
+
+Optional<PinState> VfsXAttr::pinState(const QString &folderPath)
+{
+    return pinStateInDb(folderPath);
+}
+
+Vfs::AvailabilityResult VfsXAttr::availability(const QString &folderPath)
+{
+    return availabilityInDb(folderPath);
+}
+
+void VfsXAttr::fileStatusChanged(const QString &, SyncFileStatus)
+{
+}
+
+} // namespace OCC
+
+OCC_DEFINE_VFS_FACTORY("xattr", OCC::VfsXAttr)

src/libsync/vfs/xattr/vfs_xattr.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) by Kevin Ottens <kevin.ottens@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+#pragma once
+
+#include <QObject>
+#include <QScopedPointer>
+
+#include "common/vfs.h"
+
+namespace OCC {
+
+class VfsXAttr : public Vfs
+{
+    Q_OBJECT
+
+public:
+    explicit VfsXAttr(QObject *parent = nullptr);
+    ~VfsXAttr();
+
+    Mode mode() const override;
+    QString fileSuffix() const override;
+
+    void stop() override;
+    void unregisterFolder() override;
+
+    bool socketApiPinStateActionsShown() const override;
+    bool isHydrating() const override;
+
+    Result<void, QString> updateMetadata(const QString &filePath, time_t modtime, qint64 size, const QByteArray &fileId) override;
+
+    Result<void, QString> createPlaceholder(const SyncFileItem &item) override;
+    Result<void, QString> dehydratePlaceholder(const SyncFileItem &item) override;
+    Result<void, QString> convertToPlaceholder(const QString &filename, const SyncFileItem &item, const QString &replacesFile) override;
+
+    bool needsMetadataUpdate(const SyncFileItem &item) override;
+    bool isDehydratedPlaceholder(const QString &filePath) override;
+    bool statTypeVirtualFile(csync_file_stat_t *stat, void *statData) override;
+
+    bool setPinState(const QString &folderPath, PinState state) override;
+    Optional<PinState> pinState(const QString &folderPath) override;
+    AvailabilityResult availability(const QString &folderPath) override;
+
+public slots:
+    void fileStatusChanged(const QString &systemFileName, SyncFileStatus fileStatus) override;
+
+protected:
+    void startImpl(const VfsSetupParams &params) override;
+};
+
+} // namespace OCC

src/libsync/vfs/xattr/xattrwrapper.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) by Kevin Ottens <kevin.ottens@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+#pragma once
+
+#include <QString>
+
+#include "owncloudlib.h"
+#include "common/result.h"
+
+namespace OCC {
+
+namespace XAttrWrapper
+{
+
+OWNCLOUDSYNC_EXPORT bool hasNextcloudPlaceholderAttributes(const QString &path);
+OWNCLOUDSYNC_EXPORT Result<void, QString> addNextcloudPlaceholderAttributes(const QString &path);
+
+}
+
+} // namespace OCC

src/libsync/vfs/xattr/xattrwrapper_linux.cpp

@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) by Kevin Ottens <kevin.ottens@nextcloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#include "xattrwrapper.h"
+
+#include "config.h"
+
+#include <QLoggingCategory>
+
+#include <sys/xattr.h>
+
+Q_LOGGING_CATEGORY(lcXAttrWrapper, "nextcloud.sync.vfs.xattr.wrapper", QtInfoMsg)
+
+namespace {
+constexpr auto hydrateExecAttributeName = "user.nextcloud.hydrate_exec";
+
+OCC::Optional<QByteArray> xattrGet(const QByteArray &path, const QByteArray &name)
+{
+    constexpr auto bufferSize = 256;
+    QByteArray result;
+    result.resize(bufferSize);
+    const auto count = getxattr(path.constData(), name.constData(), result.data(), bufferSize);
+    if (count >= 0) {
+        result.resize(static_cast<int>(count) - 1);
+        return result;
+    } else {
+        return {};
+    }
+}
+
+bool xattrSet(const QByteArray &path, const QByteArray &name, const QByteArray &value)
+{
+    const auto returnCode = setxattr(path.constData(), name.constData(), value.constData(), value.size() + 1, 0);
+    return returnCode == 0;
+}
+
+}
+
+
+bool OCC::XAttrWrapper::hasNextcloudPlaceholderAttributes(const QString &path)
+{
+    const auto value = xattrGet(path.toUtf8(), hydrateExecAttributeName);
+    if (value) {
+        return *value == QByteArrayLiteral(APPLICATION_EXECUTABLE);
+    } else {
+        return false;
+    }
+}
+
+OCC::Result<void, QString> OCC::XAttrWrapper::addNextcloudPlaceholderAttributes(const QString &path)
+{
+    const auto success = xattrSet(path.toUtf8(), hydrateExecAttributeName, APPLICATION_EXECUTABLE);
+    if (!success) {
+        return QStringLiteral("Failed to set the extended attribute");
+    } else {
+        return {};
+    }
+}

test/CMakeLists.txt

@@ -1,3 +1,4 @@
+include(DefinePlatformDefaults)
 find_package(SQLite3 3.8.0 REQUIRED)
 include_directories(${CMAKE_SOURCE_DIR}/src
                     ${CMAKE_SOURCE_DIR}/src/3rdparty/qtokenizer
@@ -76,6 +77,8 @@ endif(UNIX AND NOT APPLE)
 if (WIN32)
     nextcloud_add_test(LongWinPath "")
     nextcloud_add_test(SyncCfApi "")
+elseif(LINUX) # elseif(LINUX OR APPLE)
+    nextcloud_add_test(SyncXAttr "")
 endif()
 
 nextcloud_add_benchmark(LargeSync "")