FolderWatcher: Use csync exclude code #3805

Introduce a global ExcludedFiles instance to avoid loading the global
exclude lists several times.

One could still add per-folder exclude lists by checking these after
the global ones.

src/gui/application.cpp

@@ -35,6 +35,7 @@
 #include "accountmanager.h"
 #include "creds/abstractcredentials.h"
 #include "updater/ocupdater.h"
+#include "excludedfiles.h"
 
 #include "config.h"
 
@@ -135,6 +136,13 @@ Application::Application(int &argc, char **argv) :
     setupLogging();
     setupTranslations();
 
+    // Setup global excludes
+    ConfigFile cfg;
+    ExcludedFiles& excludes = ExcludedFiles::instance();
+    excludes.addExcludeFilePath( cfg.excludeFile(ConfigFile::SystemScope) );
+    excludes.addExcludeFilePath( cfg.excludeFile(ConfigFile::UserScope) );
+    excludes.reloadExcludes();
+
     _folderManager.reset(new FolderMan);
 
     connect(this, SIGNAL(messageReceived(QString, QObject*)), SLOT(slotParseMessage(QString, QObject*)));
@@ -145,7 +153,6 @@ Application::Application(int &argc, char **argv) :
 
     setQuitOnLastWindowClosed(false);
 
-    ConfigFile cfg;
     _theme->setSystrayUseMonoIcons(cfg.monoIcons());
     connect (_theme, SIGNAL(systrayUseMonoIconsChanged(bool)), SLOT(slotUseMonoIconsChanged(bool)));
 

src/gui/folder.cpp

@@ -30,7 +30,7 @@
 #include "syncrunfilelog.h"
 #include "theme.h"
 #include "filesystem.h"
-
+#include "excludedfiles.h"
 
 #include "creds/abstractcredentials.h"
 
@@ -707,6 +707,23 @@ void Folder::removeFromSettings() const
     settings->remove(_definition.alias);
 }
 
+bool Folder::isFileExcluded(const QString& fullPath) const
+{
+    QString myFullPath = fullPath;
+    if (myFullPath.endsWith(QLatin1Char('/'))) {
+        myFullPath.chop(1);
+    }
+
+    if (!myFullPath.startsWith(path())) {
+        // Mark paths we're not responsible for as excluded...
+        return true;
+    }
+
+    QString relativePath = myFullPath.mid(path().size());
+    auto excl = ExcludedFiles::instance().isExcluded(myFullPath, relativePath, _definition.ignoreHiddenFiles);
+    return excl != CSYNC_NOT_EXCLUDED;
+}
+
 void Folder::watcherSlot(QString fn)
 {
     // FIXME: On OS X we could not do this "if" since on OS X the file watcher ignores events for ourselves

src/gui/folder.h

@@ -197,6 +197,11 @@ public:
      /// Removes the folder from the account's settings.
      void removeFromSettings() const;
 
+     /**
+      * Returns whether a file inside this folder should be excluded.
+      */
+     bool isFileExcluded(const QString& fullPath) const;
+
 signals:
     void syncStateChange();
     void syncStarted();

src/gui/folderman.cpp

@@ -60,7 +60,6 @@ FolderMan::FolderMan(QObject *parent) :
     _instance = this;
 
     _socketApi = new SocketApi(this);
-    _socketApi->slotReadExcludes();
 
     ConfigFile cfg;
     int polltime = cfg.remotePollInterval();
@@ -149,10 +148,6 @@ void FolderMan::registerFolderMonitor( Folder *folder )
 
     if( !_folderWatchers.contains(folder->alias() ) ) {
         FolderWatcher *fw = new FolderWatcher(folder->path(), folder);
-        ConfigFile cfg;
-        fw->addIgnoreListFile( cfg.excludeFile(ConfigFile::SystemScope) );
-        fw->addIgnoreListFile( cfg.excludeFile(ConfigFile::UserScope) );
-        fw->setIgnoreHidden( folder->ignoreHiddenFiles() );
 
         // Connect the pathChanged signal, which comes with the changed path,
         // to the signal mapper which maps to the folder alias. The changed path
@@ -696,13 +691,6 @@ void FolderMan::slotStartScheduledFolderSync()
         _currentSyncFolder = f;
 
         f->startSync( QStringList() );
-
-        // reread the excludes of the socket api
-        // FIXME: the excludes need rework.
-        if( _socketApi ) {
-            _socketApi->slotClearExcludesList();
-            _socketApi->slotReadExcludes();
-        }
     }
 }
 

src/gui/folderwatcher.cpp

@@ -32,12 +32,14 @@
 #include "folderwatcher_linux.h"
 #endif
 
+#include "excludedfiles.h"
+#include "folder.h"
 
 namespace OCC {
 
-FolderWatcher::FolderWatcher(const QString &root, QObject *parent)
-    : QObject(parent),
-      _ignoreHidden(true)
+FolderWatcher::FolderWatcher(const QString &root, Folder* folder)
+    : QObject(folder),
+      _folder(folder)
 {
     _d.reset(new FolderWatcherPrivate(this, root));
 
@@ -47,81 +49,17 @@ FolderWatcher::FolderWatcher(const QString &root, QObject *parent)
 FolderWatcher::~FolderWatcher()
 { }
 
-void FolderWatcher::setIgnoreHidden(bool ignore)
-{
-    _ignoreHidden = ignore;
-}
-
-bool FolderWatcher::ignoreHidden()
-{
-    return _ignoreHidden;
-}
-
-void FolderWatcher::addIgnoreListFile( const QString& file )
-{
-    if( file.isEmpty() ) return;
-
-    QFile infile( file );
-    if (!infile.open(QIODevice::ReadOnly | QIODevice::Text))
-        return;
-
-    while (!infile.atEnd()) {
-        QString line = QString::fromLocal8Bit( infile.readLine() ).trimmed();
-        if( !(line.startsWith( QLatin1Char('#') ) || line.isEmpty()) ) {
-            _ignores.append(line);
-        }
-    }
-}
-
-QStringList FolderWatcher::ignores() const
-{
-    return _ignores;
-}
-
 bool FolderWatcher::pathIsIgnored( const QString& path )
 {
     if( path.isEmpty() ) return true;
+    if( !_folder ) return false;
 
-    // if events caused by changes to hidden files should be ignored, a QFileInfo
-    // object will tell us if the file is hidden
-    if( _ignoreHidden ) {
-        QFileInfo fInfo(path);
-        if( fInfo.isHidden() ) {
-            qDebug() << "* Discarded as is hidden!" << fInfo.filePath();
-            return true;
-        }
-    }
-
-    // TODO: Best use csync_excluded_no_ctx() here somehow!
-    foreach (QString pattern, _ignores) {
-        // The leading ] is a tag and not part of the pattern.
-        if (pattern.startsWith(']')) {
-            pattern.remove(0, 1);
-        }
-
-        if(pattern.endsWith('/')) {
-            // directory only pattern. But since path components are
-            // checked later, we cut off the trailing dir.
-            pattern.chop(1);
-        }
-
-        QRegExp regexp(pattern);
-        regexp.setPatternSyntax(QRegExp::Wildcard);
-
-        // if the pattern contains / it needs to match the entire path
-        if (pattern.contains('/') && regexp.exactMatch(path)) {
-            qDebug() << "* Discarded by ignore pattern: " << path;
-            return true;
-        }
-
-        QStringList components = path.split('/');
-        foreach (const QString& comp, components) {
-            if(regexp.exactMatch(comp)) {
-                qDebug() << "* Discarded by component ignore pattern " << comp;
-                return true;
-            }
-        }
+#ifndef OWNCLOUD_TEST
+    if (_folder->isFileExcluded(path)) {
+        qDebug() << "* Ignoring file" << path;
+        return true;
     }
+#endif
     return false;
 }
 

src/gui/folderwatcher.h

@@ -30,6 +30,7 @@ class QTimer;
 namespace OCC {
 
 class FolderWatcherPrivate;
+class Folder;
 
 /**
  * @brief Montiors a directory recursively for changes
@@ -53,19 +54,9 @@ public:
     /**
      * @param root Path of the root of the folder
      */
-    FolderWatcher(const QString &root, QObject *parent = 0L);
+    FolderWatcher(const QString &root, Folder* folder = 0L);
     virtual ~FolderWatcher();
 
-    /**
-      * Set a file name to load a file with ignore patterns.
-      *
-      * Valid entries do not start with a hash sign (#)
-      * and may contain wildcards
-      */
-    void addIgnoreListFile( const QString& );
-
-    QStringList ignores() const;
-
     /**
      * Not all backends are recursive by default.
      * Those need to be notified when a directory is added or removed while the watcher is disabled.
@@ -77,10 +68,6 @@ public:
     /* Check if the path is ignored. */
     bool pathIsIgnored( const QString& path );
 
-    /* set if the folderwatcher ignores events of hidden files */
-    void setIgnoreHidden(bool ignore);
-    bool ignoreHidden();
-
 signals:
     /** Emitted when one of the watched directories or one
      *  of the contained files is changed. */
@@ -99,10 +86,9 @@ protected:
 
 private:
     QScopedPointer<FolderWatcherPrivate> _d;
-    QStringList _ignores;
     QTime _timer;
     QSet<QString> _lastPaths;
-    bool  _ignoreHidden;
+    Folder* _folder;
 
     friend class FolderWatcherPrivate;
 };

src/gui/ignorelisteditor.cpp

@@ -16,6 +16,7 @@
 #include "ignorelisteditor.h"
 #include "folderman.h"
 #include "ui_ignorelisteditor.h"
+#include "excludedfiles.h"
 
 #include <QFile>
 #include <QDir>
@@ -126,6 +127,8 @@ void IgnoreListEditor::slotUpdateLocalIgnoreList()
         folder->journalDb()->forceRemoteDiscoveryNextSync();
         folderMan->slotScheduleSync(folder);
     }
+
+    ExcludedFiles::instance().reloadExcludes();
 }
 
 void IgnoreListEditor::slotAddPattern()

src/gui/socketapi.cpp

@@ -52,30 +52,12 @@
 // The second number should be changed when there are new features.
 #define MIRALL_SOCKET_API_VERSION "1.0"
 
-extern "C" {
-
-enum csync_exclude_type_e {
-  CSYNC_NOT_EXCLUDED   = 0,
-  CSYNC_FILE_SILENTLY_EXCLUDED,
-  CSYNC_FILE_EXCLUDE_AND_REMOVE,
-  CSYNC_FILE_EXCLUDE_LIST,
-  CSYNC_FILE_EXCLUDE_INVALID_CHAR,
-  CSYNC_FILE_EXCLUDE_LONG_FILENAME,
-  CSYNC_FILE_EXCLUDE_HIDDEN
-};
-typedef enum csync_exclude_type_e CSYNC_EXCLUDE_TYPE;
-
-CSYNC_EXCLUDE_TYPE csync_excluded_no_ctx(c_strlist_t *excludes, const char *path, int filetype);
-int csync_exclude_load(const char *fname, c_strlist_t **list);
-}
-
 namespace OCC {
 
 #define DEBUG qDebug() << "SocketApi: "
 
 SocketApi::SocketApi(QObject* parent)
     : QObject(parent)
-    , _excludes(0)
 {
     QString socketPath;
 
@@ -141,29 +123,6 @@ SocketApi::~SocketApi()
     // All remaining sockets will be destroyed with _localServer, their parent
     Q_ASSERT(_listeners.isEmpty() || _listeners.first()->parent() == &_localServer);
     _listeners.clear();
-    slotClearExcludesList();
-    c_strlist_destroy(_excludes);
-}
-
-void SocketApi::slotClearExcludesList()
-{
-    c_strlist_clear(_excludes);
-}
-
-void SocketApi::slotReadExcludes()
-{
-    ConfigFile cfgFile;
-    slotClearExcludesList();
-    QString excludeList = cfgFile.excludeFile( ConfigFile::SystemScope );
-    if( !excludeList.isEmpty() ) {
-        qDebug() << "==== added system ignore list to socketapi:" << excludeList.toUtf8();
-        csync_exclude_load(excludeList.toUtf8(), &_excludes);
-    }
-    excludeList = cfgFile.excludeFile( ConfigFile::UserScope );
-    if( !excludeList.isEmpty() ) {
-        qDebug() << "==== added user defined ignore list to csync:" << excludeList.toUtf8();
-        csync_exclude_load(excludeList.toUtf8(), &_excludes);
-    }
 }
 
 void SocketApi::slotNewConnection()
@@ -268,7 +227,7 @@ void SocketApi::slotUpdateFolderView(Folder *f)
                 f->syncResult().status() == SyncResult::SetupError ) {
 
             broadcastMessage(QLatin1String("STATUS"), f->path() ,
-                             this->fileStatus(f, "", _excludes).toSocketAPIString());
+                             this->fileStatus(f, "").toSocketAPIString());
 
             broadcastMessage(QLatin1String("UPDATE_VIEW"), f->path() );
         } else {
@@ -387,7 +346,7 @@ void SocketApi::command_RETRIEVE_FILE_STATUS(const QString& argument, QIODevice*
         statusString = QLatin1String("NOP");
     } else {
         const QString file = QDir::cleanPath(argument).mid(syncFolder->cleanPath().length()+1);
-        SyncFileStatus fileStatus = this->fileStatus(syncFolder, file, _excludes);
+        SyncFileStatus fileStatus = this->fileStatus(syncFolder, file);
 
         statusString = fileStatus.toSocketAPIString();
     }
@@ -541,7 +500,7 @@ SyncJournalFileRecord SocketApi::dbFileRecord_capi( Folder *folder, QString file
 /**
  * Get status about a single file.
  */
-SyncFileStatus SocketApi::fileStatus(Folder *folder, const QString& systemFileName, c_strlist_t *excludes )
+SyncFileStatus SocketApi::fileStatus(Folder *folder, const QString& systemFileName)
 {
     QString file = folder->path();
     QString fileName = systemFileName.normalized(QString::NormalizationForm_C);
@@ -581,13 +540,7 @@ SyncFileStatus SocketApi::fileStatus(Folder *folder, const QString& systemFileNa
     }
 
     // Is it excluded?
-    CSYNC_EXCLUDE_TYPE excl = csync_excluded_no_ctx(excludes, fileName.toUtf8(), type);
-    if( folder->ignoreHiddenFiles()
-            && (fi.isHidden()
-                || fi.fileName().startsWith(QLatin1Char('.'))) ) {
-        excl = CSYNC_FILE_EXCLUDE_HIDDEN;
-    }
-    if( excl != CSYNC_NOT_EXCLUDED ) {
+    if( folder->isFileExcluded(file) ) {
         return SyncFileStatus(SyncFileStatus::STATUS_IGNORE);
     }
 

src/gui/socketapi.h

@@ -16,10 +16,6 @@
 #ifndef SOCKETAPI_H
 #define SOCKETAPI_H
 
-extern "C" {
-#include <std/c_string.h>
-}
-
 #include "syncfileitem.h"
 #include "syncjournalfilerecord.h"
 #include "ownsql.h"
@@ -56,8 +52,6 @@ public slots:
     void slotUpdateFolderView(Folder *f);
     void slotUnregisterPath( const QString& alias );
     void slotRegisterPath( const QString& alias );
-    void slotReadExcludes();
-    void slotClearExcludesList();
 
 signals:
     void shareCommandReceived(const QString &sharePath, const QString &localPath, bool resharingAllowed);
@@ -70,7 +64,7 @@ private slots:
     void slotSyncItemDiscovered(const QString &, const SyncFileItem &);
 
 private:
-    SyncFileStatus fileStatus(Folder *folder, const QString& systemFileName, c_strlist_t *excludes );
+    SyncFileStatus fileStatus(Folder *folder, const QString& systemFileName);
     SyncJournalFileRecord dbFileRecord_capi( Folder *folder, QString fileName );
     SqlQuery *getSqlQuery( Folder *folder );
 
@@ -88,7 +82,6 @@ private:
 
     QList<QIODevice*> _listeners;
     SocketApiServer _localServer;
-    c_strlist_t *_excludes;
     QHash<Folder*, QSharedPointer<SqlQuery>> _dbQueries;
     QHash<Folder*, QSharedPointer<SqlDatabase>> _openDbs;
 };

src/libsync/CMakeLists.txt

@@ -64,6 +64,7 @@ set(libsync_SRCS
     utility.cpp
     ownsql.cpp
     transmissionchecksumvalidator.cpp
+    excludedfiles.cpp
     creds/dummycredentials.cpp
     creds/abstractcredentials.cpp
     creds/credentialscommon.cpp

src/libsync/excludedfiles.cpp

@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) by Christian Kamm <mail@ckamm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#include "excludedfiles.h"
+
+#include <QFileInfo>
+#include <QReadLocker>
+#include <QWriteLocker>
+
+extern "C" {
+#include "std/c_string.h"
+#include "csync.h"
+#include "csync_exclude.h"
+}
+
+using namespace OCC;
+
+ExcludedFiles::ExcludedFiles()
+    : _excludes(NULL)
+{
+}
+
+ExcludedFiles::~ExcludedFiles()
+{
+    c_strlist_destroy(_excludes);
+}
+
+ExcludedFiles& ExcludedFiles::instance()
+{
+    static ExcludedFiles inst;
+    return inst;
+}
+
+void ExcludedFiles::addExcludeFilePath(const QString& path)
+{
+    QWriteLocker locker(&_mutex);
+    _excludeFiles.append(path);
+}
+
+void ExcludedFiles::reloadExcludes()
+{
+    QWriteLocker locker(&_mutex);
+    c_strlist_destroy(_excludes);
+    _excludes = NULL;
+
+    foreach (const QString& file, _excludeFiles) {
+        csync_exclude_load(file.toUtf8(), &_excludes);
+    }
+}
+
+CSYNC_EXCLUDE_TYPE ExcludedFiles::isExcluded(
+        const QString& fullPath,
+        const QString& relativePath,
+        bool excludeHidden) const
+{
+    QFileInfo fi(fullPath);
+
+    if( excludeHidden ) {
+        if( fi.isHidden() || fi.fileName().startsWith(QLatin1Char('.')) ) {
+            return CSYNC_FILE_EXCLUDE_HIDDEN;
+        }
+    }
+
+    csync_ftw_type_e type = CSYNC_FTW_TYPE_FILE;
+    if (fi.isDir()) {
+        type = CSYNC_FTW_TYPE_DIR;
+    }
+    QReadLocker lock(&_mutex);
+    return csync_excluded_no_ctx(_excludes, relativePath.toUtf8(), type);
+}

src/libsync/excludedfiles.h

@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) by Christian Kamm <mail@ckamm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#pragma once
+
+#include "owncloudlib.h"
+
+#include <QObject>
+#include <QReadWriteLock>
+#include <QStringList>
+
+extern "C" {
+#include "std/c_string.h"
+#include "csync.h"
+#include "csync_exclude.h" // for CSYNC_EXCLUDE_TYPE
+}
+
+namespace OCC {
+
+/**
+ * Manages the global system and user exclude lists.
+ */
+class OWNCLOUDSYNC_EXPORT ExcludedFiles : public QObject
+{
+    Q_OBJECT
+public:
+    static ExcludedFiles & instance();
+
+    /**
+     * Adds a new path to a file containing exclude patterns.
+     *
+     * Does not load the file. Use reloadExcludes() afterwards.
+     */
+    void addExcludeFilePath(const QString& path);
+
+    /**
+     * Checks whether a file or directory should be excluded.
+     *
+     * @param fullPath     the absolute path to the file
+     * @param relativePath path relative to the folder
+     *
+     * For directories, the paths must not contain a trailing /.
+     */
+    CSYNC_EXCLUDE_TYPE isExcluded(
+            const QString& fullPath,
+            const QString& relativePath,
+            bool excludeHidden) const;
+
+public slots:
+    /**
+     * Reloads the exclude patterns from the registered paths.
+     */
+    void reloadExcludes();
+
+private:
+    ExcludedFiles();
+    ~ExcludedFiles();
+
+    c_strlist_t* _excludes;
+    QStringList _excludeFiles;
+    mutable QReadWriteLock _mutex;
+};
+
+} // namespace OCC

test/owncloud_add_test.cmake

@@ -23,5 +23,6 @@ macro(owncloud_add_test test_class additional_cpp)
         ${QT_QTCORE_LIBRARY}
     )
 
+    add_definitions(-DOWNCLOUD_TEST)
     add_test(NAME ${OWNCLOUD_TEST_CLASS}Test COMMAND ${OWNCLOUD_TEST_CLASS}Test)
 endmacro()