Remove qtokenizer in favour of Qt6 QStringTokenizer

Signed-off-by: Claudio Cambra <claudio.cambra@nextcloud.com>

src/3rdparty/qtokenizer/qtokenizer.h

@@ -1,261 +0,0 @@
-/****************************************************************************
-**
-** Copyright (C) 2014 Daniel Molkentin <daniel@molkentin.de>
-** Contact: http://www.qt-project.org/legal
-**
-** This file is part of the QtNetwork module of the Qt Toolkit.
-**
-** $QT_BEGIN_LICENSE:LGPL$
-** Commercial License Usage
-** Licensees holding valid commercial Qt licenses may use this file in
-** accordance with the commercial license agreement provided with the
-** Software or, alternatively, in accordance with the terms contained in
-** a written agreement between you and Digia.  For licensing terms and
-** conditions see http://qt.digia.com/licensing.  For further information
-** use the contact form at http://qt.digia.com/contact-us.
-**
-** GNU Lesser General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU Lesser
-** General Public License version 2.1 as published by the Free Software
-** Foundation and appearing in the file LICENSE.LGPL included in the
-** packaging of this file.  Please review the following information to
-** ensure the GNU Lesser General Public License version 2.1 requirements
-** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
-**
-** In addition, as a special exception, Digia gives you certain additional
-** rights.  These rights are described in the Digia Qt LGPL Exception
-** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
-**
-** GNU General Public License Usage
-** Alternatively, this file may be used under the terms of the GNU
-** General Public License version 3.0 as published by the Free Software
-** Foundation and appearing in the file LICENSE.GPL included in the
-** packaging of this file.  Please review the following information to
-** ensure the GNU General Public License version 3.0 requirements will be
-** met: http://www.gnu.org/copyleft/gpl.html.
-**
-**
-** $QT_END_LICENSE$
-**
-****************************************************************************/
-
-#ifndef TOKENIZER_H
-#define TOKENIZER_H
-
-#include <QString>
-#include <QByteArray>
-#include <QSharedPointer>
-
-QT_BEGIN_NAMESPACE
-
-template <class T, class const_iterator>
-struct QTokenizerPrivate {
-    using char_type = typename T::value_type;
-
-    struct State {
-        bool inQuote = false;
-        bool inEscape = false;
-        char_type quoteChar = '\0';
-    };
-
-    QTokenizerPrivate(const T& _string, const T& _delims) :
-        string(_string)
-      , begin(string.begin())
-      , end(string.end())
-      , tokenBegin(end)
-      , tokenEnd(begin)
-      , delimiters(_delims)
-    {
-    }
-
-    [[nodiscard]] bool isDelimiter(char_type c) const {
-        return delimiters.contains(c);
-    }
-
-    [[nodiscard]] bool isQuote(char_type c) const {
-        return quotes.contains(c);
-    }
-
-    // Returns true if a delimiter was not hit
-    bool nextChar(State* state, char_type c) {
-        if (state->inQuote) {
-            if (state->inEscape) {
-                state->inEscape = false;
-            } else if (c == '\\') {
-                state->inEscape = true;
-            } else if (c == state->quoteChar) {
-                state->inQuote = false;
-            }
-        } else {
-            if (isDelimiter(c))
-                return false;
-            state->inQuote = isQuote(state->quoteChar = c);
-        }
-        return true;
-    }
-
-    T string;
-    // ### copies begin and end for performance, premature optimization?
-    const_iterator begin;
-    const_iterator end;
-    const_iterator tokenBegin;
-    const_iterator tokenEnd;
-    T delimiters;
-    T quotes;
-    bool isDelim = false;
-    bool returnDelimiters = false;
-    bool returnQuotes = false;
-};
-
-template <class T, class const_iterator = typename T::const_iterator>
-class QTokenizer {
-public:
-    using char_type = typename T::value_type;
-
-    /*!
-       \class QTokenizer
-       \inmodule QtNetwork
-       \brief QTokenizer tokenizes Strings on QString, QByteArray,
-              std::string or std::wstring
-
-       Example Usage:
-
-       \code
-         QString str = ...;
-         QByteArrayTokenizer tokenizer(str, "; ");
-         tokenizer.setQuoteCharacters("\"'");
-         tokenizer.setReturnDelimiters(true);
-         while (tokenizer.hasNext()) {
-           QByteArray token = tokenizer.next();
-           bool isDelimiter = tokenizer.isDelimiter();
-           ...
-         }
-       \endcode
-
-       \param string The string to tokenize
-       \param delimiters A string containing delimiters
-
-       \sa QStringTokenizer, QByteArrayTokenizer, StringTokenizer, WStringTokenizer
-     */
-    QTokenizer(const T& string, const T& delimiters)
-        : d(new QTokenizerPrivate<T, const_iterator>(string, delimiters))
-    { }
-
-    /*!
-       Whether or not to return delimiters as tokens
-       \see setQuoteCharacters
-     */
-    void setReturnDelimiters(bool enable) { d->returnDelimiters = enable; }
-
-
-    /*!
-       Sets characters that are considered to start and end quotes.
-
-       When between two characters considered a quote, delimiters will
-       be ignored.
-
-       When between quotes, blackslash characters will cause the QTokenizer
-       to skip the next character.
-
-       \param quotes Characters that delimit quotes.
-     */
-    void setQuoteCharacters(const T& quotes) { d->quotes = quotes; }
-
-
-    /*!
-       Whether or not to return delimiters as tokens
-       \see setQuoteCharacters
-     */
-    void setReturnQuoteCharacters(bool enable) { d->returnQuotes = enable; }
-
-
-    /*!
-       Retrieve next token.
-
-       Returns true if there are more tokens, false otherwise.
-
-       \sa next()
-     */
-    bool hasNext()
-    {
-        typename QTokenizerPrivate<T, const_iterator>::State state;
-        d->isDelim = false;
-        for (;;) {
-            d->tokenBegin = d->tokenEnd;
-            if (d->tokenEnd == d->end)
-                return false;
-            d->tokenEnd++;
-            if (d->nextChar(&state, *d->tokenBegin))
-                break;
-            if (d->returnDelimiters) {
-                d->isDelim = true;
-                return true;
-            }
-        }
-        while (d->tokenEnd != d->end && d->nextChar(&state, *d->tokenEnd)) {
-            d->tokenEnd++;
-        }
-        return true;
-    }
-
-    /*!
-       Resets the tokenizer to the starting position.
-     */
-    void reset() {
-        d->tokenEnd = d->begin;
-    }
-
-    /*!
-       Returns true if the current token is a delimiter,
-       if one more more delimiting characters have been set.
-     */
-    [[nodiscard]] bool isDelimiter() const { return d->isDelim; }
-
-    /*!
-       Returns the current token.
-
-       Use \c hasNext() to fetch the next token.
-     */
-    [[nodiscard]] T next() const {
-        int len = std::distance(d->tokenBegin, d->tokenEnd);
-        const_iterator tmpStart = d->tokenBegin;
-        if (!d->returnQuotes && len > 1 && d->isQuote(*d->tokenBegin)) {
-            tmpStart++;
-            len -= 2;
-        }
-        return T(tmpStart, len);
-    }
-
-private:
-    friend class QStringTokenizer;
-    QSharedPointer<QTokenizerPrivate<T, const_iterator> > d;
-};
-
-class QStringTokenizer : public QTokenizer<QString> {
-public:
-    QStringTokenizer(const QString &string, const QString &delim) :
-        QTokenizer<QString, QString::const_iterator>(string, delim) {}
-    /**
-     * @brief Like \see next(), but returns a lightweight string reference
-     * @return A reference to the token within the string
-     */
-    QStringRef stringRef() {
-        // If those differences overflow an int we'd have a veeeeeery long string in memory
-        int begin = std::distance(d->begin, d->tokenBegin);
-        int end = std::distance(d->tokenBegin, d->tokenEnd);
-        if (!d->returnQuotes && d->isQuote(*d->tokenBegin)) {
-            begin++;
-            end -= 2;
-        }
-        return QStringRef(&d->string, begin, end);
-    }
-};
-
-using QByteArrayTokenizer = QTokenizer<QByteArray>;
-using StringTokenizer = QTokenizer<std::string>;
-using WStringTokenizer = QTokenizer<std::wstring>;
-
-QT_END_NAMESPACE
-
-#endif // TOKENIZER_H
-

src/3rdparty/qtokenizer/qtokenizer.pro

@@ -1,2 +0,0 @@
-TEMPLATE = subdirs
-SUBDIRS = test

src/3rdparty/qtokenizer/test/test.pro

@@ -1,8 +0,0 @@
-TEMPLATE = app
-QT += testlib
-CONFIG += testlib
-TARGET = test
-INCLUDEPATH += . ..
-
-# Input
-SOURCES += tst_qtokenizer.cpp

src/3rdparty/qtokenizer/test/tst_qtokenizer.cpp

@@ -1,139 +0,0 @@
-#include <QtTest>
-
-#include "qtokenizer.h"
-
-namespace {
-  const QString simple     = QLatin1String("A simple tokenizer test");
-  const QString quoted     = QLatin1String("\"Wait for me!\" he shouted");
-}
-
-class TestTokenizer : public QObject
-{
-  Q_OBJECT
-private slots:
-  void tokenizeQStringSimple() {
-    QStringTokenizer tokenizer(simple, " ");
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.next(), QLatin1String("A"));
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.next(), QLatin1String("simple"));
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.next(), QLatin1String("tokenizer"));
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.next(), QLatin1String("test"));
-
-    QCOMPARE(tokenizer.hasNext(), false);
-  }
-
-  void tokenizeQStringSimpleRef() {
-    QStringTokenizer tokenizer(simple, " ");
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QVERIFY(tokenizer.stringRef() == QLatin1String("A"));
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QVERIFY(tokenizer.stringRef() == QLatin1String("simple"));
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QVERIFY(tokenizer.stringRef() == QLatin1String("tokenizer"));
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QVERIFY(tokenizer.stringRef() == QLatin1String("test"));
-
-    QCOMPARE(tokenizer.hasNext(), false);
-  }
-
-  void tokenizeQStringQuoted() {
-    const QString multiquote(QLatin1String("\"'Billy - the Kid' is dead!\""));
-    QStringTokenizer tokenizer(multiquote, " -");
-    tokenizer.setQuoteCharacters("\"");
-    tokenizer.setReturnQuoteCharacters(true);
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.next(), QLatin1String("\"'Billy - the Kid' is dead!\""));
-
-    QCOMPARE(tokenizer.hasNext(), false);
-  }
-
-  void tokenizeQStringSkipQuotes() {
-    const QString multiquote(QLatin1String("\"'Billy - the Kid' is dead!\""));
-    QStringTokenizer tokenizer(multiquote, " ");
-    tokenizer.setQuoteCharacters("\"");
-    tokenizer.setReturnQuoteCharacters(false);
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.next(), QLatin1String("'Billy - the Kid' is dead!"));
-    QCOMPARE(tokenizer.stringRef().toString(), QLatin1String("'Billy - the Kid' is dead!"));
-
-    QCOMPARE(tokenizer.hasNext(), false);
-  }
-
-
-  void tokenizeQStringWithDelims() {
-    const QString delims(QLatin1String("I;Insist,On/a-Delimiter"));
-    QStringTokenizer tokenizer(delims, ";,/-");
-    tokenizer.setReturnDelimiters(true);
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.isDelimiter(), false);
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.isDelimiter(), true);
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.isDelimiter(), false);
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.isDelimiter(), true);
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.isDelimiter(), false);
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.isDelimiter(), true);
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.isDelimiter(), false);
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.isDelimiter(), true);
-
-    QCOMPARE(tokenizer.hasNext(), true);
-    QCOMPARE(tokenizer.isDelimiter(), false);
-
-    QCOMPARE(tokenizer.hasNext(), false);
-  }
-
-  void resetTokenizer() {
-    for (int i = 0; i < 2; i++) {
-      QStringTokenizer tokenizer(simple, " ");
-
-      QCOMPARE(tokenizer.hasNext(), true);
-      QCOMPARE(tokenizer.next(), QLatin1String("A"));
-
-      QCOMPARE(tokenizer.hasNext(), true);
-      QCOMPARE(tokenizer.next(), QLatin1String("simple"));
-
-      QCOMPARE(tokenizer.hasNext(), true);
-      QCOMPARE(tokenizer.next(), QLatin1String("tokenizer"));
-
-      QCOMPARE(tokenizer.hasNext(), true);
-      QCOMPARE(tokenizer.next(), QLatin1String("test"));
-
-      QCOMPARE(tokenizer.hasNext(), false);
-
-      tokenizer.reset();
-    }
-  }
-
-  // ### QByteArray, other types
-};
-
-QTEST_APPLESS_MAIN(TestTokenizer)
-
-#include "tst_qtokenizer.moc"
-

src/cmd/CMakeLists.txt

@@ -14,9 +14,6 @@ target_link_libraries(cmdCore
   Qt::Network
   )
 
-# Need tokenizer for netrc parser
-target_include_directories(cmdCore PRIVATE ${CMAKE_SOURCE_DIR}/src/3rdparty/qtokenizer)
-
 if(UNIX AND NOT APPLE)
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIE")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIE")

src/cmd/netrcparser.cpp

@@ -15,8 +15,7 @@
 #include <QDir>
 #include <QFile>
 #include <QTextStream>
-
-#include <qtokenizer.h>
+#include <QStringTokenizer>
 
 #include <QDebug>
 
@@ -59,33 +58,32 @@ bool NetrcParser::parse()
     }
     QString content = netrc.readAll();
 
-    QStringTokenizer tokenizer(content, " \n\t");
-    tokenizer.setQuoteCharacters("\"'");
+    auto tokenizer = QStringTokenizer{content, u" \n\t"};
 
     LoginPair pair;
     QString machine;
     bool isDefault = false;
-    while (tokenizer.hasNext()) {
-        QString key = tokenizer.next();
+    for(auto itToken = tokenizer.cbegin(); itToken != tokenizer.cend(); ++itToken) {
+        const auto key = *itToken;
         if (key == defaultKeyword) {
             tryAddEntryAndClear(machine, pair, isDefault);
             isDefault = true;
             continue; // don't read a value
         }
 
-        if (!tokenizer.hasNext()) {
+        if (itToken != tokenizer.cend()) {
             qDebug() << "error fetching value for" << key;
             return false;
         }
-        QString value = tokenizer.next();
+        auto value = *(++itToken);
 
         if (key == machineKeyword) {
             tryAddEntryAndClear(machine, pair, isDefault);
-            machine = value;
+            machine = value.toString();
         } else if (key == loginKeyword) {
-            pair.first = value;
+            pair.first = value.toString();
         } else if (key == passwordKeyword) {
-            pair.second = value;
+            pair.second = value.toString();
         } // ignore unsupported tokens
     }
     tryAddEntryAndClear(machine, pair, isDefault);