Tokenizer: moved test-only tokenize() to SimpleTokenizer / several cleanups

lib/tokenize.cpp

@@ -3439,16 +3439,6 @@ bool Tokenizer::simplifyTokens1(const std::string &configuration)
     return true;
 }
 
-// cppcheck-suppress unusedFunction - used in tests only
-bool Tokenizer::tokenize(std::istream &code,
-                         const char FileName[],
-                         const std::string &configuration)
-{
-    if (!list.createTokens(code, FileName))
-        return false;
-
-    return simplifyTokens1(configuration);
-}
 //---------------------------------------------------------------------------
 
 void Tokenizer::findComplicatedSyntaxErrorsInTemplates()
@@ -10659,6 +10649,7 @@ void Tokenizer::simplifyNamespaceAliases()
     }
 }
 
+// TODO: how to move the Preprocessor dependency out of here?
 bool Tokenizer::hasIfdef(const Token *start, const Token *end) const
 {
     assert(mPreprocessor);

lib/tokenize.h

@@ -79,30 +79,6 @@ class CPPCHECKLIB Tokenizer {
     bool isScopeNoReturn(const Token *endScopeToken, bool *unknown = nullptr) const;
 
     bool simplifyTokens1(const std::string &configuration);
-    /**
-     * Tokenize code
-     * @param code input stream for code, e.g.
-     * \code
-     * #file "p.h"
-     * class Foo
-     * {
-     * private:
-     * void Bar();
-     * };
-     *
-     * #endfile
-     * void Foo::Bar()
-     * {
-     * }
-     * \endcode
-     *
-     * @param FileName The filename
-     * @param configuration E.g. "A" for code where "#ifdef A" is true
-     * @return false if source code contains syntax errors
-     */
-    bool tokenize(std::istream &code,
-                  const char FileName[],
-                  const std::string &configuration = emptyString);
 
 private:
     /** Set variable id */

test/helpers.cpp

@@ -44,6 +44,9 @@
 
 class SuppressionList;
 
+const Settings SimpleTokenizer::s_settings;
+const Preprocessor SimpleTokenizer::s_preprocessor{s_settings, nullptr}; // TODO: provide ErrorLogger
+
 // TODO: better path-only usage
 ScopedFile::ScopedFile(std::string name, const std::string &content, std::string path)
     : mName(std::move(name))

test/helpers.h

@@ -19,6 +19,7 @@
 #ifndef helpersH
 #define helpersH
 
+#include "preprocessor.h"
 #include "settings.h"
 #include "standards.h"
 #include "tokenize.h"
@@ -31,34 +32,62 @@
 #include <vector>
 
 class Token;
-class Preprocessor;
 class SuppressionList;
 class ErrorLogger;
 namespace simplecpp {
     struct DUI;
 }
 
-class SimpleTokenizer {
+// TODO: make Tokenizer private
+class SimpleTokenizer : public Tokenizer {
 public:
-    SimpleTokenizer(ErrorLogger& errorlogger, const char sample[], bool cpp = true)
-        : tokenizer{settings, &errorlogger}
+    SimpleTokenizer(ErrorLogger& errorlogger, const char code[], bool cpp = true)
+        : Tokenizer{s_settings, &errorlogger, &s_preprocessor}
     {
-        std::istringstream iss(sample);
-        if (!tokenizer.tokenize(iss, cpp ? "test.cpp" : "test.c"))
+        if (!tokenize(code, cpp))
             throw std::runtime_error("creating tokens failed");
     }
 
-    Token* tokens() {
-        return tokenizer.tokens();
-    }
+    SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger)
+        : Tokenizer{settings, &errorlogger, &s_preprocessor}
+    {}
+
+    SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, const Preprocessor* preprocessor)
+        : Tokenizer{settings, &errorlogger, preprocessor}
+    {}
+
+    /*
+        Token* tokens() {
+        return Tokenizer::tokens();
+        }
+
+        const Token* tokens() const {
+        return Tokenizer::tokens();
+        }
+     */
+
+    /**
+     * Tokenize code
+     * @param code The code
+     * @param cpp Indicates if the code is C++
+     * @param configuration E.g. "A" for code where "#ifdef A" is true
+     * @return false if source code contains syntax errors
+     */
+    bool tokenize(const char code[],
+                  bool cpp = true,
+                  const std::string &configuration = emptyString)
+    {
+        std::istringstream istr(code);
+        if (!list.createTokens(istr, cpp ? "test.cpp" : "test.c"))
+            return false;
 
-    const Token* tokens() const {
-        return tokenizer.tokens();
+        return simplifyTokens1(configuration);
     }
 
 private:
-    const Settings settings;
-    Tokenizer tokenizer;
+    // TODO. find a better solution
+    static const Settings s_settings;
+    static const Preprocessor s_preprocessor;
 };
 
 class SimpleTokenList

test/test64bit.cpp

@@ -16,14 +16,11 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-
 #include "check64bit.h"
 #include "errortypes.h"
-#include "settings.h"
 #include "fixture.h"
-#include "tokenize.h"
-
-#include <sstream>
+#include "helpers.h"
+#include "settings.h"
 
 class Test64BitPortability : public TestFixture {
 public:
@@ -45,9 +42,8 @@ class Test64BitPortability : public TestFixture {
 #define check(code) check_(code, __FILE__, __LINE__)
     void check_(const char code[], const char* file, int line) {
         // Tokenize..
-        Tokenizer tokenizer(settings, this);
-        std::istringstream istr(code);
-        ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
+        SimpleTokenizer tokenizer(settings, *this);
+        ASSERT_LOC(tokenizer.tokenize(code), file, line);
 
         // Check char variable usage..
         Check64BitPortability check64BitPortability(&tokenizer, &settings, this);

test/testassert.cpp

@@ -19,12 +19,9 @@
 
 #include "checkassert.h"
 #include "errortypes.h"
-#include "settings.h"
 #include "fixture.h"
-#include "tokenize.h"
-
-#include <sstream>
-
+#include "helpers.h"
+#include "settings.h"
 
 class TestAssert : public TestFixture {
 public:
@@ -34,11 +31,10 @@ class TestAssert : public TestFixture {
     const Settings settings = settingsBuilder().severity(Severity::warning).build();
 
 #define check(...) check_(__FILE__, __LINE__, __VA_ARGS__)
-    void check_(const char* file, int line, const char code[], const char *filename = "test.cpp") {
+    void check_(const char* file, int line, const char code[]) {
         // Tokenize..
-        Tokenizer tokenizer(settings, this);
-        std::istringstream istr(code);
-        ASSERT_LOC(tokenizer.tokenize(istr, filename), file, line);
+        SimpleTokenizer tokenizer(settings, *this);
+        ASSERT_LOC(tokenizer.tokenize(code), file, line);
 
         // Check..
         runChecks<CheckAssert>(tokenizer, this);

test/testastutils.cpp

@@ -16,18 +16,16 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-
 #include "astutils.h"
+#include "fixture.h"
+#include "helpers.h"
 #include "library.h"
 #include "settings.h"
-#include "fixture.h"
 #include "symboldatabase.h"
 #include "token.h"
-#include "tokenize.h"
 #include "tokenlist.h"
 
 #include <cstring>
-#include <sstream>
 
 class TestAstUtils : public TestFixture {
 public:
@@ -54,9 +52,8 @@ class TestAstUtils : public TestFixture {
 #define findLambdaEndToken(...) findLambdaEndToken_(__FILE__, __LINE__, __VA_ARGS__)
     bool findLambdaEndToken_(const char* file, int line, const char code[], const char pattern[] = nullptr, bool checkNext = true) {
         const Settings settings;
-        Tokenizer tokenizer(settings, this);
-        std::istringstream istr(code);
-        ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
+        SimpleTokenizer tokenizer(settings, *this);
+        ASSERT_LOC(tokenizer.tokenize(code), file, line);
         const Token* const tokStart = pattern ? Token::findsimplematch(tokenizer.tokens(), pattern, strlen(pattern)) : tokenizer.tokens();
         const Token * const tokEnd = (::findLambdaEndToken)(tokStart);
         return tokEnd && (!checkNext || tokEnd->next() == nullptr);
@@ -92,9 +89,8 @@ class TestAstUtils : public TestFixture {
 
 #define findLambdaStartToken(code) findLambdaStartToken_(code, __FILE__, __LINE__)
     bool findLambdaStartToken_(const char code[], const char* file, int line) {
-        Tokenizer tokenizer(settingsDefault, this);
-        std::istringstream istr(code);
-        ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
+        SimpleTokenizer tokenizer(settingsDefault, *this);
+        ASSERT_LOC(tokenizer.tokenize(code), file, line);
         const Token * const tokStart = (::findLambdaStartToken)(tokenizer.list.back());
         return tokStart && tokStart == tokenizer.list.front();
     }
@@ -124,9 +120,8 @@ class TestAstUtils : public TestFixture {
 
 #define isNullOperand(code) isNullOperand_(code, __FILE__, __LINE__)
     bool isNullOperand_(const char code[], const char* file, int line) {
-        Tokenizer tokenizer(settingsDefault, this);
-        std::istringstream istr(code);
-        ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
+        SimpleTokenizer tokenizer(settingsDefault, *this);
+        ASSERT_LOC(tokenizer.tokenize(code), file, line);
         return (::isNullOperand)(tokenizer.tokens());
     }
 
@@ -145,9 +140,8 @@ class TestAstUtils : public TestFixture {
 
 #define isReturnScope(code, offset) isReturnScope_(code, offset, __FILE__, __LINE__)
     bool isReturnScope_(const char code[], int offset, const char* file, int line) {
-        Tokenizer tokenizer(settingsDefault, this);
-        std::istringstream istr(code);
-        ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
+        SimpleTokenizer tokenizer(settingsDefault, *this);
+        ASSERT_LOC(tokenizer.tokenize(code), file, line);
         const Token * const tok = (offset < 0)
                                   ? tokenizer.list.back()->tokAt(1+offset)
                                   : tokenizer.tokens()->tokAt(offset);
@@ -176,9 +170,8 @@ class TestAstUtils : public TestFixture {
 #define isSameExpression(...) isSameExpression_(__FILE__, __LINE__, __VA_ARGS__)
     bool isSameExpression_(const char* file, int line, const char code[], const char tokStr1[], const char tokStr2[], bool cpp) {
         Library library;
-        Tokenizer tokenizer(settingsDefault, this);
-        std::istringstream istr(code);
-        ASSERT_LOC(tokenizer.tokenize(istr, cpp ? "test.cpp" : "test.c"), file, line);
+        SimpleTokenizer tokenizer(settingsDefault, *this);
+        ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);
         const Token * const tok1 = Token::findsimplematch(tokenizer.tokens(), tokStr1, strlen(tokStr1));
         const Token * const tok2 = Token::findsimplematch(tok1->next(), tokStr2, strlen(tokStr2));
         return (isSameExpression)(false, tok1, tok2, library, false, true, nullptr);
@@ -224,9 +217,8 @@ class TestAstUtils : public TestFixture {
 
 #define isVariableChanged(code, startPattern, endPattern) isVariableChanged_(code, startPattern, endPattern, __FILE__, __LINE__)
     bool isVariableChanged_(const char code[], const char startPattern[], const char endPattern[], const char* file, int line) {
-        Tokenizer tokenizer(settingsDefault, this);
-        std::istringstream istr(code);
-        ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
+        SimpleTokenizer tokenizer(settingsDefault, *this);
+        ASSERT_LOC(tokenizer.tokenize(code), file, line);
         const Token * const tok1 = Token::findsimplematch(tokenizer.tokens(), startPattern, strlen(startPattern));
         const Token * const tok2 = Token::findsimplematch(tokenizer.tokens(), endPattern, strlen(endPattern));
         return (isVariableChanged)(tok1, tok2, 1, false, &settingsDefault);
@@ -258,9 +250,8 @@ class TestAstUtils : public TestFixture {
 
 #define isVariableChangedByFunctionCall(code, pattern, inconclusive) isVariableChangedByFunctionCall_(code, pattern, inconclusive, __FILE__, __LINE__)
     bool isVariableChangedByFunctionCall_(const char code[], const char pattern[], bool *inconclusive, const char* file, int line) {
-        Tokenizer tokenizer(settingsDefault, this);
-        std::istringstream istr(code);
-        ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
+        SimpleTokenizer tokenizer(settingsDefault, *this);
+        ASSERT_LOC(tokenizer.tokenize(code), file, line);
         const Token * const argtok = Token::findmatch(tokenizer.tokens(), pattern);
         ASSERT_LOC(argtok, file, line);
         int indirect = (argtok->variable() && argtok->variable()->isArray());
@@ -401,9 +392,8 @@ class TestAstUtils : public TestFixture {
                               int line)
     {
         const Settings settings = settingsBuilder().library("std.cfg").build();
-        Tokenizer tokenizer(settings, this);
-        std::istringstream istr(code);
-        ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
+        SimpleTokenizer tokenizer(settings, *this);
+        ASSERT_LOC(tokenizer.tokenize(code), file, line);
         const Token* const start = Token::findsimplematch(tokenizer.tokens(), startPattern, strlen(startPattern));
         const Token* const end = Token::findsimplematch(start, endPattern, strlen(endPattern));
         const Token* const expr = Token::findsimplematch(tokenizer.tokens(), var, strlen(var));
@@ -432,9 +422,8 @@ class TestAstUtils : public TestFixture {
 
 #define nextAfterAstRightmostLeaf(code, parentPattern, rightPattern) nextAfterAstRightmostLeaf_(code, parentPattern, rightPattern, __FILE__, __LINE__)
     bool nextAfterAstRightmostLeaf_(const char code[], const char parentPattern[], const char rightPattern[], const char* file, int line) {
-        Tokenizer tokenizer(settingsDefault, this);
-        std::istringstream istr(code);
-        ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
+        SimpleTokenizer tokenizer(settingsDefault, *this);
+        ASSERT_LOC(tokenizer.tokenize(code), file, line);
         const Token * tok = Token::findsimplematch(tokenizer.tokens(), parentPattern, strlen(parentPattern));
         return Token::simpleMatch((::nextAfterAstRightmostLeaf)(tok), rightPattern, strlen(rightPattern));
     }
@@ -456,9 +445,8 @@ class TestAstUtils : public TestFixture {
     enum class Result {False, True, Fail};
 
     Result isUsedAsBool(const char code[], const char pattern[]) {
-        Tokenizer tokenizer(settingsDefault, this);
-        std::istringstream istr(code);
-        if (!tokenizer.tokenize(istr, "test.cpp"))
+        SimpleTokenizer tokenizer(settingsDefault, *this);
+        if (!tokenizer.tokenize(code))
             return Result::Fail;
         const Token * const argtok = Token::findmatch(tokenizer.tokens(), pattern);
         if (!argtok)

test/testautovariables.cpp

@@ -16,14 +16,11 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-
 #include "checkautovariables.h"
 #include "errortypes.h"
-#include "settings.h"
 #include "fixture.h"
-#include "tokenize.h"
-
-#include <sstream>
+#include "helpers.h"
+#include "settings.h"
 
 class TestAutoVariables : public TestFixture {
 public:
@@ -33,13 +30,12 @@ class TestAutoVariables : public TestFixture {
     const Settings settings = settingsBuilder().severity(Severity::warning).severity(Severity::style).library("std.cfg").library("qt.cfg").build();
 
 #define check(...) check_(__FILE__, __LINE__, __VA_ARGS__)
-    void check_(const char* file, int line, const char code[], bool inconclusive = true, const char* filename = "test.cpp") {
+    void check_(const char* file, int line, const char code[], bool inconclusive = true, bool cpp = true) {
         const Settings settings1 = settingsBuilder(settings).certainty(Certainty::inconclusive, inconclusive).build();
 
         // Tokenize..
-        Tokenizer tokenizer(settings1, this);
-        std::istringstream istr(code);
-        ASSERT_LOC(tokenizer.tokenize(istr, filename), file, line);
+        SimpleTokenizer tokenizer(settings1, *this);
+        ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);
 
         runChecks<CheckAutoVariables>(tokenizer, this);
     }
@@ -916,7 +912,7 @@ class TestAutoVariables : public TestFixture {
         check("void svn_repos_dir_delta2() {\n"
               "  struct context c;\n"
               "      SVN_ERR(delete(&c, root_baton, src_entry, pool));\n"
-              "}\n", false, "test.c");
+              "}\n", false, /* cpp= */ false);
         ASSERT_EQUALS("", errout_str());
     }