Tokenizer: removed Preprocessor dependency (#6265)

danmar · Apr 11, 2024 · 423617a · 423617a
1 parent d43a1b3
commit 423617a
Show file tree

Hide file tree

Showing 15 changed files with 193 additions and 255 deletions.
diff --git a/lib/cppcheck.cpp b/lib/cppcheck.cpp
@@ -506,8 +506,7 @@ unsigned int CppCheck::checkClang(const std::string &path)
     }
 
     try {
-        Preprocessor preprocessor(mSettings, this);
-        Tokenizer tokenizer(mSettings, this, &preprocessor);
+        Tokenizer tokenizer(mSettings, this);
         tokenizer.list.appendFileIfNew(path);
         std::istringstream ast(output2);
         clangimport::parseClangAstDump(tokenizer, ast);
@@ -796,7 +795,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
         }
 
         // Get directives
-        preprocessor.setDirectives(tokens1);
+        std::list<Directive> directives = preprocessor.createDirectives(tokens1);
         preprocessor.simplifyPragmaAsm(&tokens1);
 
         preprocessor.setPlatformInfo(&tokens1);
@@ -821,7 +820,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
         // Run define rules on raw code
         if (hasRule("define")) {
             std::string code;
-            for (const Directive &dir : preprocessor.getDirectives()) {
+            for (const Directive &dir : directives) {
                 if (startsWith(dir.str,"#define ") || startsWith(dir.str,"#include "))
                     code += "#line " + std::to_string(dir.linenr) + " \"" + dir.file + "\"\n" + dir.str + '\n';
             }
@@ -887,9 +886,10 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
                 continue;
             }
 
-            Tokenizer tokenizer(mSettings, this, &preprocessor);
+            Tokenizer tokenizer(mSettings, this);
             if (mSettings.showtime != SHOWTIME_MODES::SHOWTIME_NONE)
                 tokenizer.setTimerResults(&s_timerResults);
+            tokenizer.setDirectives(directives); // TODO: how to avoid repeated copies?
 
             try {
                 // Create tokens, skip rest of iteration if failed

diff --git a/lib/preprocessor.cpp b/lib/preprocessor.cpp
@@ -308,10 +308,10 @@ void Preprocessor::inlineSuppressions(const simplecpp::TokenList &tokens, Suppre
     }
 }
 
-void Preprocessor::setDirectives(const simplecpp::TokenList &tokens)
+std::list<Directive> Preprocessor::createDirectives(const simplecpp::TokenList &tokens) const
 {
     // directive list..
-    mDirectives.clear();
+    std::list<Directive> directives;
 
     std::vector<const simplecpp::TokenList *> list;
     list.reserve(1U + mTokenLists.size());
@@ -337,9 +337,11 @@ void Preprocessor::setDirectives(const simplecpp::TokenList &tokens)
                 else
                     directive.str += tok2->str();
             }
-            mDirectives.push_back(std::move(directive));
+            directives.push_back(std::move(directive));
         }
     }
+
+    return directives;
 }
 
 static std::string readcondition(const simplecpp::Token *iftok, const std::set<std::string> &defined, const std::set<std::string> &undefined)
@@ -912,17 +914,6 @@ void Preprocessor::dump(std::ostream &out) const
 {
     // Create a xml dump.
 
-    out << "  <directivelist>" << std::endl;
-    for (const Directive &dir : mDirectives) {
-        out << "    <directive "
-            << "file=\"" << ErrorLogger::toxml(dir.file) << "\" "
-            << "linenr=\"" << dir.linenr << "\" "
-            // str might contain characters such as '"', '<' or '>' which
-            // could result in invalid XML, so run it through toxml().
-            << "str=\"" << ErrorLogger::toxml(dir.str) << "\"/>" << std::endl;
-    }
-    out << "  </directivelist>" << std::endl;
-
     if (!mMacroUsage.empty()) {
         out << "  <macro-usage>" << std::endl;
         for (const simplecpp::MacroUsage &macroUsage: mMacroUsage) {

diff --git a/lib/preprocessor.h b/lib/preprocessor.h
@@ -68,7 +68,7 @@ struct CPPCHECKLIB Directive {
  * The preprocessor has special functionality for extracting the various ifdef
  * configurations that exist in a source file.
  */
-class CPPCHECKLIB Preprocessor {
+class CPPCHECKLIB WARN_UNUSED Preprocessor {
     // TODO: get rid of this
     friend class PreprocessorHelper;
     friend class TestPreprocessor;
@@ -92,12 +92,7 @@ class CPPCHECKLIB Preprocessor {
 
     void inlineSuppressions(const simplecpp::TokenList &tokens, SuppressionList &suppressions);
 
-    void setDirectives(const simplecpp::TokenList &tokens);
-
-    /** list of all directives met while preprocessing file */
-    const std::list<Directive> &getDirectives() const {
-        return mDirectives;
-    }
+    std::list<Directive> createDirectives(const simplecpp::TokenList &tokens) const;
 
     std::set<std::string> getConfigs(const simplecpp::TokenList &tokens) const;
 
@@ -143,15 +138,10 @@ class CPPCHECKLIB Preprocessor {
 
     static bool hasErrors(const simplecpp::OutputList &outputList);
 
-    void setDirectives(const std::list<Directive> &directives) {
-        mDirectives = directives;
-    }
-
     const Settings& mSettings;
     ErrorLogger *mErrorLogger;
 
     /** list of all directives met while preprocessing file */
-    std::list<Directive> mDirectives;
 
     std::map<std::string, simplecpp::TokenList *> mTokenLists;
 

diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp
@@ -152,12 +152,11 @@ static bool isClassStructUnionEnumStart(const Token * tok)
 
 //---------------------------------------------------------------------------
 
-Tokenizer::Tokenizer(const Settings &settings, ErrorLogger *errorLogger, const Preprocessor *preprocessor) :
+Tokenizer::Tokenizer(const Settings &settings, ErrorLogger *errorLogger) :
     list(&settings),
     mSettings(settings),
     mErrorLogger(errorLogger),
-    mTemplateSimplifier(new TemplateSimplifier(*this)),
-    mPreprocessor(preprocessor)
+    mTemplateSimplifier(new TemplateSimplifier(*this))
 {}
 
 Tokenizer::~Tokenizer()
@@ -5913,6 +5912,26 @@ void Tokenizer::dump(std::ostream &out) const
 
     std::set<const Library::Container*> containers;
 
+    outs += "  <directivelist>";
+    outs += '\n';
+    for (const Directive &dir : mDirectives) {
+        outs += "    <directive ";
+        outs += "file=\"";
+        outs += ErrorLogger::toxml(dir.file);
+        outs += "\" ";
+        outs += "linenr=\"";
+        outs += std::to_string(dir.linenr);
+        outs += "\" ";
+        // str might contain characters such as '"', '<' or '>' which
+        // could result in invalid XML, so run it through toxml().
+        outs += "str=\"";
+        outs += ErrorLogger::toxml(dir.str);
+        outs +="\"/>";
+        outs += '\n';
+    }
+    outs += "  </directivelist>";
+    outs += '\n';
+
     // tokens..
     outs += "  <tokenlist>";
     outs += '\n';
@@ -10652,12 +10671,15 @@ void Tokenizer::simplifyNamespaceAliases()
     }
 }
 
-// TODO: how to move the Preprocessor dependency out of here?
-bool Tokenizer::hasIfdef(const Token *start, const Token *end) const
+void Tokenizer::setDirectives(std::list<Directive> directives)
 {
-    assert(mPreprocessor);
+    mDirectives = std::move(directives);
+}
 
-    return std::any_of(mPreprocessor->getDirectives().cbegin(), mPreprocessor->getDirectives().cend(), [&](const Directive& d) {
+bool Tokenizer::hasIfdef(const Token *start, const Token *end) const
+{
+    const auto& directives = mDirectives;
+    return std::any_of(directives.cbegin(), directives.cend(), [&](const Directive& d) {
         return startsWith(d.str, "#if") &&
         d.linenr >= start->linenr() &&
         d.linenr <= end->linenr() &&
@@ -10668,9 +10690,7 @@ bool Tokenizer::hasIfdef(const Token *start, const Token *end) const
 
 bool Tokenizer::isPacked(const Token * bodyStart) const
 {
-    assert(mPreprocessor);
-
-    const auto& directives = mPreprocessor->getDirectives();
+    const auto& directives = mDirectives;
     // TODO: should this return true if the #pragma exists in any line before the start token?
     return std::any_of(directives.cbegin(), directives.cend(), [&](const Directive& d) {
         return d.linenr < bodyStart->linenr() && d.str == "#pragma pack(1)" && d.file == list.getFiles().front();

diff --git a/lib/tokenize.h b/lib/tokenize.h
@@ -36,7 +36,7 @@ class TimerResults;
 class Token;
 class TemplateSimplifier;
 class ErrorLogger;
-class Preprocessor;
+struct Directive;
 enum class Severity;
 
 /// @addtogroup Core
@@ -53,7 +53,7 @@ class CPPCHECKLIB Tokenizer {
     friend class TestTokenizer;
 
 public:
-    explicit Tokenizer(const Settings & settings, ErrorLogger *errorLogger, const Preprocessor *preprocessor = nullptr);
+    explicit Tokenizer(const Settings & settings, ErrorLogger *errorLogger);
     ~Tokenizer();
 
     void setTimerResults(TimerResults *tr) {
@@ -624,6 +624,8 @@ class CPPCHECKLIB Tokenizer {
     /** Disable assignment operator */
     Tokenizer &operator=(const Tokenizer &) = delete;
 
+    void setDirectives(std::list<Directive> directives);
+
 private:
     const Token *processFunc(const Token *tok2, bool inOperator) const;
     Token *processFunc(Token *tok2, bool inOperator);
@@ -666,6 +668,8 @@ class CPPCHECKLIB Tokenizer {
     };
     std::vector<TypedefInfo> mTypedefInfo;
 
+    std::list<Directive> mDirectives;
+
     /** variable count */
     nonneg int mVarId{};
 
@@ -676,8 +680,6 @@ class CPPCHECKLIB Tokenizer {
      * TimerResults
      */
     TimerResults* mTimerResults{};
-
-    const Preprocessor * const mPreprocessor;
 };
 
 /// @}

diff --git a/test/helpers.cpp b/test/helpers.cpp
@@ -45,7 +45,6 @@
 class SuppressionList;
 
 const Settings SimpleTokenizer::s_settings;
-const Preprocessor SimpleTokenizer::s_preprocessor{s_settings, nullptr}; // TODO: provide ErrorLogger
 
 // TODO: better path-only usage
 ScopedFile::ScopedFile(std::string name, const std::string &content, std::string path)
@@ -124,7 +123,6 @@ std::string PreprocessorHelper::getcode(Preprocessor &preprocessor, const std::s
     tokens1.removeComments();
     preprocessor.simplifyPragmaAsm(&tokens1);
     preprocessor.removeComments();
-    preprocessor.setDirectives(tokens1);
 
     preprocessor.reportOutput(outputList, true);
 
@@ -148,25 +146,10 @@ std::string PreprocessorHelper::getcode(Preprocessor &preprocessor, const std::s
 
 void PreprocessorHelper::preprocess(const char code[], std::vector<std::string> &files, Tokenizer& tokenizer)
 {
-    // Raw Tokens..
-    std::istringstream istr(code);
-    const simplecpp::TokenList tokens1(istr, files, files[0]);
-
-    // Preprocess..
-    simplecpp::TokenList tokens2(files);
-    std::map<std::string, simplecpp::TokenList*> filedata;
-    simplecpp::preprocess(tokens2, tokens1, files, filedata, simplecpp::DUI());
-
-    // Tokenizer..
-    tokenizer.list.createTokens(std::move(tokens2));
-}
-
-void PreprocessorHelper::preprocess(Preprocessor &preprocessor, const char code[], std::vector<std::string> &files, Tokenizer& tokenizer)
-{
-    preprocess(preprocessor, code, files, tokenizer, simplecpp::DUI());
+    preprocess(code, files, tokenizer, simplecpp::DUI());
 }
 
-void PreprocessorHelper::preprocess(Preprocessor &preprocessor, const char code[], std::vector<std::string> &files, Tokenizer& tokenizer, const simplecpp::DUI& dui)
+void PreprocessorHelper::preprocess(const char code[], std::vector<std::string> &files, Tokenizer& tokenizer, const simplecpp::DUI& dui)
 {
     std::istringstream istr(code);
     const simplecpp::TokenList tokens1(istr, files, files[0]);
@@ -179,5 +162,7 @@ void PreprocessorHelper::preprocess(Preprocessor &preprocessor, const char code[
     // Tokenizer..
     tokenizer.list.createTokens(std::move(tokens2));
 
-    preprocessor.setDirectives(tokens1);
+    const Preprocessor preprocessor(tokenizer.getSettings());
+    std::list<Directive> directives = preprocessor.createDirectives(tokens1);
+    tokenizer.setDirectives(std::move(directives));
 }
diff --git a/test/helpers.h b/test/helpers.h
@@ -42,18 +42,14 @@ namespace simplecpp {
 class SimpleTokenizer : public Tokenizer {
 public:
     SimpleTokenizer(ErrorLogger& errorlogger, const char code[], bool cpp = true)
-        : Tokenizer{s_settings, &errorlogger, &s_preprocessor}
+        : Tokenizer{s_settings, &errorlogger}
     {
         if (!tokenize(code, cpp))
             throw std::runtime_error("creating tokens failed");
     }
 
     SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger)
-        : Tokenizer{settings, &errorlogger, &s_preprocessor}
-    {}
-
-    SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, const Preprocessor* preprocessor)
-        : Tokenizer{settings, &errorlogger, preprocessor}
+        : Tokenizer{settings, &errorlogger}
     {}
 
     /*
@@ -87,7 +83,6 @@ class SimpleTokenizer : public Tokenizer {
 private:
     // TODO. find a better solution
     static const Settings s_settings;
-    static const Preprocessor s_preprocessor;
 };
 
 class SimpleTokenList
@@ -156,8 +151,7 @@ class PreprocessorHelper
     static std::string getcode(Preprocessor &preprocessor, const std::string &filedata, const std::string &cfg, const std::string &filename, SuppressionList *inlineSuppression = nullptr);
 
     static void preprocess(const char code[], std::vector<std::string> &files, Tokenizer& tokenizer);
-    static void preprocess(Preprocessor &preprocessor, const char code[], std::vector<std::string> &files, Tokenizer& tokenizer);
-    static void preprocess(Preprocessor &preprocessor, const char code[], std::vector<std::string> &files, Tokenizer& tokenizer, const simplecpp::DUI& dui);
+    static void preprocess(const char code[], std::vector<std::string> &files, Tokenizer& tokenizer, const simplecpp::DUI& dui);
 };
 
 namespace cppcheck {