From 400878422982f7f965faafc3ac829f9b6d05ead9 Mon Sep 17 00:00:00 2001
From: Oleksandr Labetskyi <o.labetskyi@beetroot.se>
Date: Wed, 19 Jun 2024 12:17:58 +0300
Subject: [PATCH] Small refactor

---
 addons/cppcheckdata.py |   1 +
 lib/preprocessor.cpp   |  22 ++++--
 lib/preprocessor.h     |   3 +-
 lib/tokenize.cpp       |   3 -
 test/testtokenize.cpp  | 170 ++++++++++++++++++++---------------------
 5 files changed, 102 insertions(+), 97 deletions(-)

diff --git a/addons/cppcheckdata.py b/addons/cppcheckdata.py
index 38f9a0f3169..2de7a3512e3 100755
--- a/addons/cppcheckdata.py
+++ b/addons/cppcheckdata.py
@@ -1281,6 +1281,7 @@ def iterconfigurations(self):
         # Iterating <typedef-info>
         iter_typedef_info = False
 
+        # Iterating <directive>
         iter_directive = False
 
         # Use iterable objects to traverse XML tree for dump files incrementally.
diff --git a/lib/preprocessor.cpp b/lib/preprocessor.cpp
index f3227d9f62a..473d5c658ec 100644
--- a/lib/preprocessor.cpp
+++ b/lib/preprocessor.cpp
@@ -43,16 +43,22 @@ static bool sameline(const simplecpp::Token *tok1, const simplecpp::Token *tok2)
     return tok1 && tok2 && tok1->location.sameline(tok2->location);
 }
 
-Directive::Directive(std::string _file, const int _linenr, const std::string &_str) :
+Directive::Directive(const simplecpp::Location & _loc, const std::string & _str) :
+    file(_loc.file()),
+    linenr(_loc.line),
+    str(_str)
+{}
+
+Directive::Directive(std::string _file, const int _linenr, const std::string & _str) :
     file(std::move(_file)),
     linenr(_linenr),
-    str(trim(_str))
+    str(_str)
 {}
 
-Directive::DirectiveToken::DirectiveToken(std::string _str, int _line, int _column) :
-    line(_line),
-    column(_column),
-    tokStr(std::move(_str))
+Directive::DirectiveToken::DirectiveToken(const simplecpp::Token & _tok) :
+    line(_tok.location.line),
+    column(_tok.location.col),
+    tokStr(_tok.str())
 {}
 
 char Preprocessor::macroChar = char(1);
@@ -334,7 +340,7 @@ std::list<Directive> Preprocessor::createDirectives(const simplecpp::TokenList &
                 continue;
             if (tok->next && tok->next->str() == "endfile")
                 continue;
-            Directive directive(tok->location.file(), tok->location.line, emptyString);
+            Directive directive(tok->location, emptyString);
             for (const simplecpp::Token *tok2 = tok; tok2 && tok2->location.line == directive.linenr; tok2 = tok2->next) {
                 if (tok2->comment)
                     continue;
@@ -345,7 +351,7 @@ std::list<Directive> Preprocessor::createDirectives(const simplecpp::TokenList &
                 else
                     directive.str += tok2->str();
 
-                directive.strTokens.emplace_back(tok2->str(), tok2->location.line, tok2->location.col);
+                directive.strTokens.emplace_back(*tok2);
             }
             directives.push_back(std::move(directive));
         }
diff --git a/lib/preprocessor.h b/lib/preprocessor.h
index bd05d0fcf65..ad6d98f9f5c 100644
--- a/lib/preprocessor.h
+++ b/lib/preprocessor.h
@@ -57,7 +57,7 @@ struct CPPCHECKLIB Directive {
     std::string str;
 
     struct DirectiveToken {
-        explicit DirectiveToken(std::string _str, int _line, int _column);
+        explicit DirectiveToken(const simplecpp::Token & _tok);
         int line;
         int column;
         std::string tokStr;
@@ -66,6 +66,7 @@ struct CPPCHECKLIB Directive {
     std::vector<DirectiveToken> strTokens;
 
     /** record a directive (possibly filtering src) */
+    Directive(const simplecpp::Location & _loc, const std::string & _str);
     Directive(std::string _file, const int _linenr, const std::string &_str);
 };
 
diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp
index 0efe8a49db4..aab4a4913b1 100644
--- a/lib/tokenize.cpp
+++ b/lib/tokenize.cpp
@@ -5961,9 +5961,6 @@ void Tokenizer::dump(std::ostream &out) const
         outs += '\n';
         for (const auto & strToken : dir.strTokens) {
             outs += "      <token ";
-            outs += "line=\"";
-            outs += std::to_string(strToken.line);
-            outs += "\" ";
             outs += "column=\"";
             outs += std::to_string(strToken.column);
             outs += "\" ";
diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp
index 2321958b572..4345090f565 100644
--- a/test/testtokenize.cpp
+++ b/test/testtokenize.cpp
@@ -8076,78 +8076,78 @@ class TestTokenizer : public TestFixture {
                                 "#error some error message\n";
         const char dumpdata[] = "  <directivelist>\n"
                                 "    <directive file=\"test.c\" linenr=\"1\" str=\"#define macro some definition\">\n"
-                                "      <token line=\"1\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"1\" column=\"2\" str=\"define\"/>\n"
-                                "      <token line=\"1\" column=\"9\" str=\"macro\"/>\n"
-                                "      <token line=\"1\" column=\"15\" str=\"some\"/>\n"
-                                "      <token line=\"1\" column=\"20\" str=\"definition\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"define\"/>\n"
+                                "      <token column=\"9\" str=\"macro\"/>\n"
+                                "      <token column=\"15\" str=\"some\"/>\n"
+                                "      <token column=\"20\" str=\"definition\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"2\" str=\"#undef macro\">\n"
-                                "      <token line=\"2\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"2\" column=\"2\" str=\"undef\"/>\n"
-                                "      <token line=\"2\" column=\"8\" str=\"macro\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"undef\"/>\n"
+                                "      <token column=\"8\" str=\"macro\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"3\" str=\"#ifdef macro\">\n"
-                                "      <token line=\"3\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"3\" column=\"2\" str=\"ifdef\"/>\n"
-                                "      <token line=\"3\" column=\"8\" str=\"macro\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"ifdef\"/>\n"
+                                "      <token column=\"8\" str=\"macro\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"4\" str=\"#elif some (complex) condition\">\n"
-                                "      <token line=\"4\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"4\" column=\"2\" str=\"elif\"/>\n"
-                                "      <token line=\"4\" column=\"7\" str=\"some\"/>\n"
-                                "      <token line=\"4\" column=\"12\" str=\"(\"/>\n"
-                                "      <token line=\"4\" column=\"13\" str=\"complex\"/>\n"
-                                "      <token line=\"4\" column=\"20\" str=\")\"/>\n"
-                                "      <token line=\"4\" column=\"22\" str=\"condition\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"elif\"/>\n"
+                                "      <token column=\"7\" str=\"some\"/>\n"
+                                "      <token column=\"12\" str=\"(\"/>\n"
+                                "      <token column=\"13\" str=\"complex\"/>\n"
+                                "      <token column=\"20\" str=\")\"/>\n"
+                                "      <token column=\"22\" str=\"condition\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"5\" str=\"#else\">\n"
-                                "      <token line=\"5\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"5\" column=\"2\" str=\"else\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"else\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"6\" str=\"#endif\">\n"
-                                "      <token line=\"6\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"6\" column=\"2\" str=\"endif\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"endif\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"7\" str=\"#if some other condition\">\n"
-                                "      <token line=\"7\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"7\" column=\"2\" str=\"if\"/>\n"
-                                "      <token line=\"7\" column=\"5\" str=\"some\"/>\n"
-                                "      <token line=\"7\" column=\"10\" str=\"other\"/>\n"
-                                "      <token line=\"7\" column=\"16\" str=\"condition\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"if\"/>\n"
+                                "      <token column=\"5\" str=\"some\"/>\n"
+                                "      <token column=\"10\" str=\"other\"/>\n"
+                                "      <token column=\"16\" str=\"condition\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"8\" str=\"#pragma some proprietary content\">\n"
-                                "      <token line=\"8\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"8\" column=\"2\" str=\"pragma\"/>\n"
-                                "      <token line=\"8\" column=\"9\" str=\"some\"/>\n"
-                                "      <token line=\"8\" column=\"14\" str=\"proprietary\"/>\n"
-                                "      <token line=\"8\" column=\"26\" str=\"content\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"pragma\"/>\n"
+                                "      <token column=\"9\" str=\"some\"/>\n"
+                                "      <token column=\"14\" str=\"proprietary\"/>\n"
+                                "      <token column=\"26\" str=\"content\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"9\" str=\"#\">\n"
-                                "      <token line=\"9\" column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"10\" str=\"#ident some text\">\n"
-                                "      <token line=\"10\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"10\" column=\"2\" str=\"ident\"/>\n"
-                                "      <token line=\"10\" column=\"8\" str=\"some\"/>\n"
-                                "      <token line=\"10\" column=\"13\" str=\"text\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"ident\"/>\n"
+                                "      <token column=\"8\" str=\"some\"/>\n"
+                                "      <token column=\"13\" str=\"text\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"11\" str=\"#unknownmacro some unpredictable text\">\n"
-                                "      <token line=\"11\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"11\" column=\"2\" str=\"unknownmacro\"/>\n"
-                                "      <token line=\"11\" column=\"15\" str=\"some\"/>\n"
-                                "      <token line=\"11\" column=\"20\" str=\"unpredictable\"/>\n"
-                                "      <token line=\"11\" column=\"34\" str=\"text\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"unknownmacro\"/>\n"
+                                "      <token column=\"15\" str=\"some\"/>\n"
+                                "      <token column=\"20\" str=\"unpredictable\"/>\n"
+                                "      <token column=\"34\" str=\"text\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"12\" str=\"#warning some warning message\">\n"
-                                "      <token line=\"12\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"12\" column=\"2\" str=\"warning\"/>\n"
-                                "      <token line=\"12\" column=\"10\" str=\"some warning message\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"warning\"/>\n"
+                                "      <token column=\"10\" str=\"some warning message\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"13\" str=\"#error some error message\">\n"
-                                "      <token line=\"13\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"13\" column=\"2\" str=\"error\"/>\n"
-                                "      <token line=\"13\" column=\"8\" str=\"some error message\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"error\"/>\n"
+                                "      <token column=\"8\" str=\"some error message\"/>\n"
                                 "    </directive>\n"
                                 "  </directivelist>\n"
                                 "  <tokenlist>\n"
@@ -8170,44 +8170,44 @@ class TestTokenizer : public TestFixture {
                                 "#define macro5 val\n";
         const char dumpdata[] = "  <directivelist>\n"
                                 "    <directive file=\"test.c\" linenr=\"1\" str=\"#define macro1 val\">\n"
-                                "      <token line=\"1\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"1\" column=\"2\" str=\"define\"/>\n"
-                                "      <token line=\"1\" column=\"9\" str=\"macro1\"/>\n"
-                                "      <token line=\"1\" column=\"16\" str=\"val\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"define\"/>\n"
+                                "      <token column=\"9\" str=\"macro1\"/>\n"
+                                "      <token column=\"16\" str=\"val\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"2\" str=\"#include &quot;inc1.h&quot;\">\n"
-                                "      <token line=\"2\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"2\" column=\"2\" str=\"file\"/>\n"
-                                "      <token line=\"2\" column=\"7\" str=\"&quot;inc1.h&quot;\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"file\"/>\n"
+                                "      <token column=\"7\" str=\"&quot;inc1.h&quot;\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"inc1.h\" linenr=\"1\" str=\"#define macro2 val\">\n"
-                                "      <token line=\"1\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"1\" column=\"2\" str=\"define\"/>\n"
-                                "      <token line=\"1\" column=\"9\" str=\"macro2\"/>\n"
-                                "      <token line=\"1\" column=\"16\" str=\"val\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"define\"/>\n"
+                                "      <token column=\"9\" str=\"macro2\"/>\n"
+                                "      <token column=\"16\" str=\"val\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"inc1.h\" linenr=\"2\" str=\"#include &quot;inc2.h&quot;\">\n"
-                                "      <token line=\"2\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"2\" column=\"2\" str=\"file\"/>\n"
-                                "      <token line=\"2\" column=\"7\" str=\"&quot;inc2.h&quot;\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"file\"/>\n"
+                                "      <token column=\"7\" str=\"&quot;inc2.h&quot;\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"inc2.h\" linenr=\"1\" str=\"#define macro3 val\">\n"
-                                "      <token line=\"1\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"1\" column=\"2\" str=\"define\"/>\n"
-                                "      <token line=\"1\" column=\"9\" str=\"macro3\"/>\n"
-                                "      <token line=\"1\" column=\"16\" str=\"val\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"define\"/>\n"
+                                "      <token column=\"9\" str=\"macro3\"/>\n"
+                                "      <token column=\"16\" str=\"val\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"inc1.h\" linenr=\"3\" str=\"#define macro4 val\">\n"
-                                "      <token line=\"3\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"3\" column=\"2\" str=\"define\"/>\n"
-                                "      <token line=\"3\" column=\"9\" str=\"macro4\"/>\n"
-                                "      <token line=\"3\" column=\"16\" str=\"val\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"define\"/>\n"
+                                "      <token column=\"9\" str=\"macro4\"/>\n"
+                                "      <token column=\"16\" str=\"val\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"3\" str=\"#define macro5 val\">\n"
-                                "      <token line=\"3\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"3\" column=\"2\" str=\"define\"/>\n"
-                                "      <token line=\"3\" column=\"9\" str=\"macro5\"/>\n"
-                                "      <token line=\"3\" column=\"16\" str=\"val\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"define\"/>\n"
+                                "      <token column=\"9\" str=\"macro5\"/>\n"
+                                "      <token column=\"16\" str=\"val\"/>\n"
                                 "    </directive>\n"
                                 "  </directivelist>\n"
                                 "  <tokenlist>\n"
@@ -8223,17 +8223,17 @@ class TestTokenizer : public TestFixture {
                                 "#endif /* this will also be removed */\n";
         const char dumpdata[] = "  <directivelist>\n"
                                 "    <directive file=\"test.c\" linenr=\"1\" str=\"#ifdef macro2\">\n"
-                                "      <token line=\"1\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"1\" column=\"2\" str=\"ifdef\"/>\n"
-                                "      <token line=\"1\" column=\"8\" str=\"macro2\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"ifdef\"/>\n"
+                                "      <token column=\"8\" str=\"macro2\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"2\" str=\"#else\">\n"
-                                "      <token line=\"2\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"2\" column=\"2\" str=\"else\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"else\"/>\n"
                                 "    </directive>\n"
                                 "    <directive file=\"test.c\" linenr=\"3\" str=\"#endif\">\n"
-                                "      <token line=\"3\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"3\" column=\"2\" str=\"endif\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"endif\"/>\n"
                                 "    </directive>\n"
                                 "  </directivelist>\n"
                                 "  <tokenlist>\n"
@@ -8248,10 +8248,10 @@ class TestTokenizer : public TestFixture {
         const char filedata[] = "#define macro 1\n";
         const char dumpdata[] = "  <directivelist>\n"
                                 "    <directive file=\"test.c\" linenr=\"1\" str=\"#define macro 1\">\n"
-                                "      <token line=\"1\" column=\"1\" str=\"#\"/>\n"
-                                "      <token line=\"1\" column=\"2\" str=\"define\"/>\n"
-                                "      <token line=\"1\" column=\"9\" str=\"macro\"/>\n"
-                                "      <token line=\"1\" column=\"15\" str=\"1\"/>\n"
+                                "      <token column=\"1\" str=\"#\"/>\n"
+                                "      <token column=\"2\" str=\"define\"/>\n"
+                                "      <token column=\"9\" str=\"macro\"/>\n"
+                                "      <token column=\"15\" str=\"1\"/>\n"
                                 "    </directive>\n"
                                 "  </directivelist>\n"
                                 "  <tokenlist>\n"