Skip to content

Commit

Permalink
moved directives from Preprocessor to Tokenizer
Browse files Browse the repository at this point in the history
  • Loading branch information
@firewave
firewave committed Apr 10, 2024
1 parent 789be1c commit 8eb5295
Show file tree
Hide file tree
Showing 9 changed files with 147 additions and 121 deletions.
5 changes: 3 additions & 2 deletions lib/cppcheck.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -796,7 +796,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
}

// Get directives
preprocessor.setDirectives(tokens1);
std::list<Directive> directives = preprocessor.createDirectives(tokens1);
preprocessor.simplifyPragmaAsm(&tokens1);

preprocessor.setPlatformInfo(&tokens1);
Expand All @@ -821,7 +821,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
// Run define rules on raw code
if (hasRule("define")) {
std::string code;
for (const Directive &dir : preprocessor.getDirectives()) {
for (const Directive &dir : directives) {
if (startsWith(dir.str,"#define ") || startsWith(dir.str,"#include "))
code += "#line " + std::to_string(dir.linenr) + " \"" + dir.file + "\"\n" + dir.str + '\n';
}
Expand Down Expand Up @@ -890,6 +890,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
Tokenizer tokenizer(mSettings, this, &preprocessor);
if (mSettings.showtime != SHOWTIME_MODES::SHOWTIME_NONE)
tokenizer.setTimerResults(&s_timerResults);
tokenizer.setDirectives(directives); // TODO: how to avoid repeated copies?

try {
// Create tokens, skip rest of iteration if failed
Expand Down
19 changes: 5 additions & 14 deletions lib/preprocessor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,10 +308,10 @@ void Preprocessor::inlineSuppressions(const simplecpp::TokenList &tokens, Suppre
}
}

void Preprocessor::setDirectives(const simplecpp::TokenList &tokens)
std::list<Directive> Preprocessor::createDirectives(const simplecpp::TokenList &tokens) const
{
// directive list..
mDirectives.clear();
std::list<Directive> directives;

std::vector<const simplecpp::TokenList *> list;
list.reserve(1U + mTokenLists.size());
Expand All @@ -337,9 +337,11 @@ void Preprocessor::setDirectives(const simplecpp::TokenList &tokens)
else
directive.str += tok2->str();
}
mDirectives.push_back(std::move(directive));
directives.push_back(std::move(directive));
}
}

return directives;
}

static std::string readcondition(const simplecpp::Token *iftok, const std::set<std::string> &defined, const std::set<std::string> &undefined)
Expand Down Expand Up @@ -912,17 +914,6 @@ void Preprocessor::dump(std::ostream &out) const
{
// Create a xml dump.

out << " <directivelist>" << std::endl;
for (const Directive &dir : mDirectives) {
out << " <directive "
<< "file=\"" << ErrorLogger::toxml(dir.file) << "\" "
<< "linenr=\"" << dir.linenr << "\" "
// str might contain characters such as '"', '<' or '>' which
// could result in invalid XML, so run it through toxml().
<< "str=\"" << ErrorLogger::toxml(dir.str) << "\"/>" << std::endl;
}
out << " </directivelist>" << std::endl;

if (!mMacroUsage.empty()) {
out << " <macro-usage>" << std::endl;
for (const simplecpp::MacroUsage &macroUsage: mMacroUsage) {
Expand Down
12 changes: 1 addition & 11 deletions lib/preprocessor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,7 @@ class CPPCHECKLIB Preprocessor {

void inlineSuppressions(const simplecpp::TokenList &tokens, SuppressionList &suppressions);

void setDirectives(const simplecpp::TokenList &tokens);

/** list of all directives met while preprocessing file */
const std::list<Directive> &getDirectives() const {
return mDirectives;
}
std::list<Directive> createDirectives(const simplecpp::TokenList &tokens) const;

std::set<std::string> getConfigs(const simplecpp::TokenList &tokens) const;

Expand Down Expand Up @@ -143,15 +138,10 @@ class CPPCHECKLIB Preprocessor {

static bool hasErrors(const simplecpp::OutputList &outputList);

void setDirectives(const std::list<Directive> &directives) {
mDirectives = directives;
}

const Settings& mSettings;
ErrorLogger *mErrorLogger;

/** list of all directives met while preprocessing file */
std::list<Directive> mDirectives;

std::map<std::string, simplecpp::TokenList *> mTokenLists;

Expand Down
31 changes: 28 additions & 3 deletions lib/tokenize.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5913,6 +5913,26 @@ void Tokenizer::dump(std::ostream &out) const

std::set<const Library::Container*> containers;

outs += " <directivelist>";
outs += '\n';
for (const Directive &dir : mDirectives) {
outs += " <directive ";
outs += "file=\"";
outs += ErrorLogger::toxml(dir.file);
outs += "\" ";
outs += "linenr=\"";
outs += std::to_string(dir.linenr);
outs += "\" ";
// str might contain characters such as '"', '<' or '>' which
// could result in invalid XML, so run it through toxml().
outs += "str=\"";
outs += ErrorLogger::toxml(dir.str);
outs +="\"/>";
outs += '\n';
}
outs += " </directivelist>";
outs += '\n';

// tokens..
outs += " <tokenlist>";
outs += '\n';
Expand Down Expand Up @@ -10649,12 +10669,17 @@ void Tokenizer::simplifyNamespaceAliases()
}
}

// TODO: how to move the Preprocessor dependency out of here?
void Tokenizer::setDirectives(std::list<Directive> directives)
{
mDirectives = std::move(directives);
}

bool Tokenizer::hasIfdef(const Token *start, const Token *end) const
{
assert(mPreprocessor);

return std::any_of(mPreprocessor->getDirectives().cbegin(), mPreprocessor->getDirectives().cend(), [&](const Directive& d) {
const auto& directives = mDirectives;
return std::any_of(directives.cbegin(), directives.cend(), [&](const Directive& d) {
return startsWith(d.str, "#if") &&
d.linenr >= start->linenr() &&
d.linenr <= end->linenr() &&
Expand All @@ -10667,7 +10692,7 @@ bool Tokenizer::isPacked(const Token * bodyStart) const
{
assert(mPreprocessor);

const auto& directives = mPreprocessor->getDirectives();
const auto& directives = mDirectives;
// TODO: should this return true if the #pragma exists in any line before the start token?
return std::any_of(directives.cbegin(), directives.cend(), [&](const Directive& d) {
return d.linenr < bodyStart->linenr() && d.str == "#pragma pack(1)" && d.file == list.getFiles().front();
Expand Down
5 changes: 5 additions & 0 deletions lib/tokenize.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ class Token;
class TemplateSimplifier;
class ErrorLogger;
class Preprocessor;
struct Directive;
enum class Severity;

/// @addtogroup Core
Expand Down Expand Up @@ -624,6 +625,8 @@ class CPPCHECKLIB Tokenizer {
/** Disable assignment operator */
Tokenizer &operator=(const Tokenizer &) = delete;

void setDirectives(std::list<Directive> directives);

private:
const Token *processFunc(const Token *tok2, bool inOperator) const;
Token *processFunc(Token *tok2, bool inOperator);
Expand Down Expand Up @@ -666,6 +669,8 @@ class CPPCHECKLIB Tokenizer {
};
std::vector<TypedefInfo> mTypedefInfo;

std::list<Directive> mDirectives;

/** variable count */
nonneg int mVarId{};

Expand Down
4 changes: 2 additions & 2 deletions test/helpers.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,6 @@ std::string PreprocessorHelper::getcode(Preprocessor &preprocessor, const std::s
tokens1.removeComments();
preprocessor.simplifyPragmaAsm(&tokens1);
preprocessor.removeComments();
preprocessor.setDirectives(tokens1);

preprocessor.reportOutput(outputList, true);

Expand Down Expand Up @@ -179,5 +178,6 @@ void PreprocessorHelper::preprocess(Preprocessor &preprocessor, const char code[
// Tokenizer..
tokenizer.list.createTokens(std::move(tokens2));

preprocessor.setDirectives(tokens1);
std::list<Directive> directives = preprocessor.createDirectives(tokens1);
tokenizer.setDirectives(std::move(directives));
}
87 changes: 0 additions & 87 deletions test/testpreprocessor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,10 +245,6 @@ class TestPreprocessor : public TestFixture {

TEST_CASE(wrongPathOnErrorDirective);

TEST_CASE(testDirectiveIncludeTypes);
TEST_CASE(testDirectiveIncludeLocations);
TEST_CASE(testDirectiveIncludeComments);

TEST_CASE(testMissingInclude);
TEST_CASE(testMissingInclude2);
TEST_CASE(testMissingInclude3);
Expand All @@ -275,7 +271,6 @@ class TestPreprocessor : public TestFixture {
tokens.removeComments();
preprocessor0.simplifyPragmaAsm(&tokens);
preprocessor0.removeComments();
preprocessor0.setDirectives(tokens);

preprocessor0.reportOutput(outputList, true);

Expand Down Expand Up @@ -2313,88 +2308,6 @@ class TestPreprocessor : public TestFixture {
ASSERT_EQUALS("[test.c:1]: (error) #error hello world!\n", errout_str());
}

void testDirectiveIncludeTypes() {
const char filedata[] = "#define macro some definition\n"
"#undef macro\n"
"#ifdef macro\n"
"#elif some (complex) condition\n"
"#else\n"
"#endif\n"
"#if some other condition\n"
"#pragma some proprietary content\n"
"#\n" /* may appear in old C code */
"#ident some text\n" /* may appear in old C code */
"#unknownmacro some unpredictable text\n"
"#warning some warning message\n"
"#error some error message\n";
const char dumpdata[] = " <directivelist>\n"

" <directive file=\"test.c\" linenr=\"1\" str=\"#define macro some definition\"/>\n"
" <directive file=\"test.c\" linenr=\"2\" str=\"#undef macro\"/>\n"
" <directive file=\"test.c\" linenr=\"3\" str=\"#ifdef macro\"/>\n"
" <directive file=\"test.c\" linenr=\"4\" str=\"#elif some (complex) condition\"/>\n"
" <directive file=\"test.c\" linenr=\"5\" str=\"#else\"/>\n"
" <directive file=\"test.c\" linenr=\"6\" str=\"#endif\"/>\n"
" <directive file=\"test.c\" linenr=\"7\" str=\"#if some other condition\"/>\n"
" <directive file=\"test.c\" linenr=\"8\" str=\"#pragma some proprietary content\"/>\n"
" <directive file=\"test.c\" linenr=\"9\" str=\"#\"/>\n"
" <directive file=\"test.c\" linenr=\"10\" str=\"#ident some text\"/>\n"
" <directive file=\"test.c\" linenr=\"11\" str=\"#unknownmacro some unpredictable text\"/>\n"
" <directive file=\"test.c\" linenr=\"12\" str=\"#warning some warning message\"/>\n"
" <directive file=\"test.c\" linenr=\"13\" str=\"#error some error message\"/>\n"
" </directivelist>\n";

std::ostringstream ostr;
Preprocessor preprocessor(settings0, this);
PreprocessorHelper::getcode(preprocessor, filedata, "", "test.c");
preprocessor.dump(ostr);
ASSERT_EQUALS(dumpdata, ostr.str());
}

void testDirectiveIncludeLocations() {
const char filedata[] = "#define macro1 val\n"
"#file \"inc1.h\"\n"
"#define macro2 val\n"
"#file \"inc2.h\"\n"
"#define macro3 val\n"
"#endfile\n"
"#define macro4 val\n"
"#endfile\n"
"#define macro5 val\n";
const char dumpdata[] = " <directivelist>\n"
" <directive file=\"test.c\" linenr=\"1\" str=\"#define macro1 val\"/>\n"
" <directive file=\"test.c\" linenr=\"2\" str=\"#include &quot;inc1.h&quot;\"/>\n"
" <directive file=\"inc1.h\" linenr=\"1\" str=\"#define macro2 val\"/>\n"
" <directive file=\"inc1.h\" linenr=\"2\" str=\"#include &quot;inc2.h&quot;\"/>\n"
" <directive file=\"inc2.h\" linenr=\"1\" str=\"#define macro3 val\"/>\n"
" <directive file=\"inc1.h\" linenr=\"3\" str=\"#define macro4 val\"/>\n"
" <directive file=\"test.c\" linenr=\"3\" str=\"#define macro5 val\"/>\n"
" </directivelist>\n";

std::ostringstream ostr;
Preprocessor preprocessor(settings0, this);
PreprocessorHelper::getcode(preprocessor, filedata, "", "test.c");
preprocessor.dump(ostr);
ASSERT_EQUALS(dumpdata, ostr.str());
}

void testDirectiveIncludeComments() {
const char filedata[] = "#ifdef macro2 /* this will be removed */\n"
"#else /* this will be removed too */\n"
"#endif /* this will also be removed */\n";
const char dumpdata[] = " <directivelist>\n"
" <directive file=\"test.c\" linenr=\"1\" str=\"#ifdef macro2\"/>\n"
" <directive file=\"test.c\" linenr=\"2\" str=\"#else\"/>\n"
" <directive file=\"test.c\" linenr=\"3\" str=\"#endif\"/>\n"
" </directivelist>\n";

std::ostringstream ostr;
Preprocessor preprocessor(settings0, this);
PreprocessorHelper::getcode(preprocessor, filedata, "", "test.c");
preprocessor.dump(ostr);
ASSERT_EQUALS(dumpdata, ostr.str());
}

// test for existing local include
void testMissingInclude() {
/*const*/ Settings settings;
Expand Down
Loading

0 comments on commit 8eb5295

Please sign in to comment.