Skip to content

Commit

Permalink
Tokenizer: removed Preprocessor dependency
Browse files Browse the repository at this point in the history
  • Loading branch information
@firewave
firewave committed Apr 10, 2024
1 parent 8eb5295 commit e09bcc6
Show file tree
Hide file tree
Showing 12 changed files with 36 additions and 100 deletions.
5 changes: 2 additions & 3 deletions lib/cppcheck.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -506,8 +506,7 @@ unsigned int CppCheck::checkClang(const std::string &path)
}

try {
Preprocessor preprocessor(mSettings, this);
Tokenizer tokenizer(mSettings, this, &preprocessor);
Tokenizer tokenizer(mSettings, this);
tokenizer.list.appendFileIfNew(path);
std::istringstream ast(output2);
clangimport::parseClangAstDump(tokenizer, ast);
Expand Down Expand Up @@ -887,7 +886,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
continue;
}

Tokenizer tokenizer(mSettings, this, &preprocessor);
Tokenizer tokenizer(mSettings, this);
if (mSettings.showtime != SHOWTIME_MODES::SHOWTIME_NONE)
tokenizer.setTimerResults(&s_timerResults);
tokenizer.setDirectives(directives); // TODO: how to avoid repeated copies?
Expand Down
2 changes: 1 addition & 1 deletion lib/preprocessor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ struct CPPCHECKLIB Directive {
* The preprocessor has special functionality for extracting the various ifdef
* configurations that exist in a source file.
*/
class CPPCHECKLIB Preprocessor {
class CPPCHECKLIB WARN_UNUSED Preprocessor {
// TODO: get rid of this
friend class PreprocessorHelper;
friend class TestPreprocessor;
Expand Down
9 changes: 2 additions & 7 deletions lib/tokenize.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,12 +152,11 @@ static bool isClassStructUnionEnumStart(const Token * tok)

//---------------------------------------------------------------------------

Tokenizer::Tokenizer(const Settings &settings, ErrorLogger *errorLogger, const Preprocessor *preprocessor) :
Tokenizer::Tokenizer(const Settings &settings, ErrorLogger *errorLogger) :
list(&settings),
mSettings(settings),
mErrorLogger(errorLogger),
mTemplateSimplifier(new TemplateSimplifier(*this)),
mPreprocessor(preprocessor)
mTemplateSimplifier(new TemplateSimplifier(*this))
{}

Tokenizer::~Tokenizer()
Expand Down Expand Up @@ -10676,8 +10675,6 @@ void Tokenizer::setDirectives(std::list<Directive> directives)

bool Tokenizer::hasIfdef(const Token *start, const Token *end) const
{
assert(mPreprocessor);

const auto& directives = mDirectives;
return std::any_of(directives.cbegin(), directives.cend(), [&](const Directive& d) {
return startsWith(d.str, "#if") &&
Expand All @@ -10690,8 +10687,6 @@ bool Tokenizer::hasIfdef(const Token *start, const Token *end) const

bool Tokenizer::isPacked(const Token * bodyStart) const
{
assert(mPreprocessor);

const auto& directives = mDirectives;
// TODO: should this return true if the #pragma exists in any line before the start token?
return std::any_of(directives.cbegin(), directives.cend(), [&](const Directive& d) {
Expand Down
5 changes: 1 addition & 4 deletions lib/tokenize.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ class TimerResults;
class Token;
class TemplateSimplifier;
class ErrorLogger;
class Preprocessor;
struct Directive;
enum class Severity;

Expand All @@ -54,7 +53,7 @@ class CPPCHECKLIB Tokenizer {
friend class TestTokenizer;

public:
explicit Tokenizer(const Settings & settings, ErrorLogger *errorLogger, const Preprocessor *preprocessor = nullptr);
explicit Tokenizer(const Settings & settings, ErrorLogger *errorLogger);
~Tokenizer();

void setTimerResults(TimerResults *tr) {
Expand Down Expand Up @@ -681,8 +680,6 @@ class CPPCHECKLIB Tokenizer {
* TimerResults
*/
TimerResults* mTimerResults{};

const Preprocessor * const mPreprocessor;
};

/// @}
Expand Down
1 change: 0 additions & 1 deletion test/helpers.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
class SuppressionList;

const Settings SimpleTokenizer::s_settings;
const Preprocessor SimpleTokenizer::s_preprocessor{s_settings, nullptr}; // TODO: provide ErrorLogger

// TODO: better path-only usage
ScopedFile::ScopedFile(std::string name, const std::string &content, std::string path)
Expand Down
9 changes: 2 additions & 7 deletions test/helpers.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,18 +42,14 @@ namespace simplecpp {
class SimpleTokenizer : public Tokenizer {
public:
SimpleTokenizer(ErrorLogger& errorlogger, const char code[], bool cpp = true)
: Tokenizer{s_settings, &errorlogger, &s_preprocessor}
: Tokenizer{s_settings, &errorlogger}
{
if (!tokenize(code, cpp))
throw std::runtime_error("creating tokens failed");
}

SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger)
: Tokenizer{settings, &errorlogger, &s_preprocessor}
{}

SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, const Preprocessor* preprocessor)
: Tokenizer{settings, &errorlogger, preprocessor}
: Tokenizer{settings, &errorlogger}
{}

/*
Expand Down Expand Up @@ -87,7 +83,6 @@ class SimpleTokenizer : public Tokenizer {
private:
// TODO. find a better solution
static const Settings s_settings;
static const Preprocessor s_preprocessor;
};

class SimpleTokenList
Expand Down
74 changes: 18 additions & 56 deletions test/testclass.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
#include "errortypes.h"
#include "fixture.h"
#include "helpers.h"
#include "preprocessor.h"
#include "settings.h"
#include "tokenize.h"

Expand Down Expand Up @@ -248,10 +247,8 @@ class TestClass : public TestFixture {
void checkCopyCtorAndEqOperator_(const char code[], const char* file, int line) {
const Settings settings = settingsBuilder().severity(Severity::warning).build();

Preprocessor preprocessor(settings);

// Tokenize..
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
Expand Down Expand Up @@ -350,10 +347,8 @@ class TestClass : public TestFixture {

#define checkExplicitConstructors(code) checkExplicitConstructors_(code, __FILE__, __LINE__)
void checkExplicitConstructors_(const char code[], const char* file, int line) {
Preprocessor preprocessor(settings0);

// Tokenize..
SimpleTokenizer tokenizer(settings0, *this, &preprocessor);
SimpleTokenizer tokenizer(settings0, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
Expand Down Expand Up @@ -500,10 +495,8 @@ class TestClass : public TestFixture {

#define checkDuplInheritedMembers(code) checkDuplInheritedMembers_(code, __FILE__, __LINE__)
void checkDuplInheritedMembers_(const char code[], const char* file, int line) {
Preprocessor preprocessor(settings1);

// Tokenize..
SimpleTokenizer tokenizer(settings1, *this, &preprocessor);
SimpleTokenizer tokenizer(settings1, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
Expand Down Expand Up @@ -718,10 +711,8 @@ class TestClass : public TestFixture {

#define checkCopyConstructor(code) checkCopyConstructor_(code, __FILE__, __LINE__)
void checkCopyConstructor_(const char code[], const char* file, int line) {
Preprocessor preprocessor(settings3);

// Tokenize..
SimpleTokenizer tokenizer(settings3, *this, &preprocessor);
SimpleTokenizer tokenizer(settings3, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
Expand Down Expand Up @@ -1163,10 +1154,8 @@ class TestClass : public TestFixture {
// Check that operator Equal returns reference to this
#define checkOpertorEqRetRefThis(code) checkOpertorEqRetRefThis_(code, __FILE__, __LINE__)
void checkOpertorEqRetRefThis_(const char code[], const char* file, int line) {
Preprocessor preprocessor(settings0);

// Tokenize..
SimpleTokenizer tokenizer(settings0, *this, &preprocessor);
SimpleTokenizer tokenizer(settings0, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
Expand Down Expand Up @@ -1635,10 +1624,8 @@ class TestClass : public TestFixture {
// Check that operator Equal checks for assignment to self
#define checkOpertorEqToSelf(code) checkOpertorEqToSelf_(code, __FILE__, __LINE__)
void checkOpertorEqToSelf_(const char code[], const char* file, int line) {
Preprocessor preprocessor(settings1);

// Tokenize..
SimpleTokenizer tokenizer(settings1, *this, &preprocessor);
SimpleTokenizer tokenizer(settings1, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
Expand Down Expand Up @@ -2594,10 +2581,8 @@ class TestClass : public TestFixture {
void checkVirtualDestructor_(const char* file, int line, const char code[], bool inconclusive = false) {
const Settings s = settingsBuilder(settings0).certainty(Certainty::inconclusive, inconclusive).severity(Severity::warning).build();

Preprocessor preprocessor(s);

// Tokenize..
SimpleTokenizer tokenizer(s, *this, &preprocessor);
SimpleTokenizer tokenizer(s, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
Expand Down Expand Up @@ -2932,10 +2917,8 @@ class TestClass : public TestFixture {
}

void checkNoMemset_(const char* file, int line, const char code[], const Settings &settings) {
Preprocessor preprocessor(settings);

// Tokenize..
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
Expand Down Expand Up @@ -3579,10 +3562,8 @@ class TestClass : public TestFixture {

#define checkThisSubtraction(code) checkThisSubtraction_(code, __FILE__, __LINE__)
void checkThisSubtraction_(const char code[], const char* file, int line) {
Preprocessor preprocessor(settings1);

// Tokenize..
SimpleTokenizer tokenizer(settings1, *this, &preprocessor);
SimpleTokenizer tokenizer(settings1, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
Expand Down Expand Up @@ -3612,10 +3593,8 @@ class TestClass : public TestFixture {
void checkConst_(const char* file, int line, const char code[], const Settings *s = nullptr, bool inconclusive = true) {
const Settings settings = settingsBuilder(s ? *s : settings0).certainty(Certainty::inconclusive, inconclusive).build();

Preprocessor preprocessor(settings);

// Tokenize..
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

CheckClass checkClass(&tokenizer, &settings, this);
Expand Down Expand Up @@ -7532,11 +7511,8 @@ class TestClass : public TestFixture {

#define checkInitializerListOrder(code) checkInitializerListOrder_(code, __FILE__, __LINE__)
void checkInitializerListOrder_(const char code[], const char* file, int line) {
// Check..
Preprocessor preprocessor(settings2);

// Tokenize..
SimpleTokenizer tokenizer(settings2, *this, &preprocessor);
SimpleTokenizer tokenizer(settings2, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

CheckClass checkClass(&tokenizer, &settings2, this);
Expand Down Expand Up @@ -7663,10 +7639,8 @@ class TestClass : public TestFixture {
// Check..
const Settings settings = settingsBuilder().severity(Severity::performance).build();

Preprocessor preprocessor(settings);

// Tokenize..
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

CheckClass checkClass(&tokenizer, &settings, this);
Expand Down Expand Up @@ -7873,10 +7847,8 @@ class TestClass : public TestFixture {

#define checkSelfInitialization(code) checkSelfInitialization_(code, __FILE__, __LINE__)
void checkSelfInitialization_(const char code[], const char* file, int line) {
Preprocessor preprocessor(settings0);

// Tokenize..
SimpleTokenizer tokenizer(settings0, *this, &preprocessor);
SimpleTokenizer tokenizer(settings0, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

CheckClass checkClass(&tokenizer, &settings0, this);
Expand Down Expand Up @@ -7985,10 +7957,8 @@ class TestClass : public TestFixture {
// Check..
const Settings settings = settingsBuilder().severity(Severity::warning).severity(Severity::style).certainty(Certainty::inconclusive, inconclusive).build();

Preprocessor preprocessor(settings);

// Tokenize..
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

CheckClass checkClass(&tokenizer, &settings, this);
Expand Down Expand Up @@ -8330,10 +8300,8 @@ class TestClass : public TestFixture {
void checkOverride_(const char code[], const char* file, int line) {
const Settings settings = settingsBuilder().severity(Severity::style).build();

Preprocessor preprocessor(settings);

// Tokenize..
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
Expand Down Expand Up @@ -8704,10 +8672,8 @@ class TestClass : public TestFixture {
/*const*/ Settings settings = settingsBuilder().severity(Severity::warning).build();
settings.safeChecks.classes = true;

Preprocessor preprocessor(settings);

// Tokenize..
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
Expand All @@ -8723,10 +8689,8 @@ class TestClass : public TestFixture {

#define checkThisUseAfterFree(code) checkThisUseAfterFree_(code, __FILE__, __LINE__)
void checkThisUseAfterFree_(const char code[], const char* file, int line) {
Preprocessor preprocessor(settings1);

// Tokenize..
SimpleTokenizer tokenizer(settings1, *this, &preprocessor);
SimpleTokenizer tokenizer(settings1, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
Expand Down Expand Up @@ -8924,10 +8888,8 @@ class TestClass : public TestFixture {

#define getFileInfo(code) getFileInfo_(code, __FILE__, __LINE__)
void getFileInfo_(const char code[], const char* file, int line) {
Preprocessor preprocessor(settings1);

// Tokenize..
SimpleTokenizer tokenizer(settings1, *this, &preprocessor);
SimpleTokenizer tokenizer(settings1, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
Expand Down
2 changes: 1 addition & 1 deletion test/testcondition.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ class TestCondition : public TestFixture {
void check_(const char* file, int line, const char code[], const Settings &settings, const char* filename = "test.cpp") {
Preprocessor preprocessor(settings);
std::vector<std::string> files(1, filename);
Tokenizer tokenizer(settings, this, &preprocessor);
Tokenizer tokenizer(settings, this);
PreprocessorHelper::preprocess(preprocessor, code, files, tokenizer);

// Tokenizer..
Expand Down
5 changes: 1 addition & 4 deletions test/testgarbage.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
#include "errortypes.h"
#include "fixture.h"
#include "helpers.h"
#include "preprocessor.h"
#include "settings.h"
#include "token.h"

Expand Down Expand Up @@ -283,10 +282,8 @@ class TestGarbage : public TestFixture {
}

std::string checkCodeInternal_(const char code[], bool cpp, const char* file, int line) {
Preprocessor preprocessor(settings);

// tokenize..
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);

// call all "runChecks" in all registered Check classes
Expand Down
Loading

0 comments on commit e09bcc6

Please sign in to comment.