From 2014086bc8eabff1b764268cce15d098ea98c6f3 Mon Sep 17 00:00:00 2001 From: firewave Date: Fri, 5 Jan 2024 22:21:55 +0100 Subject: [PATCH 1/5] TokenList: added missing standards checks --- lib/tokenlist.cpp | 6 +++--- test/testtokenize.cpp | 15 ++++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp index 1756d112b1a..f8f4d1db1a4 100644 --- a/lib/tokenlist.cpp +++ b/lib/tokenlist.cpp @@ -1854,7 +1854,7 @@ void TokenList::simplifyPlatformTypes() if (!mSettings) return; - const bool isCPP11 = mSettings->standards.cpp >= Standards::CPP11; + const bool isCPP11 = mIsCpp && (mSettings->standards.cpp >= Standards::CPP11); enum { isLongLong, isLong, isInt } type; @@ -1996,7 +1996,7 @@ void TokenList::simplifyStdType() continue; } - if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (mSettings->standards.c >= Standards::C99 && Token::Match(tok, "complex|_Complex"))) { + if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (mIsC && (mSettings->standards.c >= Standards::C99) && Token::Match(tok, "complex|_Complex"))) { bool isFloat= false; bool isSigned = false; bool isUnsigned = false; @@ -2019,7 +2019,7 @@ void TokenList::simplifyStdType() else if (Token::Match(tok2, "float|double")) { isFloat = true; typeSpec = tok2; - } else if (mSettings->standards.c >= Standards::C99 && Token::Match(tok2, "complex|_Complex")) + } else if (mIsC && (mSettings->standards.c >= Standards::C99) && Token::Match(tok2, "complex|_Complex")) isComplex = !isFloat || tok2->str() == "_Complex" || Token::Match(tok2->next(), "*|&|%name%"); // Ensure that "complex" is not the variables name else if (Token::Match(tok2, "char|int")) { if (!typeSpec) diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index 8dfaba26000..6dcb875c33a 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -2873,28 +2873,33 @@ class TestTokenizer : public TestFixture { } { const char code[] = "float complex x;"; - const char expected[] = "_Complex float x ;"; + const char expected[] = "float complex x ;"; ASSERT_EQUALS(expected, tokenizeAndStringify(code)); } + { + const char code[] = "float complex x;"; + const char expected[] = "_Complex float x ;"; + ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c")); + } { const char code[] = "complex float x;"; const char expected[] = "_Complex float x ;"; - ASSERT_EQUALS(expected, tokenizeAndStringify(code)); + ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c")); } { const char code[] = "complex long double x;"; const char expected[] = "_Complex long double x ;"; - ASSERT_EQUALS(expected, tokenizeAndStringify(code)); + ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c")); } { const char code[] = "long double complex x;"; const char expected[] = "_Complex long double x ;"; - ASSERT_EQUALS(expected, tokenizeAndStringify(code)); + ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c")); } { const char code[] = "double complex;"; const char expected[] = "double complex ;"; - ASSERT_EQUALS(expected, tokenizeAndStringify(code)); + ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c")); } } From 52d6c8997f95cd64b03120bffd36f4a405050eed Mon Sep 17 00:00:00 2001 From: firewave Date: Fri, 5 Jan 2024 22:22:16 +0100 Subject: [PATCH 2/5] TokenList: reduced dependency on settings --- lib/tokenize.cpp | 4 ++-- lib/tokenlist.cpp | 4 ++-- lib/tokenlist.h | 2 +- test/testtokenize.cpp | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp index 7cab82b9509..15c747b7fe0 100644 --- a/lib/tokenize.cpp +++ b/lib/tokenize.cpp @@ -3361,10 +3361,10 @@ bool Tokenizer::simplifyTokens1(const std::string &configuration) if (mTimerResults) { Timer t("Tokenizer::simplifyTokens1::createAst", mSettings->showtime, mTimerResults); list.createAst(); - list.validateAst(); + list.validateAst(mSettings->debugnormal); } else { list.createAst(); - list.validateAst(); + list.validateAst(mSettings->debugnormal); } if (mTimerResults) { diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp index f8f4d1db1a4..bab7dc7cf37 100644 --- a/lib/tokenlist.cpp +++ b/lib/tokenlist.cpp @@ -1722,10 +1722,10 @@ namespace { }; } -void TokenList::validateAst() const +void TokenList::validateAst(bool print) const { OnException oe{[&] { - if (mSettings && mSettings->debugnormal) + if (print) mTokensFrontBack.front->printOut(); }}; // Check for some known issues in AST to avoid crash/hang later on diff --git a/lib/tokenlist.h b/lib/tokenlist.h index 19dfe51e2c6..217dc14f0ec 100644 --- a/lib/tokenlist.h +++ b/lib/tokenlist.h @@ -171,7 +171,7 @@ class CPPCHECKLIB TokenList { * Check abstract syntax tree. * Throws InternalError on failure */ - void validateAst() const; + void validateAst(bool print) const; /** * Verify that the given token is an element of the tokenlist. diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index 6dcb875c33a..43d534c3823 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -5980,7 +5980,7 @@ class TestTokenizer : public TestFixture { tokenList.prepareTernaryOpForAST(); tokenList.list.createAst(); - tokenList.list.validateAst(); + tokenList.list.validateAst(false); // Basic AST validation for (const Token *tok = tokenList.list.front(); tok; tok = tok->next()) { From 3b2a5c6cc2b2b9178599b1974a7f8b853a54945c Mon Sep 17 00:00:00 2001 From: firewave Date: Fri, 5 Jan 2024 22:29:38 +0100 Subject: [PATCH 3/5] TokenList: store `Settings::Language` instead of individual booleans --- Makefile | 6 +++--- lib/tokenlist.cpp | 19 ++++++++++--------- lib/tokenlist.h | 9 ++++----- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 92608b760a0..60c16d87b2a 100644 --- a/Makefile +++ b/Makefile @@ -491,7 +491,7 @@ $(libcppdir)/checkautovariables.o: lib/checkautovariables.cpp lib/addoninfo.h li $(libcppdir)/checkbool.o: lib/checkbool.cpp lib/addoninfo.h lib/astutils.h lib/check.h lib/checkbool.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h $(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/checkbool.cpp -$(libcppdir)/checkboost.o: lib/checkboost.cpp lib/check.h lib/checkboost.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h +$(libcppdir)/checkboost.o: lib/checkboost.cpp lib/addoninfo.h lib/check.h lib/checkboost.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h $(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/checkboost.cpp $(libcppdir)/checkbufferoverrun.o: lib/checkbufferoverrun.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/astutils.h lib/check.h lib/checkbufferoverrun.h lib/color.h lib/config.h lib/ctu.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h @@ -593,7 +593,7 @@ $(libcppdir)/infer.o: lib/infer.cpp lib/calculate.h lib/config.h lib/errortypes. $(libcppdir)/keywords.o: lib/keywords.cpp lib/config.h lib/keywords.h lib/standards.h lib/utils.h $(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/keywords.cpp -$(libcppdir)/library.o: lib/library.cpp externals/tinyxml2/tinyxml2.h lib/astutils.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h +$(libcppdir)/library.o: lib/library.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/astutils.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h $(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/library.cpp $(libcppdir)/mathlib.o: lib/mathlib.cpp externals/simplecpp/simplecpp.h lib/config.h lib/errortypes.h lib/mathlib.h lib/utils.h @@ -626,7 +626,7 @@ $(libcppdir)/settings.o: lib/settings.cpp externals/picojson/picojson.h lib/addo $(libcppdir)/summaries.o: lib/summaries.cpp lib/addoninfo.h lib/analyzerinfo.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/summaries.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h $(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/summaries.cpp -$(libcppdir)/suppressions.o: lib/suppressions.cpp externals/tinyxml2/tinyxml2.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/mathlib.h lib/path.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h lib/xml.h +$(libcppdir)/suppressions.o: lib/suppressions.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h lib/xml.h $(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/suppressions.cpp $(libcppdir)/templatesimplifier.o: lib/templatesimplifier.cpp lib/addoninfo.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp index bab7dc7cf37..e1fca97c9c4 100644 --- a/lib/tokenlist.cpp +++ b/lib/tokenlist.cpp @@ -80,12 +80,13 @@ void TokenList::deallocateTokens() void TokenList::determineCppC() { - if (!mSettings) { - mIsC = Path::isC(getSourceFilePath()); - mIsCpp = Path::isCPP(getSourceFilePath()); + if (mSettings && (mSettings->enforcedLang != Settings::Language::None)) { + mLang = mSettings->enforcedLang; } else { - mIsC = mSettings->enforcedLang == Settings::Language::C || (mSettings->enforcedLang == Settings::Language::None && Path::isC(getSourceFilePath())); - mIsCpp = mSettings->enforcedLang == Settings::Language::CPP || (mSettings->enforcedLang == Settings::Language::None && Path::isCPP(getSourceFilePath())); + if (Path::isC(getSourceFilePath())) + mLang = Settings::Language::C; + else if (Path::isCPP(getSourceFilePath())) + mLang = Settings::Language::CPP; } } @@ -1854,7 +1855,7 @@ void TokenList::simplifyPlatformTypes() if (!mSettings) return; - const bool isCPP11 = mIsCpp && (mSettings->standards.cpp >= Standards::CPP11); + const bool isCPP11 = isCPP() && (mSettings->standards.cpp >= Standards::CPP11); enum { isLongLong, isLong, isInt } type; @@ -1996,7 +1997,7 @@ void TokenList::simplifyStdType() continue; } - if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (mIsC && (mSettings->standards.c >= Standards::C99) && Token::Match(tok, "complex|_Complex"))) { + if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (isC() && (mSettings->standards.c >= Standards::C99) && Token::Match(tok, "complex|_Complex"))) { bool isFloat= false; bool isSigned = false; bool isUnsigned = false; @@ -2019,7 +2020,7 @@ void TokenList::simplifyStdType() else if (Token::Match(tok2, "float|double")) { isFloat = true; typeSpec = tok2; - } else if (mIsC && (mSettings->standards.c >= Standards::C99) && Token::Match(tok2, "complex|_Complex")) + } else if (isC() && (mSettings->standards.c >= Standards::C99) && Token::Match(tok2, "complex|_Complex")) isComplex = !isFloat || tok2->str() == "_Complex" || Token::Match(tok2->next(), "*|&|%name%"); // Ensure that "complex" is not the variables name else if (Token::Match(tok2, "char|int")) { if (!typeSpec) @@ -2057,7 +2058,7 @@ void TokenList::simplifyStdType() bool TokenList::isKeyword(const std::string &str) const { - if (mIsCpp) { + if (isCPP()) { // TODO: integrate into keywords? // types and literals are not handled as keywords static const std::unordered_set cpp_types = {"bool", "false", "true"}; diff --git a/lib/tokenlist.h b/lib/tokenlist.h index 217dc14f0ec..39c58255fb4 100644 --- a/lib/tokenlist.h +++ b/lib/tokenlist.h @@ -22,13 +22,13 @@ //--------------------------------------------------------------------------- #include "config.h" +#include "settings.h" #include #include #include #include -class Settings; class Token; class TokenList; @@ -61,12 +61,12 @@ class CPPCHECKLIB TokenList { /** Is the code C. Used for bailouts */ bool isC() const { - return mIsC; + return mLang == Settings::Language::C; } /** Is the code CPP. Used for bailouts */ bool isCPP() const { - return mIsCpp; + return mLang == Settings::Language::CPP; } /** @@ -214,8 +214,7 @@ class CPPCHECKLIB TokenList { const Settings* const mSettings{}; /** File is known to be C/C++ code */ - bool mIsC{}; - bool mIsCpp{}; + Settings::Language mLang{Settings::Language::None}; }; /// @} From bf2dd0cc8ebb7cb776b480524b7a6b90311a589a Mon Sep 17 00:00:00 2001 From: firewave Date: Sat, 6 Jan 2024 09:19:24 +0100 Subject: [PATCH 4/5] TokenList: added missing settings checks --- lib/tokenlist.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp index e1fca97c9c4..28c43ffe8c2 100644 --- a/lib/tokenlist.cpp +++ b/lib/tokenlist.cpp @@ -1997,7 +1997,7 @@ void TokenList::simplifyStdType() continue; } - if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (isC() && (mSettings->standards.c >= Standards::C99) && Token::Match(tok, "complex|_Complex"))) { + if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (isC() && (!mSettings || (mSettings->standards.c >= Standards::C99)) && Token::Match(tok, "complex|_Complex"))) { bool isFloat= false; bool isSigned = false; bool isUnsigned = false; @@ -2020,7 +2020,7 @@ void TokenList::simplifyStdType() else if (Token::Match(tok2, "float|double")) { isFloat = true; typeSpec = tok2; - } else if (isC() && (mSettings->standards.c >= Standards::C99) && Token::Match(tok2, "complex|_Complex")) + } else if (isC() && (!mSettings || (mSettings->standards.c >= Standards::C99)) && Token::Match(tok2, "complex|_Complex")) isComplex = !isFloat || tok2->str() == "_Complex" || Token::Match(tok2->next(), "*|&|%name%"); // Ensure that "complex" is not the variables name else if (Token::Match(tok2, "char|int")) { if (!typeSpec) From 19bf024306b3b09a4275c8925dddc68ed8bcb373 Mon Sep 17 00:00:00 2001 From: firewave Date: Sat, 6 Jan 2024 10:38:19 +0100 Subject: [PATCH 5/5] TokenList: apply enforced language early --- lib/tokenlist.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp index 28c43ffe8c2..5e45b8d57d3 100644 --- a/lib/tokenlist.cpp +++ b/lib/tokenlist.cpp @@ -50,6 +50,9 @@ TokenList::TokenList(const Settings* settings) : mSettings(settings) { mTokensFrontBack.list = this; + if (mSettings && (mSettings->enforcedLang != Settings::Language::None)) { + mLang = mSettings->enforcedLang; + } } TokenList::~TokenList() @@ -80,9 +83,8 @@ void TokenList::deallocateTokens() void TokenList::determineCppC() { - if (mSettings && (mSettings->enforcedLang != Settings::Language::None)) { - mLang = mSettings->enforcedLang; - } else { + // only try to determine it if it wasn't enforced + if (mLang == Settings::Language::None) { if (Path::isC(getSourceFilePath())) mLang = Settings::Language::C; else if (Path::isCPP(getSourceFilePath()))