From e1ec95b69c0d47e669512fed819a835c5705628a Mon Sep 17 00:00:00 2001 From: firewave Date: Sat, 6 Jan 2024 21:19:17 +0100 Subject: [PATCH] make sure the language for a `TokenList` is always determined --- lib/clangimport.cpp | 7 ++++- lib/cppcheck.cpp | 7 +++-- lib/importproject.cpp | 8 +++--- lib/library.cpp | 11 ++++---- lib/library.h | 1 + lib/programmemory.cpp | 7 ++--- lib/programmemory.h | 3 ++- lib/standards.h | 2 +- lib/symboldatabase.cpp | 4 +-- lib/token.cpp | 1 + lib/token.h | 2 +- lib/tokenlist.cpp | 51 ++++++++++++++++++++++++++++++++++-- lib/tokenlist.h | 15 ++++++----- lib/valueflow.cpp | 20 +++++++------- test/testlibrary.cpp | 32 +++++++++++----------- test/testsimplifytypedef.cpp | 2 +- 16 files changed, 117 insertions(+), 56 deletions(-) diff --git a/lib/clangimport.cpp b/lib/clangimport.cpp index 1833e27e9f15..810a59794645 100644 --- a/lib/clangimport.cpp +++ b/lib/clangimport.cpp @@ -330,8 +330,12 @@ namespace clangimport { void dumpAst(int num = 0, int indent = 0) const; void createTokens1(TokenList &tokenList) { //dumpAst(); - if (!tokenList.back()) + if (!tokenList.back()) { setLocations(tokenList, 0, 1, 1); + // FIXME: treat as C++ if no filename (i.e. no lang) is specified for now + if (tokenList.getSourceFilePath().empty()) + tokenList.setLang(Standards::Language::CPP); + } else setLocations(tokenList, tokenList.back()->fileIndex(), tokenList.back()->linenr(), 1); createTokens(tokenList); @@ -626,6 +630,7 @@ void clangimport::AstNode::setValueType(Token *tok) continue; TokenList decl(nullptr); + decl.setLang(tok->isCpp() ? Standards::Language::CPP : Standards::Language::C); addTypeTokens(decl, type, tok->scope()); if (!decl.front()) break; diff --git a/lib/cppcheck.cpp b/lib/cppcheck.cpp index a93ac6c4ad10..3ac8b0cf15d8 100644 --- a/lib/cppcheck.cpp +++ b/lib/cppcheck.cpp @@ -194,7 +194,7 @@ static void createDumpFile(const Settings& settings, case Standards::Language::CPP: language = " language=\"cpp\""; break; - case Standards::Language::None: + case Standards::Language::None: { // TODO: error out on unknown language? const Standards::Language lang = Path::identify(filename); if (lang == Standards::Language::CPP) @@ -203,6 +203,9 @@ static void createDumpFile(const Settings& settings, language = " language=\"c\""; break; } + case Standards::Language::Markup: + break; + } fdump << "\n"; fdump << "\n"; @@ -797,7 +800,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string } Tokenizer tokenizer2(mSettings, this); std::istringstream istr2(code); - tokenizer2.list.createTokens(istr2); + tokenizer2.list.createTokens(istr2, Path::identify(*files.begin())); executeRules("define", tokenizer2); } #endif diff --git a/lib/importproject.cpp b/lib/importproject.cpp index d43d490d792c..a921ed24b6a3 100644 --- a/lib/importproject.cpp +++ b/lib/importproject.cpp @@ -560,11 +560,11 @@ namespace { replaceAll(c, "$(Configuration)", p.configuration); replaceAll(c, "$(Platform)", p.platformStr); - // TODO : Better evaluation - Settings s; - std::istringstream istr(c); + // TODO: evaluate without using the Tokenizer + const Settings s; Tokenizer tokenizer(s); - tokenizer.tokenize(istr,"vcxproj"); + std::istringstream istr(c); + tokenizer.tokenize(istr,"vcxproj.c"); for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next()) { if (tok->str() == "(" && tok->astOperand1() && tok->astOperand2()) { if (tok->astOperand1()->expressionString() == "Configuration.Contains") diff --git a/lib/library.cpp b/lib/library.cpp index d815b3ac6159..1f0c4e88952c 100644 --- a/lib/library.cpp +++ b/lib/library.cpp @@ -53,10 +53,10 @@ static std::vector getnames(const char *names) return ret; } -static void gettokenlistfromvalid(const std::string& valid, TokenList& tokenList) +static void gettokenlistfromvalid(const std::string& valid, bool cpp, TokenList& tokenList) { std::istringstream istr(valid + ','); - tokenList.createTokens(istr); + tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C); for (Token *tok = tokenList.front(); tok; tok = tok->next()) { if (Token::Match(tok,"- %num%")) { tok->str("-" + tok->strAt(1)); @@ -920,7 +920,7 @@ bool Library::isIntArgValid(const Token *ftok, int argnr, const MathLib::bigint if (ac->valid.find('.') != std::string::npos) return isFloatArgValid(ftok, argnr, argvalue); TokenList tokenList(nullptr); - gettokenlistfromvalid(ac->valid, tokenList); + gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList); for (const Token *tok = tokenList.front(); tok; tok = tok->next()) { if (tok->isNumber() && argvalue == MathLib::toBigNumber(tok->str())) return true; @@ -940,7 +940,7 @@ bool Library::isFloatArgValid(const Token *ftok, int argnr, double argvalue) con if (!ac || ac->valid.empty()) return true; TokenList tokenList(nullptr); - gettokenlistfromvalid(ac->valid, tokenList); + gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList); for (const Token *tok = tokenList.front(); tok; tok = tok->next()) { if (Token::Match(tok, "%num% : %num%") && argvalue >= MathLib::toDoubleNumber(tok->str()) && argvalue <= MathLib::toDoubleNumber(tok->strAt(2))) return true; @@ -1729,13 +1729,14 @@ bool Library::hasAnyTypeCheck(const std::string& typeName) const std::shared_ptr createTokenFromExpression(const std::string& returnValue, const Settings* settings, + bool cpp, std::unordered_map* lookupVarId) { std::shared_ptr tokenList = std::make_shared(settings); { const std::string code = "return " + returnValue + ";"; std::istringstream istr(code); - if (!tokenList->createTokens(istr)) + if (!tokenList->createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C)) return nullptr; } diff --git a/lib/library.h b/lib/library.h index 1f20964b8ce6..0065b446ec79 100644 --- a/lib/library.h +++ b/lib/library.h @@ -603,6 +603,7 @@ CPPCHECKLIB const Library::Container * getLibraryContainer(const Token * tok); std::shared_ptr createTokenFromExpression(const std::string& returnValue, const Settings* settings, + bool cpp, std::unordered_map* lookupVarId = nullptr); /// @} diff --git a/lib/programmemory.cpp b/lib/programmemory.cpp index 96b79098aed9..d5957321071f 100644 --- a/lib/programmemory.cpp +++ b/lib/programmemory.cpp @@ -1595,7 +1595,7 @@ namespace { arg_map[argn] = v; argn++; } - return evaluateLibraryFunction(arg_map, returnValue, settings); + return evaluateLibraryFunction(arg_map, returnValue, settings, ftok->isCpp()); } } } @@ -1751,7 +1751,8 @@ std::vector execute(const Scope* scope, ProgramMemory& pm, con ValueFlow::Value evaluateLibraryFunction(const std::unordered_map& args, const std::string& returnValue, - const Settings* settings) + const Settings* settings, + bool cpp) { thread_local static std::unordered_map& arg)>> @@ -1759,7 +1760,7 @@ ValueFlow::Value evaluateLibraryFunction(const std::unordered_map lookupVarId; - std::shared_ptr expr = createTokenFromExpression(returnValue, settings, &lookupVarId); + std::shared_ptr expr = createTokenFromExpression(returnValue, settings, cpp, &lookupVarId); functions[returnValue] = [lookupVarId, expr, settings](const std::unordered_map& xargs) { diff --git a/lib/programmemory.h b/lib/programmemory.h index f81ef7b35be6..32fce3a1392c 100644 --- a/lib/programmemory.h +++ b/lib/programmemory.h @@ -211,7 +211,8 @@ ProgramMemory getProgramMemory(const Token* tok, const Token* expr, const ValueF ValueFlow::Value evaluateLibraryFunction(const std::unordered_map& args, const std::string& returnValue, - const Settings* settings); + const Settings* settings, + bool cpp); #endif diff --git a/lib/standards.h b/lib/standards.h index 4b1469164ca5..d8a1a5ccab5b 100644 --- a/lib/standards.h +++ b/lib/standards.h @@ -34,7 +34,7 @@ * This struct contains all possible standards that cppcheck recognize. */ struct Standards { - enum Language { None, C, CPP }; + enum Language { None, C, CPP, Markup }; /** C code standard */ enum cstd_t { C89, C99, C11, CLatest = C11 } c = CLatest; diff --git a/lib/symboldatabase.cpp b/lib/symboldatabase.cpp index b8c1e73342d2..3e8b7e0e0981 100644 --- a/lib/symboldatabase.cpp +++ b/lib/symboldatabase.cpp @@ -7479,7 +7479,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to ValueType valuetype; TokenList tokenList(&mSettings); std::istringstream istr(typestr+";"); - tokenList.createTokens(istr); + tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C); tokenList.simplifyStdType(); if (parsedecl(tokenList.front(), &valuetype, mDefaultSignedness, mSettings, mIsCpp)) { valuetype.originalTypeName = typestr; @@ -7569,7 +7569,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to } TokenList tokenList(&mSettings); std::istringstream istr(typestr+";"); - if (tokenList.createTokens(istr)) { + if (tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C)) { ValueType vt; tokenList.simplifyPlatformTypes(); tokenList.simplifyStdType(); diff --git a/lib/token.cpp b/lib/token.cpp index 5be97b9b426e..0f22d8b0bea5 100644 --- a/lib/token.cpp +++ b/lib/token.cpp @@ -2741,5 +2741,6 @@ bool Token::isCpp() const if (mTokensFrontBack && mTokensFrontBack->list) { return mTokensFrontBack->list->isCPP(); } + assert(false); return true; // assume C++ by default } diff --git a/lib/token.h b/lib/token.h index 91d8281be760..36ad9622893c 100644 --- a/lib/token.h +++ b/lib/token.h @@ -151,7 +151,7 @@ class CPPCHECKLIB Token { friend class TestToken; private: - TokensFrontBack* mTokensFrontBack{}; + TokensFrontBack* const mTokensFrontBack{}; public: Token(const Token &) = delete; diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp index 14b560193f69..8cd6cfe6c89e 100644 --- a/lib/tokenlist.cpp +++ b/lib/tokenlist.cpp @@ -30,6 +30,7 @@ #include "standards.h" #include "token.h" +#include #include #include #include @@ -83,9 +84,14 @@ void TokenList::deallocateTokens() void TokenList::determineCppC() { - // only try to determine it if it wasn't enforced + // only try to determine if it wasn't enforced if (mLang == Standards::Language::None) { - mLang = Path::identify(getSourceFilePath()); + if (mSettings->library.markupFile(getSourceFilePath())) { + mLang = Standards::Language::Markup; + } + else { + mLang = Path::identify(getSourceFilePath()); + } } } @@ -317,8 +323,27 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n) bool TokenList::createTokens(std::istream &code, const std::string& file0) { + assert(!file0.empty()); + appendFileIfNew(file0); + return createTokensInternal(code, file0); +} + +//--------------------------------------------------------------------------- + +bool TokenList::createTokens(std::istream &code, Standards::Language lang) +{ + if (mLang == Standards::Language::None) + mLang = lang; + + return createTokensInternal(code, ""); +} + +//--------------------------------------------------------------------------- + +bool TokenList::createTokensInternal(std::istream &code, const std::string& file0) +{ simplecpp::OutputList outputList; simplecpp::TokenList tokens(code, mFiles, file0, &outputList); @@ -2096,3 +2121,25 @@ bool TokenList::isKeyword(const std::string &str) const static const auto& latest_c_keywords = Keywords::getAll(Standards::cstd_t::CLatest); return latest_c_keywords.find(str) != latest_c_keywords.end(); } + +bool TokenList::isC() const +{ + assert(mLang != Standards::Language::None); + + return mLang == Standards::Language::C; +} + +bool TokenList::isCPP() const +{ + assert(mLang != Standards::Language::None); + + return mLang == Standards::Language::CPP; +} + +void TokenList::setLang(Standards::Language lang) +{ + assert(lang != Standards::Language::None); + assert(mLang == Standards::Language::None); + + mLang = lang; +} diff --git a/lib/tokenlist.h b/lib/tokenlist.h index 3b71d07d1566..fcbb8f4efa56 100644 --- a/lib/tokenlist.h +++ b/lib/tokenlist.h @@ -61,14 +61,12 @@ class CPPCHECKLIB TokenList { const std::string& getSourceFilePath() const; /** Is the code C. Used for bailouts */ - bool isC() const { - return mLang == Standards::Language::C; - } + bool isC() const; /** Is the code CPP. Used for bailouts */ - bool isCPP() const { - return mLang == Standards::Language::CPP; - } + bool isCPP() const; + + void setLang(Standards::Language lang); /** * Delete all tokens in given token list @@ -104,7 +102,8 @@ class CPPCHECKLIB TokenList { * @param code input stream for code * @param file0 source file name */ - bool createTokens(std::istream &code, const std::string& file0 = emptyString); + bool createTokens(std::istream &code, const std::string& file0); + bool createTokens(std::istream &code, Standards::Language lang); void createTokens(simplecpp::TokenList&& tokenList); @@ -202,6 +201,8 @@ class CPPCHECKLIB TokenList { private: void determineCppC(); + bool createTokensInternal(std::istream &code, const std::string& file0); + /** Token list */ TokensFrontBack mTokensFrontBack; diff --git a/lib/valueflow.cpp b/lib/valueflow.cpp index 63d538dd3530..10a79ebdc5c3 100644 --- a/lib/valueflow.cpp +++ b/lib/valueflow.cpp @@ -3738,20 +3738,20 @@ static bool isNotEqual(std::pair x, std::pair x, const std::string& y) +static bool isNotEqual(std::pair x, const std::string& y, bool cpp) { TokenList tokenList(nullptr); std::istringstream istr(y); - tokenList.createTokens(istr); + tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C); return isNotEqual(x, std::make_pair(tokenList.front(), tokenList.back())); } -static bool isNotEqual(std::pair x, const ValueType* y) +static bool isNotEqual(std::pair x, const ValueType* y, bool cpp) { if (y == nullptr) return false; if (y->originalTypeName.empty()) return false; - return isNotEqual(x, y->originalTypeName); + return isNotEqual(x, y->originalTypeName, cpp); } static bool isDifferentType(const Token* src, const Token* dst) @@ -3766,9 +3766,9 @@ static bool isDifferentType(const Token* src, const Token* dst) std::pair parentdecl = Token::typeDecl(dst); if (isNotEqual(decl, parentdecl)) return true; - if (isNotEqual(decl, dst->valueType())) + if (isNotEqual(decl, dst->valueType(), dst->isCpp())) return true; - if (isNotEqual(parentdecl, src->valueType())) + if (isNotEqual(parentdecl, src->valueType(), src->isCpp())) return true; } return false; @@ -7592,7 +7592,7 @@ static void valueFlowLibraryFunction(Token *tok, const std::string &returnValue, if (returnValue.find("arg") != std::string::npos && argValues.empty()) return; productParams(settings, argValues, [&](const std::unordered_map& arg) { - ValueFlow::Value value = evaluateLibraryFunction(arg, returnValue, settings); + ValueFlow::Value value = evaluateLibraryFunction(arg, returnValue, settings, tok->isCpp()); if (value.isUninitValue()) return; ValueFlow::Value::ValueKind kind = ValueFlow::Value::ValueKind::Known; @@ -9086,11 +9086,11 @@ static bool getMinMaxValues(const ValueType *vt, const Platform &platform, MathL return true; } -static bool getMinMaxValues(const std::string &typestr, const Settings *settings, MathLib::bigint &minvalue, MathLib::bigint &maxvalue) +static bool getMinMaxValues(const std::string &typestr, const Settings *settings, bool cpp, MathLib::bigint &minvalue, MathLib::bigint &maxvalue) { TokenList typeTokens(settings); std::istringstream istr(typestr+";"); - if (!typeTokens.createTokens(istr)) + if (!typeTokens.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C)) return false; typeTokens.simplifyPlatformTypes(); typeTokens.simplifyStdType(); @@ -9210,7 +9210,7 @@ static void valueFlowUnknownFunctionReturn(TokenList &tokenlist, const Settings // Get min/max values for return type const std::string &typestr = settings->library.returnValueType(tok->previous()); MathLib::bigint minvalue, maxvalue; - if (!getMinMaxValues(typestr, settings, minvalue, maxvalue)) + if (!getMinMaxValues(typestr, settings, tok->isCpp(), minvalue, maxvalue)) continue; for (MathLib::bigint value : unknownValues) { diff --git a/test/testlibrary.cpp b/test/testlibrary.cpp index fc5b2f1fda0d..a2cb05cbbf5e 100644 --- a/test/testlibrary.cpp +++ b/test/testlibrary.cpp @@ -118,7 +118,7 @@ class TestLibrary : public TestFixture { TokenList tokenList(nullptr); std::istringstream istr("foo();"); - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); tokenList.front()->next()->astOperand1(tokenList.front()); Library library; @@ -141,14 +141,14 @@ class TestLibrary : public TestFixture { { TokenList tokenList(nullptr); std::istringstream istr("fred.foo(123);"); // <- wrong scope, not library function - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); ASSERT(library.isNotLibraryFunction(tokenList.front()->tokAt(2))); } { TokenList tokenList(nullptr); std::istringstream istr("Fred::foo(123);"); // <- wrong scope, not library function - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); ASSERT(library.isNotLibraryFunction(tokenList.front()->tokAt(2))); } @@ -164,7 +164,7 @@ class TestLibrary : public TestFixture { TokenList tokenList(nullptr); std::istringstream istr("foo();"); // <- too few arguments, not library function - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous()); tokenList.createAst(); @@ -188,7 +188,7 @@ class TestLibrary : public TestFixture { { TokenList tokenList(nullptr); std::istringstream istr("foo();"); // <- too few arguments, not library function - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous()); tokenList.createAst(); @@ -197,7 +197,7 @@ class TestLibrary : public TestFixture { { TokenList tokenList(nullptr); std::istringstream istr("foo(a);"); // <- library function - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous()); tokenList.createAst(); @@ -206,7 +206,7 @@ class TestLibrary : public TestFixture { { TokenList tokenList(nullptr); std::istringstream istr("foo(a, b);"); // <- library function - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous()); tokenList.createAst(); @@ -215,7 +215,7 @@ class TestLibrary : public TestFixture { { TokenList tokenList(nullptr); std::istringstream istr("foo(a, b, c);"); // <- too much arguments, not library function - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous()); tokenList.createAst(); @@ -233,7 +233,7 @@ class TestLibrary : public TestFixture { TokenList tokenList(nullptr); std::istringstream istr("Fred foo(123);"); // <- Variable declaration, not library function - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); tokenList.front()->next()->astOperand1(tokenList.front()); tokenList.front()->next()->varId(1); @@ -293,7 +293,7 @@ class TestLibrary : public TestFixture { TokenList tokenList(nullptr); std::istringstream istr("foo(a,b,c,d,e);"); - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); tokenList.front()->next()->astOperand1(tokenList.front()); ASSERT_EQUALS(false, library.isuninitargbad(tokenList.front(), 1)); @@ -318,7 +318,7 @@ class TestLibrary : public TestFixture { TokenList tokenList(nullptr); std::istringstream istr("foo(a,b,c,d);"); - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); tokenList.front()->next()->astOperand1(tokenList.front()); ASSERT(Library::ArgumentChecks::Direction::DIR_IN == library.getArgDirection(tokenList.front(), 1)); @@ -350,7 +350,7 @@ class TestLibrary : public TestFixture { TokenList tokenList(nullptr); std::istringstream istr("foo(a,b,c,d,e,f,g,h,i,j,k);"); - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); tokenList.front()->next()->astOperand1(tokenList.front()); // 1- @@ -492,7 +492,7 @@ class TestLibrary : public TestFixture { TokenList tokenList(nullptr); std::istringstream istr("foo(a,b,c,d,e);"); - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); tokenList.front()->next()->astOperand1(tokenList.front()); // arg1: type=strlen arg2 @@ -555,14 +555,14 @@ class TestLibrary : public TestFixture { { TokenList tokenList(nullptr); std::istringstream istr("Foo::foo();"); - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); ASSERT(library.isnotnoreturn(tokenList.front()->tokAt(2))); } { TokenList tokenList(nullptr); std::istringstream istr("bar();"); - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); ASSERT(library.isnotnoreturn(tokenList.front())); } } @@ -636,7 +636,7 @@ class TestLibrary : public TestFixture { TokenList tokenList(nullptr); std::istringstream istr("a(); b();"); - tokenList.createTokens(istr); + tokenList.createTokens(istr, Standards::Language::CPP); const Library::WarnInfo* a = library.getWarnInfo(tokenList.front()); const Library::WarnInfo* b = library.getWarnInfo(tokenList.front()->tokAt(4)); diff --git a/test/testsimplifytypedef.cpp b/test/testsimplifytypedef.cpp index 3f68230420f7..7e193ee912d7 100644 --- a/test/testsimplifytypedef.cpp +++ b/test/testsimplifytypedef.cpp @@ -252,7 +252,7 @@ class TestSimplifyTypedef : public TestFixture { Tokenizer tokenizer(settings1, this); std::istringstream istr(code); - tokenizer.list.createTokens(istr); + tokenizer.list.createTokens(istr, Standards::Language::CPP); tokenizer.createLinks(); tokenizer.simplifyTypedef();