make sure the language for a TokenList is always determined [skip ci]

lib/clangimport.cpp

@@ -334,6 +334,7 @@ namespace clangimport {
                 setLocations(tokenList, 0, 1, 1);
             else
                 setLocations(tokenList, tokenList->back()->fileIndex(), tokenList->back()->linenr(), 1);
+            // TODO: language might not be set for the tokens if no file is specified in the AST dump
             createTokens(tokenList);
             if (nodeType == VarDecl || nodeType == RecordDecl || nodeType == TypedefDecl)
                 addtoken(tokenList, ";");
@@ -626,6 +627,7 @@ void clangimport::AstNode::setValueType(Token *tok)
             continue;
 
         TokenList decl(nullptr);
+        decl.setLang(tok->isCpp() ? Settings::Language::CPP : Settings::Language::C);
         addTypeTokens(&decl, type, tok->scope());
         if (!decl.front())
             break;

lib/cppcheck.cpp

@@ -794,7 +794,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
             }
             Tokenizer tokenizer2(&mSettings, this);
             std::istringstream istr2(code);
-            tokenizer2.list.createTokens(istr2);
+            tokenizer2.list.createTokens(istr2, Path::isCPP(*files.begin()) ? Settings::Language::CPP : Settings::Language::C);
             executeRules("define", tokenizer2);
         }
 #endif

lib/library.cpp

@@ -53,10 +53,10 @@ static std::vector<std::string> getnames(const char *names)
     return ret;
 }
 
-static void gettokenlistfromvalid(const std::string& valid, TokenList& tokenList)
+static void gettokenlistfromvalid(const std::string& valid, bool cpp, TokenList& tokenList)
 {
     std::istringstream istr(valid + ',');
-    tokenList.createTokens(istr);
+    tokenList.createTokens(istr, cpp ? Settings::Language::CPP : Settings::Language::C);
     for (Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (Token::Match(tok,"- %num%")) {
             tok->str("-" + tok->strAt(1));
@@ -920,7 +920,7 @@ bool Library::isIntArgValid(const Token *ftok, int argnr, const MathLib::bigint
     if (ac->valid.find('.') != std::string::npos)
         return isFloatArgValid(ftok, argnr, argvalue);
     TokenList tokenList(nullptr);
-    gettokenlistfromvalid(ac->valid, tokenList);
+    gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
     for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (tok->isNumber() && argvalue == MathLib::toBigNumber(tok->str()))
             return true;
@@ -940,7 +940,7 @@ bool Library::isFloatArgValid(const Token *ftok, int argnr, double argvalue) con
     if (!ac || ac->valid.empty())
         return true;
     TokenList tokenList(nullptr);
-    gettokenlistfromvalid(ac->valid, tokenList);
+    gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
     for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (Token::Match(tok, "%num% : %num%") && argvalue >= MathLib::toDoubleNumber(tok->str()) && argvalue <= MathLib::toDoubleNumber(tok->strAt(2)))
             return true;
@@ -1725,13 +1725,14 @@ bool Library::hasAnyTypeCheck(const std::string& typeName) const
 
 std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
                                                  const Settings* settings,
+                                                 bool cpp,
                                                  std::unordered_map<nonneg int, const Token*>* lookupVarId)
 {
     std::shared_ptr<TokenList> tokenList = std::make_shared<TokenList>(settings);
     {
         const std::string code = "return " + returnValue + ";";
         std::istringstream istr(code);
-        if (!tokenList->createTokens(istr))
+        if (!tokenList->createTokens(istr, cpp ? Settings::Language::CPP : Settings::Language::C))
             return nullptr;
     }
 

lib/library.h

@@ -603,6 +603,7 @@ CPPCHECKLIB const Library::Container * getLibraryContainer(const Token * tok);
 
 std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
                                                  const Settings* settings,
+                                                 bool cpp,
                                                  std::unordered_map<nonneg int, const Token*>* lookupVarId = nullptr);
 
 /// @}

lib/programmemory.cpp

@@ -1596,7 +1596,7 @@ namespace {
                                     arg_map[argn] = v;
                                 argn++;
                             }
-                            return evaluateLibraryFunction(arg_map, returnValue, settings);
+                            return evaluateLibraryFunction(arg_map, returnValue, settings, ftok->isCpp());
                         }
                     }
                 }
@@ -1752,15 +1752,16 @@ std::vector<ValueFlow::Value> execute(const Scope* scope, ProgramMemory& pm, con
 
 ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
                                          const std::string& returnValue,
-                                         const Settings* settings)
+                                         const Settings* settings,
+                                         bool cpp)
 {
     thread_local static std::unordered_map<std::string,
                                            std::function<ValueFlow::Value(const std::unordered_map<nonneg int, ValueFlow::Value>& arg)>>
     functions = {};
     if (functions.count(returnValue) == 0) {
 
         std::unordered_map<nonneg int, const Token*> lookupVarId;
-        std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, &lookupVarId);
+        std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, cpp, &lookupVarId);
 
         functions[returnValue] =
             [lookupVarId, expr, settings](const std::unordered_map<nonneg int, ValueFlow::Value>& xargs) {

lib/programmemory.h

@@ -211,7 +211,8 @@ ProgramMemory getProgramMemory(const Token* tok, const Token* expr, const ValueF
 
 ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
                                          const std::string& returnValue,
-                                         const Settings* settings);
+                                         const Settings* settings,
+                                         bool cpp);
 
 #endif
 

lib/symboldatabase.cpp

@@ -7479,7 +7479,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
                     ValueType valuetype;
                     TokenList tokenList(&mSettings);
                     std::istringstream istr(typestr+";");
-                    tokenList.createTokens(istr);
+                    tokenList.createTokens(istr, tok->isCpp() ? Settings::Language::CPP : Settings::Language::C);
                     tokenList.simplifyStdType();
                     if (parsedecl(tokenList.front(), &valuetype, mDefaultSignedness, mSettings, mIsCpp)) {
                         valuetype.originalTypeName = typestr;
@@ -7569,7 +7569,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
                 }
                 TokenList tokenList(&mSettings);
                 std::istringstream istr(typestr+";");
-                if (tokenList.createTokens(istr)) {
+                if (tokenList.createTokens(istr, tok->isCpp() ? Settings::Language::CPP : Settings::Language::C)) {
                     ValueType vt;
                     tokenList.simplifyPlatformTypes();
                     tokenList.simplifyStdType();

lib/tokenlist.cpp

@@ -30,6 +30,7 @@
 #include "standards.h"
 #include "token.h"
 
+#include <cassert>
 #include <cctype>
 #include <exception>
 #include <functional>
@@ -89,6 +90,8 @@ void TokenList::determineCppC()
             mLang = Settings::Language::C;
         else if (Path::isCPP(getSourceFilePath()))
             mLang = Settings::Language::CPP;
+
+        assert(mLang != Settings::Language::None);
     }
 }
 
@@ -320,8 +323,27 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n)
 
 bool TokenList::createTokens(std::istream &code, const std::string& file0)
 {
+    assert(!file0.empty());
+
     appendFileIfNew(file0);
 
+    return createTokensInternal(code, file0);
+}
+
+//---------------------------------------------------------------------------
+
+bool TokenList::createTokens(std::istream &code, Settings::Language lang)
+{
+    if (mLang == Settings::Language::None)
+        mLang = lang;
+
+    return createTokensInternal(code, "");
+}
+
+//---------------------------------------------------------------------------
+
+bool TokenList::createTokensInternal(std::istream &code, const std::string& file0)
+{
     simplecpp::OutputList outputList;
     simplecpp::TokenList tokens(code, mFiles, file0, &outputList);
 
@@ -2090,3 +2112,25 @@ bool TokenList::isKeyword(const std::string &str) const
     static const auto& latest_c_keywords = Keywords::getAll(Standards::cstd_t::CLatest);
     return latest_c_keywords.find(str) != latest_c_keywords.end();
 }
+
+bool TokenList::isC() const
+{
+    assert(mLang != Settings::Language::None);
+
+    return mLang == Settings::Language::C;
+}
+
+bool TokenList::isCPP() const
+{
+    assert(mLang != Settings::Language::None);
+
+    return mLang == Settings::Language::CPP;
+}
+
+void TokenList::setLang(Settings::Language lang)
+{
+    assert(lang != Settings::Language::None);
+    assert(mLang == Settings::Language::None);
+
+    mLang = lang;
+}

lib/tokenlist.h

@@ -60,14 +60,12 @@ class CPPCHECKLIB TokenList {
     const std::string& getSourceFilePath() const;
 
     /** Is the code C. Used for bailouts */
-    bool isC() const {
-        return mLang == Settings::Language::C;
-    }
+    bool isC() const;
 
     /** Is the code CPP. Used for bailouts */
-    bool isCPP() const {
-        return mLang == Settings::Language::CPP;
-    }
+    bool isCPP() const;
+
+    void setLang(Settings::Language lang);
 
     /**
      * Delete all tokens in given token list
@@ -103,7 +101,8 @@ class CPPCHECKLIB TokenList {
      * @param code input stream for code
      * @param file0 source file name
      */
-    bool createTokens(std::istream &code, const std::string& file0 = emptyString);
+    bool createTokens(std::istream &code, const std::string& file0);
+    bool createTokens(std::istream &code, Settings::Language lang);
 
     void createTokens(simplecpp::TokenList&& tokenList);
 
@@ -201,6 +200,8 @@ class CPPCHECKLIB TokenList {
 private:
     void determineCppC();
 
+    bool createTokensInternal(std::istream &code, const std::string& file0);
+
     /** Token list */
     TokensFrontBack mTokensFrontBack;
 

lib/valueflow.cpp

@@ -3738,20 +3738,20 @@ static bool isNotEqual(std::pair<const Token*, const Token*> x, std::pair<const
     start2 = skipCVRefs(start2, y.second);
     return !(start1 == x.second && start2 == y.second);
 }
-static bool isNotEqual(std::pair<const Token*, const Token*> x, const std::string& y)
+static bool isNotEqual(std::pair<const Token*, const Token*> x, const std::string& y, bool cpp)
 {
     TokenList tokenList(nullptr);
     std::istringstream istr(y);
-    tokenList.createTokens(istr);
+    tokenList.createTokens(istr, cpp ? Settings::Language::CPP : Settings::Language::C);
     return isNotEqual(x, std::make_pair(tokenList.front(), tokenList.back()));
 }
-static bool isNotEqual(std::pair<const Token*, const Token*> x, const ValueType* y)
+static bool isNotEqual(std::pair<const Token*, const Token*> x, const ValueType* y, bool cpp)
 {
     if (y == nullptr)
         return false;
     if (y->originalTypeName.empty())
         return false;
-    return isNotEqual(x, y->originalTypeName);
+    return isNotEqual(x, y->originalTypeName, cpp);
 }
 
 static bool isDifferentType(const Token* src, const Token* dst)
@@ -3766,9 +3766,9 @@ static bool isDifferentType(const Token* src, const Token* dst)
         std::pair<const Token*, const Token*> parentdecl = Token::typeDecl(dst);
         if (isNotEqual(decl, parentdecl))
             return true;
-        if (isNotEqual(decl, dst->valueType()))
+        if (isNotEqual(decl, dst->valueType(), dst->isCpp()))
             return true;
-        if (isNotEqual(parentdecl, src->valueType()))
+        if (isNotEqual(parentdecl, src->valueType(), src->isCpp()))
             return true;
     }
     return false;
@@ -7592,7 +7592,7 @@ static void valueFlowLibraryFunction(Token *tok, const std::string &returnValue,
     if (returnValue.find("arg") != std::string::npos && argValues.empty())
         return;
     productParams(settings, argValues, [&](const std::unordered_map<nonneg int, ValueFlow::Value>& arg) {
-        ValueFlow::Value value = evaluateLibraryFunction(arg, returnValue, settings);
+        ValueFlow::Value value = evaluateLibraryFunction(arg, returnValue, settings, tok->isCpp());
         if (value.isUninitValue())
             return;
         ValueFlow::Value::ValueKind kind = ValueFlow::Value::ValueKind::Known;
@@ -9086,11 +9086,11 @@ static bool getMinMaxValues(const ValueType *vt, const Platform &platform, MathL
     return true;
 }
 
-static bool getMinMaxValues(const std::string &typestr, const Settings *settings, MathLib::bigint &minvalue, MathLib::bigint &maxvalue)
+static bool getMinMaxValues(const std::string &typestr, const Settings *settings, bool cpp, MathLib::bigint &minvalue, MathLib::bigint &maxvalue)
 {
     TokenList typeTokens(settings);
     std::istringstream istr(typestr+";");
-    if (!typeTokens.createTokens(istr))
+    if (!typeTokens.createTokens(istr, cpp ? Settings::Language::CPP : Settings::Language::C))
         return false;
     typeTokens.simplifyPlatformTypes();
     typeTokens.simplifyStdType();
@@ -9210,7 +9210,7 @@ static void valueFlowUnknownFunctionReturn(TokenList &tokenlist, const Settings
         // Get min/max values for return type
         const std::string &typestr = settings->library.returnValueType(tok->previous());
         MathLib::bigint minvalue, maxvalue;
-        if (!getMinMaxValues(typestr, settings, minvalue, maxvalue))
+        if (!getMinMaxValues(typestr, settings, tok->isCpp(), minvalue, maxvalue))
             continue;
 
         for (MathLib::bigint value : unknownValues) {