make sure the language for a TokenList/Token is always determined

lib/clangimport.cpp

@@ -330,8 +330,12 @@ namespace clangimport {
         void dumpAst(int num = 0, int indent = 0) const;
         void createTokens1(TokenList &tokenList) {
             //dumpAst();
-            if (!tokenList.back())
+            if (!tokenList.back()) {
                 setLocations(tokenList, 0, 1, 1);
+                // FIXME: treat as C++ if no filename (i.e. no lang) is specified for now
+                if (tokenList.getSourceFilePath().empty())
+                    tokenList.setLang(Standards::Language::CPP);
+            }
             else
                 setLocations(tokenList, tokenList.back()->fileIndex(), tokenList.back()->linenr(), 1);
             createTokens(tokenList);
@@ -626,6 +630,7 @@ void clangimport::AstNode::setValueType(Token *tok)
             continue;
 
         TokenList decl(nullptr);
+        decl.setLang(tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
         addTypeTokens(decl, type, tok->scope());
         if (!decl.front())
             break;

lib/cppcheck.cpp

@@ -633,10 +633,14 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
 
     try {
         if (mSettings.library.markupFile(filename)) {
-            if (mSettings.isUnusedFunctionCheckEnabled() && mSettings.buildDir.empty()) {
+            if (mSettings.isUnusedFunctionCheckEnabled() &&
+                mSettings.buildDir.empty()) {
+                // this is not a real source file - we just want to tokenize it. treat it as C anyways as the language needs to be determined.
                 Tokenizer tokenizer(mSettings, this);
-                if (fileStream)
+                tokenizer.list.setLang(Standards::Language::C);
+                if (fileStream) {
                     tokenizer.list.createTokens(*fileStream, filename);
+                }
                 else {
                     std::ifstream in(filename);
                     tokenizer.list.createTokens(in, filename);
@@ -804,7 +808,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
             }
             Tokenizer tokenizer2(mSettings, this);
             std::istringstream istr2(code);
-            tokenizer2.list.createTokens(istr2);
+            tokenizer2.list.createTokens(istr2, Path::identify(*files.begin()));
             executeRules("define", tokenizer2);
         }
 #endif

lib/importproject.cpp

@@ -560,11 +560,11 @@ namespace {
             replaceAll(c, "$(Configuration)", p.configuration);
             replaceAll(c, "$(Platform)", p.platformStr);
 
-            // TODO : Better evaluation
-            Settings s;
-            std::istringstream istr(c);
+            // TODO: evaluate without using the Tokenizer
+            const Settings s;
             Tokenizer tokenizer(s);
-            tokenizer.tokenize(istr,"vcxproj");
+            std::istringstream istr(c);
+            tokenizer.tokenize(istr,"vcxproj.c");
             for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next()) {
                 if (tok->str() == "(" && tok->astOperand1() && tok->astOperand2()) {
                     if (tok->astOperand1()->expressionString() == "Configuration.Contains")

lib/library.cpp

@@ -53,10 +53,10 @@ static std::vector<std::string> getnames(const char *names)
     return ret;
 }
 
-static void gettokenlistfromvalid(const std::string& valid, TokenList& tokenList)
+static void gettokenlistfromvalid(const std::string& valid, bool cpp, TokenList& tokenList)
 {
     std::istringstream istr(valid + ',');
-    tokenList.createTokens(istr);
+    tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C);
     for (Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (Token::Match(tok,"- %num%")) {
             tok->str("-" + tok->strAt(1));
@@ -929,7 +929,7 @@ bool Library::isIntArgValid(const Token *ftok, int argnr, const MathLib::bigint
     if (ac->valid.find('.') != std::string::npos)
         return isFloatArgValid(ftok, argnr, argvalue);
     TokenList tokenList(nullptr);
-    gettokenlistfromvalid(ac->valid, tokenList);
+    gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
     for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (tok->isNumber() && argvalue == MathLib::toBigNumber(tok->str()))
             return true;
@@ -949,7 +949,7 @@ bool Library::isFloatArgValid(const Token *ftok, int argnr, double argvalue) con
     if (!ac || ac->valid.empty())
         return true;
     TokenList tokenList(nullptr);
-    gettokenlistfromvalid(ac->valid, tokenList);
+    gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
     for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (Token::Match(tok, "%num% : %num%") && argvalue >= MathLib::toDoubleNumber(tok->str()) && argvalue <= MathLib::toDoubleNumber(tok->strAt(2)))
             return true;
@@ -1738,13 +1738,14 @@ bool Library::hasAnyTypeCheck(const std::string& typeName) const
 
 std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
                                                  const Settings* settings,
+                                                 bool cpp,
                                                  std::unordered_map<nonneg int, const Token*>* lookupVarId)
 {
     std::shared_ptr<TokenList> tokenList = std::make_shared<TokenList>(settings);
     {
         const std::string code = "return " + returnValue + ";";
         std::istringstream istr(code);
-        if (!tokenList->createTokens(istr))
+        if (!tokenList->createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C))
             return nullptr;
     }
 

lib/library.h

@@ -607,6 +607,7 @@ CPPCHECKLIB const Library::Container * getLibraryContainer(const Token * tok);
 
 std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
                                                  const Settings* settings,
+                                                 bool cpp,
                                                  std::unordered_map<nonneg int, const Token*>* lookupVarId = nullptr);
 
 /// @}

lib/programmemory.cpp

@@ -1596,7 +1596,7 @@ namespace {
                                     arg_map[argn] = v;
                                 argn++;
                             }
-                            return evaluateLibraryFunction(arg_map, returnValue, settings);
+                            return evaluateLibraryFunction(arg_map, returnValue, settings, ftok->isCpp());
                         }
                     }
                 }
@@ -1752,15 +1752,16 @@ std::vector<ValueFlow::Value> execute(const Scope* scope, ProgramMemory& pm, con
 
 ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
                                          const std::string& returnValue,
-                                         const Settings* settings)
+                                         const Settings* settings,
+                                         bool cpp)
 {
     thread_local static std::unordered_map<std::string,
                                            std::function<ValueFlow::Value(const std::unordered_map<nonneg int, ValueFlow::Value>& arg)>>
     functions = {};
     if (functions.count(returnValue) == 0) {
 
         std::unordered_map<nonneg int, const Token*> lookupVarId;
-        std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, &lookupVarId);
+        std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, cpp, &lookupVarId);
 
         functions[returnValue] =
             [lookupVarId, expr, settings](const std::unordered_map<nonneg int, ValueFlow::Value>& xargs) {

lib/programmemory.h

@@ -211,7 +211,8 @@ ProgramMemory getProgramMemory(const Token* tok, const Token* expr, const ValueF
 
 ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
                                          const std::string& returnValue,
-                                         const Settings* settings);
+                                         const Settings* settings,
+                                         bool cpp);
 
 #endif
 

lib/symboldatabase.cpp

@@ -7585,7 +7585,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
                     ValueType valuetype;
                     TokenList tokenList(&mSettings);
                     std::istringstream istr(typestr+";");
-                    tokenList.createTokens(istr);
+                    tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
                     tokenList.simplifyStdType();
                     if (parsedecl(tokenList.front(), &valuetype, mDefaultSignedness, mSettings, mIsCpp)) {
                         valuetype.originalTypeName = typestr;
@@ -7675,7 +7675,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
                 }
                 TokenList tokenList(&mSettings);
                 std::istringstream istr(typestr+";");
-                if (tokenList.createTokens(istr)) {
+                if (tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C)) {
                     ValueType vt;
                     tokenList.simplifyPlatformTypes();
                     tokenList.simplifyStdType();

lib/token.h

@@ -1479,7 +1479,6 @@ class CPPCHECKLIB Token {
         mImpl->mDebug = td;
     }
 
-    /** defaults to C++ if it cannot be determined */
     bool isCpp() const;
 };
 

lib/tokenlist.cpp

@@ -17,6 +17,7 @@
  */
 
 //---------------------------------------------------------------------------
+
 #include "tokenlist.h"
 
 #include "astutils.h"
@@ -40,6 +41,15 @@
 
 #include <simplecpp.h>
 
+//#define N_ASSERT_LANG
+
+#ifndef N_ASSERT_LANG
+#include <cassert>
+#define ASSERT_LANG(x) assert(x)
+#else
+#define ASSERT_LANG(x)
+#endif
+
 // How many compileExpression recursions are allowed?
 // For practical code this could be endless. But in some special torture test
 // there needs to be a limit.
@@ -83,7 +93,7 @@ void TokenList::deallocateTokens()
 
 void TokenList::determineCppC()
 {
-    // only try to determine it if it wasn't enforced
+    // only try to determine if it wasn't enforced
     if (mLang == Standards::Language::None) {
         mLang = Path::identify(getSourceFilePath());
     }
@@ -317,8 +327,31 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n)
 
 bool TokenList::createTokens(std::istream &code, const std::string& file0)
 {
+    ASSERT_LANG(!file0.empty());
+
     appendFileIfNew(file0);
 
+    return createTokensInternal(code, file0);
+}
+
+//---------------------------------------------------------------------------
+
+bool TokenList::createTokens(std::istream &code, Standards::Language lang)
+{
+    ASSERT_LANG(lang != Standards::Language::None);
+    if (mLang == Standards::Language::None) {
+        mLang = lang;
+    } else {
+        ASSERT_LANG(lang == mLang);
+    }
+
+    return createTokensInternal(code, "");
+}
+
+//---------------------------------------------------------------------------
+
+bool TokenList::createTokensInternal(std::istream &code, const std::string& file0)
+{
     simplecpp::OutputList outputList;
     simplecpp::TokenList tokens(code, mFiles, file0, &outputList);
 
@@ -2096,3 +2129,33 @@ bool TokenList::isKeyword(const std::string &str) const
     static const auto& latest_c_keywords = Keywords::getAll(Standards::cstd_t::CLatest);
     return latest_c_keywords.find(str) != latest_c_keywords.end();
 }
+
+bool TokenList::isC() const
+{
+    ASSERT_LANG(mLang != Standards::Language::None);
+
+    // TODO: remove the fallback
+    if (mLang == Standards::Language::None)
+        return false; // treat as C++ by default
+
+    return mLang == Standards::Language::C;
+}
+
+bool TokenList::isCPP() const
+{
+    ASSERT_LANG(mLang != Standards::Language::None);
+
+    // TODO: remove the fallback
+    if (mLang == Standards::Language::None)
+        return true; // treat as C++ by default
+
+    return mLang == Standards::Language::CPP;
+}
+
+void TokenList::setLang(Standards::Language lang)
+{
+    ASSERT_LANG(lang != Standards::Language::None);
+    ASSERT_LANG(mLang == Standards::Language::None);
+
+    mLang = lang;
+}

lib/tokenlist.h

@@ -61,15 +61,13 @@ class CPPCHECKLIB TokenList {
     /** @return the source file path. e.g. "file.cpp" */
     const std::string& getSourceFilePath() const;
 
-    /** Is the code C. Used for bailouts */
-    bool isC() const {
-        return mLang == Standards::Language::C;
-    }
+    /** @return true if the code is C */
+    bool isC() const;
 
-    /** Is the code CPP. Used for bailouts */
-    bool isCPP() const {
-        return mLang == Standards::Language::CPP;
-    }
+    /** @return true if the code is C++ */
+    bool isCPP() const;
+
+    void setLang(Standards::Language lang);
 
     /**
      * Delete all tokens in given token list
@@ -105,7 +103,8 @@ class CPPCHECKLIB TokenList {
      * @param code input stream for code
      * @param file0 source file name
      */
-    bool createTokens(std::istream &code, const std::string& file0 = emptyString);
+    bool createTokens(std::istream &code, const std::string& file0);
+    bool createTokens(std::istream &code, Standards::Language lang);
 
     void createTokens(simplecpp::TokenList&& tokenList);
 
@@ -203,6 +202,8 @@ class CPPCHECKLIB TokenList {
 private:
     void determineCppC();
 
+    bool createTokensInternal(std::istream &code, const std::string& file0);
+
     /** Token list */
     TokensFrontBack mTokensFrontBack;
 

lib/valueflow.cpp

@@ -3810,20 +3810,20 @@ static bool isNotEqual(std::pair<const Token*, const Token*> x, std::pair<const
     start2 = skipCVRefs(start2, y.second);
     return !(start1 == x.second && start2 == y.second);
 }
-static bool isNotEqual(std::pair<const Token*, const Token*> x, const std::string& y)
+static bool isNotEqual(std::pair<const Token*, const Token*> x, const std::string& y, bool cpp)
 {
     TokenList tokenList(nullptr);
     std::istringstream istr(y);
-    tokenList.createTokens(istr);
+    tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C);
     return isNotEqual(x, std::make_pair(tokenList.front(), tokenList.back()));
 }
-static bool isNotEqual(std::pair<const Token*, const Token*> x, const ValueType* y)
+static bool isNotEqual(std::pair<const Token*, const Token*> x, const ValueType* y, bool cpp)
 {
     if (y == nullptr)
         return false;
     if (y->originalTypeName.empty())
         return false;
-    return isNotEqual(x, y->originalTypeName);
+    return isNotEqual(x, y->originalTypeName, cpp);
 }
 
 static bool isDifferentType(const Token* src, const Token* dst)
@@ -3838,9 +3838,9 @@ static bool isDifferentType(const Token* src, const Token* dst)
         std::pair<const Token*, const Token*> parentdecl = Token::typeDecl(dst);
         if (isNotEqual(decl, parentdecl))
             return true;
-        if (isNotEqual(decl, dst->valueType()))
+        if (isNotEqual(decl, dst->valueType(), dst->isCpp()))
             return true;
-        if (isNotEqual(parentdecl, src->valueType()))
+        if (isNotEqual(parentdecl, src->valueType(), src->isCpp()))
             return true;
     }
     return false;
@@ -7680,7 +7680,7 @@ static void valueFlowLibraryFunction(Token *tok, const std::string &returnValue,
     if (returnValue.find("arg") != std::string::npos && argValues.empty())
         return;
     productParams(settings, argValues, [&](const std::unordered_map<nonneg int, ValueFlow::Value>& arg) {
-        ValueFlow::Value value = evaluateLibraryFunction(arg, returnValue, &settings);
+        ValueFlow::Value value = evaluateLibraryFunction(arg, returnValue, &settings, tok->isCpp());
         if (value.isUninitValue())
             return;
         ValueFlow::Value::ValueKind kind = ValueFlow::Value::ValueKind::Known;
@@ -9174,11 +9174,11 @@ static bool getMinMaxValues(const ValueType *vt, const Platform &platform, MathL
     return true;
 }
 
-static bool getMinMaxValues(const std::string &typestr, const Settings &settings, MathLib::bigint &minvalue, MathLib::bigint &maxvalue)
+static bool getMinMaxValues(const std::string &typestr, const Settings &settings, bool cpp, MathLib::bigint &minvalue, MathLib::bigint &maxvalue)
 {
     TokenList typeTokens(&settings);
     std::istringstream istr(typestr+";");
-    if (!typeTokens.createTokens(istr))
+    if (!typeTokens.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C))
         return false;
     typeTokens.simplifyPlatformTypes();
     typeTokens.simplifyStdType();
@@ -9300,7 +9300,7 @@ static void valueFlowUnknownFunctionReturn(TokenList &tokenlist, const Settings
         // Get min/max values for return type
         const std::string &typestr = settings.library.returnValueType(tok->previous());
         MathLib::bigint minvalue, maxvalue;
-        if (!getMinMaxValues(typestr, settings, minvalue, maxvalue))
+        if (!getMinMaxValues(typestr, settings, tok->isCpp(), minvalue, maxvalue))
             continue;
 
         for (MathLib::bigint value : unknownValues) {