Skip to content

Commit

Permalink
Tokenizer: moved test-only tokenize() to SimpleTokenizer / severa…
Browse files Browse the repository at this point in the history
…l cleanups
  • Loading branch information
firewave committed Apr 10, 2024
1 parent 9e7578e commit 78052ba
Show file tree
Hide file tree
Showing 45 changed files with 1,224 additions and 1,467 deletions.
90 changes: 45 additions & 45 deletions Makefile

Large diffs are not rendered by default.

11 changes: 1 addition & 10 deletions lib/tokenize.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3439,16 +3439,6 @@ bool Tokenizer::simplifyTokens1(const std::string &configuration)
return true;
}

// cppcheck-suppress unusedFunction - used in tests only
bool Tokenizer::tokenize(std::istream &code,
const char FileName[],
const std::string &configuration)
{
if (!list.createTokens(code, FileName))
return false;

return simplifyTokens1(configuration);
}
//---------------------------------------------------------------------------

void Tokenizer::findComplicatedSyntaxErrorsInTemplates()
Expand Down Expand Up @@ -10659,6 +10649,7 @@ void Tokenizer::simplifyNamespaceAliases()
}
}

// TODO: how to move the Preprocessor dependency out of here?
bool Tokenizer::hasIfdef(const Token *start, const Token *end) const
{
assert(mPreprocessor);
Expand Down
24 changes: 0 additions & 24 deletions lib/tokenize.h
Original file line number Diff line number Diff line change
Expand Up @@ -79,30 +79,6 @@ class CPPCHECKLIB Tokenizer {
bool isScopeNoReturn(const Token *endScopeToken, bool *unknown = nullptr) const;

bool simplifyTokens1(const std::string &configuration);
/**
* Tokenize code
* @param code input stream for code, e.g.
* \code
* #file "p.h"
* class Foo
* {
* private:
* void Bar();
* };
*
* #endfile
* void Foo::Bar()
* {
* }
* \endcode
*
* @param FileName The filename
* @param configuration E.g. "A" for code where "#ifdef A" is true
* @return false if source code contains syntax errors
*/
bool tokenize(std::istream &code,
const char FileName[],
const std::string &configuration = emptyString);

private:
/** Set variable id */
Expand Down
3 changes: 3 additions & 0 deletions test/helpers.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@

class SuppressionList;

const Settings SimpleTokenizer::s_settings;
const Preprocessor SimpleTokenizer::s_preprocessor{s_settings, nullptr}; // TODO: provide ErrorLogger

// TODO: better path-only usage
ScopedFile::ScopedFile(std::string name, const std::string &content, std::string path)
: mName(std::move(name))
Expand Down
47 changes: 37 additions & 10 deletions test/helpers.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#ifndef helpersH
#define helpersH

#include "preprocessor.h"
#include "settings.h"
#include "standards.h"
#include "tokenize.h"
Expand All @@ -31,34 +32,60 @@
#include <vector>

class Token;
class Preprocessor;
class SuppressionList;
class ErrorLogger;
namespace simplecpp {
struct DUI;
}

class SimpleTokenizer {
// TODO: make Tokenizer private
class SimpleTokenizer : public Tokenizer {
public:
SimpleTokenizer(ErrorLogger& errorlogger, const char sample[], bool cpp = true)
: tokenizer{settings, &errorlogger}
SimpleTokenizer(ErrorLogger& errorlogger, const char code[], bool cpp = true)
: Tokenizer{s_settings, &errorlogger, &s_preprocessor}
{
std::istringstream iss(sample);
if (!tokenizer.tokenize(iss, cpp ? "test.cpp" : "test.c"))
if (!tokenize(code, cpp))
throw std::runtime_error("creating tokens failed");
}

SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger)
: Tokenizer{settings, &errorlogger, &s_preprocessor}
{}

SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, const Preprocessor* preprocessor)
: Tokenizer{settings, &errorlogger, preprocessor}
{}

Token* tokens() {
return tokenizer.tokens();
return Tokenizer::tokens();
}

const Token* tokens() const {
return tokenizer.tokens();
return Tokenizer::tokens();
}

/**
* Tokenize code
* @param code The code
* @param cpp Indicates if the code is C++
* @param configuration E.g. "A" for code where "#ifdef A" is true
* @return false if source code contains syntax errors
*/
bool tokenize(const char code[],
bool cpp = true,
const std::string &configuration = emptyString)
{
std::istringstream istr(code);
if (!list.createTokens(istr, cpp ? "test.cpp" : "test.c"))
return false;

return simplifyTokens1(configuration);
}

private:
const Settings settings;
Tokenizer tokenizer;
// TODO. find a better solution
static const Settings s_settings;
static const Preprocessor s_preprocessor;
};

class SimpleTokenList
Expand Down
12 changes: 4 additions & 8 deletions test/test64bit.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,11 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/


#include "check64bit.h"
#include "errortypes.h"
#include "settings.h"
#include "fixture.h"
#include "tokenize.h"

#include <sstream>
#include "helpers.h"
#include "settings.h"

class Test64BitPortability : public TestFixture {
public:
Expand All @@ -45,9 +42,8 @@ class Test64BitPortability : public TestFixture {
#define check(code) check_(code, __FILE__, __LINE__)
void check_(const char code[], const char* file, int line) {
// Tokenize..
Tokenizer tokenizer(settings, this);
std::istringstream istr(code);
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check char variable usage..
Check64BitPortability check64BitPortability(&tokenizer, &settings, this);
Expand Down
14 changes: 5 additions & 9 deletions test/testassert.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,9 @@

#include "checkassert.h"
#include "errortypes.h"
#include "settings.h"
#include "fixture.h"
#include "tokenize.h"

#include <sstream>

#include "helpers.h"
#include "settings.h"

class TestAssert : public TestFixture {
public:
Expand All @@ -34,11 +31,10 @@ class TestAssert : public TestFixture {
const Settings settings = settingsBuilder().severity(Severity::warning).build();

#define check(...) check_(__FILE__, __LINE__, __VA_ARGS__)
void check_(const char* file, int line, const char code[], const char *filename = "test.cpp") {
void check_(const char* file, int line, const char code[]) {
// Tokenize..
Tokenizer tokenizer(settings, this);
std::istringstream istr(code);
ASSERT_LOC(tokenizer.tokenize(istr, filename), file, line);
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);

// Check..
runChecks<CheckAssert>(tokenizer, this);
Expand Down
56 changes: 22 additions & 34 deletions test/testastutils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,16 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/


#include "astutils.h"
#include "fixture.h"
#include "helpers.h"
#include "library.h"
#include "settings.h"
#include "fixture.h"
#include "symboldatabase.h"
#include "token.h"
#include "tokenize.h"
#include "tokenlist.h"

#include <cstring>
#include <sstream>

class TestAstUtils : public TestFixture {
public:
Expand All @@ -54,9 +52,8 @@ class TestAstUtils : public TestFixture {
#define findLambdaEndToken(...) findLambdaEndToken_(__FILE__, __LINE__, __VA_ARGS__)
bool findLambdaEndToken_(const char* file, int line, const char code[], const char pattern[] = nullptr, bool checkNext = true) {
const Settings settings;
Tokenizer tokenizer(settings, this);
std::istringstream istr(code);
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);
const Token* const tokStart = pattern ? Token::findsimplematch(tokenizer.tokens(), pattern, strlen(pattern)) : tokenizer.tokens();
const Token * const tokEnd = (::findLambdaEndToken)(tokStart);
return tokEnd && (!checkNext || tokEnd->next() == nullptr);
Expand Down Expand Up @@ -92,9 +89,8 @@ class TestAstUtils : public TestFixture {

#define findLambdaStartToken(code) findLambdaStartToken_(code, __FILE__, __LINE__)
bool findLambdaStartToken_(const char code[], const char* file, int line) {
Tokenizer tokenizer(settingsDefault, this);
std::istringstream istr(code);
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
SimpleTokenizer tokenizer(settingsDefault, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);
const Token * const tokStart = (::findLambdaStartToken)(tokenizer.list.back());
return tokStart && tokStart == tokenizer.list.front();
}
Expand Down Expand Up @@ -124,9 +120,8 @@ class TestAstUtils : public TestFixture {

#define isNullOperand(code) isNullOperand_(code, __FILE__, __LINE__)
bool isNullOperand_(const char code[], const char* file, int line) {
Tokenizer tokenizer(settingsDefault, this);
std::istringstream istr(code);
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
SimpleTokenizer tokenizer(settingsDefault, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);
return (::isNullOperand)(tokenizer.tokens());
}

Expand All @@ -145,9 +140,8 @@ class TestAstUtils : public TestFixture {

#define isReturnScope(code, offset) isReturnScope_(code, offset, __FILE__, __LINE__)
bool isReturnScope_(const char code[], int offset, const char* file, int line) {
Tokenizer tokenizer(settingsDefault, this);
std::istringstream istr(code);
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
SimpleTokenizer tokenizer(settingsDefault, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);
const Token * const tok = (offset < 0)
? tokenizer.list.back()->tokAt(1+offset)
: tokenizer.tokens()->tokAt(offset);
Expand Down Expand Up @@ -176,9 +170,8 @@ class TestAstUtils : public TestFixture {
#define isSameExpression(...) isSameExpression_(__FILE__, __LINE__, __VA_ARGS__)
bool isSameExpression_(const char* file, int line, const char code[], const char tokStr1[], const char tokStr2[], bool cpp) {
Library library;
Tokenizer tokenizer(settingsDefault, this);
std::istringstream istr(code);
ASSERT_LOC(tokenizer.tokenize(istr, cpp ? "test.cpp" : "test.c"), file, line);
SimpleTokenizer tokenizer(settingsDefault, *this);
ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);
const Token * const tok1 = Token::findsimplematch(tokenizer.tokens(), tokStr1, strlen(tokStr1));
const Token * const tok2 = Token::findsimplematch(tok1->next(), tokStr2, strlen(tokStr2));
return (isSameExpression)(false, tok1, tok2, library, false, true, nullptr);
Expand Down Expand Up @@ -224,9 +217,8 @@ class TestAstUtils : public TestFixture {

#define isVariableChanged(code, startPattern, endPattern) isVariableChanged_(code, startPattern, endPattern, __FILE__, __LINE__)
bool isVariableChanged_(const char code[], const char startPattern[], const char endPattern[], const char* file, int line) {
Tokenizer tokenizer(settingsDefault, this);
std::istringstream istr(code);
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
SimpleTokenizer tokenizer(settingsDefault, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);
const Token * const tok1 = Token::findsimplematch(tokenizer.tokens(), startPattern, strlen(startPattern));
const Token * const tok2 = Token::findsimplematch(tokenizer.tokens(), endPattern, strlen(endPattern));
return (isVariableChanged)(tok1, tok2, 1, false, &settingsDefault);
Expand Down Expand Up @@ -258,9 +250,8 @@ class TestAstUtils : public TestFixture {

#define isVariableChangedByFunctionCall(code, pattern, inconclusive) isVariableChangedByFunctionCall_(code, pattern, inconclusive, __FILE__, __LINE__)
bool isVariableChangedByFunctionCall_(const char code[], const char pattern[], bool *inconclusive, const char* file, int line) {
Tokenizer tokenizer(settingsDefault, this);
std::istringstream istr(code);
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
SimpleTokenizer tokenizer(settingsDefault, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);
const Token * const argtok = Token::findmatch(tokenizer.tokens(), pattern);
ASSERT_LOC(argtok, file, line);
int indirect = (argtok->variable() && argtok->variable()->isArray());
Expand Down Expand Up @@ -401,9 +392,8 @@ class TestAstUtils : public TestFixture {
int line)
{
const Settings settings = settingsBuilder().library("std.cfg").build();
Tokenizer tokenizer(settings, this);
std::istringstream istr(code);
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
SimpleTokenizer tokenizer(settings, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);
const Token* const start = Token::findsimplematch(tokenizer.tokens(), startPattern, strlen(startPattern));
const Token* const end = Token::findsimplematch(start, endPattern, strlen(endPattern));
const Token* const expr = Token::findsimplematch(tokenizer.tokens(), var, strlen(var));
Expand Down Expand Up @@ -432,9 +422,8 @@ class TestAstUtils : public TestFixture {

#define nextAfterAstRightmostLeaf(code, parentPattern, rightPattern) nextAfterAstRightmostLeaf_(code, parentPattern, rightPattern, __FILE__, __LINE__)
bool nextAfterAstRightmostLeaf_(const char code[], const char parentPattern[], const char rightPattern[], const char* file, int line) {
Tokenizer tokenizer(settingsDefault, this);
std::istringstream istr(code);
ASSERT_LOC(tokenizer.tokenize(istr, "test.cpp"), file, line);
SimpleTokenizer tokenizer(settingsDefault, *this);
ASSERT_LOC(tokenizer.tokenize(code), file, line);
const Token * tok = Token::findsimplematch(tokenizer.tokens(), parentPattern, strlen(parentPattern));
return Token::simpleMatch((::nextAfterAstRightmostLeaf)(tok), rightPattern, strlen(rightPattern));
}
Expand All @@ -456,9 +445,8 @@ class TestAstUtils : public TestFixture {
enum class Result {False, True, Fail};

Result isUsedAsBool(const char code[], const char pattern[]) {
Tokenizer tokenizer(settingsDefault, this);
std::istringstream istr(code);
if (!tokenizer.tokenize(istr, "test.cpp"))
SimpleTokenizer tokenizer(settingsDefault, *this);
if (!tokenizer.tokenize(code))
return Result::Fail;
const Token * const argtok = Token::findmatch(tokenizer.tokens(), pattern);
if (!argtok)
Expand Down
16 changes: 6 additions & 10 deletions test/testautovariables.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,11 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/


#include "checkautovariables.h"
#include "errortypes.h"
#include "settings.h"
#include "fixture.h"
#include "tokenize.h"

#include <sstream>
#include "helpers.h"
#include "settings.h"

class TestAutoVariables : public TestFixture {
public:
Expand All @@ -33,13 +30,12 @@ class TestAutoVariables : public TestFixture {
const Settings settings = settingsBuilder().severity(Severity::warning).severity(Severity::style).library("std.cfg").library("qt.cfg").build();

#define check(...) check_(__FILE__, __LINE__, __VA_ARGS__)
void check_(const char* file, int line, const char code[], bool inconclusive = true, const char* filename = "test.cpp") {
void check_(const char* file, int line, const char code[], bool inconclusive = true, bool cpp = true) {
const Settings settings1 = settingsBuilder(settings).certainty(Certainty::inconclusive, inconclusive).build();

// Tokenize..
Tokenizer tokenizer(settings1, this);
std::istringstream istr(code);
ASSERT_LOC(tokenizer.tokenize(istr, filename), file, line);
SimpleTokenizer tokenizer(settings1, *this);
ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);

runChecks<CheckAutoVariables>(tokenizer, this);
}
Expand Down Expand Up @@ -916,7 +912,7 @@ class TestAutoVariables : public TestFixture {
check("void svn_repos_dir_delta2() {\n"
" struct context c;\n"
" SVN_ERR(delete(&c, root_baton, src_entry, pool));\n"
"}\n", false, "test.c");
"}\n", false, /* cpp= */ false);
ASSERT_EQUALS("", errout_str());
}

Expand Down
Loading

0 comments on commit 78052ba

Please sign in to comment.