Skip to content

Commit

Permalink
testrunner: refactored givenACodeSampleToTokenize into `SimpleToken…
Browse files Browse the repository at this point in the history 
…izer` and `SimpleTokenList` / several related cleanups (#6084)
  • Loading branch information
@firewave
firewave committed Mar 10, 2024
1 parent f9f3fe0 commit fadc93f
Show file tree
Hide file tree
Showing 9 changed files with 289 additions and 287 deletions.
4 changes: 2 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -856,10 +856,10 @@ test/testtoken.o: test/testtoken.cpp lib/addoninfo.h lib/check.h lib/color.h lib
test/testtokenize.o: test/testtokenize.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testtokenize.cpp

test/testtokenlist.o: test/testtokenlist.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
test/testtokenlist.o: test/testtokenlist.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testtokenlist.cpp

test/testtokenrange.o: test/testtokenrange.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/tokenrange.h lib/utils.h lib/vfvalue.h test/fixture.h
test/testtokenrange.o: test/testtokenrange.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/tokenrange.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testtokenrange.cpp

test/testtype.o: test/testtype.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/checktype.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h test/helpers.h
Expand Down
48 changes: 37 additions & 11 deletions test/helpers.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,34 +20,32 @@
#define helpersH

#include "settings.h"
#include "standards.h"
#include "tokenize.h"
#include "tokenlist.h"

#include <cstddef>
#include <stdexcept>
#include <sstream>
#include <string>
#include <vector>

class Token;
class Preprocessor;
class SuppressionList;
class ErrorLogger;
namespace simplecpp {
struct DUI;
}

class givenACodeSampleToTokenize {
private:
const Settings settings;
Tokenizer tokenizer;

class SimpleTokenizer {
public:
explicit givenACodeSampleToTokenize(const char sample[], bool createOnly = false, bool cpp = true)
: tokenizer(settings, nullptr) {
SimpleTokenizer(ErrorLogger& errorlogger, const char sample[], bool cpp = true)
: tokenizer{settings, &errorlogger}
{
std::istringstream iss(sample);
if (createOnly)
tokenizer.list.createTokens(iss, cpp ? "test.cpp" : "test.c");
else
tokenizer.tokenize(iss, cpp ? "test.cpp" : "test.c");
if (!tokenizer.tokenize(iss, cpp ? "test.cpp" : "test.c"))
throw std::runtime_error("creating tokens failed");
}

Token* tokens() {
Expand All @@ -57,6 +55,34 @@ class givenACodeSampleToTokenize {
const Token* tokens() const {
return tokenizer.tokens();
}

private:
const Settings settings;
Tokenizer tokenizer;
};

class SimpleTokenList
{
public:

explicit SimpleTokenList(const char code[], Standards::Language lang = Standards::Language::CPP)
{
std::istringstream iss(code);
if (!list.createTokens(iss, lang))
throw std::runtime_error("creating tokens failed");
}

Token* front() {
return list.front();
}

const Token* front() const {
return list.front();
}

private:
const Settings settings;
TokenList list{&settings};
};


Expand Down
83 changes: 36 additions & 47 deletions test/testlibrary.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,8 @@ class TestLibrary : public TestFixture {
" </function>\n"
"</def>";

TokenList tokenList(&settings);
std::istringstream istr("foo();");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "foo();";
SimpleTokenList tokenList(code);
tokenList.front()->next()->astOperand1(tokenList.front());

Library library;
Expand All @@ -140,16 +139,14 @@ class TestLibrary : public TestFixture {
Library library;
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));
{
TokenList tokenList(&settings);
std::istringstream istr("fred.foo(123);"); // <- wrong scope, not library function
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "fred.foo(123);"; // <- wrong scope, not library function
const SimpleTokenList tokenList(code);

ASSERT(library.isNotLibraryFunction(tokenList.front()->tokAt(2)));
}
{
TokenList tokenList(&settings);
std::istringstream istr("Fred::foo(123);"); // <- wrong scope, not library function
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "Fred::foo(123);"; // <- wrong scope, not library function
const SimpleTokenList tokenList(code);

ASSERT(library.isNotLibraryFunction(tokenList.front()->tokAt(2)));
}
Expand All @@ -165,7 +162,7 @@ class TestLibrary : public TestFixture {

TokenList tokenList(&settings);
std::istringstream istr("foo();"); // <- too few arguments, not library function
tokenList.createTokens(istr, Standards::Language::CPP);
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
tokenList.createAst();

Expand All @@ -189,7 +186,7 @@ class TestLibrary : public TestFixture {
{
TokenList tokenList(&settings);
std::istringstream istr("foo();"); // <- too few arguments, not library function
tokenList.createTokens(istr, Standards::Language::CPP);
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
tokenList.createAst();

Expand All @@ -198,7 +195,7 @@ class TestLibrary : public TestFixture {
{
TokenList tokenList(&settings);
std::istringstream istr("foo(a);"); // <- library function
tokenList.createTokens(istr, Standards::Language::CPP);
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
tokenList.createAst();

Expand All @@ -207,7 +204,7 @@ class TestLibrary : public TestFixture {
{
TokenList tokenList(&settings);
std::istringstream istr("foo(a, b);"); // <- library function
tokenList.createTokens(istr, Standards::Language::CPP);
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
tokenList.createAst();

Expand All @@ -216,7 +213,7 @@ class TestLibrary : public TestFixture {
{
TokenList tokenList(&settings);
std::istringstream istr("foo(a, b, c);"); // <- too much arguments, not library function
tokenList.createTokens(istr, Standards::Language::CPP);
ASSERT(tokenList.createTokens(istr, Standards::Language::CPP));
Token::createMutualLinks(tokenList.front()->next(), tokenList.back()->previous());
tokenList.createAst();

Expand All @@ -232,9 +229,8 @@ class TestLibrary : public TestFixture {
" </function>\n"
"</def>";

TokenList tokenList(&settings);
std::istringstream istr("Fred foo(123);"); // <- Variable declaration, not library function
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "Fred foo(123);"; // <- Variable declaration, not library function
SimpleTokenList tokenList(code);
tokenList.front()->next()->astOperand1(tokenList.front());
tokenList.front()->next()->varId(1);

Expand Down Expand Up @@ -292,9 +288,8 @@ class TestLibrary : public TestFixture {
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));
ASSERT_EQUALS(0, library.functions["foo"].argumentChecks[-1].notuninit);

TokenList tokenList(&settings);
std::istringstream istr("foo(a,b,c,d,e);");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "foo(a,b,c,d,e);";
SimpleTokenList tokenList(code);
tokenList.front()->next()->astOperand1(tokenList.front());

ASSERT_EQUALS(false, library.isuninitargbad(tokenList.front(), 1));
Expand All @@ -317,9 +312,8 @@ class TestLibrary : public TestFixture {
Library library;
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));

TokenList tokenList(&settings);
std::istringstream istr("foo(a,b,c,d);");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "foo(a,b,c,d);";
SimpleTokenList tokenList(code);
tokenList.front()->next()->astOperand1(tokenList.front());

ASSERT(Library::ArgumentChecks::Direction::DIR_IN == library.getArgDirection(tokenList.front(), 1));
Expand Down Expand Up @@ -349,9 +343,8 @@ class TestLibrary : public TestFixture {
Library library;
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));

TokenList tokenList(&settings);
std::istringstream istr("foo(a,b,c,d,e,f,g,h,i,j,k);");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "foo(a,b,c,d,e,f,g,h,i,j,k);";
SimpleTokenList tokenList(code);
tokenList.front()->next()->astOperand1(tokenList.front());

// 1-
Expand Down Expand Up @@ -491,9 +484,8 @@ class TestLibrary : public TestFixture {
Library library;
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));

TokenList tokenList(&settings);
std::istringstream istr("foo(a,b,c,d,e);");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "foo(a,b,c,d,e);";
SimpleTokenList tokenList(code);
tokenList.front()->next()->astOperand1(tokenList.front());

// arg1: type=strlen arg2
Expand Down Expand Up @@ -554,16 +546,14 @@ class TestLibrary : public TestFixture {
ASSERT(library.functions.at("bar").argumentChecks.empty());

{
TokenList tokenList(&settings);
std::istringstream istr("Foo::foo();");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "Foo::foo();";
const SimpleTokenList tokenList(code);
ASSERT(library.isnotnoreturn(tokenList.front()->tokAt(2)));
}

{
TokenList tokenList(&settings);
std::istringstream istr("bar();");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "bar();";
const SimpleTokenList tokenList(code);
ASSERT(library.isnotnoreturn(tokenList.front()));
}
}
Expand Down Expand Up @@ -635,9 +625,8 @@ class TestLibrary : public TestFixture {
Library library;
ASSERT(loadxmldata(library, xmldata, sizeof(xmldata)));

TokenList tokenList(&settings);
std::istringstream istr("a(); b();");
tokenList.createTokens(istr, Standards::Language::CPP);
const char code[] = "a(); b();";
const SimpleTokenList tokenList(code);

const Library::WarnInfo* a = library.getWarnInfo(tokenList.front());
const Library::WarnInfo* b = library.getWarnInfo(tokenList.front()->tokAt(4));
Expand Down Expand Up @@ -791,7 +780,7 @@ class TestLibrary : public TestFixture {
}
}

void container() const {
void container() {
constexpr char xmldata[] = "<?xml version=\"1.0\"?>\n"
"<def>\n"
" <container id=\"A\" startPattern=\"std :: A &lt;\" endPattern=\"&gt; !!::\" itEndPattern=\"&gt; :: iterator\">\n"
Expand Down Expand Up @@ -871,7 +860,7 @@ class TestLibrary : public TestFixture {
ASSERT_EQUALS(C.arrayLike_indexOp, true);

{
givenACodeSampleToTokenize var("std::A<int> a;");
const SimpleTokenizer var(*this, "std::A<int> a;");
ASSERT_EQUALS(&A, library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
bool isIterator;
Expand All @@ -880,14 +869,14 @@ class TestLibrary : public TestFixture {
}

{
givenACodeSampleToTokenize var("std::A<int>::size_type a_s;");
const SimpleTokenizer var(*this, "std::A<int>::size_type a_s;");
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
}

{
givenACodeSampleToTokenize var("std::A<int>::iterator a_it;");
const SimpleTokenizer var(*this, "std::A<int>::iterator a_it;");
ASSERT(!library.detectContainer(var.tokens()));
ASSERT_EQUALS(&A, library.detectIterator(var.tokens()));
bool isIterator;
Expand All @@ -896,7 +885,7 @@ class TestLibrary : public TestFixture {
}

{
givenACodeSampleToTokenize var("std::B<int> b;");
const SimpleTokenizer var(*this, "std::B<int> b;");
ASSERT_EQUALS(&B, library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
bool isIterator;
Expand All @@ -905,14 +894,14 @@ class TestLibrary : public TestFixture {
}

{
givenACodeSampleToTokenize var("std::B<int>::size_type b_s;");
const SimpleTokenizer var(*this, "std::B<int>::size_type b_s;");
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
}

{
givenACodeSampleToTokenize var("std::B<int>::iterator b_it;");
const SimpleTokenizer var(*this, "std::B<int>::iterator b_it;");
ASSERT(!library.detectContainer(var.tokens()));
ASSERT_EQUALS(&B, library.detectIterator(var.tokens()));
bool isIterator;
Expand All @@ -921,14 +910,14 @@ class TestLibrary : public TestFixture {
}

{
givenACodeSampleToTokenize var("C c;");
const SimpleTokenizer var(*this, "C c;");
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
}

{
givenACodeSampleToTokenize var("D d;");
const SimpleTokenizer var(*this, "D d;");
ASSERT(!library.detectContainer(var.tokens()));
ASSERT(!library.detectIterator(var.tokens()));
ASSERT(!library.detectContainerOrIterator(var.tokens()));
Expand Down
12 changes: 8 additions & 4 deletions test/testsimplifytemplate.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5290,7 +5290,8 @@ class TestSimplifyTemplate : public TestFixture {
Tokenizer tokenizer(settings, this);

std::istringstream istr(code);
tokenizer.list.createTokens(istr, "test.cpp");
if (!tokenizer.list.createTokens(istr, "test.cpp"))
return false;
tokenizer.createLinks();
tokenizer.splitTemplateRightAngleBrackets(false);

Expand Down Expand Up @@ -5357,7 +5358,8 @@ class TestSimplifyTemplate : public TestFixture {
Tokenizer tokenizer(settings, this);

std::istringstream istr(code);
tokenizer.list.createTokens(istr, "test.cpp");
if (!tokenizer.list.createTokens(istr, "test.cpp"))
return false;
tokenizer.createLinks();
tokenizer.splitTemplateRightAngleBrackets(false);

Expand Down Expand Up @@ -5427,7 +5429,8 @@ class TestSimplifyTemplate : public TestFixture {
Tokenizer tokenizer(settings, this);

std::istringstream istr(code);
tokenizer.list.createTokens(istr, "test.cpp");
if (!tokenizer.list.createTokens(istr, "test.cpp"))
return false;
tokenizer.createLinks();
tokenizer.splitTemplateRightAngleBrackets(false);

Expand Down Expand Up @@ -5456,7 +5459,8 @@ class TestSimplifyTemplate : public TestFixture {
Tokenizer tokenizer(settings, this);

std::istringstream istr(code);
tokenizer.list.createTokens(istr, "test.cpp");
if (!tokenizer.list.createTokens(istr, "test.cpp"))
return false;
tokenizer.createLinks();
tokenizer.splitTemplateRightAngleBrackets(false);

Expand Down
Loading

0 comments on commit fadc93f

Please sign in to comment.