Skip to content

Commit

Permalink
testrunner: make sure that more redirects/outputs are actually being …
Browse files Browse the repository at this point in the history 
…consumed (danmar#5721)
  • Loading branch information
@firewave
firewave committed Dec 6, 2023
1 parent 613bbe7 commit 4538002
Show file tree
Hide file tree
Showing 8 changed files with 50 additions and 14 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -758,7 +758,7 @@ test/testfunctions.o: test/testfunctions.cpp lib/addoninfo.h lib/check.h lib/che
test/testgarbage.o: test/testgarbage.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h test/fixture.h
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testgarbage.cpp

test/testimportproject.o: test/testimportproject.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/filesettings.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/utils.h test/fixture.h
test/testimportproject.o: test/testimportproject.cpp lib/addoninfo.h lib/check.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/filesettings.h lib/importproject.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/utils.h test/fixture.h test/redirect.h
$(CXX) ${INCLUDE_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/testimportproject.cpp

test/testincompletestatement.o: test/testincompletestatement.cpp lib/addoninfo.h lib/check.h lib/checkother.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/tokenize.h lib/tokenlist.h lib/utils.h test/fixture.h test/helpers.h
Expand Down
17 changes: 16 additions & 1 deletion test/fixture.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,20 @@ bool TestFixture::prepareTest(const char testname[])
void TestFixture::teardownTest()
{
teardownTestInternal();

// TODO: enable
/*
{
const std::string s = errout.str();
if (!s.empty())
throw std::runtime_error("unconsumed ErrorLogger err: " + s);
}
*/
{
const std::string s = output_str();
if (!s.empty())
throw std::runtime_error("unconsumed ErrorLogger out: " + s);
}
}

std::string TestFixture::getLocationStr(const char * const filename, const unsigned int linenr) const
Expand Down Expand Up @@ -381,14 +395,15 @@ std::size_t TestFixture::runTests(const options& args)

void TestFixture::reportOut(const std::string & outmsg, Color /*c*/)
{
output << outmsg << std::endl;
mOutput << outmsg << std::endl;
}

void TestFixture::reportErr(const ErrorMessage &msg)
{
if (msg.severity == Severity::none && msg.id == "logChecker")
return;
const std::string errormessage(msg.toString(mVerbose, mTemplateFormat, mTemplateLocation));
// TODO: remove the unique error handling?
if (errout.str().find(errormessage) == std::string::npos)
errout << errormessage << std::endl;
}
Expand Down
10 changes: 8 additions & 2 deletions test/fixture.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,11 +236,17 @@ class TestFixture : public ErrorLogger {
return SettingsBuilder(*this, std::move(settings));
}

// TODO: make sure the output has been consumed in the test
std::string output_str() {
std::string s = mOutput.str();
mOutput.str("");
return s;
}

std::ostringstream errout;
std::ostringstream output;

private:
std::ostringstream mOutput;

void reportOut(const std::string &outmsg, Color c = Color::Reset) override;
void reportErr(const ErrorMessage &msg) override;
void run(const std::string &str);
Expand Down
16 changes: 16 additions & 0 deletions test/testimportproject.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include "settings.h"
#include "filesettings.h"
#include "fixture.h"
#include "redirect.h"

#include <list>
#include <map>
Expand Down Expand Up @@ -112,6 +113,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands1() const {
REDIRECT;
constexpr char json[] = R"([{
"directory": "/tmp",
"command": "gcc -DTEST1 -DTEST2=2 -o /tmp/src.o -c /tmp/src.c",
Expand All @@ -125,6 +127,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands2() const {
REDIRECT;
// Absolute file path
#ifdef _WIN32
const char json[] = R"([{
Expand Down Expand Up @@ -152,6 +155,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands3() const {
REDIRECT;
const char json[] = R"([{
"directory": "/tmp/",
"command": "gcc -c src.c",
Expand All @@ -165,6 +169,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands4() const {
REDIRECT;
constexpr char json[] = R"([{
"directory": "/tmp/",
"command": "gcc -c src.mm",
Expand All @@ -177,6 +182,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands5() const {
REDIRECT;
constexpr char json[] =
R"([{
"directory": "C:/Users/dan/git/build-test-cppcheck-Desktop_Qt_5_15_0_MSVC2019_64bit-Debug",
Expand All @@ -196,6 +202,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands6() const {
REDIRECT;
constexpr char json[] =
R"([{
"directory": "C:/Users/dan/git/build-test-cppcheck-Desktop_Qt_5_15_0_MSVC2019_64bit-Debug",
Expand All @@ -217,6 +224,7 @@ class TestImportProject : public TestFixture {


void importCompileCommands7() const {
REDIRECT;
// cmake -DFILESDIR="/some/path" ..
constexpr char json[] =
R"([{
Expand All @@ -237,6 +245,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands8() const {
REDIRECT;
// cmake -DFILESDIR="C:\Program Files\Cppcheck" -G"NMake Makefiles" ..
constexpr char json[] =
R"([{
Expand All @@ -250,6 +259,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands9() const {
REDIRECT;
// IAR output (https://sourceforge.net/p/cppcheck/discussion/general/thread/608af51e0a/)
constexpr char json[] =
R"([{
Expand All @@ -266,6 +276,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands10() const { // #10887
REDIRECT;
constexpr char json[] =
R"([{
"file": "/home/danielm/cppcheck/1/test folder/1.c" ,
Expand All @@ -285,6 +296,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommands11() const { // include path order
REDIRECT;
constexpr char json[] =
R"([{
"file": "1.c" ,
Expand All @@ -307,6 +319,7 @@ class TestImportProject : public TestFixture {
}

void importCompileCommandsArgumentsSection() const {
REDIRECT;
constexpr char json[] = "[ { \"directory\": \"/tmp/\","
"\"arguments\": [\"gcc\", \"-c\", \"src.c\"],"
"\"file\": \"src.c\" } ]";
Expand All @@ -318,15 +331,18 @@ class TestImportProject : public TestFixture {
}

void importCompileCommandsNoCommandSection() const {
REDIRECT;
constexpr char json[] = "[ { \"directory\": \"/tmp/\","
"\"file\": \"src.mm\" } ]";
std::istringstream istr(json);
TestImporter importer;
ASSERT_EQUALS(false, importer.importCompileCommands(istr));
ASSERT_EQUALS(0, importer.fileSettings.size());
ASSERT_EQUALS("cppcheck: error: no 'arguments' or 'command' field found in compilation database entry\n", GET_REDIRECT_OUTPUT);
}

void importCppcheckGuiProject() const {
REDIRECT;
constexpr char xml[] = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<project version=\"1\">\n"
" <root name=\".\"/>\n"
Expand Down
3 changes: 1 addition & 2 deletions test/testprocessexecutor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ class TestProcessExecutorBase : public TestFixture {
*/
void check(unsigned int jobs, int files, int result, const std::string &data, const CheckOptions& opt = make_default_obj{}) {
errout.str("");
output.str("");

std::list<FileSettings> fileSettings;

Expand Down Expand Up @@ -265,7 +264,7 @@ class TestProcessExecutorBase : public TestFixture {
$.executeCommandCalled = true,
$.exe = exe,
$.args = {"-quiet", "-checks=*,-clang-analyzer-*,-llvm*", file, "--"}*/));
ASSERT_EQUALS("Checking " + file + " ...\n", output.str());
ASSERT_EQUALS("Checking " + file + " ...\n", output_str());
}

// TODO: provide data which actually shows values above 0
Expand Down
5 changes: 2 additions & 3 deletions test/testsingleexecutor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ class TestSingleExecutorBase : public TestFixture {

void check(int files, int result, const std::string &data, const CheckOptions& opt = make_default_obj{}) {
errout.str("");
output.str("");

std::list<FileSettings> fileSettings;

Expand Down Expand Up @@ -169,7 +168,7 @@ class TestSingleExecutorBase : public TestFixture {
expected += "Checking " + fprefix() + "_" + zpad3(i) + ".cpp ...\n";
expected += std::to_string(i) + "/100 files checked " + std::to_string(i) + "% done\n";
}
ASSERT_EQUALS(expected, output.str());
ASSERT_EQUALS(expected, output_str());
}

void many_files_showtime() {
Expand Down Expand Up @@ -259,7 +258,7 @@ class TestSingleExecutorBase : public TestFixture {
$.executeCommandCalled = true,
$.exe = exe,
$.args = {"-quiet", "-checks=*,-clang-analyzer-*,-llvm*", file, "--"}));
ASSERT_EQUALS("Checking " + file + " ...\n", output.str());
ASSERT_EQUALS("Checking " + file + " ...\n", output_str());
}

// TODO: provide data which actually shows values above 0
Expand Down
8 changes: 5 additions & 3 deletions test/testsuppressions.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,6 @@ class TestSuppressions : public TestFixture {
unsigned int checkSuppression(std::map<std::string, std::string> &f, const std::string &suppression = emptyString) {
// Clear the error log
errout.str("");
output.str("");

std::list<std::pair<std::string, std::size_t>> files;
for (std::map<std::string, std::string>::const_iterator i = f.cbegin(); i != f.cend(); ++i) {
Expand All @@ -210,6 +209,7 @@ class TestSuppressions : public TestFixture {
CppCheck cppCheck(*this, true, nullptr);
Settings& settings = cppCheck.settings();
settings.jobs = 1;
settings.quiet = true;
settings.inlineSuppressions = true;
settings.severity.enable(Severity::information);
if (suppression == "unusedFunction")
Expand All @@ -234,13 +234,13 @@ class TestSuppressions : public TestFixture {

unsigned int checkSuppressionThreads(const char code[], const std::string &suppression = emptyString) {
errout.str("");
output.str("");

std::list<std::pair<std::string, std::size_t>> files;
files.emplace_back("test.cpp", strlen(code));

Settings settings;
settings.jobs = 2;
settings.quiet = true;
settings.inlineSuppressions = true;
settings.severity.enable(Severity::information);
if (!suppression.empty()) {
Expand All @@ -264,13 +264,13 @@ class TestSuppressions : public TestFixture {
#if !defined(WIN32) && !defined(__MINGW32__) && !defined(__CYGWIN__)
unsigned int checkSuppressionProcesses(const char code[], const std::string &suppression = emptyString) {
errout.str("");
output.str("");

std::list<std::pair<std::string, std::size_t>> files;
files.emplace_back("test.cpp", strlen(code));

Settings settings;
settings.jobs = 2;
settings.quiet = true;
settings.inlineSuppressions = true;
settings.severity.enable(Severity::information);
if (!suppression.empty()) {
Expand Down Expand Up @@ -1089,6 +1089,7 @@ class TestSuppressions : public TestFixture {

CppCheck cppCheck(*this, false, nullptr); // <- do not "use global suppressions". pretend this is a thread that just checks a file.
Settings& settings = cppCheck.settings();
settings.quiet = true;
settings.nomsg.addSuppressionLine("uninitvar");
settings.exitCode = 1;

Expand Down Expand Up @@ -1123,6 +1124,7 @@ class TestSuppressions : public TestFixture {

CppCheck cppCheck(*this, true, nullptr);
Settings& settings = cppCheck.settings();
settings.quiet = true;
settings.severity.enable(Severity::style);
settings.inlineSuppressions = true;
settings.relativePaths = true;
Expand Down
3 changes: 1 addition & 2 deletions test/testthreadexecutor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ class TestThreadExecutorBase : public TestFixture {
*/
void check(unsigned int jobs, int files, int result, const std::string &data, const CheckOptions& opt = make_default_obj{}) {
errout.str("");
output.str("");

std::list<FileSettings> fileSettings;

Expand Down Expand Up @@ -262,7 +261,7 @@ class TestThreadExecutorBase : public TestFixture {
$.executeCommandCalled = true,
$.exe = exe,
$.args = {"-quiet", "-checks=*,-clang-analyzer-*,-llvm*", file, "--"}));
ASSERT_EQUALS("Checking " + file + " ...\n", output.str());
ASSERT_EQUALS("Checking " + file + " ...\n", output_str());
}

// TODO: provide data which actually shows values above 0
Expand Down

0 comments on commit 4538002

Please sign in to comment.