From ec9dbb31f4abda2625b52184fa9087db38bb1cf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 1 Dec 2023 20:56:30 +0100 Subject: [PATCH] testrunner: make sure all redirected output is being consumed / some cleanups (#5714) --- test/fixture.cpp | 3 --- test/fixture.h | 14 +++++----- test/redirect.h | 46 +++++++++++++++----------------- test/testcmdlineparser.cpp | 52 +++++-------------------------------- test/testsymboldatabase.cpp | 2 +- test/testtokenlist.cpp | 2 +- 6 files changed, 36 insertions(+), 83 deletions(-) diff --git a/test/fixture.cpp b/test/fixture.cpp index a99a80a0d93..32863bbc637 100644 --- a/test/fixture.cpp +++ b/test/fixture.cpp @@ -34,9 +34,6 @@ #include "xml.h" -std::ostringstream errout; -std::ostringstream output; - /** * TestRegistry **/ diff --git a/test/fixture.h b/test/fixture.h index 1224e388dc1..dc1c62dcfe8 100644 --- a/test/fixture.h +++ b/test/fixture.h @@ -236,10 +236,16 @@ class TestFixture : public ErrorLogger { return SettingsBuilder(*this, std::move(settings)); } -public: + // TODO: make sure the output has been consumed in the test + std::ostringstream errout; + std::ostringstream output; + +private: void reportOut(const std::string &outmsg, Color c = Color::Reset) override; void reportErr(const ErrorMessage &msg) override; void run(const std::string &str); + +public: static void printHelp(); const std::string classname; @@ -248,12 +254,6 @@ class TestFixture : public ErrorLogger { static std::size_t runTests(const options& args); }; -// TODO: fix these -// NOLINTNEXTLINE(readability-redundant-declaration) -extern std::ostringstream errout; -// NOLINTNEXTLINE(readability-redundant-declaration) -extern std::ostringstream output; - // TODO: most asserts do not actually assert i.e. do not return #define TEST_CASE( NAME ) do { if (prepareTest(#NAME)) { setVerbose(false); NAME(); teardownTest(); } } while (false) #define ASSERT( CONDITION ) if (!assert_(__FILE__, __LINE__, (CONDITION))) return diff --git a/test/redirect.h b/test/redirect.h index 092e2c9bdd9..10e101c6101 100644 --- a/test/redirect.h +++ b/test/redirect.h @@ -19,12 +19,9 @@ #include #include +#include #include -// NOLINTNEXTLINE(readability-redundant-declaration) - TODO: fix this -extern std::ostringstream errout; -// NOLINTNEXTLINE(readability-redundant-declaration) - TODO: fix this -extern std::ostringstream output; /** * @brief Utility class for capturing cout and cerr to ostringstream buffers * for later use. Uses RAII to stop redirection when the object goes out of @@ -47,34 +44,35 @@ class RedirectOutputError { } /** Revert cout and cerr behaviour */ - ~RedirectOutputError() { + ~RedirectOutputError() noexcept(false) { std::cout.rdbuf(_oldCout); // restore cout's original streambuf std::cerr.rdbuf(_oldCerr); // restore cerrs's original streambuf - errout << _err.str(); - output << _out.str(); + { + const std::string s = _out.str(); + if (!s.empty()) + throw std::runtime_error("unconsumed stdout: " + s); // cppcheck-suppress exceptThrowInDestructor - FP #11031 + } + + { + const std::string s = _err.str(); + if (!s.empty()) + throw std::runtime_error("consumed stderr: " + s); + } } - /** Return what would be printed to cout. See also clearOutput() */ - std::string getOutput() const { - return _out.str(); - } - - /** Normally called after getOutput() to prevent same text to be returned - twice. */ - void clearOutput() { + /** Return what would be printed to cout. */ + std::string getOutput() { + std::string s = _out.str(); _out.str(""); + return s; } - /** Return what would be printed to cerr. See also clearErrout() */ - std::string getErrout() const { - return _err.str(); - } - - /** Normally called after getErrout() to prevent same text to be returned - twice. */ - void clearErrout() { + /** Return what would be printed to cerr. */ + std::string getErrout() { + std::string s = _err.str(); _err.str(""); + return s; } private: @@ -112,9 +110,7 @@ class SuppressOutput { #define REDIRECT RedirectOutputError redir #define GET_REDIRECT_OUTPUT redir.getOutput() -#define CLEAR_REDIRECT_OUTPUT redir.clearOutput() #define GET_REDIRECT_ERROUT redir.getErrout() -#define CLEAR_REDIRECT_ERROUT redir.clearErrout() #define SUPPRESS SuppressOutput supprout diff --git a/test/testcmdlineparser.cpp b/test/testcmdlineparser.cpp index d5998c078b9..9c493b8aa34 100644 --- a/test/testcmdlineparser.cpp +++ b/test/testcmdlineparser.cpp @@ -102,7 +102,6 @@ class TestCmdlineParser : public TestFixture { void teardownTestInternal() override { logger->destroy(); - // TODO: verify that the redirect output is empty } // add overload so the enums can be compared without a cast @@ -373,7 +372,6 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck"}; ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(1, argv)); ASSERT(startsWith(logger->str(), "Cppcheck - A tool for static C/C++ code analysis")); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void helpshort() { @@ -381,7 +379,6 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck", "-h"}; ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(2, argv)); ASSERT(startsWith(logger->str(), "Cppcheck - A tool for static C/C++ code analysis")); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void helpshortExclusive() { @@ -389,7 +386,6 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck", "--library=missing", "-h"}; ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(3, argv)); ASSERT(startsWith(logger->str(), "Cppcheck - A tool for static C/C++ code analysis")); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void helplong() { @@ -397,7 +393,6 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck", "--help"}; ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(2, argv)); ASSERT(startsWith(logger->str(), "Cppcheck - A tool for static C/C++ code analysis")); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void helplongExclusive() { @@ -405,7 +400,6 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck", "--library=missing", "--help"}; ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(3, argv)); ASSERT(startsWith(logger->str(), "Cppcheck - A tool for static C/C++ code analysis")); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void version() { @@ -413,7 +407,6 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck", "--version"}; ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(2, argv)); ASSERT_EQUALS("Cppcheck 2.13 dev\n", logger->str()); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void versionWithCfg() { @@ -426,7 +419,6 @@ class TestCmdlineParser : public TestFixture { ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(2, argv)); // TODO: somehow the config is not loaded on some systems (void)logger->str(); //ASSERT_EQUALS("The Product\n", logger->str()); // TODO: include version? - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } // TODO: test --version with extraVersion @@ -436,7 +428,6 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck", "--library=missing", "--version"}; ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(3, argv)); ASSERT_EQUALS("Cppcheck 2.13 dev\n", logger->str()); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void versionWithInvalidCfg() { @@ -446,7 +437,6 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck", "--version"}; ASSERT_EQUALS(CmdLineParser::Result::Fail, parser->parseFromArgs(2, argv)); ASSERT_EQUALS("cppcheck: error: could not load cppcheck.cfg - not a valid JSON - syntax error at line 1 near: \n", logger->str()); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void onefile() { @@ -455,7 +445,6 @@ class TestCmdlineParser : public TestFixture { ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(2, argv)); ASSERT_EQUALS(1, (int)parser->getPathNames().size()); ASSERT_EQUALS("file.cpp", parser->getPathNames().at(0)); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void onepath() { @@ -464,7 +453,6 @@ class TestCmdlineParser : public TestFixture { ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(2, argv)); ASSERT_EQUALS(1, (int)parser->getPathNames().size()); ASSERT_EQUALS("src", parser->getPathNames().at(0)); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void optionwithoutfile() { @@ -473,7 +461,6 @@ class TestCmdlineParser : public TestFixture { ASSERT_EQUALS(CmdLineParser::Result::Fail, parser->parseFromArgs(2, argv)); ASSERT_EQUALS(0, (int)parser->getPathNames().size()); ASSERT_EQUALS("cppcheck: error: no C or C++ source files found.\n", logger->str()); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void verboseshort() { @@ -482,7 +469,6 @@ class TestCmdlineParser : public TestFixture { settings->verbose = false; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS(true, settings->verbose); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void verboselong() { @@ -491,7 +477,6 @@ class TestCmdlineParser : public TestFixture { settings->verbose = false; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS(true, settings->verbose); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void debugSimplified() { @@ -500,7 +485,6 @@ class TestCmdlineParser : public TestFixture { settings->debugSimplified = false; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS(true, settings->debugSimplified); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void debugwarnings() { @@ -509,7 +493,6 @@ class TestCmdlineParser : public TestFixture { settings->debugwarnings = false; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS(true, settings->debugwarnings); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void forceshort() { @@ -518,7 +501,6 @@ class TestCmdlineParser : public TestFixture { settings->force = false; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS(true, settings->force); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void forcelong() { @@ -527,7 +509,6 @@ class TestCmdlineParser : public TestFixture { settings->force = false; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS(true, settings->force); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void relativePaths1() { @@ -536,7 +517,6 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck", "-rp", "file.cpp"}; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS(true, settings->relativePaths); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void relativePaths2() { @@ -545,7 +525,6 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck", "--relative-paths", "file.cpp"}; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS(true, settings->relativePaths); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void relativePaths3() { @@ -558,7 +537,6 @@ class TestCmdlineParser : public TestFixture { ASSERT_EQUALS(2, settings->basePaths.size()); ASSERT_EQUALS("C:/foo", settings->basePaths[0]); ASSERT_EQUALS("C:/bar", settings->basePaths[1]); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void relativePaths4() { @@ -572,7 +550,6 @@ class TestCmdlineParser : public TestFixture { ASSERT_EQUALS(2, settings->basePaths.size()); ASSERT_EQUALS("C:/foo", settings->basePaths[0]); ASSERT_EQUALS("C:/bar", settings->basePaths[1]); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void quietshort() { @@ -581,7 +558,6 @@ class TestCmdlineParser : public TestFixture { settings->quiet = false; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS(true, settings->quiet); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void quietlong() { @@ -590,7 +566,6 @@ class TestCmdlineParser : public TestFixture { settings->quiet = false; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS(true, settings->quiet); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void defines_noarg() { @@ -623,7 +598,6 @@ class TestCmdlineParser : public TestFixture { settings->userDefines.clear(); ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS("_WIN32=1", settings->userDefines); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void defines2() { @@ -632,7 +606,6 @@ class TestCmdlineParser : public TestFixture { settings->userDefines.clear(); ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(4, argv)); ASSERT_EQUALS("_WIN32=1;NODEBUG=1", settings->userDefines); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void defines3() { @@ -641,7 +614,6 @@ class TestCmdlineParser : public TestFixture { settings->userDefines.clear(); ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(4, argv)); ASSERT_EQUALS("DEBUG=1", settings->userDefines); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void defines4() { @@ -650,7 +622,6 @@ class TestCmdlineParser : public TestFixture { settings->userDefines.clear(); ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS("DEBUG=", settings->userDefines); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void enforceLanguage1() { @@ -1619,7 +1590,6 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck", "--doc"}; ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(2, argv)); ASSERT(startsWith(logger->str(), "## ")); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void docExclusive() { @@ -1627,7 +1597,6 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck", "--library=missing", "--doc"}; ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(3, argv)); ASSERT(startsWith(logger->str(), "## ")); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void showtimeSummary() { @@ -1706,8 +1675,9 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck", "--errorlist"}; ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(2, argv)); ASSERT_EQUALS("", logger->str()); // empty since it is logged via ErrorLogger - ASSERT(startsWith(GET_REDIRECT_OUTPUT, ErrorMessage::getXMLHeader(""))); - ASSERT(endsWith(GET_REDIRECT_OUTPUT, "\n")); + const std::string errout_s = GET_REDIRECT_OUTPUT; + ASSERT(startsWith(errout_s, ErrorMessage::getXMLHeader(""))); + ASSERT(endsWith(errout_s, "\n")); } void errorlistWithCfg() { @@ -1725,8 +1695,9 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck", "--library=missing", "--errorlist"}; ASSERT_EQUALS(CmdLineParser::Result::Exit, parser->parseFromArgs(3, argv)); ASSERT_EQUALS("", logger->str()); // empty since it is logged via ErrorLogger - ASSERT(startsWith(GET_REDIRECT_OUTPUT, "\n")); + const std::string errout_s = GET_REDIRECT_OUTPUT; + ASSERT(startsWith(errout_s, ErrorMessage::getXMLHeader(""))); + ASSERT(endsWith(errout_s, "\n")); } void errorlistWithInvalidCfg() { @@ -1736,7 +1707,6 @@ class TestCmdlineParser : public TestFixture { const char * const argv[] = {"cppcheck", "--errorlist"}; ASSERT_EQUALS(CmdLineParser::Result::Fail, parser->parseFromArgs(2, argv)); ASSERT_EQUALS("cppcheck: error: could not load cppcheck.cfg - not a valid JSON - syntax error at line 1 near: \n", logger->str()); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void ignorepathsnopath() { @@ -1998,7 +1968,6 @@ class TestCmdlineParser : public TestFixture { settings->typedefMaxTime = 0; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS(12, settings->typedefMaxTime); - ASSERT_EQUALS("", logger->str()); } void typedefMaxTimeInvalid() { @@ -2074,7 +2043,6 @@ class TestCmdlineParser : public TestFixture { ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS(1, settings->addons.size()); ASSERT_EQUALS("misra", *settings->addons.cbegin()); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void addonMissing() { @@ -2085,7 +2053,6 @@ class TestCmdlineParser : public TestFixture { ASSERT_EQUALS(1, settings->addons.size()); ASSERT_EQUALS("misra2", *settings->addons.cbegin()); ASSERT_EQUALS("Did not find addon misra2.py\n", logger->str()); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void signedChar() { @@ -2094,7 +2061,6 @@ class TestCmdlineParser : public TestFixture { settings->platform.defaultSign = '\0'; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS('s', settings->platform.defaultSign); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void signedChar2() { @@ -2103,7 +2069,6 @@ class TestCmdlineParser : public TestFixture { settings->platform.defaultSign = '\0'; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(4, argv)); ASSERT_EQUALS('s', settings->platform.defaultSign); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void unsignedChar() { @@ -2112,7 +2077,6 @@ class TestCmdlineParser : public TestFixture { settings->platform.defaultSign = '\0'; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS('u', settings->platform.defaultSign); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void unsignedChar2() { @@ -2121,7 +2085,6 @@ class TestCmdlineParser : public TestFixture { settings->platform.defaultSign = '\0'; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(4, argv)); ASSERT_EQUALS('u', settings->platform.defaultSign); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void signedCharUnsignedChar() { @@ -2130,7 +2093,6 @@ class TestCmdlineParser : public TestFixture { settings->platform.defaultSign = '\0'; ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(4, argv)); ASSERT_EQUALS('u', settings->platform.defaultSign); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } #ifdef HAVE_RULES @@ -2204,7 +2166,6 @@ class TestCmdlineParser : public TestFixture { ASSERT_EQUALS(CmdLineParser::Result::Success, parser->parseFromArgs(3, argv)); ASSERT_EQUALS(1, settings->libraries.size()); ASSERT_EQUALS("posix", *settings->libraries.cbegin()); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void libraryMissing() { @@ -2215,7 +2176,6 @@ class TestCmdlineParser : public TestFixture { ASSERT_EQUALS(1, settings->libraries.size()); ASSERT_EQUALS("posix2", *settings->libraries.cbegin()); ASSERT_EQUALS("cppcheck: Failed to load library configuration file 'posix2'. File not found\n", logger->str()); - ASSERT_EQUALS("", GET_REDIRECT_OUTPUT); } void ignorepaths1() { diff --git a/test/testsymboldatabase.cpp b/test/testsymboldatabase.cpp index d363f32454f..9a6a6f07884 100644 --- a/test/testsymboldatabase.cpp +++ b/test/testsymboldatabase.cpp @@ -76,7 +76,7 @@ class TestSymbolDatabase : public TestFixture { typetok = nullptr; } - const static SymbolDatabase* getSymbolDB_inner(Tokenizer& tokenizer, const char* code, const char* filename) { + const SymbolDatabase* getSymbolDB_inner(Tokenizer& tokenizer, const char* code, const char* filename) { errout.str(""); std::istringstream istr(code); return tokenizer.tokenize(istr, filename) ? tokenizer.getSymbolDatabase() : nullptr; diff --git a/test/testtokenlist.cpp b/test/testtokenlist.cpp index 2e764608ee4..0b979aa2df6 100644 --- a/test/testtokenlist.cpp +++ b/test/testtokenlist.cpp @@ -57,7 +57,7 @@ class TestTokenList : public TestFixture { ASSERT_EQUALS("0xF0000000", tokenlist.front()->str()); } - void inc() const { + void inc() { const char code[] = "a++1;1++b;"; errout.str("");