diff --git a/include/internal/catch_interfaces_testcase.h b/include/internal/catch_interfaces_testcase.h index f57cc8fe7c..2492c07de6 100644 --- a/include/internal/catch_interfaces_testcase.h +++ b/include/internal/catch_interfaces_testcase.h @@ -28,6 +28,7 @@ namespace Catch { virtual std::vector const& getAllTestsSorted( IConfig const& config ) const = 0; }; + bool isThrowSafe( TestCase const& testCase, IConfig const& config ); bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ); std::vector filterTests( std::vector const& testCases, TestSpec const& testSpec, IConfig const& config ); std::vector const& getAllTestCasesSorted( IConfig const& config ); diff --git a/include/internal/catch_session.cpp b/include/internal/catch_session.cpp index f9818f74ac..13aaeec4ce 100644 --- a/include/internal/catch_session.cpp +++ b/include/internal/catch_session.cpp @@ -25,6 +25,8 @@ #include #include +#include +#include namespace Catch { @@ -58,46 +60,53 @@ namespace Catch { return ret; } - - Catch::Totals runTests(std::shared_ptr const& config) { - auto reporter = makeReporter(config); - - RunContext context(config, std::move(reporter)); - - Totals totals; - - context.testGroupStarting(config->name(), 1, 1); - - TestSpec testSpec = config->testSpec(); - - auto const& allTestCases = getAllTestCasesSorted(*config); - for (auto const& testCase : allTestCases) { - bool matching = (!testSpec.hasFilters() && !testCase.isHidden()) || - (testSpec.hasFilters() && matchTest(testCase, testSpec, *config)); - - if (!context.aborting() && matching) - totals += context.runTest(testCase); - else - context.reporter().skipTest(testCase); + class TestGroup { + public: + explicit TestGroup(std::shared_ptr const& config) + : m_config{config} + , m_context{config, makeReporter(config)} + { + auto const& allTestCases = getAllTestCasesSorted(*m_config); + m_matches = m_config->testSpec().matchesByFilter(allTestCases, *m_config); + + if (m_matches.empty()) { + for (auto const& test : allTestCases) + if (!test.isHidden()) + m_tests.emplace(&test); + } else { + for (auto const& match : m_matches) + m_tests.insert(match.tests.begin(), match.tests.end()); + } } - if (config->warnAboutNoTests() && totals.testCases.total() == 0) { - ReusableStringStream testConfig; - - bool first = true; - for (const auto& input : config->getTestsOrTags()) { - if (!first) { testConfig << ' '; } - first = false; - testConfig << input; + Totals execute() { + Totals totals; + m_context.testGroupStarting(m_config->name(), 1, 1); + for (auto const& testCase : m_tests) { + if (!m_context.aborting()) + totals += m_context.runTest(*testCase); + else + m_context.reporter().skipTest(*testCase); } - context.reporter().noMatchingTestCases(testConfig.str()); - totals.error = -1; + for (auto const& match : m_matches) { + if (match.tests.empty()) { + m_context.reporter().noMatchingTestCases(match.name); + totals.error = -1; + } + } + m_context.testGroupEnded(m_config->name(), totals, 1, 1); + return totals; } - context.testGroupEnded(config->name(), totals, 1, 1); - return totals; - } + private: + using Tests = std::set; + + std::shared_ptr m_config; + RunContext m_context; + Tests m_tests; + TestSpec::Matches m_matches; + }; void applyFilenamesAsTags(Catch::IConfig const& config) { auto& tests = const_cast&>(getAllTestCasesSorted(config)); @@ -274,7 +283,12 @@ namespace Catch { if( Option listed = list( m_config ) ) return static_cast( *listed ); - auto totals = runTests( m_config ); + TestGroup tests { m_config }; + auto const totals = tests.execute(); + + if( m_config->warnAboutNoTests() && totals.error == -1 ) + return 2; + // Note that on unices only the lower 8 bits are usually used, clamping // the return value to 255 prevents false negative when some multiple // of 256 tests has failed diff --git a/include/internal/catch_test_case_registry_impl.cpp b/include/internal/catch_test_case_registry_impl.cpp index a85d0edf6c..bb59680423 100644 --- a/include/internal/catch_test_case_registry_impl.cpp +++ b/include/internal/catch_test_case_registry_impl.cpp @@ -36,8 +36,13 @@ namespace Catch { } return sorted; } + + bool isThrowSafe( TestCase const& testCase, IConfig const& config ) { + return !testCase.throws() || config.allowThrows(); + } + bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ) { - return testSpec.matches( testCase ) && ( config.allowThrows() || !testCase.throws() ); + return testSpec.matches( testCase ) && isThrowSafe( testCase, config ); } void enforceNoDuplicateTestCases( std::vector const& functions ) { diff --git a/include/internal/catch_test_case_registry_impl.h b/include/internal/catch_test_case_registry_impl.h index 8dc5b0fe3b..359ac3e380 100644 --- a/include/internal/catch_test_case_registry_impl.h +++ b/include/internal/catch_test_case_registry_impl.h @@ -23,6 +23,8 @@ namespace Catch { struct IConfig; std::vector sortTests( IConfig const& config, std::vector const& unsortedTestCases ); + + bool isThrowSafe( TestCase const& testCase, IConfig const& config ); bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ); void enforceNoDuplicateTestCases( std::vector const& functions ); diff --git a/include/internal/catch_test_spec.cpp b/include/internal/catch_test_spec.cpp index d9c149d501..54b638c4a1 100644 --- a/include/internal/catch_test_spec.cpp +++ b/include/internal/catch_test_spec.cpp @@ -7,6 +7,7 @@ #include "catch_test_spec.h" #include "catch_string_manip.h" +#include "catch_interfaces_config.h" #include #include @@ -15,45 +16,80 @@ namespace Catch { + TestSpec::Pattern::Pattern( std::string const& name ) + : m_name( name ) + {} + TestSpec::Pattern::~Pattern() = default; - TestSpec::NamePattern::~NamePattern() = default; - TestSpec::TagPattern::~TagPattern() = default; - TestSpec::ExcludedPattern::~ExcludedPattern() = default; - TestSpec::NamePattern::NamePattern( std::string const& name ) - : m_wildcardPattern( toLower( name ), CaseSensitive::No ) + std::string const& TestSpec::Pattern::name() const { + return m_name; + } + + + TestSpec::NamePattern::NamePattern( std::string const& name, std::string const& filterString ) + : Pattern( filterString ) + , m_wildcardPattern( toLower( name ), CaseSensitive::No ) {} + bool TestSpec::NamePattern::matches( TestCaseInfo const& testCase ) const { return m_wildcardPattern.matches( toLower( testCase.name ) ); } - TestSpec::TagPattern::TagPattern( std::string const& tag ) : m_tag( toLower( tag ) ) {} + + TestSpec::TagPattern::TagPattern( std::string const& tag, std::string const& filterString ) + : Pattern( filterString ) + , m_tag( toLower( tag ) ) + {} + bool TestSpec::TagPattern::matches( TestCaseInfo const& testCase ) const { return std::find(begin(testCase.lcaseTags), end(testCase.lcaseTags), m_tag) != end(testCase.lcaseTags); } - TestSpec::ExcludedPattern::ExcludedPattern( PatternPtr const& underlyingPattern ) : m_underlyingPattern( underlyingPattern ) {} - bool TestSpec::ExcludedPattern::matches( TestCaseInfo const& testCase ) const { return !m_underlyingPattern->matches( testCase ); } + + TestSpec::ExcludedPattern::ExcludedPattern( PatternPtr const& underlyingPattern ) + : Pattern( underlyingPattern->name() ) + , m_underlyingPattern( underlyingPattern ) + {} + + bool TestSpec::ExcludedPattern::matches( TestCaseInfo const& testCase ) const { + return !m_underlyingPattern->matches( testCase ); + } + bool TestSpec::Filter::matches( TestCaseInfo const& testCase ) const { - // All patterns in a filter must match for the filter to be a match - for( auto const& pattern : m_patterns ) { - if( !pattern->matches( testCase ) ) - return false; - } - return true; + return std::all_of( m_patterns.begin(), m_patterns.end(), [&]( PatternPtr const& p ){ return p->matches( testCase ); } ); + } + + std::string TestSpec::Filter::name() const { + std::string name; + for( auto const& p : m_patterns ) + name += p->name(); + return name; } + bool TestSpec::hasFilters() const { return !m_filters.empty(); } + bool TestSpec::matches( TestCaseInfo const& testCase ) const { - // A TestSpec matches if any filter matches - for( auto const& filter : m_filters ) - if( filter.matches( testCase ) ) - return true; - return false; + return std::any_of( m_filters.begin(), m_filters.end(), [&]( Filter const& f ){ return f.matches( testCase ); } ); } + + TestSpec::Matches TestSpec::matchesByFilter( std::vector const& testCases, IConfig const& config ) const + { + Matches matches( m_filters.size() ); + std::transform( m_filters.begin(), m_filters.end(), matches.begin(), [&]( Filter const& filter ){ + std::vector currentMatches; + for( auto const& test : testCases ) + if( isThrowSafe( test, config ) && filter.matches( test ) ) + currentMatches.emplace_back( &test ); + return FilterMatch{ filter.name(), currentMatches }; + } ); + return matches; + } + } diff --git a/include/internal/catch_test_spec.h b/include/internal/catch_test_spec.h index d2565187ba..d0e7ea9f70 100644 --- a/include/internal/catch_test_spec.h +++ b/include/internal/catch_test_spec.h @@ -22,17 +22,23 @@ namespace Catch { + struct IConfig; + class TestSpec { - struct Pattern { + class Pattern { + public: + explicit Pattern( std::string const& name ); virtual ~Pattern(); virtual bool matches( TestCaseInfo const& testCase ) const = 0; + std::string const& name() const; + private: + std::string const m_name; }; using PatternPtr = std::shared_ptr; class NamePattern : public Pattern { public: - NamePattern( std::string const& name ); - virtual ~NamePattern(); + explicit NamePattern( std::string const& name, std::string const& filterString ); bool matches( TestCaseInfo const& testCase ) const override; private: WildcardPattern m_wildcardPattern; @@ -40,8 +46,7 @@ namespace Catch { class TagPattern : public Pattern { public: - TagPattern( std::string const& tag ); - virtual ~TagPattern(); + explicit TagPattern( std::string const& tag, std::string const& filterString ); bool matches( TestCaseInfo const& testCase ) const override; private: std::string m_tag; @@ -49,8 +54,7 @@ namespace Catch { class ExcludedPattern : public Pattern { public: - ExcludedPattern( PatternPtr const& underlyingPattern ); - virtual ~ExcludedPattern(); + explicit ExcludedPattern( PatternPtr const& underlyingPattern ); bool matches( TestCaseInfo const& testCase ) const override; private: PatternPtr m_underlyingPattern; @@ -60,11 +64,19 @@ namespace Catch { std::vector m_patterns; bool matches( TestCaseInfo const& testCase ) const; + std::string name() const; }; public: + struct FilterMatch { + std::string name; + std::vector tests; + }; + using Matches = std::vector; + bool hasFilters() const; bool matches( TestCaseInfo const& testCase ) const; + Matches matchesByFilter( std::vector const& testCases, IConfig const& config ) const; private: std::vector m_filters; diff --git a/include/internal/catch_test_spec_parser.cpp b/include/internal/catch_test_spec_parser.cpp index 61c9e4df02..a910ac7e07 100644 --- a/include/internal/catch_test_spec_parser.cpp +++ b/include/internal/catch_test_spec_parser.cpp @@ -14,64 +14,125 @@ namespace Catch { TestSpecParser& TestSpecParser::parse( std::string const& arg ) { m_mode = None; m_exclusion = false; - m_start = std::string::npos; m_arg = m_tagAliases->expandAliases( arg ); m_escapeChars.clear(); + m_substring.reserve(m_arg.size()); + m_patternName.reserve(m_arg.size()); for( m_pos = 0; m_pos < m_arg.size(); ++m_pos ) visitChar( m_arg[m_pos] ); - if( m_mode == Name ) - addPattern(); + endMode(); return *this; } TestSpec TestSpecParser::testSpec() { addFilter(); return m_testSpec; } - void TestSpecParser::visitChar( char c ) { - if( m_mode == None ) { - switch( c ) { - case ' ': return; - case '~': m_exclusion = true; return; - case '[': return startNewMode( Tag, ++m_pos ); - case '"': return startNewMode( QuotedName, ++m_pos ); - case '\\': return escape(); - default: startNewMode( Name, m_pos ); break; - } + if( c == ',' ) { + endMode(); + addFilter(); + return; + } + + switch( m_mode ) { + case None: + if( processNoneChar( c ) ) + return; + break; + case Name: + processNameChar( c ); + break; + case EscapedName: + endMode(); + break; + default: + case Tag: + case QuotedName: + if( processOtherChar( c ) ) + return; + break; + } + + m_substring += c; + if( !isControlChar( c ) ) + m_patternName += c; + } + // Two of the processing methods return true to signal the caller to return + // without adding the given character to the current pattern strings + bool TestSpecParser::processNoneChar( char c ) { + switch( c ) { + case ' ': + return true; + case '~': + m_exclusion = true; + return false; + case '[': + startNewMode( Tag ); + return false; + case '"': + startNewMode( QuotedName ); + return false; + case '\\': + escape(); + return true; + default: + startNewMode( Name ); + return false; } - if( m_mode == Name ) { - if( c == ',' ) { - addPattern(); - addFilter(); - } - else if( c == '[' ) { - if( subString() == "exclude:" ) - m_exclusion = true; - else - addPattern(); - startNewMode( Tag, ++m_pos ); - } - else if( c == '\\' ) - escape(); + } + void TestSpecParser::processNameChar( char c ) { + if( c == '[' ) { + if( m_substring == "exclude:" ) + m_exclusion = true; + else + endMode(); + startNewMode( Tag ); } - else if( m_mode == EscapedName ) - m_mode = Name; - else if( m_mode == QuotedName && c == '"' ) - addPattern(); - else if( m_mode == Tag && c == ']' ) - addPattern(); } - void TestSpecParser::startNewMode( Mode mode, std::size_t start ) { + bool TestSpecParser::processOtherChar( char c ) { + if( !isControlChar( c ) ) + return false; + m_substring += c; + endMode(); + return true; + } + void TestSpecParser::startNewMode( Mode mode ) { m_mode = mode; - m_start = start; + } + void TestSpecParser::endMode() { + switch( m_mode ) { + case Name: + case QuotedName: + return addPattern(); + case Tag: + return addPattern(); + case EscapedName: + return startNewMode( Name ); + case None: + default: + return startNewMode( None ); + } } void TestSpecParser::escape() { - if( m_mode == None ) - m_start = m_pos; m_mode = EscapedName; m_escapeChars.push_back( m_pos ); } - std::string TestSpecParser::subString() const { return m_arg.substr( m_start, m_pos - m_start ); } + bool TestSpecParser::isControlChar( char c ) const { + switch( m_mode ) { + default: + return false; + case None: + return c == '~'; + case Name: + return c == '['; + case EscapedName: + return true; + case QuotedName: + return c == '"'; + case Tag: + return c == '[' || c == ']'; + } + } void TestSpecParser::addFilter() { if( !m_currentFilter.m_patterns.empty() ) { diff --git a/include/internal/catch_test_spec_parser.h b/include/internal/catch_test_spec_parser.h index 79ce889885..5b02bf6dc9 100644 --- a/include/internal/catch_test_spec_parser.h +++ b/include/internal/catch_test_spec_parser.h @@ -23,8 +23,10 @@ namespace Catch { enum Mode{ None, Name, QuotedName, Tag, EscapedName }; Mode m_mode = None; bool m_exclusion = false; - std::size_t m_start = std::string::npos, m_pos = 0; + std::size_t m_pos = 0; std::string m_arg; + std::string m_substring; + std::string m_patternName; std::vector m_escapeChars; TestSpec::Filter m_currentFilter; TestSpec m_testSpec; @@ -38,26 +40,32 @@ namespace Catch { private: void visitChar( char c ); - void startNewMode( Mode mode, std::size_t start ); + void startNewMode( Mode mode ); + bool processNoneChar( char c ); + void processNameChar( char c ); + bool processOtherChar( char c ); + void endMode(); void escape(); - std::string subString() const; + bool isControlChar( char c ) const; template void addPattern() { - std::string token = subString(); + std::string token = m_patternName; for( std::size_t i = 0; i < m_escapeChars.size(); ++i ) - token = token.substr( 0, m_escapeChars[i]-m_start-i ) + token.substr( m_escapeChars[i]-m_start-i+1 ); + token = token.substr( 0, m_escapeChars[i] - i ) + token.substr( m_escapeChars[i] -i +1 ); m_escapeChars.clear(); if( startsWith( token, "exclude:" ) ) { m_exclusion = true; token = token.substr( 8 ); } if( !token.empty() ) { - TestSpec::PatternPtr pattern = std::make_shared( token ); + TestSpec::PatternPtr pattern = std::make_shared( token, m_substring ); if( m_exclusion ) pattern = std::make_shared( pattern ); m_currentFilter.m_patterns.push_back( pattern ); } + m_substring.clear(); + m_patternName.clear(); m_exclusion = false; m_mode = None; } diff --git a/projects/CMakeLists.txt b/projects/CMakeLists.txt index ee8f1013de..4f76d94ab1 100644 --- a/projects/CMakeLists.txt +++ b/projects/CMakeLists.txt @@ -393,8 +393,19 @@ set_tests_properties(ListTestNamesOnly PROPERTIES add_test(NAME NoAssertions COMMAND $ -w NoAssertions) set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case") -add_test(NAME NoTest COMMAND $ -w NoTests "___nonexistent_test___") -set_tests_properties(NoTest PROPERTIES PASS_REGULAR_EXPRESSION "No test cases matched") +add_test(NAME NoTest COMMAND $ Tracker, "___nonexistent_test___") +set_tests_properties(NoTest PROPERTIES + PASS_REGULAR_EXPRESSION "No test cases matched '___nonexistent_test___'" + FAIL_REGULAR_EXPRESSION "No tests ran" +) + +add_test(NAME WarnAboutNoTests COMMAND ${CMAKE_COMMAND} -P ${CATCH_DIR}/projects/SelfTest/WarnAboutNoTests.cmake $) + +add_test(NAME UnmatchedOutputFilter COMMAND $ [this-tag-does-not-exist] -w NoTests) +set_tests_properties(UnmatchedOutputFilter + PROPERTIES + PASS_REGULAR_EXPRESSION "No test cases matched '\\[this-tag-does-not-exist\\]'" +) add_test(NAME FilteredSection-1 COMMAND $ \#1394 -c RunSection) set_tests_properties(FilteredSection-1 PROPERTIES FAIL_REGULAR_EXPRESSION "No tests ran") diff --git a/projects/SelfTest/WarnAboutNoTests.cmake b/projects/SelfTest/WarnAboutNoTests.cmake new file mode 100644 index 0000000000..4637e3f3c7 --- /dev/null +++ b/projects/SelfTest/WarnAboutNoTests.cmake @@ -0,0 +1,19 @@ +# Workaround for a peculiarity where CTest disregards the return code from a +# test command if a PASS_REGULAR_EXPRESSION is also set +execute_process( + COMMAND ${CMAKE_ARGV3} -w NoTests "___nonexistent_test___" + RESULT_VARIABLE ret + OUTPUT_VARIABLE out +) + +message("${out}") + +if(NOT ${ret} MATCHES "^[0-9]+$") + message(FATAL_ERROR "${ret}") +endif() + +if(${ret} EQUAL 0) + message(FATAL_ERROR "Expected nonzero return code") +elseif(${out} MATCHES "Helper failed with") + message(FATAL_ERROR "Helper failed") +endif()