1 #include "catch_run_context.h" 2 #include "catch_compiler_capabilities.h" 3 #include "catch_context.h" 4 #include "catch_enforce.h" 5 #include "catch_random_number_generator.h" 6 #include "catch_stream.h" 7 #include "catch_output_redirect.h" 8 9 #include <cassert> 10 #include <algorithm> 11 #include <sstream> 12 13 namespace Catch { 14 15 namespace Generators { 16 struct GeneratorTracker : TestCaseTracking::TrackerBase, IGeneratorTracker { 17 GeneratorBasePtr m_generator; 18 GeneratorTrackerCatch::Generators::GeneratorTracker19 GeneratorTracker( TestCaseTracking::NameAndLocation const& nameAndLocation, TrackerContext& ctx, ITracker* parent ) 20 : TrackerBase( nameAndLocation, ctx, parent ) 21 {} 22 ~GeneratorTracker(); 23 acquireCatch::Generators::GeneratorTracker24 static GeneratorTracker& acquire( TrackerContext& ctx, TestCaseTracking::NameAndLocation const& nameAndLocation ) { 25 std::shared_ptr<GeneratorTracker> tracker; 26 27 ITracker& currentTracker = ctx.currentTracker(); 28 if( TestCaseTracking::ITrackerPtr childTracker = currentTracker.findChild( nameAndLocation ) ) { 29 assert( childTracker ); 30 assert( childTracker->isGeneratorTracker() ); 31 tracker = std::static_pointer_cast<GeneratorTracker>( childTracker ); 32 } 33 else { 34 tracker = std::make_shared<GeneratorTracker>( nameAndLocation, ctx, ¤tTracker ); 35 currentTracker.addChild( tracker ); 36 } 37 38 if( !ctx.completedCycle() && !tracker->isComplete() ) { 39 tracker->open(); 40 } 41 42 return *tracker; 43 } 44 45 // TrackerBase interface isGeneratorTrackerCatch::Generators::GeneratorTracker46 bool isGeneratorTracker() const override { return true; } hasGeneratorCatch::Generators::GeneratorTracker47 auto hasGenerator() const -> bool override { 48 return !!m_generator; 49 } closeCatch::Generators::GeneratorTracker50 void close() override { 51 TrackerBase::close(); 52 // Generator interface only finds out if it has another item on atual move 53 if (m_runState == CompletedSuccessfully && m_generator->next()) { 54 m_children.clear(); 55 m_runState = Executing; 56 } 57 } 58 59 // IGeneratorTracker interface getGeneratorCatch::Generators::GeneratorTracker60 auto getGenerator() const -> GeneratorBasePtr const& override { 61 return m_generator; 62 } setGeneratorCatch::Generators::GeneratorTracker63 void setGenerator( GeneratorBasePtr&& generator ) override { 64 m_generator = std::move( generator ); 65 } 66 }; ~GeneratorTracker()67 GeneratorTracker::~GeneratorTracker() {} 68 } 69 RunContext(IConfigPtr const & _config,IStreamingReporterPtr && reporter)70 RunContext::RunContext(IConfigPtr const& _config, IStreamingReporterPtr&& reporter) 71 : m_runInfo(_config->name()), 72 m_context(getCurrentMutableContext()), 73 m_config(_config), 74 m_reporter(std::move(reporter)), 75 m_lastAssertionInfo{ StringRef(), SourceLineInfo("",0), StringRef(), ResultDisposition::Normal }, 76 m_includeSuccessfulResults( m_config->includeSuccessfulResults() || m_reporter->getPreferences().shouldReportAllAssertions ) 77 { 78 m_context.setRunner(this); 79 m_context.setConfig(m_config); 80 m_context.setResultCapture(this); 81 m_reporter->testRunStarting(m_runInfo); 82 } 83 ~RunContext()84 RunContext::~RunContext() { 85 m_reporter->testRunEnded(TestRunStats(m_runInfo, m_totals, aborting())); 86 } 87 testGroupStarting(std::string const & testSpec,std::size_t groupIndex,std::size_t groupsCount)88 void RunContext::testGroupStarting(std::string const& testSpec, std::size_t groupIndex, std::size_t groupsCount) { 89 m_reporter->testGroupStarting(GroupInfo(testSpec, groupIndex, groupsCount)); 90 } 91 testGroupEnded(std::string const & testSpec,Totals const & totals,std::size_t groupIndex,std::size_t groupsCount)92 void RunContext::testGroupEnded(std::string const& testSpec, Totals const& totals, std::size_t groupIndex, std::size_t groupsCount) { 93 m_reporter->testGroupEnded(TestGroupStats(GroupInfo(testSpec, groupIndex, groupsCount), totals, aborting())); 94 } 95 runTest(TestCase const & testCase)96 Totals RunContext::runTest(TestCase const& testCase) { 97 Totals prevTotals = m_totals; 98 99 std::string redirectedCout; 100 std::string redirectedCerr; 101 102 auto const& testInfo = testCase.getTestCaseInfo(); 103 104 m_reporter->testCaseStarting(testInfo); 105 106 m_activeTestCase = &testCase; 107 108 109 ITracker& rootTracker = m_trackerContext.startRun(); 110 assert(rootTracker.isSectionTracker()); 111 static_cast<SectionTracker&>(rootTracker).addInitialFilters(m_config->getSectionsToRun()); 112 do { 113 m_trackerContext.startCycle(); 114 m_testCaseTracker = &SectionTracker::acquire(m_trackerContext, TestCaseTracking::NameAndLocation(testInfo.name, testInfo.lineInfo)); 115 runCurrentTest(redirectedCout, redirectedCerr); 116 } while (!m_testCaseTracker->isSuccessfullyCompleted() && !aborting()); 117 118 Totals deltaTotals = m_totals.delta(prevTotals); 119 if (testInfo.expectedToFail() && deltaTotals.testCases.passed > 0) { 120 deltaTotals.assertions.failed++; 121 deltaTotals.testCases.passed--; 122 deltaTotals.testCases.failed++; 123 } 124 m_totals.testCases += deltaTotals.testCases; 125 m_reporter->testCaseEnded(TestCaseStats(testInfo, 126 deltaTotals, 127 redirectedCout, 128 redirectedCerr, 129 aborting())); 130 131 m_activeTestCase = nullptr; 132 m_testCaseTracker = nullptr; 133 134 return deltaTotals; 135 } 136 config() const137 IConfigPtr RunContext::config() const { 138 return m_config; 139 } 140 reporter() const141 IStreamingReporter& RunContext::reporter() const { 142 return *m_reporter; 143 } 144 assertionEnded(AssertionResult const & result)145 void RunContext::assertionEnded(AssertionResult const & result) { 146 if (result.getResultType() == ResultWas::Ok) { 147 m_totals.assertions.passed++; 148 m_lastAssertionPassed = true; 149 } else if (!result.isOk()) { 150 m_lastAssertionPassed = false; 151 if( m_activeTestCase->getTestCaseInfo().okToFail() ) 152 m_totals.assertions.failedButOk++; 153 else 154 m_totals.assertions.failed++; 155 } 156 else { 157 m_lastAssertionPassed = true; 158 } 159 160 // We have no use for the return value (whether messages should be cleared), because messages were made scoped 161 // and should be let to clear themselves out. 162 static_cast<void>(m_reporter->assertionEnded(AssertionStats(result, m_messages, m_totals))); 163 164 if (result.getResultType() != ResultWas::Warning) 165 m_messageScopes.clear(); 166 167 // Reset working state 168 resetAssertionInfo(); 169 m_lastResult = result; 170 } resetAssertionInfo()171 void RunContext::resetAssertionInfo() { 172 m_lastAssertionInfo.macroName = StringRef(); 173 m_lastAssertionInfo.capturedExpression = "{Unknown expression after the reported line}"_sr; 174 } 175 sectionStarted(SectionInfo const & sectionInfo,Counts & assertions)176 bool RunContext::sectionStarted(SectionInfo const & sectionInfo, Counts & assertions) { 177 ITracker& sectionTracker = SectionTracker::acquire(m_trackerContext, TestCaseTracking::NameAndLocation(sectionInfo.name, sectionInfo.lineInfo)); 178 if (!sectionTracker.isOpen()) 179 return false; 180 m_activeSections.push_back(§ionTracker); 181 182 m_lastAssertionInfo.lineInfo = sectionInfo.lineInfo; 183 184 m_reporter->sectionStarting(sectionInfo); 185 186 assertions = m_totals.assertions; 187 188 return true; 189 } acquireGeneratorTracker(SourceLineInfo const & lineInfo)190 auto RunContext::acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& { 191 using namespace Generators; 192 GeneratorTracker& tracker = GeneratorTracker::acquire( m_trackerContext, TestCaseTracking::NameAndLocation( "generator", lineInfo ) ); 193 assert( tracker.isOpen() ); 194 m_lastAssertionInfo.lineInfo = lineInfo; 195 return tracker; 196 } 197 testForMissingAssertions(Counts & assertions)198 bool RunContext::testForMissingAssertions(Counts& assertions) { 199 if (assertions.total() != 0) 200 return false; 201 if (!m_config->warnAboutMissingAssertions()) 202 return false; 203 if (m_trackerContext.currentTracker().hasChildren()) 204 return false; 205 m_totals.assertions.failed++; 206 assertions.failed++; 207 return true; 208 } 209 sectionEnded(SectionEndInfo const & endInfo)210 void RunContext::sectionEnded(SectionEndInfo const & endInfo) { 211 Counts assertions = m_totals.assertions - endInfo.prevAssertions; 212 bool missingAssertions = testForMissingAssertions(assertions); 213 214 if (!m_activeSections.empty()) { 215 m_activeSections.back()->close(); 216 m_activeSections.pop_back(); 217 } 218 219 m_reporter->sectionEnded(SectionStats(endInfo.sectionInfo, assertions, endInfo.durationInSeconds, missingAssertions)); 220 m_messages.clear(); 221 m_messageScopes.clear(); 222 } 223 sectionEndedEarly(SectionEndInfo const & endInfo)224 void RunContext::sectionEndedEarly(SectionEndInfo const & endInfo) { 225 if (m_unfinishedSections.empty()) 226 m_activeSections.back()->fail(); 227 else 228 m_activeSections.back()->close(); 229 m_activeSections.pop_back(); 230 231 m_unfinishedSections.push_back(endInfo); 232 } 233 234 #if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) benchmarkPreparing(std::string const & name)235 void RunContext::benchmarkPreparing(std::string const& name) { 236 m_reporter->benchmarkPreparing(name); 237 } benchmarkStarting(BenchmarkInfo const & info)238 void RunContext::benchmarkStarting( BenchmarkInfo const& info ) { 239 m_reporter->benchmarkStarting( info ); 240 } benchmarkEnded(BenchmarkStats<> const & stats)241 void RunContext::benchmarkEnded( BenchmarkStats<> const& stats ) { 242 m_reporter->benchmarkEnded( stats ); 243 } benchmarkFailed(std::string const & error)244 void RunContext::benchmarkFailed(std::string const & error) { 245 m_reporter->benchmarkFailed(error); 246 } 247 #endif // CATCH_CONFIG_ENABLE_BENCHMARKING 248 pushScopedMessage(MessageInfo const & message)249 void RunContext::pushScopedMessage(MessageInfo const & message) { 250 m_messages.push_back(message); 251 } 252 popScopedMessage(MessageInfo const & message)253 void RunContext::popScopedMessage(MessageInfo const & message) { 254 m_messages.erase(std::remove(m_messages.begin(), m_messages.end(), message), m_messages.end()); 255 } 256 emplaceUnscopedMessage(MessageBuilder const & builder)257 void RunContext::emplaceUnscopedMessage( MessageBuilder const& builder ) { 258 m_messageScopes.emplace_back( builder ); 259 } 260 getCurrentTestName() const261 std::string RunContext::getCurrentTestName() const { 262 return m_activeTestCase 263 ? m_activeTestCase->getTestCaseInfo().name 264 : std::string(); 265 } 266 getLastResult() const267 const AssertionResult * RunContext::getLastResult() const { 268 return &(*m_lastResult); 269 } 270 exceptionEarlyReported()271 void RunContext::exceptionEarlyReported() { 272 m_shouldReportUnexpected = false; 273 } 274 handleFatalErrorCondition(StringRef message)275 void RunContext::handleFatalErrorCondition( StringRef message ) { 276 // First notify reporter that bad things happened 277 m_reporter->fatalErrorEncountered(message); 278 279 // Don't rebuild the result -- the stringification itself can cause more fatal errors 280 // Instead, fake a result data. 281 AssertionResultData tempResult( ResultWas::FatalErrorCondition, { false } ); 282 tempResult.message = static_cast<std::string>(message); 283 AssertionResult result(m_lastAssertionInfo, tempResult); 284 285 assertionEnded(result); 286 287 handleUnfinishedSections(); 288 289 // Recreate section for test case (as we will lose the one that was in scope) 290 auto const& testCaseInfo = m_activeTestCase->getTestCaseInfo(); 291 SectionInfo testCaseSection(testCaseInfo.lineInfo, testCaseInfo.name); 292 293 Counts assertions; 294 assertions.failed = 1; 295 SectionStats testCaseSectionStats(testCaseSection, assertions, 0, false); 296 m_reporter->sectionEnded(testCaseSectionStats); 297 298 auto const& testInfo = m_activeTestCase->getTestCaseInfo(); 299 300 Totals deltaTotals; 301 deltaTotals.testCases.failed = 1; 302 deltaTotals.assertions.failed = 1; 303 m_reporter->testCaseEnded(TestCaseStats(testInfo, 304 deltaTotals, 305 std::string(), 306 std::string(), 307 false)); 308 m_totals.testCases.failed++; 309 testGroupEnded(std::string(), m_totals, 1, 1); 310 m_reporter->testRunEnded(TestRunStats(m_runInfo, m_totals, false)); 311 } 312 lastAssertionPassed()313 bool RunContext::lastAssertionPassed() { 314 return m_lastAssertionPassed; 315 } 316 assertionPassed()317 void RunContext::assertionPassed() { 318 m_lastAssertionPassed = true; 319 ++m_totals.assertions.passed; 320 resetAssertionInfo(); 321 m_messageScopes.clear(); 322 } 323 aborting() const324 bool RunContext::aborting() const { 325 return m_totals.assertions.failed >= static_cast<std::size_t>(m_config->abortAfter()); 326 } 327 runCurrentTest(std::string & redirectedCout,std::string & redirectedCerr)328 void RunContext::runCurrentTest(std::string & redirectedCout, std::string & redirectedCerr) { 329 auto const& testCaseInfo = m_activeTestCase->getTestCaseInfo(); 330 SectionInfo testCaseSection(testCaseInfo.lineInfo, testCaseInfo.name); 331 m_reporter->sectionStarting(testCaseSection); 332 Counts prevAssertions = m_totals.assertions; 333 double duration = 0; 334 m_shouldReportUnexpected = true; 335 m_lastAssertionInfo = { "TEST_CASE"_sr, testCaseInfo.lineInfo, StringRef(), ResultDisposition::Normal }; 336 337 seedRng(*m_config); 338 339 Timer timer; 340 CATCH_TRY { 341 if (m_reporter->getPreferences().shouldRedirectStdOut) { 342 #if !defined(CATCH_CONFIG_EXPERIMENTAL_REDIRECT) 343 RedirectedStreams redirectedStreams(redirectedCout, redirectedCerr); 344 345 timer.start(); 346 invokeActiveTestCase(); 347 #else 348 OutputRedirect r(redirectedCout, redirectedCerr); 349 timer.start(); 350 invokeActiveTestCase(); 351 #endif 352 } else { 353 timer.start(); 354 invokeActiveTestCase(); 355 } 356 duration = timer.getElapsedSeconds(); 357 } CATCH_CATCH_ANON (TestFailureException&) { 358 // This just means the test was aborted due to failure 359 } CATCH_CATCH_ALL { 360 // Under CATCH_CONFIG_FAST_COMPILE, unexpected exceptions under REQUIRE assertions 361 // are reported without translation at the point of origin. 362 if( m_shouldReportUnexpected ) { 363 AssertionReaction dummyReaction; 364 handleUnexpectedInflightException( m_lastAssertionInfo, translateActiveException(), dummyReaction ); 365 } 366 } 367 Counts assertions = m_totals.assertions - prevAssertions; 368 bool missingAssertions = testForMissingAssertions(assertions); 369 370 m_testCaseTracker->close(); 371 handleUnfinishedSections(); 372 m_messages.clear(); 373 m_messageScopes.clear(); 374 375 SectionStats testCaseSectionStats(testCaseSection, assertions, duration, missingAssertions); 376 m_reporter->sectionEnded(testCaseSectionStats); 377 } 378 invokeActiveTestCase()379 void RunContext::invokeActiveTestCase() { 380 FatalConditionHandler fatalConditionHandler; // Handle signals 381 m_activeTestCase->invoke(); 382 fatalConditionHandler.reset(); 383 } 384 handleUnfinishedSections()385 void RunContext::handleUnfinishedSections() { 386 // If sections ended prematurely due to an exception we stored their 387 // infos here so we can tear them down outside the unwind process. 388 for (auto it = m_unfinishedSections.rbegin(), 389 itEnd = m_unfinishedSections.rend(); 390 it != itEnd; 391 ++it) 392 sectionEnded(*it); 393 m_unfinishedSections.clear(); 394 } 395 handleExpr(AssertionInfo const & info,ITransientExpression const & expr,AssertionReaction & reaction)396 void RunContext::handleExpr( 397 AssertionInfo const& info, 398 ITransientExpression const& expr, 399 AssertionReaction& reaction 400 ) { 401 m_reporter->assertionStarting( info ); 402 403 bool negated = isFalseTest( info.resultDisposition ); 404 bool result = expr.getResult() != negated; 405 406 if( result ) { 407 if (!m_includeSuccessfulResults) { 408 assertionPassed(); 409 } 410 else { 411 reportExpr(info, ResultWas::Ok, &expr, negated); 412 } 413 } 414 else { 415 reportExpr(info, ResultWas::ExpressionFailed, &expr, negated ); 416 populateReaction( reaction ); 417 } 418 } reportExpr(AssertionInfo const & info,ResultWas::OfType resultType,ITransientExpression const * expr,bool negated)419 void RunContext::reportExpr( 420 AssertionInfo const &info, 421 ResultWas::OfType resultType, 422 ITransientExpression const *expr, 423 bool negated ) { 424 425 m_lastAssertionInfo = info; 426 AssertionResultData data( resultType, LazyExpression( negated ) ); 427 428 AssertionResult assertionResult{ info, data }; 429 assertionResult.m_resultData.lazyExpression.m_transientExpression = expr; 430 431 assertionEnded( assertionResult ); 432 } 433 handleMessage(AssertionInfo const & info,ResultWas::OfType resultType,StringRef const & message,AssertionReaction & reaction)434 void RunContext::handleMessage( 435 AssertionInfo const& info, 436 ResultWas::OfType resultType, 437 StringRef const& message, 438 AssertionReaction& reaction 439 ) { 440 m_reporter->assertionStarting( info ); 441 442 m_lastAssertionInfo = info; 443 444 AssertionResultData data( resultType, LazyExpression( false ) ); 445 data.message = static_cast<std::string>(message); 446 AssertionResult assertionResult{ m_lastAssertionInfo, data }; 447 assertionEnded( assertionResult ); 448 if( !assertionResult.isOk() ) 449 populateReaction( reaction ); 450 } handleUnexpectedExceptionNotThrown(AssertionInfo const & info,AssertionReaction & reaction)451 void RunContext::handleUnexpectedExceptionNotThrown( 452 AssertionInfo const& info, 453 AssertionReaction& reaction 454 ) { 455 handleNonExpr(info, Catch::ResultWas::DidntThrowException, reaction); 456 } 457 handleUnexpectedInflightException(AssertionInfo const & info,std::string const & message,AssertionReaction & reaction)458 void RunContext::handleUnexpectedInflightException( 459 AssertionInfo const& info, 460 std::string const& message, 461 AssertionReaction& reaction 462 ) { 463 m_lastAssertionInfo = info; 464 465 AssertionResultData data( ResultWas::ThrewException, LazyExpression( false ) ); 466 data.message = message; 467 AssertionResult assertionResult{ info, data }; 468 assertionEnded( assertionResult ); 469 populateReaction( reaction ); 470 } 471 populateReaction(AssertionReaction & reaction)472 void RunContext::populateReaction( AssertionReaction& reaction ) { 473 reaction.shouldDebugBreak = m_config->shouldDebugBreak(); 474 reaction.shouldThrow = aborting() || (m_lastAssertionInfo.resultDisposition & ResultDisposition::Normal); 475 } 476 handleIncomplete(AssertionInfo const & info)477 void RunContext::handleIncomplete( 478 AssertionInfo const& info 479 ) { 480 m_lastAssertionInfo = info; 481 482 AssertionResultData data( ResultWas::ThrewException, LazyExpression( false ) ); 483 data.message = "Exception translation was disabled by CATCH_CONFIG_FAST_COMPILE"; 484 AssertionResult assertionResult{ info, data }; 485 assertionEnded( assertionResult ); 486 } handleNonExpr(AssertionInfo const & info,ResultWas::OfType resultType,AssertionReaction & reaction)487 void RunContext::handleNonExpr( 488 AssertionInfo const &info, 489 ResultWas::OfType resultType, 490 AssertionReaction &reaction 491 ) { 492 m_lastAssertionInfo = info; 493 494 AssertionResultData data( resultType, LazyExpression( false ) ); 495 AssertionResult assertionResult{ info, data }; 496 assertionEnded( assertionResult ); 497 498 if( !assertionResult.isOk() ) 499 populateReaction( reaction ); 500 } 501 502 getResultCapture()503 IResultCapture& getResultCapture() { 504 if (auto* capture = getCurrentContext().getResultCapture()) 505 return *capture; 506 else 507 CATCH_INTERNAL_ERROR("No result capture instance"); 508 } 509 seedRng(IConfig const & config)510 void seedRng(IConfig const& config) { 511 if (config.rngSeed() != 0) { 512 std::srand(config.rngSeed()); 513 rng().seed(config.rngSeed()); 514 } 515 } 516 rngSeed()517 unsigned int rngSeed() { 518 return getCurrentContext().getConfig()->rngSeed(); 519 } 520 521 } 522