1 #include "catch_run_context.h" 2 #include "catch_context.h" 3 #include "catch_enforce.h" 4 #include "catch_random_number_generator.h" 5 #include "catch_stream.h" 6 7 #include <cassert> 8 #include <algorithm> 9 #include <sstream> 10 11 namespace Catch { 12 13 class RedirectedStream { 14 std::ostream& m_originalStream; 15 std::ostream& m_redirectionStream; 16 std::streambuf* m_prevBuf; 17 18 public: RedirectedStream(std::ostream & originalStream,std::ostream & redirectionStream)19 RedirectedStream( std::ostream& originalStream, std::ostream& redirectionStream ) 20 : m_originalStream( originalStream ), 21 m_redirectionStream( redirectionStream ), 22 m_prevBuf( m_originalStream.rdbuf() ) 23 { 24 m_originalStream.rdbuf( m_redirectionStream.rdbuf() ); 25 } ~RedirectedStream()26 ~RedirectedStream() { 27 m_originalStream.rdbuf( m_prevBuf ); 28 } 29 }; 30 31 class RedirectedStdOut { 32 ReusableStringStream m_rss; 33 RedirectedStream m_cout; 34 public: RedirectedStdOut()35 RedirectedStdOut() : m_cout( Catch::cout(), m_rss.get() ) {} str() const36 auto str() const -> std::string { return m_rss.str(); } 37 }; 38 39 // StdErr has two constituent streams in C++, std::cerr and std::clog 40 // This means that we need to redirect 2 streams into 1 to keep proper 41 // order of writes 42 class RedirectedStdErr { 43 ReusableStringStream m_rss; 44 RedirectedStream m_cerr; 45 RedirectedStream m_clog; 46 public: RedirectedStdErr()47 RedirectedStdErr() 48 : m_cerr( Catch::cerr(), m_rss.get() ), 49 m_clog( Catch::clog(), m_rss.get() ) 50 {} str() const51 auto str() const -> std::string { return m_rss.str(); } 52 }; 53 54 RunContext(IConfigPtr const & _config,IStreamingReporterPtr && reporter)55 RunContext::RunContext(IConfigPtr const& _config, IStreamingReporterPtr&& reporter) 56 : m_runInfo(_config->name()), 57 m_context(getCurrentMutableContext()), 58 m_config(_config), 59 m_reporter(std::move(reporter)), 60 m_lastAssertionInfo{ StringRef(), SourceLineInfo("",0), StringRef(), ResultDisposition::Normal }, 61 m_includeSuccessfulResults( m_config->includeSuccessfulResults() ) 62 { 63 m_context.setRunner(this); 64 m_context.setConfig(m_config); 65 m_context.setResultCapture(this); 66 m_reporter->testRunStarting(m_runInfo); 67 } 68 ~RunContext()69 RunContext::~RunContext() { 70 m_reporter->testRunEnded(TestRunStats(m_runInfo, m_totals, aborting())); 71 } 72 testGroupStarting(std::string const & testSpec,std::size_t groupIndex,std::size_t groupsCount)73 void RunContext::testGroupStarting(std::string const& testSpec, std::size_t groupIndex, std::size_t groupsCount) { 74 m_reporter->testGroupStarting(GroupInfo(testSpec, groupIndex, groupsCount)); 75 } 76 testGroupEnded(std::string const & testSpec,Totals const & totals,std::size_t groupIndex,std::size_t groupsCount)77 void RunContext::testGroupEnded(std::string const& testSpec, Totals const& totals, std::size_t groupIndex, std::size_t groupsCount) { 78 m_reporter->testGroupEnded(TestGroupStats(GroupInfo(testSpec, groupIndex, groupsCount), totals, aborting())); 79 } 80 runTest(TestCase const & testCase)81 Totals RunContext::runTest(TestCase const& testCase) { 82 Totals prevTotals = m_totals; 83 84 std::string redirectedCout; 85 std::string redirectedCerr; 86 87 auto const& testInfo = testCase.getTestCaseInfo(); 88 89 m_reporter->testCaseStarting(testInfo); 90 91 m_activeTestCase = &testCase; 92 93 94 ITracker& rootTracker = m_trackerContext.startRun(); 95 assert(rootTracker.isSectionTracker()); 96 static_cast<SectionTracker&>(rootTracker).addInitialFilters(m_config->getSectionsToRun()); 97 do { 98 m_trackerContext.startCycle(); 99 m_testCaseTracker = &SectionTracker::acquire(m_trackerContext, TestCaseTracking::NameAndLocation(testInfo.name, testInfo.lineInfo)); 100 runCurrentTest(redirectedCout, redirectedCerr); 101 } while (!m_testCaseTracker->isSuccessfullyCompleted() && !aborting()); 102 103 Totals deltaTotals = m_totals.delta(prevTotals); 104 if (testInfo.expectedToFail() && deltaTotals.testCases.passed > 0) { 105 deltaTotals.assertions.failed++; 106 deltaTotals.testCases.passed--; 107 deltaTotals.testCases.failed++; 108 } 109 m_totals.testCases += deltaTotals.testCases; 110 m_reporter->testCaseEnded(TestCaseStats(testInfo, 111 deltaTotals, 112 redirectedCout, 113 redirectedCerr, 114 aborting())); 115 116 m_activeTestCase = nullptr; 117 m_testCaseTracker = nullptr; 118 119 return deltaTotals; 120 } 121 config() const122 IConfigPtr RunContext::config() const { 123 return m_config; 124 } 125 reporter() const126 IStreamingReporter& RunContext::reporter() const { 127 return *m_reporter; 128 } 129 assertionEnded(AssertionResult const & result)130 void RunContext::assertionEnded(AssertionResult const & result) { 131 if (result.getResultType() == ResultWas::Ok) { 132 m_totals.assertions.passed++; 133 m_lastAssertionPassed = true; 134 } else if (!result.isOk()) { 135 m_lastAssertionPassed = false; 136 if( m_activeTestCase->getTestCaseInfo().okToFail() ) 137 m_totals.assertions.failedButOk++; 138 else 139 m_totals.assertions.failed++; 140 } 141 else { 142 m_lastAssertionPassed = true; 143 } 144 145 // We have no use for the return value (whether messages should be cleared), because messages were made scoped 146 // and should be let to clear themselves out. 147 static_cast<void>(m_reporter->assertionEnded(AssertionStats(result, m_messages, m_totals))); 148 149 // Reset working state 150 resetAssertionInfo(); 151 m_lastResult = result; 152 } resetAssertionInfo()153 void RunContext::resetAssertionInfo() { 154 m_lastAssertionInfo.macroName = StringRef(); 155 m_lastAssertionInfo.capturedExpression = "{Unknown expression after the reported line}"_sr; 156 } 157 sectionStarted(SectionInfo const & sectionInfo,Counts & assertions)158 bool RunContext::sectionStarted(SectionInfo const & sectionInfo, Counts & assertions) { 159 ITracker& sectionTracker = SectionTracker::acquire(m_trackerContext, TestCaseTracking::NameAndLocation(sectionInfo.name, sectionInfo.lineInfo)); 160 if (!sectionTracker.isOpen()) 161 return false; 162 m_activeSections.push_back(§ionTracker); 163 164 m_lastAssertionInfo.lineInfo = sectionInfo.lineInfo; 165 166 m_reporter->sectionStarting(sectionInfo); 167 168 assertions = m_totals.assertions; 169 170 return true; 171 } 172 testForMissingAssertions(Counts & assertions)173 bool RunContext::testForMissingAssertions(Counts& assertions) { 174 if (assertions.total() != 0) 175 return false; 176 if (!m_config->warnAboutMissingAssertions()) 177 return false; 178 if (m_trackerContext.currentTracker().hasChildren()) 179 return false; 180 m_totals.assertions.failed++; 181 assertions.failed++; 182 return true; 183 } 184 sectionEnded(SectionEndInfo const & endInfo)185 void RunContext::sectionEnded(SectionEndInfo const & endInfo) { 186 Counts assertions = m_totals.assertions - endInfo.prevAssertions; 187 bool missingAssertions = testForMissingAssertions(assertions); 188 189 if (!m_activeSections.empty()) { 190 m_activeSections.back()->close(); 191 m_activeSections.pop_back(); 192 } 193 194 m_reporter->sectionEnded(SectionStats(endInfo.sectionInfo, assertions, endInfo.durationInSeconds, missingAssertions)); 195 m_messages.clear(); 196 } 197 sectionEndedEarly(SectionEndInfo const & endInfo)198 void RunContext::sectionEndedEarly(SectionEndInfo const & endInfo) { 199 if (m_unfinishedSections.empty()) 200 m_activeSections.back()->fail(); 201 else 202 m_activeSections.back()->close(); 203 m_activeSections.pop_back(); 204 205 m_unfinishedSections.push_back(endInfo); 206 } benchmarkStarting(BenchmarkInfo const & info)207 void RunContext::benchmarkStarting( BenchmarkInfo const& info ) { 208 m_reporter->benchmarkStarting( info ); 209 } benchmarkEnded(BenchmarkStats const & stats)210 void RunContext::benchmarkEnded( BenchmarkStats const& stats ) { 211 m_reporter->benchmarkEnded( stats ); 212 } 213 pushScopedMessage(MessageInfo const & message)214 void RunContext::pushScopedMessage(MessageInfo const & message) { 215 m_messages.push_back(message); 216 } 217 popScopedMessage(MessageInfo const & message)218 void RunContext::popScopedMessage(MessageInfo const & message) { 219 m_messages.erase(std::remove(m_messages.begin(), m_messages.end(), message), m_messages.end()); 220 } 221 getCurrentTestName() const222 std::string RunContext::getCurrentTestName() const { 223 return m_activeTestCase 224 ? m_activeTestCase->getTestCaseInfo().name 225 : std::string(); 226 } 227 getLastResult() const228 const AssertionResult * RunContext::getLastResult() const { 229 return &(*m_lastResult); 230 } 231 exceptionEarlyReported()232 void RunContext::exceptionEarlyReported() { 233 m_shouldReportUnexpected = false; 234 } 235 handleFatalErrorCondition(StringRef message)236 void RunContext::handleFatalErrorCondition( StringRef message ) { 237 // First notify reporter that bad things happened 238 m_reporter->fatalErrorEncountered(message); 239 240 // Don't rebuild the result -- the stringification itself can cause more fatal errors 241 // Instead, fake a result data. 242 AssertionResultData tempResult( ResultWas::FatalErrorCondition, { false } ); 243 tempResult.message = message; 244 AssertionResult result(m_lastAssertionInfo, tempResult); 245 246 assertionEnded(result); 247 248 handleUnfinishedSections(); 249 250 // Recreate section for test case (as we will lose the one that was in scope) 251 auto const& testCaseInfo = m_activeTestCase->getTestCaseInfo(); 252 SectionInfo testCaseSection(testCaseInfo.lineInfo, testCaseInfo.name, testCaseInfo.description); 253 254 Counts assertions; 255 assertions.failed = 1; 256 SectionStats testCaseSectionStats(testCaseSection, assertions, 0, false); 257 m_reporter->sectionEnded(testCaseSectionStats); 258 259 auto const& testInfo = m_activeTestCase->getTestCaseInfo(); 260 261 Totals deltaTotals; 262 deltaTotals.testCases.failed = 1; 263 deltaTotals.assertions.failed = 1; 264 m_reporter->testCaseEnded(TestCaseStats(testInfo, 265 deltaTotals, 266 std::string(), 267 std::string(), 268 false)); 269 m_totals.testCases.failed++; 270 testGroupEnded(std::string(), m_totals, 1, 1); 271 m_reporter->testRunEnded(TestRunStats(m_runInfo, m_totals, false)); 272 } 273 lastAssertionPassed()274 bool RunContext::lastAssertionPassed() { 275 return m_lastAssertionPassed; 276 } 277 assertionPassed()278 void RunContext::assertionPassed() { 279 m_lastAssertionPassed = true; 280 ++m_totals.assertions.passed; 281 resetAssertionInfo(); 282 } 283 aborting() const284 bool RunContext::aborting() const { 285 return m_totals.assertions.failed == static_cast<std::size_t>(m_config->abortAfter()); 286 } 287 runCurrentTest(std::string & redirectedCout,std::string & redirectedCerr)288 void RunContext::runCurrentTest(std::string & redirectedCout, std::string & redirectedCerr) { 289 auto const& testCaseInfo = m_activeTestCase->getTestCaseInfo(); 290 SectionInfo testCaseSection(testCaseInfo.lineInfo, testCaseInfo.name, testCaseInfo.description); 291 m_reporter->sectionStarting(testCaseSection); 292 Counts prevAssertions = m_totals.assertions; 293 double duration = 0; 294 m_shouldReportUnexpected = true; 295 m_lastAssertionInfo = { "TEST_CASE"_sr, testCaseInfo.lineInfo, StringRef(), ResultDisposition::Normal }; 296 297 seedRng(*m_config); 298 299 Timer timer; 300 try { 301 if (m_reporter->getPreferences().shouldRedirectStdOut) { 302 RedirectedStdOut redirectedStdOut; 303 RedirectedStdErr redirectedStdErr; 304 timer.start(); 305 invokeActiveTestCase(); 306 redirectedCout += redirectedStdOut.str(); 307 redirectedCerr += redirectedStdErr.str(); 308 309 } else { 310 timer.start(); 311 invokeActiveTestCase(); 312 } 313 duration = timer.getElapsedSeconds(); 314 } catch (TestFailureException&) { 315 // This just means the test was aborted due to failure 316 } catch (...) { 317 // Under CATCH_CONFIG_FAST_COMPILE, unexpected exceptions under REQUIRE assertions 318 // are reported without translation at the point of origin. 319 if( m_shouldReportUnexpected ) { 320 AssertionReaction dummyReaction; 321 handleUnexpectedInflightException( m_lastAssertionInfo, translateActiveException(), dummyReaction ); 322 } 323 } 324 Counts assertions = m_totals.assertions - prevAssertions; 325 bool missingAssertions = testForMissingAssertions(assertions); 326 327 m_testCaseTracker->close(); 328 handleUnfinishedSections(); 329 m_messages.clear(); 330 331 SectionStats testCaseSectionStats(testCaseSection, assertions, duration, missingAssertions); 332 m_reporter->sectionEnded(testCaseSectionStats); 333 } 334 invokeActiveTestCase()335 void RunContext::invokeActiveTestCase() { 336 FatalConditionHandler fatalConditionHandler; // Handle signals 337 m_activeTestCase->invoke(); 338 fatalConditionHandler.reset(); 339 } 340 handleUnfinishedSections()341 void RunContext::handleUnfinishedSections() { 342 // If sections ended prematurely due to an exception we stored their 343 // infos here so we can tear them down outside the unwind process. 344 for (auto it = m_unfinishedSections.rbegin(), 345 itEnd = m_unfinishedSections.rend(); 346 it != itEnd; 347 ++it) 348 sectionEnded(*it); 349 m_unfinishedSections.clear(); 350 } 351 handleExpr(AssertionInfo const & info,ITransientExpression const & expr,AssertionReaction & reaction)352 void RunContext::handleExpr( 353 AssertionInfo const& info, 354 ITransientExpression const& expr, 355 AssertionReaction& reaction 356 ) { 357 m_reporter->assertionStarting( info ); 358 359 bool negated = isFalseTest( info.resultDisposition ); 360 bool result = expr.getResult() != negated; 361 362 if( result ) { 363 if (!m_includeSuccessfulResults) { 364 assertionPassed(); 365 } 366 else { 367 reportExpr(info, ResultWas::Ok, &expr, negated); 368 } 369 } 370 else { 371 reportExpr(info, ResultWas::ExpressionFailed, &expr, negated ); 372 populateReaction( reaction ); 373 } 374 } reportExpr(AssertionInfo const & info,ResultWas::OfType resultType,ITransientExpression const * expr,bool negated)375 void RunContext::reportExpr( 376 AssertionInfo const &info, 377 ResultWas::OfType resultType, 378 ITransientExpression const *expr, 379 bool negated ) { 380 381 m_lastAssertionInfo = info; 382 AssertionResultData data( resultType, LazyExpression( negated ) ); 383 384 AssertionResult assertionResult{ info, data }; 385 assertionResult.m_resultData.lazyExpression.m_transientExpression = expr; 386 387 assertionEnded( assertionResult ); 388 } 389 handleMessage(AssertionInfo const & info,ResultWas::OfType resultType,StringRef const & message,AssertionReaction & reaction)390 void RunContext::handleMessage( 391 AssertionInfo const& info, 392 ResultWas::OfType resultType, 393 StringRef const& message, 394 AssertionReaction& reaction 395 ) { 396 m_reporter->assertionStarting( info ); 397 398 m_lastAssertionInfo = info; 399 400 AssertionResultData data( resultType, LazyExpression( false ) ); 401 data.message = message; 402 AssertionResult assertionResult{ m_lastAssertionInfo, data }; 403 assertionEnded( assertionResult ); 404 if( !assertionResult.isOk() ) 405 populateReaction( reaction ); 406 } handleUnexpectedExceptionNotThrown(AssertionInfo const & info,AssertionReaction & reaction)407 void RunContext::handleUnexpectedExceptionNotThrown( 408 AssertionInfo const& info, 409 AssertionReaction& reaction 410 ) { 411 handleNonExpr(info, Catch::ResultWas::DidntThrowException, reaction); 412 } 413 handleUnexpectedInflightException(AssertionInfo const & info,std::string const & message,AssertionReaction & reaction)414 void RunContext::handleUnexpectedInflightException( 415 AssertionInfo const& info, 416 std::string const& message, 417 AssertionReaction& reaction 418 ) { 419 m_lastAssertionInfo = info; 420 421 AssertionResultData data( ResultWas::ThrewException, LazyExpression( false ) ); 422 data.message = message; 423 AssertionResult assertionResult{ info, data }; 424 assertionEnded( assertionResult ); 425 populateReaction( reaction ); 426 } 427 populateReaction(AssertionReaction & reaction)428 void RunContext::populateReaction( AssertionReaction& reaction ) { 429 reaction.shouldDebugBreak = m_config->shouldDebugBreak(); 430 reaction.shouldThrow = aborting() || (m_lastAssertionInfo.resultDisposition & ResultDisposition::Normal); 431 } 432 handleIncomplete(AssertionInfo const & info)433 void RunContext::handleIncomplete( 434 AssertionInfo const& info 435 ) { 436 m_lastAssertionInfo = info; 437 438 AssertionResultData data( ResultWas::ThrewException, LazyExpression( false ) ); 439 data.message = "Exception translation was disabled by CATCH_CONFIG_FAST_COMPILE"; 440 AssertionResult assertionResult{ info, data }; 441 assertionEnded( assertionResult ); 442 } handleNonExpr(AssertionInfo const & info,ResultWas::OfType resultType,AssertionReaction & reaction)443 void RunContext::handleNonExpr( 444 AssertionInfo const &info, 445 ResultWas::OfType resultType, 446 AssertionReaction &reaction 447 ) { 448 m_lastAssertionInfo = info; 449 450 AssertionResultData data( resultType, LazyExpression( false ) ); 451 AssertionResult assertionResult{ info, data }; 452 assertionEnded( assertionResult ); 453 454 if( !assertionResult.isOk() ) 455 populateReaction( reaction ); 456 } 457 458 getResultCapture()459 IResultCapture& getResultCapture() { 460 if (auto* capture = getCurrentContext().getResultCapture()) 461 return *capture; 462 else 463 CATCH_INTERNAL_ERROR("No result capture instance"); 464 } 465 } 466