1  /*
2  *  Created by Phil on 22/10/2010.
3  *  Copyright 2010 Two Blue Cubes Ltd. All rights reserved.
4  *
5  *  Distributed under the Boost Software License, Version 1.0. (See accompanying
6  *  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7  */
8 #ifndef TWOBLUECUBES_CATCH_RUNNER_IMPL_HPP_INCLUDED
9 #define TWOBLUECUBES_CATCH_RUNNER_IMPL_HPP_INCLUDED
10 
11 #include "catch_interfaces_runner.h"
12 #include "catch_interfaces_reporter.h"
13 #include "catch_interfaces_exception.h"
14 #include "catch_config.hpp"
15 #include "catch_test_registry.hpp"
16 #include "catch_test_case_info.h"
17 #include "catch_capture.hpp"
18 #include "catch_totals.hpp"
19 #include "catch_test_spec.hpp"
20 #include "catch_test_case_tracker.hpp"
21 #include "catch_timer.h"
22 #include "catch_result_builder.h"
23 #include "catch_fatal_condition.hpp"
24 
25 
26 #include <set>
27 #include <string>
28 
29 namespace Catch {
30 
31     class StreamRedirect {
32 
33     public:
StreamRedirect(std::ostream & stream,std::string & targetString)34         StreamRedirect( std::ostream& stream, std::string& targetString )
35         :   m_stream( stream ),
36             m_prevBuf( stream.rdbuf() ),
37             m_targetString( targetString )
38         {
39             stream.rdbuf( m_oss.rdbuf() );
40         }
41 
~StreamRedirect()42         ~StreamRedirect() {
43             m_targetString += m_oss.str();
44             m_stream.rdbuf( m_prevBuf );
45         }
46 
47     private:
48         std::ostream& m_stream;
49         std::streambuf* m_prevBuf;
50         std::ostringstream m_oss;
51         std::string& m_targetString;
52     };
53 
54     // StdErr has two constituent streams in C++, std::cerr and std::clog
55     // This means that we need to redirect 2 streams into 1 to keep proper
56     // order of writes and cannot use StreamRedirect on its own
57     class StdErrRedirect {
58     public:
StdErrRedirect(std::string & targetString)59         StdErrRedirect(std::string& targetString)
60         :m_cerrBuf( cerr().rdbuf() ), m_clogBuf(clog().rdbuf()),
61         m_targetString(targetString){
62             cerr().rdbuf(m_oss.rdbuf());
63             clog().rdbuf(m_oss.rdbuf());
64         }
~StdErrRedirect()65         ~StdErrRedirect() {
66             m_targetString += m_oss.str();
67             cerr().rdbuf(m_cerrBuf);
68             clog().rdbuf(m_clogBuf);
69         }
70     private:
71         std::streambuf* m_cerrBuf;
72         std::streambuf* m_clogBuf;
73         std::ostringstream m_oss;
74         std::string& m_targetString;
75     };
76 
77     ///////////////////////////////////////////////////////////////////////////
78 
79     class RunContext : public IResultCapture, public IRunner {
80 
81         RunContext( RunContext const& );
82         void operator =( RunContext const& );
83 
84     public:
85 
RunContext(Ptr<IConfig const> const & _config,Ptr<IStreamingReporter> const & reporter)86         explicit RunContext( Ptr<IConfig const> const& _config, Ptr<IStreamingReporter> const& reporter )
87         :   m_runInfo( _config->name() ),
88             m_context( getCurrentMutableContext() ),
89             m_activeTestCase( CATCH_NULL ),
90             m_config( _config ),
91             m_reporter( reporter ),
92             m_shouldReportUnexpected ( true )
93         {
94             m_context.setRunner( this );
95             m_context.setConfig( m_config );
96             m_context.setResultCapture( this );
97             m_reporter->testRunStarting( m_runInfo );
98         }
99 
~RunContext()100         virtual ~RunContext() {
101             m_reporter->testRunEnded( TestRunStats( m_runInfo, m_totals, aborting() ) );
102         }
103 
testGroupStarting(std::string const & testSpec,std::size_t groupIndex,std::size_t groupsCount)104         void testGroupStarting( std::string const& testSpec, std::size_t groupIndex, std::size_t groupsCount ) {
105             m_reporter->testGroupStarting( GroupInfo( testSpec, groupIndex, groupsCount ) );
106         }
testGroupEnded(std::string const & testSpec,Totals const & totals,std::size_t groupIndex,std::size_t groupsCount)107         void testGroupEnded( std::string const& testSpec, Totals const& totals, std::size_t groupIndex, std::size_t groupsCount ) {
108             m_reporter->testGroupEnded( TestGroupStats( GroupInfo( testSpec, groupIndex, groupsCount ), totals, aborting() ) );
109         }
110 
runTest(TestCase const & testCase)111         Totals runTest( TestCase const& testCase ) {
112             Totals prevTotals = m_totals;
113 
114             std::string redirectedCout;
115             std::string redirectedCerr;
116 
117             TestCaseInfo testInfo = testCase.getTestCaseInfo();
118 
119             m_reporter->testCaseStarting( testInfo );
120 
121             m_activeTestCase = &testCase;
122 
123 
124             do {
125                 ITracker& rootTracker = m_trackerContext.startRun();
126                 assert( rootTracker.isSectionTracker() );
127                 static_cast<SectionTracker&>( rootTracker ).addInitialFilters( m_config->getSectionsToRun() );
128                 do {
129                     m_trackerContext.startCycle();
130                     m_testCaseTracker = &SectionTracker::acquire( m_trackerContext, TestCaseTracking::NameAndLocation( testInfo.name, testInfo.lineInfo ) );
131                     runCurrentTest( redirectedCout, redirectedCerr );
132                 }
133                 while( !m_testCaseTracker->isSuccessfullyCompleted() && !aborting() );
134             }
135             // !TBD: deprecated - this will be replaced by indexed trackers
136             while( getCurrentContext().advanceGeneratorsForCurrentTest() && !aborting() );
137 
138             Totals deltaTotals = m_totals.delta( prevTotals );
139             if( testInfo.expectedToFail() && deltaTotals.testCases.passed > 0 ) {
140                 deltaTotals.assertions.failed++;
141                 deltaTotals.testCases.passed--;
142                 deltaTotals.testCases.failed++;
143             }
144             m_totals.testCases += deltaTotals.testCases;
145             m_reporter->testCaseEnded( TestCaseStats(   testInfo,
146                                                         deltaTotals,
147                                                         redirectedCout,
148                                                         redirectedCerr,
149                                                         aborting() ) );
150 
151             m_activeTestCase = CATCH_NULL;
152             m_testCaseTracker = CATCH_NULL;
153 
154             return deltaTotals;
155         }
156 
config() const157         Ptr<IConfig const> config() const {
158             return m_config;
159         }
160 
161     private: // IResultCapture
162 
163 
assertionEnded(AssertionResult const & result)164         virtual void assertionEnded( AssertionResult const& result ) {
165             if( result.getResultType() == ResultWas::Ok ) {
166                 m_totals.assertions.passed++;
167             }
168             else if( !result.isOk() ) {
169                 if( m_activeTestCase->getTestCaseInfo().okToFail() )
170                     m_totals.assertions.failedButOk++;
171                 else
172                     m_totals.assertions.failed++;
173             }
174 
175             // We have no use for the return value (whether messages should be cleared), because messages were made scoped
176             // and should be let to clear themselves out.
177             static_cast<void>(m_reporter->assertionEnded(AssertionStats(result, m_messages, m_totals)));
178 
179             // Reset working state
180             m_lastAssertionInfo = AssertionInfo( "", m_lastAssertionInfo.lineInfo, "{Unknown expression after the reported line}" , m_lastAssertionInfo.resultDisposition );
181             m_lastResult = result;
182         }
183 
lastAssertionPassed()184         virtual bool lastAssertionPassed()
185         {
186             return m_totals.assertions.passed == (m_prevPassed + 1);
187         }
188 
assertionPassed()189         virtual void assertionPassed()
190         {
191             m_totals.assertions.passed++;
192             m_lastAssertionInfo.capturedExpression = "{Unknown expression after the reported line}";
193             m_lastAssertionInfo.macroName = "";
194         }
195 
assertionRun()196         virtual void assertionRun()
197         {
198             m_prevPassed = m_totals.assertions.passed;
199         }
200 
sectionStarted(SectionInfo const & sectionInfo,Counts & assertions)201         virtual bool sectionStarted (
202             SectionInfo const& sectionInfo,
203             Counts& assertions
204         )
205         {
206             ITracker& sectionTracker = SectionTracker::acquire( m_trackerContext, TestCaseTracking::NameAndLocation( sectionInfo.name, sectionInfo.lineInfo ) );
207             if( !sectionTracker.isOpen() )
208                 return false;
209             m_activeSections.push_back( &sectionTracker );
210 
211             m_lastAssertionInfo.lineInfo = sectionInfo.lineInfo;
212 
213             m_reporter->sectionStarting( sectionInfo );
214 
215             assertions = m_totals.assertions;
216 
217             return true;
218         }
testForMissingAssertions(Counts & assertions)219         bool testForMissingAssertions( Counts& assertions ) {
220             if( assertions.total() != 0 )
221                 return false;
222             if( !m_config->warnAboutMissingAssertions() )
223                 return false;
224             if( m_trackerContext.currentTracker().hasChildren() )
225                 return false;
226             m_totals.assertions.failed++;
227             assertions.failed++;
228             return true;
229         }
230 
sectionEnded(SectionEndInfo const & endInfo)231         virtual void sectionEnded( SectionEndInfo const& endInfo ) {
232             Counts assertions = m_totals.assertions - endInfo.prevAssertions;
233             bool missingAssertions = testForMissingAssertions( assertions );
234 
235             if( !m_activeSections.empty() ) {
236                 m_activeSections.back()->close();
237                 m_activeSections.pop_back();
238             }
239 
240             m_reporter->sectionEnded( SectionStats( endInfo.sectionInfo, assertions, endInfo.durationInSeconds, missingAssertions ) );
241             m_messages.clear();
242         }
243 
sectionEndedEarly(SectionEndInfo const & endInfo)244         virtual void sectionEndedEarly( SectionEndInfo const& endInfo ) {
245             if( m_unfinishedSections.empty() )
246                 m_activeSections.back()->fail();
247             else
248                 m_activeSections.back()->close();
249             m_activeSections.pop_back();
250 
251             m_unfinishedSections.push_back( endInfo );
252         }
253 
pushScopedMessage(MessageInfo const & message)254         virtual void pushScopedMessage( MessageInfo const& message ) {
255             m_messages.push_back( message );
256         }
257 
popScopedMessage(MessageInfo const & message)258         virtual void popScopedMessage( MessageInfo const& message ) {
259             m_messages.erase( std::remove( m_messages.begin(), m_messages.end(), message ), m_messages.end() );
260         }
261 
getCurrentTestName() const262         virtual std::string getCurrentTestName() const {
263             return m_activeTestCase
264                 ? m_activeTestCase->getTestCaseInfo().name
265                 : std::string();
266         }
267 
getLastResult() const268         virtual const AssertionResult* getLastResult() const {
269             return &m_lastResult;
270         }
271 
exceptionEarlyReported()272         virtual void exceptionEarlyReported() {
273             m_shouldReportUnexpected = false;
274         }
275 
handleFatalErrorCondition(std::string const & message)276         virtual void handleFatalErrorCondition( std::string const& message ) {
277             // Don't rebuild the result -- the stringification itself can cause more fatal errors
278             // Instead, fake a result data.
279             AssertionResultData tempResult;
280             tempResult.resultType = ResultWas::FatalErrorCondition;
281             tempResult.message = message;
282             AssertionResult result(m_lastAssertionInfo, tempResult);
283 
284             getResultCapture().assertionEnded(result);
285 
286             handleUnfinishedSections();
287 
288             // Recreate section for test case (as we will lose the one that was in scope)
289             TestCaseInfo const& testCaseInfo = m_activeTestCase->getTestCaseInfo();
290             SectionInfo testCaseSection( testCaseInfo.lineInfo, testCaseInfo.name, testCaseInfo.description );
291 
292             Counts assertions;
293             assertions.failed = 1;
294             SectionStats testCaseSectionStats( testCaseSection, assertions, 0, false );
295             m_reporter->sectionEnded( testCaseSectionStats );
296 
297             TestCaseInfo testInfo = m_activeTestCase->getTestCaseInfo();
298 
299             Totals deltaTotals;
300             deltaTotals.testCases.failed = 1;
301             deltaTotals.assertions.failed = 1;
302             m_reporter->testCaseEnded( TestCaseStats(   testInfo,
303                                                         deltaTotals,
304                                                         std::string(),
305                                                         std::string(),
306                                                         false ) );
307             m_totals.testCases.failed++;
308             testGroupEnded( std::string(), m_totals, 1, 1 );
309             m_reporter->testRunEnded( TestRunStats( m_runInfo, m_totals, false ) );
310         }
311 
312     public:
313         // !TBD We need to do this another way!
aborting() const314         bool aborting() const {
315             return m_totals.assertions.failed == static_cast<std::size_t>( m_config->abortAfter() );
316         }
317 
318     private:
319 
runCurrentTest(std::string & redirectedCout,std::string & redirectedCerr)320         void runCurrentTest( std::string& redirectedCout, std::string& redirectedCerr ) {
321             TestCaseInfo const& testCaseInfo = m_activeTestCase->getTestCaseInfo();
322             SectionInfo testCaseSection( testCaseInfo.lineInfo, testCaseInfo.name, testCaseInfo.description );
323             m_reporter->sectionStarting( testCaseSection );
324             Counts prevAssertions = m_totals.assertions;
325             double duration = 0;
326             m_shouldReportUnexpected = true;
327             try {
328                 m_lastAssertionInfo = AssertionInfo( "TEST_CASE", testCaseInfo.lineInfo, "", ResultDisposition::Normal );
329 
330                 seedRng( *m_config );
331 
332                 Timer timer;
333                 timer.start();
334                 if( m_reporter->getPreferences().shouldRedirectStdOut ) {
335                     StreamRedirect coutRedir( Catch::cout(), redirectedCout );
336                     StdErrRedirect errRedir( redirectedCerr );
337                     invokeActiveTestCase();
338                 }
339                 else {
340                     invokeActiveTestCase();
341                 }
342                 duration = timer.getElapsedSeconds();
343             }
344             catch( TestFailureException& ) {
345                 // This just means the test was aborted due to failure
346             }
347             catch(...) {
348                 // Under CATCH_CONFIG_FAST_COMPILE, unexpected exceptions under REQUIRE assertions
349                 // are reported without translation at the point of origin.
350                 if (m_shouldReportUnexpected) {
351                     makeUnexpectedResultBuilder().useActiveException();
352                 }
353             }
354             m_testCaseTracker->close();
355             handleUnfinishedSections();
356             m_messages.clear();
357 
358             Counts assertions = m_totals.assertions - prevAssertions;
359             bool missingAssertions = testForMissingAssertions( assertions );
360 
361             SectionStats testCaseSectionStats( testCaseSection, assertions, duration, missingAssertions );
362             m_reporter->sectionEnded( testCaseSectionStats );
363         }
364 
invokeActiveTestCase()365         void invokeActiveTestCase() {
366             FatalConditionHandler fatalConditionHandler; // Handle signals
367             m_activeTestCase->invoke();
368             fatalConditionHandler.reset();
369         }
370 
371     private:
372 
makeUnexpectedResultBuilder() const373         ResultBuilder makeUnexpectedResultBuilder() const {
374             return ResultBuilder(   m_lastAssertionInfo.macroName,
375                                     m_lastAssertionInfo.lineInfo,
376                                     m_lastAssertionInfo.capturedExpression,
377                                     m_lastAssertionInfo.resultDisposition );
378         }
379 
handleUnfinishedSections()380         void handleUnfinishedSections() {
381             // If sections ended prematurely due to an exception we stored their
382             // infos here so we can tear them down outside the unwind process.
383             for( std::vector<SectionEndInfo>::const_reverse_iterator it = m_unfinishedSections.rbegin(),
384                         itEnd = m_unfinishedSections.rend();
385                     it != itEnd;
386                     ++it )
387                 sectionEnded( *it );
388             m_unfinishedSections.clear();
389         }
390 
391         TestRunInfo m_runInfo;
392         IMutableContext& m_context;
393         TestCase const* m_activeTestCase;
394         ITracker* m_testCaseTracker;
395         ITracker* m_currentSectionTracker;
396         AssertionResult m_lastResult;
397 
398         Ptr<IConfig const> m_config;
399         Totals m_totals;
400         Ptr<IStreamingReporter> m_reporter;
401         std::vector<MessageInfo> m_messages;
402         AssertionInfo m_lastAssertionInfo;
403         std::vector<SectionEndInfo> m_unfinishedSections;
404         std::vector<ITracker*> m_activeSections;
405         TrackerContext m_trackerContext;
406         size_t m_prevPassed;
407         bool m_shouldReportUnexpected;
408     };
409 
getResultCapture()410     IResultCapture& getResultCapture() {
411         if( IResultCapture* capture = getCurrentContext().getResultCapture() )
412             return *capture;
413         else
414             throw std::logic_error( "No result capture instance" );
415     }
416 
417 } // end namespace Catch
418 
419 #endif // TWOBLUECUBES_CATCH_RUNNER_IMPL_HPP_INCLUDED
420