1 /*
2  *  Created by Phil on 31/12/2010.
3  *  Copyright 2010 Two Blue Cubes Ltd. All rights reserved.
4  *
5  *  Distributed under the Boost Software License, Version 1.0. (See accompanying
6  *  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7  */
8 #ifndef TWOBLUECUBES_CATCH_INTERFACES_REPORTER_H_INCLUDED
9 #define TWOBLUECUBES_CATCH_INTERFACES_REPORTER_H_INCLUDED
10 
11 #include "catch_section_info.h"
12 #include "catch_common.h"
13 #include "catch_config.hpp"
14 #include "catch_totals.h"
15 #include "catch_test_case_info.h"
16 #include "catch_assertionresult.h"
17 #include "catch_message.h"
18 #include "catch_option.hpp"
19 #include "catch_stringref.h"
20 
21 #if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
22 #include "benchmark/catch_estimate.hpp"
23 #include "benchmark/catch_outlier_classification.hpp"
24 #endif // CATCH_CONFIG_ENABLE_BENCHMARKING
25 
26 
27 #include <string>
28 #include <iosfwd>
29 #include <map>
30 #include <set>
31 #include <memory>
32 #include <algorithm>
33 
34 namespace Catch {
35 
36     struct ReporterConfig {
37         explicit ReporterConfig( IConfigPtr const& _fullConfig );
38 
39         ReporterConfig( IConfigPtr const& _fullConfig, std::ostream& _stream );
40 
41         std::ostream& stream() const;
42         IConfigPtr fullConfig() const;
43 
44     private:
45         std::ostream* m_stream;
46         IConfigPtr m_fullConfig;
47     };
48 
49     struct ReporterPreferences {
50         bool shouldRedirectStdOut = false;
51         bool shouldReportAllAssertions = false;
52     };
53 
54     template<typename T>
55     struct LazyStat : Option<T> {
56         LazyStat& operator=( T const& _value ) {
57             Option<T>::operator=( _value );
58             used = false;
59             return *this;
60         }
resetLazyStat61         void reset() {
62             Option<T>::reset();
63             used = false;
64         }
65         bool used = false;
66     };
67 
68     struct TestRunInfo {
69         TestRunInfo( std::string const& _name );
70         std::string name;
71     };
72     struct GroupInfo {
73         GroupInfo(  std::string const& _name,
74                     std::size_t _groupIndex,
75                     std::size_t _groupsCount );
76 
77         std::string name;
78         std::size_t groupIndex;
79         std::size_t groupsCounts;
80     };
81 
82     struct AssertionStats {
83         AssertionStats( AssertionResult const& _assertionResult,
84                         std::vector<MessageInfo> const& _infoMessages,
85                         Totals const& _totals );
86 
87         AssertionStats( AssertionStats const& )              = default;
88         AssertionStats( AssertionStats && )                  = default;
89         AssertionStats& operator = ( AssertionStats const& ) = delete;
90         AssertionStats& operator = ( AssertionStats && )     = delete;
91         virtual ~AssertionStats();
92 
93         AssertionResult assertionResult;
94         std::vector<MessageInfo> infoMessages;
95         Totals totals;
96     };
97 
98     struct SectionStats {
99         SectionStats(   SectionInfo const& _sectionInfo,
100                         Counts const& _assertions,
101                         double _durationInSeconds,
102                         bool _missingAssertions );
103         SectionStats( SectionStats const& )              = default;
104         SectionStats( SectionStats && )                  = default;
105         SectionStats& operator = ( SectionStats const& ) = default;
106         SectionStats& operator = ( SectionStats && )     = default;
107         virtual ~SectionStats();
108 
109         SectionInfo sectionInfo;
110         Counts assertions;
111         double durationInSeconds;
112         bool missingAssertions;
113     };
114 
115     struct TestCaseStats {
116         TestCaseStats(  TestCaseInfo const& _testInfo,
117                         Totals const& _totals,
118                         std::string const& _stdOut,
119                         std::string const& _stdErr,
120                         bool _aborting );
121 
122         TestCaseStats( TestCaseStats const& )              = default;
123         TestCaseStats( TestCaseStats && )                  = default;
124         TestCaseStats& operator = ( TestCaseStats const& ) = default;
125         TestCaseStats& operator = ( TestCaseStats && )     = default;
126         virtual ~TestCaseStats();
127 
128         TestCaseInfo testInfo;
129         Totals totals;
130         std::string stdOut;
131         std::string stdErr;
132         bool aborting;
133     };
134 
135     struct TestGroupStats {
136         TestGroupStats( GroupInfo const& _groupInfo,
137                         Totals const& _totals,
138                         bool _aborting );
139         TestGroupStats( GroupInfo const& _groupInfo );
140 
141         TestGroupStats( TestGroupStats const& )              = default;
142         TestGroupStats( TestGroupStats && )                  = default;
143         TestGroupStats& operator = ( TestGroupStats const& ) = default;
144         TestGroupStats& operator = ( TestGroupStats && )     = default;
145         virtual ~TestGroupStats();
146 
147         GroupInfo groupInfo;
148         Totals totals;
149         bool aborting;
150     };
151 
152     struct TestRunStats {
153         TestRunStats(   TestRunInfo const& _runInfo,
154                         Totals const& _totals,
155                         bool _aborting );
156 
157         TestRunStats( TestRunStats const& )              = default;
158         TestRunStats( TestRunStats && )                  = default;
159         TestRunStats& operator = ( TestRunStats const& ) = default;
160         TestRunStats& operator = ( TestRunStats && )     = default;
161         virtual ~TestRunStats();
162 
163         TestRunInfo runInfo;
164         Totals totals;
165         bool aborting;
166     };
167 
168 #if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
169     struct BenchmarkInfo {
170         std::string name;
171         double estimatedDuration;
172         int iterations;
173         int samples;
174         unsigned int resamples;
175         double clockResolution;
176         double clockCost;
177     };
178 
179     template <class Duration>
180     struct BenchmarkStats {
181         BenchmarkInfo info;
182 
183         std::vector<Duration> samples;
184         Benchmark::Estimate<Duration> mean;
185         Benchmark::Estimate<Duration> standardDeviation;
186         Benchmark::OutlierClassification outliers;
187         double outlierVariance;
188 
189         template <typename Duration2>
190         operator BenchmarkStats<Duration2>() const {
191             std::vector<Duration2> samples2;
192             samples2.reserve(samples.size());
193             std::transform(samples.begin(), samples.end(), std::back_inserter(samples2), [](Duration d) { return Duration2(d); });
194             return {
195                 info,
196                 std::move(samples2),
197                 mean,
198                 standardDeviation,
199                 outliers,
200                 outlierVariance,
201             };
202         }
203     };
204 #endif // CATCH_CONFIG_ENABLE_BENCHMARKING
205 
206     struct IStreamingReporter {
207         virtual ~IStreamingReporter() = default;
208 
209         // Implementing class must also provide the following static methods:
210         // static std::string getDescription();
211         // static std::set<Verbosity> getSupportedVerbosities()
212 
213         virtual ReporterPreferences getPreferences() const = 0;
214 
215         virtual void noMatchingTestCases( std::string const& spec ) = 0;
216 
reportInvalidArgumentsIStreamingReporter217         virtual void reportInvalidArguments(std::string const&) {}
218 
219         virtual void testRunStarting( TestRunInfo const& testRunInfo ) = 0;
220         virtual void testGroupStarting( GroupInfo const& groupInfo ) = 0;
221 
222         virtual void testCaseStarting( TestCaseInfo const& testInfo ) = 0;
223         virtual void sectionStarting( SectionInfo const& sectionInfo ) = 0;
224 
225 #if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
benchmarkPreparingIStreamingReporter226         virtual void benchmarkPreparing( std::string const& ) {}
benchmarkStartingIStreamingReporter227         virtual void benchmarkStarting( BenchmarkInfo const& ) {}
benchmarkEndedIStreamingReporter228         virtual void benchmarkEnded( BenchmarkStats<> const& ) {}
benchmarkFailedIStreamingReporter229         virtual void benchmarkFailed( std::string const& ) {}
230 #endif // CATCH_CONFIG_ENABLE_BENCHMARKING
231 
232         virtual void assertionStarting( AssertionInfo const& assertionInfo ) = 0;
233 
234         // The return value indicates if the messages buffer should be cleared:
235         virtual bool assertionEnded( AssertionStats const& assertionStats ) = 0;
236 
237         virtual void sectionEnded( SectionStats const& sectionStats ) = 0;
238         virtual void testCaseEnded( TestCaseStats const& testCaseStats ) = 0;
239         virtual void testGroupEnded( TestGroupStats const& testGroupStats ) = 0;
240         virtual void testRunEnded( TestRunStats const& testRunStats ) = 0;
241 
242         virtual void skipTest( TestCaseInfo const& testInfo ) = 0;
243 
244         // Default empty implementation provided
245         virtual void fatalErrorEncountered( StringRef name );
246 
247         virtual bool isMulti() const;
248     };
249     using IStreamingReporterPtr = std::unique_ptr<IStreamingReporter>;
250 
251     struct IReporterFactory {
252         virtual ~IReporterFactory();
253         virtual IStreamingReporterPtr create( ReporterConfig const& config ) const = 0;
254         virtual std::string getDescription() const = 0;
255     };
256     using IReporterFactoryPtr = std::shared_ptr<IReporterFactory>;
257 
258     struct IReporterRegistry {
259         using FactoryMap = std::map<std::string, IReporterFactoryPtr>;
260         using Listeners = std::vector<IReporterFactoryPtr>;
261 
262         virtual ~IReporterRegistry();
263         virtual IStreamingReporterPtr create( std::string const& name, IConfigPtr const& config ) const = 0;
264         virtual FactoryMap const& getFactories() const = 0;
265         virtual Listeners const& getListeners() const = 0;
266     };
267 
268 } // end namespace Catch
269 
270 #endif // TWOBLUECUBES_CATCH_INTERFACES_REPORTER_H_INCLUDED
271