1 /*
2  *  Created by Phil on 31/12/2010.
3  *  Copyright 2010 Two Blue Cubes Ltd. All rights reserved.
4  *
5  *  Distributed under the Boost Software License, Version 1.0. (See accompanying
6  *  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7  */
8 #ifndef TWOBLUECUBES_CATCH_INTERFACES_REPORTER_H_INCLUDED
9 #define TWOBLUECUBES_CATCH_INTERFACES_REPORTER_H_INCLUDED
10 
11 #include "catch_section_info.h"
12 #include "catch_common.h"
13 #include "catch_config.hpp"
14 #include "catch_totals.h"
15 #include "catch_test_case_info.h"
16 #include "catch_assertionresult.h"
17 #include "catch_message.h"
18 #include "catch_option.hpp"
19 #include "catch_stringref.h"
20 
21 #if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
22 #include "benchmark/catch_estimate.hpp"
23 #include "benchmark/catch_outlier_classification.hpp"
24 
25 #include <iterator>
26 #endif // CATCH_CONFIG_ENABLE_BENCHMARKING
27 
28 
29 #include <string>
30 #include <iosfwd>
31 #include <map>
32 #include <set>
33 #include <memory>
34 #include <algorithm>
35 
36 namespace Catch {
37 
38     struct ReporterConfig {
39         explicit ReporterConfig( IConfigPtr const& _fullConfig );
40 
41         ReporterConfig( IConfigPtr const& _fullConfig, std::ostream& _stream );
42 
43         std::ostream& stream() const;
44         IConfigPtr fullConfig() const;
45 
46     private:
47         std::ostream* m_stream;
48         IConfigPtr m_fullConfig;
49     };
50 
51     struct ReporterPreferences {
52         bool shouldRedirectStdOut = false;
53         bool shouldReportAllAssertions = false;
54     };
55 
56     template<typename T>
57     struct LazyStat : Option<T> {
58         LazyStat& operator=( T const& _value ) {
59             Option<T>::operator=( _value );
60             used = false;
61             return *this;
62         }
resetLazyStat63         void reset() {
64             Option<T>::reset();
65             used = false;
66         }
67         bool used = false;
68     };
69 
70     struct TestRunInfo {
71         TestRunInfo( std::string const& _name );
72         std::string name;
73     };
74     struct GroupInfo {
75         GroupInfo(  std::string const& _name,
76                     std::size_t _groupIndex,
77                     std::size_t _groupsCount );
78 
79         std::string name;
80         std::size_t groupIndex;
81         std::size_t groupsCounts;
82     };
83 
84     struct AssertionStats {
85         AssertionStats( AssertionResult const& _assertionResult,
86                         std::vector<MessageInfo> const& _infoMessages,
87                         Totals const& _totals );
88 
89         AssertionStats( AssertionStats const& )              = default;
90         AssertionStats( AssertionStats && )                  = default;
91         AssertionStats& operator = ( AssertionStats const& ) = delete;
92         AssertionStats& operator = ( AssertionStats && )     = delete;
93         virtual ~AssertionStats();
94 
95         AssertionResult assertionResult;
96         std::vector<MessageInfo> infoMessages;
97         Totals totals;
98     };
99 
100     struct SectionStats {
101         SectionStats(   SectionInfo const& _sectionInfo,
102                         Counts const& _assertions,
103                         double _durationInSeconds,
104                         bool _missingAssertions );
105         SectionStats( SectionStats const& )              = default;
106         SectionStats( SectionStats && )                  = default;
107         SectionStats& operator = ( SectionStats const& ) = default;
108         SectionStats& operator = ( SectionStats && )     = default;
109         virtual ~SectionStats();
110 
111         SectionInfo sectionInfo;
112         Counts assertions;
113         double durationInSeconds;
114         bool missingAssertions;
115     };
116 
117     struct TestCaseStats {
118         TestCaseStats(  TestCaseInfo const& _testInfo,
119                         Totals const& _totals,
120                         std::string const& _stdOut,
121                         std::string const& _stdErr,
122                         bool _aborting );
123 
124         TestCaseStats( TestCaseStats const& )              = default;
125         TestCaseStats( TestCaseStats && )                  = default;
126         TestCaseStats& operator = ( TestCaseStats const& ) = default;
127         TestCaseStats& operator = ( TestCaseStats && )     = default;
128         virtual ~TestCaseStats();
129 
130         TestCaseInfo testInfo;
131         Totals totals;
132         std::string stdOut;
133         std::string stdErr;
134         bool aborting;
135     };
136 
137     struct TestGroupStats {
138         TestGroupStats( GroupInfo const& _groupInfo,
139                         Totals const& _totals,
140                         bool _aborting );
141         TestGroupStats( GroupInfo const& _groupInfo );
142 
143         TestGroupStats( TestGroupStats const& )              = default;
144         TestGroupStats( TestGroupStats && )                  = default;
145         TestGroupStats& operator = ( TestGroupStats const& ) = default;
146         TestGroupStats& operator = ( TestGroupStats && )     = default;
147         virtual ~TestGroupStats();
148 
149         GroupInfo groupInfo;
150         Totals totals;
151         bool aborting;
152     };
153 
154     struct TestRunStats {
155         TestRunStats(   TestRunInfo const& _runInfo,
156                         Totals const& _totals,
157                         bool _aborting );
158 
159         TestRunStats( TestRunStats const& )              = default;
160         TestRunStats( TestRunStats && )                  = default;
161         TestRunStats& operator = ( TestRunStats const& ) = default;
162         TestRunStats& operator = ( TestRunStats && )     = default;
163         virtual ~TestRunStats();
164 
165         TestRunInfo runInfo;
166         Totals totals;
167         bool aborting;
168     };
169 
170 #if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
171     struct BenchmarkInfo {
172         std::string name;
173         double estimatedDuration;
174         int iterations;
175         int samples;
176         unsigned int resamples;
177         double clockResolution;
178         double clockCost;
179     };
180 
181     template <class Duration>
182     struct BenchmarkStats {
183         BenchmarkInfo info;
184 
185         std::vector<Duration> samples;
186         Benchmark::Estimate<Duration> mean;
187         Benchmark::Estimate<Duration> standardDeviation;
188         Benchmark::OutlierClassification outliers;
189         double outlierVariance;
190 
191         template <typename Duration2>
192         operator BenchmarkStats<Duration2>() const {
193             std::vector<Duration2> samples2;
194             samples2.reserve(samples.size());
195             std::transform(samples.begin(), samples.end(), std::back_inserter(samples2), [](Duration d) { return Duration2(d); });
196             return {
197                 info,
198                 std::move(samples2),
199                 mean,
200                 standardDeviation,
201                 outliers,
202                 outlierVariance,
203             };
204         }
205     };
206 #endif // CATCH_CONFIG_ENABLE_BENCHMARKING
207 
208     struct IStreamingReporter {
209         virtual ~IStreamingReporter() = default;
210 
211         // Implementing class must also provide the following static methods:
212         // static std::string getDescription();
213         // static std::set<Verbosity> getSupportedVerbosities()
214 
215         virtual ReporterPreferences getPreferences() const = 0;
216 
217         virtual void noMatchingTestCases( std::string const& spec ) = 0;
218 
reportInvalidArgumentsIStreamingReporter219         virtual void reportInvalidArguments(std::string const&) {}
220 
221         virtual void testRunStarting( TestRunInfo const& testRunInfo ) = 0;
222         virtual void testGroupStarting( GroupInfo const& groupInfo ) = 0;
223 
224         virtual void testCaseStarting( TestCaseInfo const& testInfo ) = 0;
225         virtual void sectionStarting( SectionInfo const& sectionInfo ) = 0;
226 
227 #if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
benchmarkPreparingIStreamingReporter228         virtual void benchmarkPreparing( std::string const& ) {}
benchmarkStartingIStreamingReporter229         virtual void benchmarkStarting( BenchmarkInfo const& ) {}
benchmarkEndedIStreamingReporter230         virtual void benchmarkEnded( BenchmarkStats<> const& ) {}
benchmarkFailedIStreamingReporter231         virtual void benchmarkFailed( std::string const& ) {}
232 #endif // CATCH_CONFIG_ENABLE_BENCHMARKING
233 
234         virtual void assertionStarting( AssertionInfo const& assertionInfo ) = 0;
235 
236         // The return value indicates if the messages buffer should be cleared:
237         virtual bool assertionEnded( AssertionStats const& assertionStats ) = 0;
238 
239         virtual void sectionEnded( SectionStats const& sectionStats ) = 0;
240         virtual void testCaseEnded( TestCaseStats const& testCaseStats ) = 0;
241         virtual void testGroupEnded( TestGroupStats const& testGroupStats ) = 0;
242         virtual void testRunEnded( TestRunStats const& testRunStats ) = 0;
243 
244         virtual void skipTest( TestCaseInfo const& testInfo ) = 0;
245 
246         // Default empty implementation provided
247         virtual void fatalErrorEncountered( StringRef name );
248 
249         virtual bool isMulti() const;
250     };
251     using IStreamingReporterPtr = std::unique_ptr<IStreamingReporter>;
252 
253     struct IReporterFactory {
254         virtual ~IReporterFactory();
255         virtual IStreamingReporterPtr create( ReporterConfig const& config ) const = 0;
256         virtual std::string getDescription() const = 0;
257     };
258     using IReporterFactoryPtr = std::shared_ptr<IReporterFactory>;
259 
260     struct IReporterRegistry {
261         using FactoryMap = std::map<std::string, IReporterFactoryPtr>;
262         using Listeners = std::vector<IReporterFactoryPtr>;
263 
264         virtual ~IReporterRegistry();
265         virtual IStreamingReporterPtr create( std::string const& name, IConfigPtr const& config ) const = 0;
266         virtual FactoryMap const& getFactories() const = 0;
267         virtual Listeners const& getListeners() const = 0;
268     };
269 
270 } // end namespace Catch
271 
272 #endif // TWOBLUECUBES_CATCH_INTERFACES_REPORTER_H_INCLUDED
273