1 // (C) Copyright Gennadiy Rozental 2001-2015.
2 // Distributed under the Boost Software License, Version 1.0.
3 // (See accompanying file LICENSE_1_0.txt or copy at
4 // http://www.boost.org/LICENSE_1_0.txt)
5
6 // See http://www.boost.org/libs/test for the library home page.
7 //
8 // File : $RCSfile$
9 //
10 // Version : $Revision$
11 //
12 // Description : tests Unit Test Framework reporting facilities against
13 // pattern file
14 // ***************************************************************************
15
16 // Boost.Test
17 #define BOOST_TEST_MAIN
18 #include <boost/test/unit_test.hpp>
19 #include <boost/test/results_reporter.hpp>
20 #include <boost/test/tools/output_test_stream.hpp>
21 #include <boost/test/unit_test_log.hpp>
22 #include <boost/test/unit_test_suite.hpp>
23 #include <boost/test/framework.hpp>
24 #include <boost/test/unit_test_parameters.hpp>
25 #include <boost/test/utils/nullstream.hpp>
26 #include <boost/test/execution_monitor.hpp>
27 typedef boost::onullstream onullstream_type;
28
29 // BOOST
30 #include <boost/lexical_cast.hpp>
31
32 // STL
33 #include <iostream>
34
35 using boost::test_tools::output_test_stream;
36 using namespace boost::unit_test;
37
38 //____________________________________________________________________________//
39
good_foo()40 void good_foo() {}
41
almost_good_foo()42 void almost_good_foo() { BOOST_TEST_WARN( 2>3 ); }
43
bad_foo()44 void bad_foo() {
45 onullstream_type null_out;
46 unit_test_log.set_stream( null_out );
47 BOOST_ERROR( "" );
48 unit_test_log.set_stream( std::cout );
49 }
bad_foo2()50 void bad_foo2() { bad_foo(); } // preventing clashing names
51 struct log_guard {
~log_guardlog_guard52 ~log_guard()
53 {
54 unit_test_log.set_stream( std::cout );
55 }
56 };
57
very_bad_foo()58 void very_bad_foo() {
59 log_guard lg;
60 boost::ignore_unused( lg );
61 onullstream_type null_out;
62 unit_test_log.set_stream( null_out );
63 BOOST_FAIL( "" );
64 }
65
timeout_foo()66 void timeout_foo()
67 {
68 log_guard lg;
69 boost::ignore_unused( lg );
70 onullstream_type null_out;
71 unit_test_log.set_stream( null_out );
72 using boost::execution_exception;
73 execution_exception::location dummy;
74 throw execution_exception(
75 execution_exception::timeout_error,
76 "fake timeout",
77 dummy);
78 }
79
80 //____________________________________________________________________________//
81
check(output_test_stream & output,output_format report_format,test_unit_id id)82 void check( output_test_stream& output, output_format report_format, test_unit_id id )
83 {
84 results_reporter::set_format( report_format );
85
86 results_reporter::confirmation_report( id );
87 output << "*************************************************************************\n";
88 BOOST_TEST( output.match_pattern() );
89
90 results_reporter::short_report( id );
91 output << "*************************************************************************\n";
92 BOOST_TEST( output.match_pattern() );
93
94 results_reporter::detailed_report( id );
95 output << "*************************************************************************\n";
96 BOOST_TEST( output.match_pattern() );
97 }
98
99 //____________________________________________________________________________//
100
check(output_test_stream & output,test_suite * ts)101 void check( output_test_stream& output, test_suite* ts )
102 {
103 ts->p_default_status.value = test_unit::RS_ENABLED;
104
105 results_reporter::set_level( NO_REPORT );
106 framework::finalize_setup_phase( ts->p_id );
107 framework::run( ts );
108
109 check( output, OF_CLF, ts->p_id );
110 check( output, OF_XML, ts->p_id );
111 }
112
113 //____________________________________________________________________________//
114
115 struct guard {
~guardguard116 ~guard()
117 {
118 results_reporter::set_stream( std::cerr );
119 results_reporter::set_format( runtime_config::get<output_format>(
120 runtime_config::btrt_report_format ) );
121 results_reporter::set_level( NO_REPORT );
122 }
123 };
124
125 //____________________________________________________________________________//
126
BOOST_AUTO_TEST_CASE(test_result_reports)127 BOOST_AUTO_TEST_CASE( test_result_reports )
128 {
129 guard G;
130 boost::ignore_unused( G );
131
132 #define PATTERN_FILE_NAME "result_report_test.pattern"
133
134 std::string pattern_file_name(
135 framework::master_test_suite().argc == 1
136 ? (runtime_config::save_pattern() ? PATTERN_FILE_NAME : "./baseline-outputs/" PATTERN_FILE_NAME )
137 : framework::master_test_suite().argv[1] );
138
139 output_test_stream test_output( pattern_file_name, !runtime_config::save_pattern() );
140 results_reporter::set_stream( test_output );
141
142 test_suite* ts_0 = BOOST_TEST_SUITE( "0 test cases inside" );
143
144 test_suite* ts_1 = BOOST_TEST_SUITE( "1 test cases inside" );
145 ts_1->add( BOOST_TEST_CASE( good_foo ) );
146
147 test_suite* ts_1b = BOOST_TEST_SUITE( "1 bad test case inside" );
148 ts_1b->add( BOOST_TEST_CASE( bad_foo ), 1 );
149
150 test_suite* ts_1c = BOOST_TEST_SUITE( "1 almost good test case inside" );
151 ts_1c->add( BOOST_TEST_CASE( almost_good_foo ) );
152
153 test_suite* ts_2 = BOOST_TEST_SUITE( "2 test cases inside" );
154 ts_2->add( BOOST_TEST_CASE( good_foo ) );
155 ts_2->add( BOOST_TEST_CASE( bad_foo ), 1 );
156
157 test_suite* ts_3 = BOOST_TEST_SUITE( "3 test cases inside" );
158 ts_3->add( BOOST_TEST_CASE( bad_foo ) );
159 test_case* tc1 = BOOST_TEST_CASE( very_bad_foo );
160 ts_3->add( tc1 );
161 test_case* tc2 = BOOST_TEST_CASE( bad_foo2 );
162 ts_3->add( tc2 );
163 tc2->depends_on( tc1 );
164
165 test_suite* ts_main = BOOST_TEST_SUITE( "Fake Test Suite Hierarchy" );
166 ts_main->add( ts_0 );
167 ts_main->add( ts_1 );
168 ts_main->add( ts_2 );
169 ts_main->add( ts_3 );
170
171 test_suite* ts_char_escaping = BOOST_TEST_SUITE( "Char escaping" );
172 ts_char_escaping->add( BOOST_TEST_CASE( good_foo ) );
173 test_case * i_have_problems = BOOST_TEST_CASE( bad_foo );
174 i_have_problems->p_name.set("bad_foo<h>");
175 ts_char_escaping->add( i_have_problems );
176
177 test_suite* ts_timeout = BOOST_TEST_SUITE( "Timeout" );
178 ts_timeout->add( BOOST_TEST_CASE( good_foo ) );
179 test_case * tc_timeout = BOOST_TEST_CASE( timeout_foo );
180 ts_timeout->add( tc_timeout );
181
182 test_suite* ts_timeout_nested = BOOST_TEST_SUITE( "Timeout-nested" );
183 ts_timeout_nested->add( BOOST_TEST_CASE( good_foo ) );
184 test_suite* ts_timeout_internal = BOOST_TEST_SUITE( "Timeout" );
185 ts_timeout_internal->add( BOOST_TEST_CASE( good_foo ) );
186 test_case * tc_timeout_internal = BOOST_TEST_CASE( timeout_foo );
187 ts_timeout_internal->add( tc_timeout_internal );
188 ts_timeout_nested->add( ts_timeout_internal );
189 ts_timeout_nested->add( BOOST_TEST_CASE_NAME( good_foo, "good_foo2" ) );
190
191 check( test_output, ts_1 );
192
193 check( test_output, ts_1b );
194
195 check( test_output, ts_1c );
196
197 check( test_output, ts_2 );
198
199 check( test_output, ts_3 );
200 ts_1->add( BOOST_TEST_CASE( bad_foo ) );
201 ts_3->depends_on( ts_1 );
202
203 check( test_output, ts_main );
204
205 check( test_output, ts_char_escaping );
206
207 check( test_output, ts_timeout );
208
209 check( test_output, ts_timeout_nested );
210
211 results_reporter::set_stream( std::cout );
212 }
213
214 //____________________________________________________________________________//
215
216
check2(output_test_stream & output,output_format report_format,test_unit_id id)217 void check2( output_test_stream& output, output_format report_format, test_unit_id id )
218 {
219 results_reporter::set_format( report_format );
220
221 results_reporter::confirmation_report( id );
222 output << "*************************************************************************\n";
223 BOOST_TEST( output.match_pattern() );
224
225 results_reporter::short_report( id );
226 output << "*************************************************************************\n";
227 BOOST_TEST( output.match_pattern() );
228
229 results_reporter::detailed_report( id );
230 output << "*************************************************************************\n";
231 BOOST_TEST( output.match_pattern() );
232 }
233
234 //____________________________________________________________________________//
235
check2(output_test_stream & output,test_suite * ts)236 void check2( output_test_stream& output, test_suite* ts )
237 {
238 ts->p_default_status.value = test_unit::RS_ENABLED;
239
240 output << "\n* NO_REPORT *********************************************************************\n";
241 results_reporter::set_level( NO_REPORT );
242 results_reporter::set_format( OF_CLF );
243 framework::finalize_setup_phase( ts->p_id );
244 framework::run( ts );
245 BOOST_TEST( output.match_pattern() );
246
247 output << "\n* CONFIRMATION_REPORT ***********************************************************\n";
248 results_reporter::set_level( CONFIRMATION_REPORT );
249 results_reporter::set_format( OF_CLF );
250 framework::finalize_setup_phase( ts->p_id );
251 framework::run( ts );
252 BOOST_TEST( output.match_pattern() );
253
254 output << "\n* SHORT_REPORT ******************************************************************\n";
255 results_reporter::set_level( SHORT_REPORT );
256 results_reporter::set_format( OF_CLF );
257 framework::finalize_setup_phase( ts->p_id );
258 framework::run( ts );
259 BOOST_TEST( output.match_pattern() );
260
261 output << "\n* DETAILED_REPORT ***************************************************************\n";
262 results_reporter::set_level( DETAILED_REPORT );
263 results_reporter::set_format( OF_CLF );
264 framework::finalize_setup_phase( ts->p_id );
265 framework::run( ts );
266 BOOST_TEST( output.match_pattern() );
267
268 // XML
269 output << "\n* NO_REPORT *********************************************************************\n";
270 results_reporter::set_level( NO_REPORT );
271 results_reporter::set_format( OF_XML );
272 framework::finalize_setup_phase( ts->p_id );
273 framework::run( ts );
274 BOOST_TEST( output.match_pattern() );
275
276 output << "\n* CONFIRMATION_REPORT ***********************************************************\n";
277 results_reporter::set_level( CONFIRMATION_REPORT );
278 results_reporter::set_format( OF_XML );
279 framework::finalize_setup_phase( ts->p_id );
280 framework::run( ts );
281 BOOST_TEST( output.match_pattern() );
282
283 output << "\n* SHORT_REPORT ******************************************************************\n";
284 results_reporter::set_level( SHORT_REPORT );
285 results_reporter::set_format( OF_XML );
286 framework::finalize_setup_phase( ts->p_id );
287 framework::run( ts );
288 BOOST_TEST( output.match_pattern() );
289
290 output << "\n* DETAILED_REPORT ***************************************************************\n";
291 results_reporter::set_level( DETAILED_REPORT );
292 results_reporter::set_format( OF_XML );
293 framework::finalize_setup_phase( ts->p_id );
294 framework::run( ts );
295 BOOST_TEST( output.match_pattern() );
296
297 }
298
BOOST_AUTO_TEST_CASE(test_result_reports_default_behaviour)299 BOOST_AUTO_TEST_CASE( test_result_reports_default_behaviour )
300 {
301 guard G;
302 boost::ignore_unused( G );
303
304 #define PATTERN_FILE_NAME_DEFAULT_BEHAVIOUR "result_report_test.pattern.default_behaviour"
305
306 std::string pattern_file_name(
307 framework::master_test_suite().argc <= 2
308 ? (runtime_config::save_pattern() ? PATTERN_FILE_NAME_DEFAULT_BEHAVIOUR : "./baseline-outputs/" PATTERN_FILE_NAME_DEFAULT_BEHAVIOUR )
309 : framework::master_test_suite().argv[2] );
310
311 output_test_stream test_output( pattern_file_name, !runtime_config::save_pattern() );
312 results_reporter::set_stream( test_output );
313
314 test_suite* ts_0 = BOOST_TEST_SUITE( "0 test cases inside" );
315
316 test_suite* ts_1 = BOOST_TEST_SUITE( "1 test cases inside" );
317 ts_1->add( BOOST_TEST_CASE( good_foo ) );
318
319 test_suite* ts_1b = BOOST_TEST_SUITE( "1 bad test case inside" );
320 ts_1b->add( BOOST_TEST_CASE( bad_foo ), 1 );
321
322 test_suite* ts_1c = BOOST_TEST_SUITE( "1 almost good test case inside" );
323 ts_1c->add( BOOST_TEST_CASE( almost_good_foo ) );
324
325 test_suite* ts_2 = BOOST_TEST_SUITE( "2 test cases inside" );
326 ts_2->add( BOOST_TEST_CASE( good_foo ) );
327 ts_2->add( BOOST_TEST_CASE( bad_foo ), 1 );
328
329 test_suite* ts_3 = BOOST_TEST_SUITE( "3 test cases inside" );
330 ts_3->add( BOOST_TEST_CASE( bad_foo ) );
331 test_case* tc1 = BOOST_TEST_CASE( very_bad_foo );
332 ts_3->add( tc1 );
333 test_case* tc2 = BOOST_TEST_CASE( bad_foo2 );
334 ts_3->add( tc2 );
335 tc2->depends_on( tc1 );
336
337 test_suite* ts_main = BOOST_TEST_SUITE( "Fake Test Suite Hierarchy" );
338 ts_main->add( ts_0 );
339 ts_main->add( ts_1 );
340 ts_main->add( ts_2 );
341 ts_main->add( ts_3 );
342
343 test_suite* ts_char_escaping = BOOST_TEST_SUITE( "Char escaping" );
344 ts_char_escaping->add( BOOST_TEST_CASE( good_foo ) );
345 test_case * i_have_problems = BOOST_TEST_CASE( bad_foo );
346 i_have_problems->p_name.set("bad_foo<h>");
347 ts_char_escaping->add( i_have_problems );
348
349 test_suite* ts_timeout = BOOST_TEST_SUITE( "Timeout" );
350 ts_timeout->add( BOOST_TEST_CASE( good_foo ) );
351 test_case * tc_timeout = BOOST_TEST_CASE( timeout_foo );
352 ts_timeout->add( tc_timeout );
353
354 test_suite* ts_timeout_nested = BOOST_TEST_SUITE( "Timeout-nested" );
355 ts_timeout_nested->add( BOOST_TEST_CASE( good_foo ) );
356 test_suite* ts_timeout_internal = BOOST_TEST_SUITE( "Timeout" );
357 ts_timeout_internal->add( BOOST_TEST_CASE( good_foo ) );
358 test_case * tc_timeout_internal = BOOST_TEST_CASE( timeout_foo );
359 ts_timeout_internal->add( tc_timeout_internal );
360 ts_timeout_nested->add( ts_timeout_internal );
361 ts_timeout_nested->add( BOOST_TEST_CASE_NAME( good_foo, "good_foo2" ) );
362
363 check2( test_output, ts_1 );
364
365 check2( test_output, ts_1b );
366
367 check2( test_output, ts_1c );
368
369 check2( test_output, ts_2 );
370
371 check2( test_output, ts_3 );
372 ts_1->add( BOOST_TEST_CASE( bad_foo ) );
373 ts_3->depends_on( ts_1 );
374
375 check2( test_output, ts_main );
376
377 check2( test_output, ts_char_escaping );
378
379 check2( test_output, ts_timeout );
380
381 check2( test_output, ts_timeout_nested );
382
383 results_reporter::set_stream( std::cout );
384 }
385
386 // EOF
387