1 // Copyright 2011 Google Inc.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #include "cli/cmd_report.hpp"
30 #include "cli/cmd_report_tap.hpp"
31
32 #include <cstddef>
33 #include <cstdlib>
34 #include <fstream>
35 #include <map>
36 #include <vector>
37
38 #include "cli/common.ipp"
39 #include "engine/action.hpp"
40 #include "engine/context.hpp"
41 #include "engine/drivers/scan_action.hpp"
42 #include "engine/test_result.hpp"
43 #include "utils/cmdline/exceptions.hpp"
44 #include "utils/cmdline/parser.ipp"
45 #include "utils/defs.hpp"
46 #include "utils/format/macros.hpp"
47 #include "utils/optional.ipp"
48
49 namespace cmdline = utils::cmdline;
50 namespace config = utils::config;
51 namespace datetime = utils::datetime;
52 namespace fs = utils::fs;
53 namespace scan_action = engine::drivers::scan_action;
54
55 using cli::cmd_report_tap;
56 using utils::optional;
57
58
59 namespace {
60
61
62 /// Generates a plain-text report intended to be printed to the console.
63 class console_tap_hooks : public scan_action::base_hooks {
64 /// Indirection to print the output to the correct file stream.
65 cli::file_writer _writer;
66
67 /// Whether to include the runtime context in the output or not.
68 const bool _show_context;
69
70 /// Collection of result types to include in the report.
71 const cli::result_types& _results_filters;
72
73 /// The action ID loaded.
74 int64_t _action_id;
75
76 /// The total run time of the tests.
77 datetime::delta _runtime;
78
79 /// Representation of a single result.
80 struct result_data {
81 /// The relative path to the test program.
82 fs::path binary_path;
83
84 /// The name of the test case.
85 std::string test_case_name;
86
87 /// The result of the test case.
88 engine::test_result result;
89
90 /// The duration of the test case execution.
91 datetime::delta duration;
92
93 /// Constructs a new results data.
94 ///
95 /// \param binary_path_ The relative path to the test program.
96 /// \param test_case_name_ The name of the test case.
97 /// \param result_ The result of the test case.
98 /// \param duration_ The duration of the test case execution.
result_data__anonc7553b090111::console_tap_hooks::result_data99 result_data(const fs::path& binary_path_,
100 const std::string& test_case_name_,
101 const engine::test_result& result_,
102 const datetime::delta& duration_) :
103 binary_path(binary_path_), test_case_name(test_case_name_),
104 result(result_), duration(duration_)
105 {
106 }
107 };
108
109 /// Results received, broken down by their type.
110 ///
111 /// Note that this may not include all results, as keeping the whole list in
112 /// memory may be too much.
113 std::map< engine::test_result::result_type,
114 std::vector< result_data > > _results;
115
116 /// Prints the execution context to the output.
117 ///
118 /// \param context The context to dump.
119 void
print_context(const engine::context & context)120 print_context(const engine::context& context)
121 {
122 _writer(" ===> Execution context");
123
124 _writer(F(" Current directory: %s") % context.cwd());
125 const std::map< std::string, std::string >& env = context.env();
126 if (env.empty())
127 _writer(" No environment variables recorded");
128 else {
129 _writer(" Environment variables:");
130 for (std::map< std::string, std::string >::const_iterator
131 iter = env.begin(); iter != env.end(); iter++) {
132 _writer(F(" %s=%s") % (*iter).first % (*iter).second);
133 }
134 }
135 }
136
137 /// Counts how many results of a given type have been received.
138 std::size_t
count_results(const engine::test_result::result_type type)139 count_results(const engine::test_result::result_type type)
140 {
141 const std::map< engine::test_result::result_type,
142 std::vector< result_data > >::const_iterator iter =
143 _results.find(type);
144 if (iter == _results.end())
145 return 0;
146 else
147 return (*iter).second.size();
148 }
149
150 /// Prints a set of results.
151 void
print_results(const engine::test_result::result_type type,const char * title)152 print_results(const engine::test_result::result_type type,
153 const char* title)
154 {
155 const std::map< engine::test_result::result_type,
156 std::vector< result_data > >::const_iterator iter2 =
157 _results.find(type);
158 if (iter2 == _results.end())
159 return;
160 const std::vector< result_data >& all = (*iter2).second;
161
162 std::string result = "";
163 std::string comment = "";
164
165 if (type == engine::test_result::failed)
166 result = "not ";
167
168 if (type == engine::test_result::broken) {
169 result = "not ";
170 comment = " # broken";
171 }
172
173 if (type == engine::test_result::skipped)
174 comment = " # skip";
175
176 _writer(F(" ===> %s") % title);
177 for (std::vector< result_data >::const_iterator iter = all.begin();
178 iter != all.end(); iter++) {
179 _writer(F("%sok %s:%s%s") %
180 result %
181 (*iter).binary_path %
182 (*iter).test_case_name %
183 comment
184 );
185 _writer(F(" %s:%s -> %s [%s]") % (*iter).binary_path %
186 (*iter).test_case_name %
187 cli::format_result((*iter).result) %
188 cli::format_delta((*iter).duration));
189 }
190 }
191
192 public:
193 /// Constructor for the hooks.
194 ///
195 /// \param ui_ The user interface object of the caller command.
196 /// \param outfile_ The file to which to send the output.
197 /// \param show_context_ Whether to include the runtime context in
198 /// the output or not.
199 /// \param results_filters_ The result types to include in the report.
200 /// Cannot be empty.
console_tap_hooks(cmdline::ui * ui_,const fs::path & outfile_,const bool show_context_,const cli::result_types & results_filters_)201 console_tap_hooks(cmdline::ui* ui_, const fs::path& outfile_,
202 const bool show_context_,
203 const cli::result_types& results_filters_) :
204 _writer(ui_, outfile_),
205 _show_context(show_context_),
206 _results_filters(results_filters_)
207 {
208 PRE(!results_filters_.empty());
209 }
210
211 /// Callback executed when an action is found.
212 ///
213 /// \param action_id The identifier of the loaded action.
214 /// \param action The action loaded from the database.
215 void
got_action(const int64_t action_id,const engine::action & action)216 got_action(const int64_t action_id, const engine::action& action)
217 {
218 _action_id = action_id;
219 if (_show_context)
220 print_context(action.runtime_context());
221 }
222
223 /// Callback executed when a test results is found.
224 ///
225 /// \param iter Container for the test result's data.
226 void
got_result(store::results_iterator & iter)227 got_result(store::results_iterator& iter)
228 {
229 _runtime += iter.duration();
230 const engine::test_result result = iter.result();
231 _results[result.type()].push_back(
232 result_data(iter.test_program()->relative_path(),
233 iter.test_case_name(), iter.result(), iter.duration()));
234 }
235
236 /// Prints the tests summary.
237 void
print_tests(void)238 print_tests(void)
239 {
240 using engine::test_result;
241 typedef std::map< test_result::result_type, const char* > types_map;
242 typedef std::map< test_result::result_type, std::size_t > types_counts;
243
244 types_map titles;
245 titles[engine::test_result::broken] = "Broken tests";
246 titles[engine::test_result::expected_failure] = "Expected failures";
247 titles[engine::test_result::failed] = "Failed tests";
248 titles[engine::test_result::passed] = "Passed tests";
249 titles[engine::test_result::skipped] = "Skipped tests";
250
251 types_counts counts;
252 counts[engine::test_result::broken] = count_results(test_result::broken);
253 counts[engine::test_result::expected_failure] = count_results(test_result::expected_failure);
254 counts[engine::test_result::failed] = count_results(test_result::failed);
255 counts[engine::test_result::passed] = count_results(test_result::passed);
256 counts[engine::test_result::skipped] = count_results(test_result::skipped);
257
258 const std::size_t total =
259 counts[engine::test_result::broken] +
260 counts[engine::test_result::expected_failure] +
261 counts[engine::test_result::failed] +
262 counts[engine::test_result::passed] +
263 counts[engine::test_result::skipped];
264
265 std::size_t selected_types_total = 0;
266
267 for (cli::result_types::const_iterator iter = _results_filters.begin();
268 iter != _results_filters.end(); ++iter) {
269 selected_types_total += counts.find(*iter)->second;
270 }
271
272 _writer(F("1..%s") % selected_types_total);
273 for (cli::result_types::const_iterator iter = _results_filters.begin();
274 iter != _results_filters.end(); ++iter) {
275 const types_map::const_iterator match = titles.find(*iter);
276 INV_MSG(match != titles.end(), "Conditional does not match user "
277 "input validation in parse_types()");
278 print_results((*match).first, (*match).second);
279 }
280
281 _writer(" ===> Summary");
282 _writer(F(" Action: %s") % _action_id);
283 _writer(F(" Test cases: %s total, %s passed, %s skipped, %s expected failures, "
284 "%s broken, %s failed") % total %
285 counts[engine::test_result::passed] %
286 counts[engine::test_result::skipped] %
287 counts[engine::test_result::expected_failure] %
288 counts[engine::test_result::broken] %
289 counts[engine::test_result::failed]);
290 _writer(F(" Total time: %s") % cli::format_delta(_runtime));
291 }
292 };
293
294
295 } // anonymous namespace
296
297
298 /// Default constructor for cmd_report_tap.
cmd_report_tap(void)299 cmd_report_tap::cmd_report_tap(void) : cli_command(
300 "report-tap", "", 0, 0,
301 "Generates a TAP report with the result of a "
302 "previous action")
303 {
304 add_option(store_option);
305 add_option(cmdline::bool_option(
306 "show-context", "Include the execution context in the report"));
307 add_option(cmdline::int_option(
308 "action", "The action to report; if not specified, defaults to the "
309 "latest action in the database", "id"));
310 add_option(cmdline::path_option(
311 "output", "The file to which to write the report",
312 "path", "/dev/stdout"));
313 add_option(results_filter_option);
314 }
315
316
317 /// Entry point for the "report" subcommand.
318 ///
319 /// \param ui Object to interact with the I/O of the program.
320 /// \param cmdline Representation of the command line to the subcommand.
321 /// \param unused_user_config The runtime configuration of the program.
322 ///
323 /// \return 0 if everything is OK, 1 if the statement is invalid or if there is
324 /// any other problem.
325 int
run(cmdline::ui * ui,const cmdline::parsed_cmdline & cmdline,const config::tree & UTILS_UNUSED_PARAM (user_config))326 cmd_report_tap::run(cmdline::ui* ui, const cmdline::parsed_cmdline& cmdline,
327 const config::tree& UTILS_UNUSED_PARAM(user_config))
328 {
329 optional< int64_t > action_id;
330 if (cmdline.has_option("action"))
331 action_id = cmdline.get_option< cmdline::int_option >("action");
332
333 const result_types types = get_result_types(cmdline);
334 console_tap_hooks hooks(
335 ui, cmdline.get_option< cmdline::path_option >("output"),
336 cmdline.has_option("show-context"), types);
337 scan_action::drive(store_path(cmdline), action_id, hooks);
338 hooks.print_tests();
339
340 return EXIT_SUCCESS;
341 }
342