1 // Copyright 2011 The Kyua Authors.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 #include "cli/common.hpp"
30
31 #include <algorithm>
32 #include <fstream>
33 #include <iostream>
34 #include <stdexcept>
35
36 #include "engine/filters.hpp"
37 #include "model/test_program.hpp"
38 #include "model/test_result.hpp"
39 #include "store/layout.hpp"
40 #include "utils/cmdline/exceptions.hpp"
41 #include "utils/cmdline/options.hpp"
42 #include "utils/cmdline/parser.ipp"
43 #include "utils/cmdline/ui.hpp"
44 #include "utils/datetime.hpp"
45 #include "utils/env.hpp"
46 #include "utils/format/macros.hpp"
47 #include "utils/logging/macros.hpp"
48 #include "utils/fs/exceptions.hpp"
49 #include "utils/fs/operations.hpp"
50 #include "utils/fs/path.hpp"
51 #include "utils/optional.ipp"
52 #include "utils/sanity.hpp"
53
54 #if defined(HAVE_CONFIG_H)
55 # include "config.h"
56 #endif
57
58 namespace cmdline = utils::cmdline;
59 namespace datetime = utils::datetime;
60 namespace fs = utils::fs;
61 namespace layout = store::layout;
62
63 using utils::none;
64 using utils::optional;
65
66
67 /// Standard definition of the option to specify the build root.
68 const cmdline::path_option cli::build_root_option(
69 "build-root",
70 "Path to the built test programs, if different from the location of the "
71 "Kyuafile scripts",
72 "path");
73
74
75 /// Standard definition of the option to specify a Kyuafile.
76 const cmdline::path_option cli::kyuafile_option(
77 'k', "kyuafile",
78 "Path to the test suite definition",
79 "file", "Kyuafile");
80
81
82 /// Standard definition of the option to specify filters on test results.
83 const cmdline::list_option cli::results_filter_option(
84 "results-filter", "Comma-separated list of result types to include in "
85 "the report", "types", "skipped,xfail,broken,failed");
86
87
88 /// Standard definition of the option to specify the results file.
89 ///
90 /// TODO(jmmv): Should support a git-like syntax to go back in time, like
91 /// --results-file=LATEST^N where N indicates how many runs to go back to.
92 const cmdline::string_option cli::results_file_create_option(
93 'r', "results-file",
94 "Path to the results file to create; if left to the default value, the "
95 "name of the file is automatically computed for the current test suite",
96 "file", layout::results_auto_create_name);
97
98
99 /// Standard definition of the option to specify the results file.
100 ///
101 /// TODO(jmmv): Should support a git-like syntax to go back in time, like
102 /// --results-file=LATEST^N where N indicates how many runs to go back to.
103 const cmdline::string_option cli::results_file_open_option(
104 'r', "results-file",
105 "Path to the results file to open or the identifier of the current test "
106 "suite or a previous results file for automatic lookup; if left to the "
107 "default value, uses the current directory as the test suite name",
108 "file", layout::results_auto_open_name);
109
110
111 namespace {
112
113
114 /// Gets the path to the historical database if it exists.
115 ///
116 /// TODO(jmmv): This function should go away. It only exists as a temporary
117 /// transitional path to force the use of the stale ~/.kyua/store.db if it
118 /// exists.
119 ///
120 /// \return A path if the file is found; none otherwise.
121 static optional< fs::path >
get_historical_db(void)122 get_historical_db(void)
123 {
124 optional< fs::path > home = utils::get_home();
125 if (home) {
126 const fs::path old_db = home.get() / ".kyua/store.db";
127 if (fs::exists(old_db)) {
128 if (old_db.is_absolute())
129 return utils::make_optional(old_db);
130 else
131 return utils::make_optional(old_db.to_absolute());
132 } else {
133 return none;
134 }
135 } else {
136 return none;
137 }
138 }
139
140
141 /// Converts a set of result type names to identifiers.
142 ///
143 /// \param names The collection of names to process; may be empty.
144 ///
145 /// \return The result type identifiers corresponding to the input names.
146 ///
147 /// \throw std::runtime_error If any name in the input names is invalid.
148 static cli::result_types
parse_types(const std::vector<std::string> & names)149 parse_types(const std::vector< std::string >& names)
150 {
151 typedef std::map< std::string, model::test_result_type > types_map;
152 types_map valid_types;
153 valid_types["broken"] = model::test_result_broken;
154 valid_types["failed"] = model::test_result_failed;
155 valid_types["passed"] = model::test_result_passed;
156 valid_types["skipped"] = model::test_result_skipped;
157 valid_types["xfail"] = model::test_result_expected_failure;
158
159 cli::result_types types;
160 for (std::vector< std::string >::const_iterator iter = names.begin();
161 iter != names.end(); ++iter) {
162 const types_map::const_iterator match = valid_types.find(*iter);
163 if (match == valid_types.end())
164 throw std::runtime_error(F("Unknown result type '%s'") % *iter);
165 else
166 types.push_back((*match).second);
167 }
168 return types;
169 }
170
171
172 } // anonymous namespace
173
174
175 /// Gets the path to the build root, if any.
176 ///
177 /// This is just syntactic sugar to simplify quierying the 'build_root_option'.
178 ///
179 /// \param cmdline The parsed command line.
180 ///
181 /// \return The path to the build root, if specified; none otherwise.
182 optional< fs::path >
build_root_path(const cmdline::parsed_cmdline & cmdline)183 cli::build_root_path(const cmdline::parsed_cmdline& cmdline)
184 {
185 optional< fs::path > build_root;
186 if (cmdline.has_option(build_root_option.long_name()))
187 build_root = cmdline.get_option< cmdline::path_option >(
188 build_root_option.long_name());
189 return build_root;
190 }
191
192
193 /// Gets the path to the Kyuafile to be loaded.
194 ///
195 /// This is just syntactic sugar to simplify quierying the 'kyuafile_option'.
196 ///
197 /// \param cmdline The parsed command line.
198 ///
199 /// \return The path to the Kyuafile to be loaded.
200 fs::path
kyuafile_path(const cmdline::parsed_cmdline & cmdline)201 cli::kyuafile_path(const cmdline::parsed_cmdline& cmdline)
202 {
203 return cmdline.get_option< cmdline::path_option >(
204 kyuafile_option.long_name());
205 }
206
207
208 /// Gets the value of the results-file flag for the creation of a new file.
209 ///
210 /// \param cmdline The parsed command line from which to extract any possible
211 /// override for the location of the database via the --results-file flag.
212 ///
213 /// \return The path to the database to be used.
214 ///
215 /// \throw cmdline::error If the value passed to the flag is invalid.
216 std::string
results_file_create(const cmdline::parsed_cmdline & cmdline)217 cli::results_file_create(const cmdline::parsed_cmdline& cmdline)
218 {
219 std::string results_file = cmdline.get_option< cmdline::string_option >(
220 results_file_create_option.long_name());
221 if (results_file == results_file_create_option.default_value()) {
222 const optional< fs::path > historical_db = get_historical_db();
223 if (historical_db)
224 results_file = historical_db.get().str();
225 } else {
226 try {
227 (void)fs::path(results_file);
228 } catch (const fs::error& e) {
229 throw cmdline::usage_error(F("Invalid value passed to --%s") %
230 results_file_create_option.long_name());
231 }
232 }
233 return results_file;
234 }
235
236
237 /// Gets the value of the results-file flag for the lookup of the file.
238 ///
239 /// \param cmdline The parsed command line from which to extract any possible
240 /// override for the location of the database via the --results-file flag.
241 ///
242 /// \return The path to the database to be used.
243 ///
244 /// \throw cmdline::error If the value passed to the flag is invalid.
245 std::string
results_file_open(const cmdline::parsed_cmdline & cmdline)246 cli::results_file_open(const cmdline::parsed_cmdline& cmdline)
247 {
248 std::string results_file = cmdline.get_option< cmdline::string_option >(
249 results_file_open_option.long_name());
250 if (results_file == results_file_open_option.default_value()) {
251 const optional< fs::path > historical_db = get_historical_db();
252 if (historical_db)
253 results_file = historical_db.get().str();
254 } else {
255 try {
256 (void)fs::path(results_file);
257 } catch (const fs::error& e) {
258 throw cmdline::usage_error(F("Invalid value passed to --%s") %
259 results_file_open_option.long_name());
260 }
261 }
262 return results_file;
263 }
264
265
266 /// Gets the filters for the result types.
267 ///
268 /// \param cmdline The parsed command line.
269 ///
270 /// \return A collection of result types to be used for filtering.
271 ///
272 /// \throw std::runtime_error If any of the user-provided filters is invalid.
273 cli::result_types
get_result_types(const utils::cmdline::parsed_cmdline & cmdline)274 cli::get_result_types(const utils::cmdline::parsed_cmdline& cmdline)
275 {
276 result_types types = parse_types(
277 cmdline.get_option< cmdline::list_option >("results-filter"));
278 if (types.empty()) {
279 types.push_back(model::test_result_passed);
280 types.push_back(model::test_result_skipped);
281 types.push_back(model::test_result_expected_failure);
282 types.push_back(model::test_result_broken);
283 types.push_back(model::test_result_failed);
284 }
285 return types;
286 }
287
288
289 /// Parses a set of command-line arguments to construct test filters.
290 ///
291 /// \param args The command-line arguments representing test filters.
292 ///
293 /// \throw cmdline:error If any of the arguments is invalid, or if they
294 /// represent a non-disjoint collection of filters.
295 std::set< engine::test_filter >
parse_filters(const cmdline::args_vector & args)296 cli::parse_filters(const cmdline::args_vector& args)
297 {
298 std::set< engine::test_filter > filters;
299
300 try {
301 for (cmdline::args_vector::const_iterator iter = args.begin();
302 iter != args.end(); iter++) {
303 const engine::test_filter filter(engine::test_filter::parse(*iter));
304 if (filters.find(filter) != filters.end())
305 throw cmdline::error(F("Duplicate filter '%s'") % filter.str());
306 filters.insert(filter);
307 }
308 check_disjoint_filters(filters);
309 } catch (const std::runtime_error& e) {
310 throw cmdline::error(e.what());
311 }
312
313 return filters;
314 }
315
316
317 /// Reports the filters that have not matched any tests as errors.
318 ///
319 /// \param unused The collection of unused filters to report.
320 /// \param ui The user interface object through which errors are to be reported.
321 ///
322 /// \return True if there are any unused filters. The caller should report this
323 /// as an error to the user by means of a non-successful exit code.
324 bool
report_unused_filters(const std::set<engine::test_filter> & unused,cmdline::ui * ui)325 cli::report_unused_filters(const std::set< engine::test_filter >& unused,
326 cmdline::ui* ui)
327 {
328 for (std::set< engine::test_filter >::const_iterator iter = unused.begin();
329 iter != unused.end(); iter++) {
330 cmdline::print_warning(ui, F("No test cases matched by the filter "
331 "'%s'.") % (*iter).str());
332 }
333
334 return !unused.empty();
335 }
336
337
338 /// Formats a time delta for user presentation.
339 ///
340 /// \param delta The time delta to format.
341 ///
342 /// \return A user-friendly representation of the time delta.
343 std::string
format_delta(const datetime::delta & delta)344 cli::format_delta(const datetime::delta& delta)
345 {
346 return F("%.3ss") % (delta.seconds + (delta.useconds / 1000000.0));
347 }
348
349
350 /// Formats a test case result for user presentation.
351 ///
352 /// \param result The result to format.
353 ///
354 /// \return A user-friendly representation of the result.
355 std::string
format_result(const model::test_result & result)356 cli::format_result(const model::test_result& result)
357 {
358 std::string text;
359
360 switch (result.type()) {
361 case model::test_result_broken: text = "broken"; break;
362 case model::test_result_expected_failure: text = "expected_failure"; break;
363 case model::test_result_failed: text = "failed"; break;
364 case model::test_result_passed: text = "passed"; break;
365 case model::test_result_skipped: text = "skipped"; break;
366 }
367 INV(!text.empty());
368
369 if (!result.reason().empty())
370 text += ": " + result.reason();
371
372 return text;
373 }
374
375
376 /// Formats the identifier of a test case for user presentation.
377 ///
378 /// \param test_program The test program containing the test case.
379 /// \param test_case_name The name of the test case.
380 ///
381 /// \return A string representing the test case uniquely within a test suite.
382 std::string
format_test_case_id(const model::test_program & test_program,const std::string & test_case_name)383 cli::format_test_case_id(const model::test_program& test_program,
384 const std::string& test_case_name)
385 {
386 return F("%s:%s") % test_program.relative_path() % test_case_name;
387 }
388
389
390 /// Formats a filter using the same syntax of a test case.
391 ///
392 /// \param test_filter The filter to format.
393 ///
394 /// \return A string representing the test filter.
395 std::string
format_test_case_id(const engine::test_filter & test_filter)396 cli::format_test_case_id(const engine::test_filter& test_filter)
397 {
398 return F("%s:%s") % test_filter.test_program % test_filter.test_case;
399 }
400
401
402 /// Prints the version header information to the interface output.
403 ///
404 /// \param ui Interface to which to write the version details.
405 void
write_version_header(utils::cmdline::ui * ui)406 cli::write_version_header(utils::cmdline::ui* ui)
407 {
408 ui->out(PACKAGE " (" PACKAGE_NAME ") " PACKAGE_VERSION);
409 }
410