1 // Copyright 2015 The Kyua Authors.
2 // All rights reserved.
3 //
4 // Redistribution and use in source and binary forms, with or without
5 // modification, are permitted provided that the following conditions are
6 // met:
7 //
8 // * Redistributions of source code must retain the above copyright
9 //   notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 //   notice, this list of conditions and the following disclaimer in the
12 //   documentation and/or other materials provided with the distribution.
13 // * Neither the name of Google Inc. nor the names of its contributors
14 //   may be used to endorse or promote products derived from this software
15 //   without specific prior written permission.
16 //
17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 
29 #include "engine/tap_parser.hpp"
30 
31 #include <fstream>
32 
33 #include <atf-c++.hpp>
34 
35 #include "engine/exceptions.hpp"
36 #include "utils/format/containers.ipp"
37 #include "utils/format/macros.hpp"
38 #include "utils/fs/path.hpp"
39 
40 namespace fs = utils::fs;
41 
42 
43 namespace {
44 
45 
46 /// Helper to execute parse_tap_output() on inline text contents.
47 ///
48 /// \param contents The TAP output to parse.
49 ///
50 /// \return The tap_summary object resultingafter the parse.
51 ///
52 /// \throw engine::load_error If parse_tap_output() fails.
53 static engine::tap_summary
do_parse(const std::string & contents)54 do_parse(const std::string& contents)
55 {
56     std::ofstream output("tap.txt");
57     ATF_REQUIRE(output);
58     output << contents;
59     output.close();
60     return engine::parse_tap_output(fs::path("tap.txt"));
61 }
62 
63 
64 }  // anonymous namespace
65 
66 
67 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__bailed_out);
ATF_TEST_CASE_BODY(tap_summary__bailed_out)68 ATF_TEST_CASE_BODY(tap_summary__bailed_out)
69 {
70     const engine::tap_summary summary = engine::tap_summary::new_bailed_out();
71     ATF_REQUIRE(summary.bailed_out());
72 }
73 
74 
75 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__some_results);
ATF_TEST_CASE_BODY(tap_summary__some_results)76 ATF_TEST_CASE_BODY(tap_summary__some_results)
77 {
78     const engine::tap_summary summary = engine::tap_summary::new_results(
79         engine::tap_plan(1, 5), 3, 2);
80     ATF_REQUIRE(!summary.bailed_out());
81     ATF_REQUIRE_EQ(engine::tap_plan(1, 5), summary.plan());
82     ATF_REQUIRE_EQ(3, summary.ok_count());
83     ATF_REQUIRE_EQ(2, summary.not_ok_count());
84 }
85 
86 
87 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__all_skipped);
ATF_TEST_CASE_BODY(tap_summary__all_skipped)88 ATF_TEST_CASE_BODY(tap_summary__all_skipped)
89 {
90     const engine::tap_summary summary = engine::tap_summary::new_all_skipped(
91         "Skipped");
92     ATF_REQUIRE(!summary.bailed_out());
93     ATF_REQUIRE_EQ(engine::tap_plan(1, 0), summary.plan());
94     ATF_REQUIRE_EQ("Skipped", summary.all_skipped_reason());
95 }
96 
97 
98 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__equality_operators);
ATF_TEST_CASE_BODY(tap_summary__equality_operators)99 ATF_TEST_CASE_BODY(tap_summary__equality_operators)
100 {
101     const engine::tap_summary bailed_out =
102         engine::tap_summary::new_bailed_out();
103     const engine::tap_summary all_skipped_1 =
104         engine::tap_summary::new_all_skipped("Reason 1");
105     const engine::tap_summary results_1 =
106         engine::tap_summary::new_results(engine::tap_plan(1, 5), 3, 2);
107 
108     // Self-equality checks.
109     ATF_REQUIRE(  bailed_out == bailed_out);
110     ATF_REQUIRE(!(bailed_out != bailed_out));
111     ATF_REQUIRE(  all_skipped_1 == all_skipped_1);
112     ATF_REQUIRE(!(all_skipped_1 != all_skipped_1));
113     ATF_REQUIRE(  results_1 == results_1);
114     ATF_REQUIRE(!(results_1 != results_1));
115 
116     // Cross-equality checks.
117     ATF_REQUIRE(!(bailed_out == all_skipped_1));
118     ATF_REQUIRE(  bailed_out != all_skipped_1);
119     ATF_REQUIRE(!(bailed_out == results_1));
120     ATF_REQUIRE(  bailed_out != results_1);
121     ATF_REQUIRE(!(all_skipped_1 == results_1));
122     ATF_REQUIRE(  all_skipped_1 != results_1);
123 
124     // Checks for the all_skipped "type".
125     const engine::tap_summary all_skipped_2 =
126         engine::tap_summary::new_all_skipped("Reason 2");
127     ATF_REQUIRE(!(all_skipped_1 == all_skipped_2));
128     ATF_REQUIRE(  all_skipped_1 != all_skipped_2);
129 
130 
131     // Checks for the results "type", different plan.
132     const engine::tap_summary results_2 =
133         engine::tap_summary::new_results(engine::tap_plan(2, 6),
134                                          results_1.ok_count(),
135                                          results_1.not_ok_count());
136     ATF_REQUIRE(!(results_1 == results_2));
137     ATF_REQUIRE(  results_1 != results_2);
138 
139 
140     // Checks for the results "type", different counts.
141     const engine::tap_summary results_3 =
142         engine::tap_summary::new_results(results_1.plan(),
143                                          results_1.not_ok_count(),
144                                          results_1.ok_count());
145     ATF_REQUIRE(!(results_1 == results_3));
146     ATF_REQUIRE(  results_1 != results_3);
147 }
148 
149 
150 ATF_TEST_CASE_WITHOUT_HEAD(tap_summary__output);
ATF_TEST_CASE_BODY(tap_summary__output)151 ATF_TEST_CASE_BODY(tap_summary__output)
152 {
153     {
154         const engine::tap_summary summary =
155             engine::tap_summary::new_bailed_out();
156         ATF_REQUIRE_EQ(
157             "tap_summary{bailed_out=true}",
158             (F("%s") % summary).str());
159     }
160 
161     {
162         const engine::tap_summary summary =
163             engine::tap_summary::new_results(engine::tap_plan(5, 10), 2, 4);
164         ATF_REQUIRE_EQ(
165             "tap_summary{bailed_out=false, plan=5..10, ok_count=2, "
166             "not_ok_count=4}",
167             (F("%s") % summary).str());
168     }
169 
170     {
171         const engine::tap_summary summary =
172             engine::tap_summary::new_all_skipped("Who knows");
173         ATF_REQUIRE_EQ(
174             "tap_summary{bailed_out=false, plan=1..0, "
175             "all_skipped_reason=Who knows}",
176             (F("%s") % summary).str());
177     }
178 }
179 
180 
181 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__only_one_result);
ATF_TEST_CASE_BODY(parse_tap_output__only_one_result)182 ATF_TEST_CASE_BODY(parse_tap_output__only_one_result)
183 {
184     const engine::tap_summary summary = do_parse(
185         "1..1\n"
186         "ok - 1\n");
187 
188     const engine::tap_summary exp_summary =
189         engine::tap_summary::new_results(engine::tap_plan(1, 1), 1, 0);
190     ATF_REQUIRE_EQ(exp_summary, summary);
191 }
192 
193 
194 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__all_pass);
ATF_TEST_CASE_BODY(parse_tap_output__all_pass)195 ATF_TEST_CASE_BODY(parse_tap_output__all_pass)
196 {
197     const engine::tap_summary summary = do_parse(
198         "1..8\n"
199         "ok - 1\n"
200         "    Some diagnostic message\n"
201         "ok - 2 This test also passed\n"
202         "garbage line\n"
203         "ok - 3 This test passed\n"
204         "not ok 4 # SKIP Some reason\n"
205         "not ok 5 # TODO Another reason\n"
206         "ok - 6 Doesn't make a difference SKIP\n"
207         "ok - 7 Doesn't make a difference either TODO\n"
208         "ok # Also works without a number\n");
209 
210     const engine::tap_summary exp_summary =
211         engine::tap_summary::new_results(engine::tap_plan(1, 8), 8, 0);
212     ATF_REQUIRE_EQ(exp_summary, summary);
213 }
214 
215 
216 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__some_fail);
ATF_TEST_CASE_BODY(parse_tap_output__some_fail)217 ATF_TEST_CASE_BODY(parse_tap_output__some_fail)
218 {
219     const engine::tap_summary summary = do_parse(
220         "garbage line\n"
221         "not ok - 1 This test failed\n"
222         "ok - 2 This test passed\n"
223         "not ok - 3 This test failed\n"
224         "1..6\n"
225         "not ok - 4 This test failed\n"
226         "ok - 5 This test passed\n"
227         "not ok # Fails as well without a number\n");
228 
229     const engine::tap_summary exp_summary =
230         engine::tap_summary::new_results(engine::tap_plan(1, 6), 2, 4);
231     ATF_REQUIRE_EQ(exp_summary, summary);
232 }
233 
234 
235 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_and_todo_variants);
ATF_TEST_CASE_BODY(parse_tap_output__skip_and_todo_variants)236 ATF_TEST_CASE_BODY(parse_tap_output__skip_and_todo_variants)
237 {
238     const engine::tap_summary summary = do_parse(
239         "1..8\n"
240         "not ok - 1 # SKIP Some reason\n"
241         "not ok - 2 # skip Some reason\n"
242         "not ok - 3 # Skipped Some reason\n"
243         "not ok - 4 # skipped Some reason\n"
244         "not ok - 5 # Skipped: Some reason\n"
245         "not ok - 6 # skipped: Some reason\n"
246         "not ok - 7 # TODO Some reason\n"
247         "not ok - 8 # todo Some reason\n");
248 
249     const engine::tap_summary exp_summary =
250         engine::tap_summary::new_results(engine::tap_plan(1, 8), 8, 0);
251     ATF_REQUIRE_EQ(exp_summary, summary);
252 }
253 
254 
255 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_with_reason);
ATF_TEST_CASE_BODY(parse_tap_output__skip_all_with_reason)256 ATF_TEST_CASE_BODY(parse_tap_output__skip_all_with_reason)
257 {
258     const engine::tap_summary summary = do_parse(
259         "1..0 SKIP Some reason for skipping\n"
260         "ok - 1\n"
261         "    Some diagnostic message\n"
262         "ok - 6 Doesn't make a difference SKIP\n"
263         "ok - 7 Doesn't make a difference either TODO\n");
264 
265     const engine::tap_summary exp_summary =
266         engine::tap_summary::new_all_skipped("Some reason for skipping");
267     ATF_REQUIRE_EQ(exp_summary, summary);
268 }
269 
270 
271 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_without_reason);
ATF_TEST_CASE_BODY(parse_tap_output__skip_all_without_reason)272 ATF_TEST_CASE_BODY(parse_tap_output__skip_all_without_reason)
273 {
274     const engine::tap_summary summary = do_parse(
275         "1..0 unrecognized # garbage skip\n");
276 
277     const engine::tap_summary exp_summary =
278         engine::tap_summary::new_all_skipped("No reason specified");
279     ATF_REQUIRE_EQ(exp_summary, summary);
280 }
281 
282 
283 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__skip_all_invalid);
ATF_TEST_CASE_BODY(parse_tap_output__skip_all_invalid)284 ATF_TEST_CASE_BODY(parse_tap_output__skip_all_invalid)
285 {
286     ATF_REQUIRE_THROW_RE(engine::load_error,
287                          "Skipped plan must be 1\\.\\.0",
288                          do_parse("1..3 # skip\n"));
289 }
290 
291 
292 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__plan_at_end);
ATF_TEST_CASE_BODY(parse_tap_output__plan_at_end)293 ATF_TEST_CASE_BODY(parse_tap_output__plan_at_end)
294 {
295     const engine::tap_summary summary = do_parse(
296         "ok - 1\n"
297         "    Some diagnostic message\n"
298         "ok - 2 This test also passed\n"
299         "garbage line\n"
300         "ok - 3 This test passed\n"
301         "not ok 4 # SKIP Some reason\n"
302         "not ok 5 # TODO Another reason\n"
303         "ok - 6 Doesn't make a difference SKIP\n"
304         "ok - 7 Doesn't make a difference either TODO\n"
305         "1..7\n");
306 
307     const engine::tap_summary exp_summary =
308         engine::tap_summary::new_results(engine::tap_plan(1, 7), 7, 0);
309     ATF_REQUIRE_EQ(exp_summary, summary);
310 }
311 
312 
313 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__stray_oks);
ATF_TEST_CASE_BODY(parse_tap_output__stray_oks)314 ATF_TEST_CASE_BODY(parse_tap_output__stray_oks)
315 {
316     const engine::tap_summary summary = do_parse(
317         "1..3\n"
318         "ok - 1\n"
319         "ok\n"
320         "ok - 2 This test also passed\n"
321         "not ok\n"
322         "ok - 3 This test passed\n");
323 
324     const engine::tap_summary exp_summary =
325         engine::tap_summary::new_results(engine::tap_plan(1, 3), 3, 0);
326     ATF_REQUIRE_EQ(exp_summary, summary);
327 }
328 
329 
330 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__no_plan);
ATF_TEST_CASE_BODY(parse_tap_output__no_plan)331 ATF_TEST_CASE_BODY(parse_tap_output__no_plan)
332 {
333     ATF_REQUIRE_THROW_RE(
334         engine::load_error,
335         "Output did not contain any TAP plan",
336         do_parse(
337             "not ok - 1 This test failed\n"
338             "ok - 2 This test passed\n"));
339 }
340 
341 
342 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__double_plan);
ATF_TEST_CASE_BODY(parse_tap_output__double_plan)343 ATF_TEST_CASE_BODY(parse_tap_output__double_plan)
344 {
345     ATF_REQUIRE_THROW_RE(
346         engine::load_error,
347         "Found duplicate plan",
348         do_parse(
349             "garbage line\n"
350             "1..5\n"
351             "not ok - 1 This test failed\n"
352             "ok - 2 This test passed\n"
353             "1..8\n"
354             "ok\n"));
355 }
356 
357 
358 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__inconsistent_plan);
ATF_TEST_CASE_BODY(parse_tap_output__inconsistent_plan)359 ATF_TEST_CASE_BODY(parse_tap_output__inconsistent_plan)
360 {
361     ATF_REQUIRE_THROW_RE(
362         engine::load_error,
363         "Reported plan differs from actual executed tests",
364         do_parse(
365             "1..3\n"
366             "not ok - 1 This test failed\n"
367             "ok - 2 This test passed\n"));
368 }
369 
370 
371 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__inconsistent_trailing_plan);
ATF_TEST_CASE_BODY(parse_tap_output__inconsistent_trailing_plan)372 ATF_TEST_CASE_BODY(parse_tap_output__inconsistent_trailing_plan)
373 {
374     ATF_REQUIRE_THROW_RE(
375         engine::load_error,
376         "Reported plan differs from actual executed tests",
377         do_parse(
378             "not ok - 1 This test failed\n"
379             "ok - 2 This test passed\n"
380             "1..3\n"));
381 }
382 
383 
384 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__insane_plan);
ATF_TEST_CASE_BODY(parse_tap_output__insane_plan)385 ATF_TEST_CASE_BODY(parse_tap_output__insane_plan)
386 {
387     ATF_REQUIRE_THROW_RE(
388         engine::load_error, "Invalid value",
389         do_parse("120830981209831..234891793874080981092803981092312\n"));
390 }
391 
392 
393 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__reversed_plan);
ATF_TEST_CASE_BODY(parse_tap_output__reversed_plan)394 ATF_TEST_CASE_BODY(parse_tap_output__reversed_plan)
395 {
396     ATF_REQUIRE_THROW_RE(engine::load_error,
397                          "Found reversed plan 8\\.\\.5",
398                          do_parse("8..5\n"));
399 }
400 
401 
402 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__bail_out);
ATF_TEST_CASE_BODY(parse_tap_output__bail_out)403 ATF_TEST_CASE_BODY(parse_tap_output__bail_out)
404 {
405     const engine::tap_summary summary = do_parse(
406         "1..3\n"
407         "not ok - 1 This test failed\n"
408         "Bail out! There is some unknown problem\n"
409         "ok - 2 This test passed\n");
410 
411     const engine::tap_summary exp_summary =
412         engine::tap_summary::new_bailed_out();
413     ATF_REQUIRE_EQ(exp_summary, summary);
414 }
415 
416 
417 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__bail_out_wins_over_no_plan);
ATF_TEST_CASE_BODY(parse_tap_output__bail_out_wins_over_no_plan)418 ATF_TEST_CASE_BODY(parse_tap_output__bail_out_wins_over_no_plan)
419 {
420     const engine::tap_summary summary = do_parse(
421         "not ok - 1 This test failed\n"
422         "Bail out! There is some unknown problem\n"
423         "ok - 2 This test passed\n");
424 
425     const engine::tap_summary exp_summary =
426         engine::tap_summary::new_bailed_out();
427     ATF_REQUIRE_EQ(exp_summary, summary);
428 }
429 
430 
431 ATF_TEST_CASE_WITHOUT_HEAD(parse_tap_output__open_failure);
ATF_TEST_CASE_BODY(parse_tap_output__open_failure)432 ATF_TEST_CASE_BODY(parse_tap_output__open_failure)
433 {
434     ATF_REQUIRE_THROW_RE(engine::load_error, "hello.txt.*Failed to open",
435                          engine::parse_tap_output(fs::path("hello.txt")));
436 }
437 
438 
ATF_INIT_TEST_CASES(tcs)439 ATF_INIT_TEST_CASES(tcs)
440 {
441     ATF_ADD_TEST_CASE(tcs, tap_summary__bailed_out);
442     ATF_ADD_TEST_CASE(tcs, tap_summary__some_results);
443     ATF_ADD_TEST_CASE(tcs, tap_summary__all_skipped);
444     ATF_ADD_TEST_CASE(tcs, tap_summary__equality_operators);
445     ATF_ADD_TEST_CASE(tcs, tap_summary__output);
446 
447     ATF_ADD_TEST_CASE(tcs, parse_tap_output__only_one_result);
448     ATF_ADD_TEST_CASE(tcs, parse_tap_output__all_pass);
449     ATF_ADD_TEST_CASE(tcs, parse_tap_output__some_fail);
450     ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_and_todo_variants);
451     ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_without_reason);
452     ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_with_reason);
453     ATF_ADD_TEST_CASE(tcs, parse_tap_output__skip_all_invalid);
454     ATF_ADD_TEST_CASE(tcs, parse_tap_output__plan_at_end);
455     ATF_ADD_TEST_CASE(tcs, parse_tap_output__stray_oks);
456     ATF_ADD_TEST_CASE(tcs, parse_tap_output__no_plan);
457     ATF_ADD_TEST_CASE(tcs, parse_tap_output__double_plan);
458     ATF_ADD_TEST_CASE(tcs, parse_tap_output__inconsistent_plan);
459     ATF_ADD_TEST_CASE(tcs, parse_tap_output__inconsistent_trailing_plan);
460     ATF_ADD_TEST_CASE(tcs, parse_tap_output__insane_plan);
461     ATF_ADD_TEST_CASE(tcs, parse_tap_output__reversed_plan);
462     ATF_ADD_TEST_CASE(tcs, parse_tap_output__bail_out);
463     ATF_ADD_TEST_CASE(tcs, parse_tap_output__bail_out_wins_over_no_plan);
464     ATF_ADD_TEST_CASE(tcs, parse_tap_output__open_failure);
465 }
466