1 //
2 // Automated Testing Framework (atf)
3 //
4 // Copyright (c) 2008, 2009, 2010 The NetBSD Foundation, Inc.
5 // All rights reserved.
6 //
7 // Redistribution and use in source and binary forms, with or without
8 // modification, are permitted provided that the following conditions
9 // are met:
10 // 1. Redistributions of source code must retain the above copyright
11 //    notice, this list of conditions and the following disclaimer.
12 // 2. Redistributions in binary form must reproduce the above copyright
13 //    notice, this list of conditions and the following disclaimer in the
14 //    documentation and/or other materials provided with the distribution.
15 //
16 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 //
29 
30 extern "C" {
31 #include <fcntl.h>
32 #include <unistd.h>
33 }
34 
35 #include <cerrno>
36 #include <cstdlib>
37 #include <iostream>
38 #include <stdexcept>
39 
40 #include "macros.hpp"
41 
42 #include "detail/fs.hpp"
43 #include "detail/process.hpp"
44 #include "detail/sanity.hpp"
45 #include "detail/test_helpers.hpp"
46 #include "detail/text.hpp"
47 
48 // ------------------------------------------------------------------------
49 // Auxiliary functions.
50 // ------------------------------------------------------------------------
51 
52 static
53 void
54 create_ctl_file(const atf::tests::tc& tc, const char *name)
55 {
56     ATF_REQUIRE(open(name, O_CREAT | O_WRONLY | O_TRUNC, 0644) != -1);
57 }
58 
59 // ------------------------------------------------------------------------
60 // Auxiliary test cases.
61 // ------------------------------------------------------------------------
62 
63 ATF_TEST_CASE(h_pass);
64 ATF_TEST_CASE_HEAD(h_pass)
65 {
66     set_md_var("descr", "Helper test case");
67 }
68 ATF_TEST_CASE_BODY(h_pass)
69 {
70     create_ctl_file(*this, "before");
71     ATF_PASS();
72     create_ctl_file(*this, "after");
73 }
74 
75 ATF_TEST_CASE(h_fail);
76 ATF_TEST_CASE_HEAD(h_fail)
77 {
78     set_md_var("descr", "Helper test case");
79 }
80 ATF_TEST_CASE_BODY(h_fail)
81 {
82     create_ctl_file(*this, "before");
83     ATF_FAIL("Failed on purpose");
84     create_ctl_file(*this, "after");
85 }
86 
87 ATF_TEST_CASE(h_skip);
88 ATF_TEST_CASE_HEAD(h_skip)
89 {
90     set_md_var("descr", "Helper test case");
91 }
92 ATF_TEST_CASE_BODY(h_skip)
93 {
94     create_ctl_file(*this, "before");
95     ATF_SKIP("Skipped on purpose");
96     create_ctl_file(*this, "after");
97 }
98 
99 ATF_TEST_CASE(h_require);
100 ATF_TEST_CASE_HEAD(h_require)
101 {
102     set_md_var("descr", "Helper test case");
103 }
104 ATF_TEST_CASE_BODY(h_require)
105 {
106     bool condition = atf::text::to_bool(get_config_var("condition"));
107 
108     create_ctl_file(*this, "before");
109     ATF_REQUIRE(condition);
110     create_ctl_file(*this, "after");
111 }
112 
113 ATF_TEST_CASE(h_require_eq);
114 ATF_TEST_CASE_HEAD(h_require_eq)
115 {
116     set_md_var("descr", "Helper test case");
117 }
118 ATF_TEST_CASE_BODY(h_require_eq)
119 {
120     long v1 = atf::text::to_type< long >(get_config_var("v1"));
121     long v2 = atf::text::to_type< long >(get_config_var("v2"));
122 
123     create_ctl_file(*this, "before");
124     ATF_REQUIRE_EQ(v1, v2);
125     create_ctl_file(*this, "after");
126 }
127 
128 ATF_TEST_CASE(h_require_match);
129 ATF_TEST_CASE_HEAD(h_require_match)
130 {
131     set_md_var("descr", "Helper test case");
132 }
133 ATF_TEST_CASE_BODY(h_require_match)
134 {
135     const std::string regexp = get_config_var("regexp");
136     const std::string string = get_config_var("string");
137 
138     create_ctl_file(*this, "before");
139     ATF_REQUIRE_MATCH(regexp, string);
140     create_ctl_file(*this, "after");
141 }
142 
143 ATF_TEST_CASE(h_require_throw);
144 ATF_TEST_CASE_HEAD(h_require_throw)
145 {
146     set_md_var("descr", "Helper test case");
147 }
148 ATF_TEST_CASE_BODY(h_require_throw)
149 {
150     create_ctl_file(*this, "before");
151 
152     if (get_config_var("what") == "throw_int")
153         ATF_REQUIRE_THROW(std::runtime_error, if (1) throw int(5));
154     else if (get_config_var("what") == "throw_rt")
155         ATF_REQUIRE_THROW(std::runtime_error,
156                         if (1) throw std::runtime_error("e"));
157     else if (get_config_var("what") == "no_throw_rt")
158         ATF_REQUIRE_THROW(std::runtime_error,
159                         if (0) throw std::runtime_error("e"));
160 
161     create_ctl_file(*this, "after");
162 }
163 
164 ATF_TEST_CASE(h_require_throw_re);
165 ATF_TEST_CASE_HEAD(h_require_throw_re)
166 {
167     set_md_var("descr", "Helper test case");
168 }
169 ATF_TEST_CASE_BODY(h_require_throw_re)
170 {
171     create_ctl_file(*this, "before");
172 
173     if (get_config_var("what") == "throw_int")
174         ATF_REQUIRE_THROW_RE(std::runtime_error, "5", if (1) throw int(5));
175     else if (get_config_var("what") == "throw_rt_match")
176         ATF_REQUIRE_THROW_RE(std::runtime_error, "foo.*baz",
177                              if (1) throw std::runtime_error("a foo bar baz"));
178     else if (get_config_var("what") == "throw_rt_no_match")
179         ATF_REQUIRE_THROW_RE(std::runtime_error, "foo.*baz",
180                              if (1) throw std::runtime_error("baz foo bar a"));
181     else if (get_config_var("what") == "no_throw_rt")
182         ATF_REQUIRE_THROW_RE(std::runtime_error, "e",
183                              if (0) throw std::runtime_error("e"));
184 
185     create_ctl_file(*this, "after");
186 }
187 
188 static int
189 errno_fail_stub(const int raised_errno)
190 {
191     errno = raised_errno;
192     return -1;
193 }
194 
195 static int
196 errno_ok_stub(void)
197 {
198     return 0;
199 }
200 
201 ATF_TEST_CASE(h_check_errno);
202 ATF_TEST_CASE_HEAD(h_check_errno)
203 {
204     set_md_var("descr", "Helper test case");
205 }
206 ATF_TEST_CASE_BODY(h_check_errno)
207 {
208     create_ctl_file(*this, "before");
209 
210     if (get_config_var("what") == "no_error")
211         ATF_CHECK_ERRNO(-1, errno_ok_stub() == -1);
212     else if (get_config_var("what") == "errno_ok")
213         ATF_CHECK_ERRNO(2, errno_fail_stub(2) == -1);
214     else if (get_config_var("what") == "errno_fail")
215         ATF_CHECK_ERRNO(3, errno_fail_stub(4) == -1);
216     else
217         UNREACHABLE;
218 
219     create_ctl_file(*this, "after");
220 }
221 
222 ATF_TEST_CASE(h_require_errno);
223 ATF_TEST_CASE_HEAD(h_require_errno)
224 {
225     set_md_var("descr", "Helper test case");
226 }
227 ATF_TEST_CASE_BODY(h_require_errno)
228 {
229     create_ctl_file(*this, "before");
230 
231     if (get_config_var("what") == "no_error")
232         ATF_REQUIRE_ERRNO(-1, errno_ok_stub() == -1);
233     else if (get_config_var("what") == "errno_ok")
234         ATF_REQUIRE_ERRNO(2, errno_fail_stub(2) == -1);
235     else if (get_config_var("what") == "errno_fail")
236         ATF_REQUIRE_ERRNO(3, errno_fail_stub(4) == -1);
237     else
238         UNREACHABLE;
239 
240     create_ctl_file(*this, "after");
241 }
242 
243 // ------------------------------------------------------------------------
244 // Test cases for the macros.
245 // ------------------------------------------------------------------------
246 
247 ATF_TEST_CASE(pass);
248 ATF_TEST_CASE_HEAD(pass)
249 {
250     set_md_var("descr", "Tests the ATF_PASS macro");
251 }
252 ATF_TEST_CASE_BODY(pass)
253 {
254     run_h_tc< ATF_TEST_CASE_NAME(h_pass) >();
255     ATF_REQUIRE(grep_file("result", "^passed"));
256     ATF_REQUIRE(atf::fs::exists(atf::fs::path("before")));
257     ATF_REQUIRE(!atf::fs::exists(atf::fs::path("after")));
258 }
259 
260 ATF_TEST_CASE(fail);
261 ATF_TEST_CASE_HEAD(fail)
262 {
263     set_md_var("descr", "Tests the ATF_FAIL macro");
264 }
265 ATF_TEST_CASE_BODY(fail)
266 {
267     run_h_tc< ATF_TEST_CASE_NAME(h_fail) >();
268     ATF_REQUIRE(grep_file("result", "^failed: Failed on purpose"));
269     ATF_REQUIRE(atf::fs::exists(atf::fs::path("before")));
270     ATF_REQUIRE(!atf::fs::exists(atf::fs::path("after")));
271 }
272 
273 ATF_TEST_CASE(skip);
274 ATF_TEST_CASE_HEAD(skip)
275 {
276     set_md_var("descr", "Tests the ATF_SKIP macro");
277 }
278 ATF_TEST_CASE_BODY(skip)
279 {
280     run_h_tc< ATF_TEST_CASE_NAME(h_skip) >();
281     ATF_REQUIRE(grep_file("result", "^skipped: Skipped on purpose"));
282     ATF_REQUIRE(atf::fs::exists(atf::fs::path("before")));
283     ATF_REQUIRE(!atf::fs::exists(atf::fs::path("after")));
284 }
285 
286 ATF_TEST_CASE(require);
287 ATF_TEST_CASE_HEAD(require)
288 {
289     set_md_var("descr", "Tests the ATF_REQUIRE macro");
290 }
291 ATF_TEST_CASE_BODY(require)
292 {
293     struct test {
294         const char *cond;
295         bool ok;
296     } *t, tests[] = {
297         { "false", false },
298         { "true", true },
299         { NULL, false }
300     };
301 
302     const atf::fs::path before("before");
303     const atf::fs::path after("after");
304 
305     for (t = &tests[0]; t->cond != NULL; t++) {
306         atf::tests::vars_map config;
307         config["condition"] = t->cond;
308 
309         std::cout << "Checking with a " << t->cond << " value\n";
310 
311         run_h_tc< ATF_TEST_CASE_NAME(h_require) >(config);
312 
313         ATF_REQUIRE(atf::fs::exists(before));
314         if (t->ok) {
315             ATF_REQUIRE(grep_file("result", "^passed"));
316             ATF_REQUIRE(atf::fs::exists(after));
317         } else {
318             ATF_REQUIRE(grep_file("result", "^failed: .*condition not met"));
319             ATF_REQUIRE(!atf::fs::exists(after));
320         }
321 
322         atf::fs::remove(before);
323         if (t->ok)
324             atf::fs::remove(after);
325     }
326 }
327 
328 ATF_TEST_CASE(require_eq);
329 ATF_TEST_CASE_HEAD(require_eq)
330 {
331     set_md_var("descr", "Tests the ATF_REQUIRE_EQ macro");
332 }
333 ATF_TEST_CASE_BODY(require_eq)
334 {
335     struct test {
336         const char *v1;
337         const char *v2;
338         bool ok;
339     } *t, tests[] = {
340         { "1", "1", true },
341         { "1", "2", false },
342         { "2", "1", false },
343         { "2", "2", true },
344         { NULL, NULL, false }
345     };
346 
347     const atf::fs::path before("before");
348     const atf::fs::path after("after");
349 
350     for (t = &tests[0]; t->v1 != NULL; t++) {
351         atf::tests::vars_map config;
352         config["v1"] = t->v1;
353         config["v2"] = t->v2;
354 
355         std::cout << "Checking with " << t->v1 << ", " << t->v2
356                   << " and expecting " << (t->ok ? "true" : "false")
357                   << "\n";
358 
359         run_h_tc< ATF_TEST_CASE_NAME(h_require_eq) >(config);
360 
361         ATF_REQUIRE(atf::fs::exists(before));
362         if (t->ok) {
363             ATF_REQUIRE(grep_file("result", "^passed"));
364             ATF_REQUIRE(atf::fs::exists(after));
365         } else {
366             ATF_REQUIRE(grep_file("result", "^failed: .*v1 != v2"));
367             ATF_REQUIRE(!atf::fs::exists(after));
368         }
369 
370         atf::fs::remove(before);
371         if (t->ok)
372             atf::fs::remove(after);
373     }
374 }
375 
376 ATF_TEST_CASE(require_match);
377 ATF_TEST_CASE_HEAD(require_match)
378 {
379     set_md_var("descr", "Tests the ATF_REQUIRE_MATCH macro");
380 }
381 ATF_TEST_CASE_BODY(require_match)
382 {
383     struct test {
384         const char *regexp;
385         const char *string;
386         bool ok;
387     } *t, tests[] = {
388         { "foo.*bar", "this is a foo, bar, baz", true },
389         { "bar.*baz", "this is a baz, bar, foo", false },
390         { NULL, NULL, false }
391     };
392 
393     const atf::fs::path before("before");
394     const atf::fs::path after("after");
395 
396     for (t = &tests[0]; t->regexp != NULL; t++) {
397         atf::tests::vars_map config;
398         config["regexp"] = t->regexp;
399         config["string"] = t->string;
400 
401         std::cout << "Checking with " << t->regexp << ", " << t->string
402                   << " and expecting " << (t->ok ? "true" : "false")
403                   << "\n";
404 
405         run_h_tc< ATF_TEST_CASE_NAME(h_require_match) >(config);
406 
407         ATF_REQUIRE(atf::fs::exists(before));
408         if (t->ok) {
409             ATF_REQUIRE(grep_file("result", "^passed"));
410             ATF_REQUIRE(atf::fs::exists(after));
411         } else {
412             ATF_REQUIRE(grep_file("result", "^failed: "));
413             ATF_REQUIRE(!atf::fs::exists(after));
414         }
415 
416         atf::fs::remove(before);
417         if (t->ok)
418             atf::fs::remove(after);
419     }
420 }
421 
422 ATF_TEST_CASE(require_throw);
423 ATF_TEST_CASE_HEAD(require_throw)
424 {
425     set_md_var("descr", "Tests the ATF_REQUIRE_THROW macro");
426 }
427 ATF_TEST_CASE_BODY(require_throw)
428 {
429     struct test {
430         const char *what;
431         bool ok;
432         const char *msg;
433     } *t, tests[] = {
434         { "throw_int", false, "unexpected error" },
435         { "throw_rt", true, NULL },
436         { "no_throw_rt", false, "did not throw" },
437         { NULL, false, NULL }
438     };
439 
440     const atf::fs::path before("before");
441     const atf::fs::path after("after");
442 
443     for (t = &tests[0]; t->what != NULL; t++) {
444         atf::tests::vars_map config;
445         config["what"] = t->what;
446 
447         std::cout << "Checking with " << t->what << " and expecting "
448                   << (t->ok ? "true" : "false") << "\n";
449 
450         run_h_tc< ATF_TEST_CASE_NAME(h_require_throw) >(config);
451 
452         ATF_REQUIRE(atf::fs::exists(before));
453         if (t->ok) {
454             ATF_REQUIRE(grep_file("result", "^passed"));
455             ATF_REQUIRE(atf::fs::exists(after));
456         } else {
457             std::cout << "Checking that message contains '" << t->msg
458                       << "'\n";
459             std::string exp_result = std::string("^failed: .*") + t->msg;
460             ATF_REQUIRE(grep_file("result", exp_result.c_str()));
461             ATF_REQUIRE(!atf::fs::exists(after));
462         }
463 
464         atf::fs::remove(before);
465         if (t->ok)
466             atf::fs::remove(after);
467     }
468 }
469 
470 ATF_TEST_CASE(require_throw_re);
471 ATF_TEST_CASE_HEAD(require_throw_re)
472 {
473     set_md_var("descr", "Tests the ATF_REQUIRE_THROW_RE macro");
474 }
475 ATF_TEST_CASE_BODY(require_throw_re)
476 {
477     struct test {
478         const char *what;
479         bool ok;
480         const char *msg;
481     } *t, tests[] = {
482         { "throw_int", false, "unexpected error" },
483         { "throw_rt_match", true, NULL },
484         { "throw_rt_no_match", true, "threw.*runtime_error(baz foo bar a).*"
485           "does not match 'a foo bar baz'" },
486         { "no_throw_rt", false, "did not throw" },
487         { NULL, false, NULL }
488     };
489 
490     const atf::fs::path before("before");
491     const atf::fs::path after("after");
492 
493     for (t = &tests[0]; t->what != NULL; t++) {
494         atf::tests::vars_map config;
495         config["what"] = t->what;
496 
497         std::cout << "Checking with " << t->what << " and expecting "
498                   << (t->ok ? "true" : "false") << "\n";
499 
500         run_h_tc< ATF_TEST_CASE_NAME(h_require_throw) >(config);
501 
502         ATF_REQUIRE(atf::fs::exists(before));
503         if (t->ok) {
504             ATF_REQUIRE(grep_file("result", "^passed"));
505             ATF_REQUIRE(atf::fs::exists(after));
506         } else {
507             std::cout << "Checking that message contains '" << t->msg
508                       << "'\n";
509             std::string exp_result = std::string("^failed: .*") + t->msg;
510             ATF_REQUIRE(grep_file("result", exp_result.c_str()));
511             ATF_REQUIRE(!atf::fs::exists(after));
512         }
513 
514         atf::fs::remove(before);
515         if (t->ok)
516             atf::fs::remove(after);
517     }
518 }
519 
520 ATF_TEST_CASE(check_errno);
521 ATF_TEST_CASE_HEAD(check_errno)
522 {
523     set_md_var("descr", "Tests the ATF_CHECK_ERRNO macro");
524 }
525 ATF_TEST_CASE_BODY(check_errno)
526 {
527     struct test {
528         const char *what;
529         bool ok;
530         const char *msg;
531     } *t, tests[] = {
532         { "no_error", false,
533           "Expected true value in errno_ok_stub\\(\\) == -1" },
534         { "errno_ok", true, NULL },
535         { "errno_fail", false,
536           "Expected errno 3, got 4, in errno_fail_stub\\(4\\) == -1" },
537         { NULL, false, NULL }
538     };
539 
540     const atf::fs::path before("before");
541     const atf::fs::path after("after");
542 
543     for (t = &tests[0]; t->what != NULL; t++) {
544         atf::tests::vars_map config;
545         config["what"] = t->what;
546 
547         run_h_tc< ATF_TEST_CASE_NAME(h_check_errno) >(config);
548 
549         ATF_REQUIRE(atf::fs::exists(before));
550         ATF_REQUIRE(atf::fs::exists(after));
551 
552         if (t->ok) {
553             ATF_REQUIRE(grep_file("result", "^passed"));
554         } else {
555             ATF_REQUIRE(grep_file("result", "^failed"));
556 
557             std::string exp_result = "macros_test.cpp:[0-9]+: " +
558                 std::string(t->msg) + "$";
559             ATF_REQUIRE(grep_file("stderr", exp_result.c_str()));
560         }
561 
562         atf::fs::remove(before);
563         atf::fs::remove(after);
564     }
565 }
566 
567 ATF_TEST_CASE(require_errno);
568 ATF_TEST_CASE_HEAD(require_errno)
569 {
570     set_md_var("descr", "Tests the ATF_REQUIRE_ERRNO macro");
571 }
572 ATF_TEST_CASE_BODY(require_errno)
573 {
574     struct test {
575         const char *what;
576         bool ok;
577         const char *msg;
578     } *t, tests[] = {
579         { "no_error", false,
580           "Expected true value in errno_ok_stub\\(\\) == -1" },
581         { "errno_ok", true, NULL },
582         { "errno_fail", false,
583           "Expected errno 3, got 4, in errno_fail_stub\\(4\\) == -1" },
584         { NULL, false, NULL }
585     };
586 
587     const atf::fs::path before("before");
588     const atf::fs::path after("after");
589 
590     for (t = &tests[0]; t->what != NULL; t++) {
591         atf::tests::vars_map config;
592         config["what"] = t->what;
593 
594         run_h_tc< ATF_TEST_CASE_NAME(h_require_errno) >(config);
595 
596         ATF_REQUIRE(atf::fs::exists(before));
597         if (t->ok) {
598             ATF_REQUIRE(grep_file("result", "^passed"));
599             ATF_REQUIRE(atf::fs::exists(after));
600         } else {
601             std::string exp_result = "^failed: .*macros_test.cpp:[0-9]+: " +
602                 std::string(t->msg) + "$";
603             ATF_REQUIRE(grep_file("result", exp_result.c_str()));
604 
605             ATF_REQUIRE(!atf::fs::exists(after));
606         }
607 
608         atf::fs::remove(before);
609         if (t->ok)
610             atf::fs::remove(after);
611     }
612 }
613 
614 // ------------------------------------------------------------------------
615 // Tests cases for the header file.
616 // ------------------------------------------------------------------------
617 
618 HEADER_TC(include, "atf-c++/macros.hpp");
619 BUILD_TC(use, "macros_hpp_test.cpp",
620          "Tests that the macros provided by the atf-c++/macros.hpp file "
621          "do not cause syntax errors when used",
622          "Build of macros_hpp_test.cpp failed; some macros in "
623          "atf-c++/macros.hpp are broken");
624 
625 // ------------------------------------------------------------------------
626 // Main.
627 // ------------------------------------------------------------------------
628 
629 ATF_INIT_TEST_CASES(tcs)
630 {
631     // Add the test cases for the macros.
632     ATF_ADD_TEST_CASE(tcs, pass);
633     ATF_ADD_TEST_CASE(tcs, fail);
634     ATF_ADD_TEST_CASE(tcs, skip);
635     ATF_ADD_TEST_CASE(tcs, check_errno);
636     ATF_ADD_TEST_CASE(tcs, require);
637     ATF_ADD_TEST_CASE(tcs, require_eq);
638     ATF_ADD_TEST_CASE(tcs, require_match);
639     ATF_ADD_TEST_CASE(tcs, require_throw);
640     ATF_ADD_TEST_CASE(tcs, require_throw_re);
641     ATF_ADD_TEST_CASE(tcs, require_errno);
642 
643     // Add the test cases for the header file.
644     ATF_ADD_TEST_CASE(tcs, include);
645     ATF_ADD_TEST_CASE(tcs, use);
646 }
647