1 // 2 // Automated Testing Framework (atf) 3 // 4 // Copyright (c) 2007 The NetBSD Foundation, Inc. 5 // All rights reserved. 6 // 7 // Redistribution and use in source and binary forms, with or without 8 // modification, are permitted provided that the following conditions 9 // are met: 10 // 1. Redistributions of source code must retain the above copyright 11 // notice, this list of conditions and the following disclaimer. 12 // 2. Redistributions in binary form must reproduce the above copyright 13 // notice, this list of conditions and the following disclaimer in the 14 // documentation and/or other materials provided with the distribution. 15 // 16 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 // 29 30 extern "C" { 31 #include <sys/types.h> 32 #include <sys/stat.h> 33 #include <sys/time.h> 34 #include <sys/wait.h> 35 #include <signal.h> 36 #include <unistd.h> 37 } 38 39 #include <algorithm> 40 #include <cctype> 41 #include <cerrno> 42 #include <cstdlib> 43 #include <cstring> 44 #include <fstream> 45 #include <iostream> 46 #include <map> 47 #include <memory> 48 #include <sstream> 49 #include <stdexcept> 50 #include <vector> 51 52 extern "C" { 53 #include "atf-c/error.h" 54 #include "atf-c/tc.h" 55 #include "atf-c/utils.h" 56 } 57 58 #include "noncopyable.hpp" 59 #include "tests.hpp" 60 61 #include "detail/application.hpp" 62 #include "detail/auto_array.hpp" 63 #include "detail/env.hpp" 64 #include "detail/exceptions.hpp" 65 #include "detail/fs.hpp" 66 #include "detail/parser.hpp" 67 #include "detail/sanity.hpp" 68 #include "detail/text.hpp" 69 70 namespace impl = atf::tests; 71 namespace detail = atf::tests::detail; 72 #define IMPL_NAME "atf::tests" 73 74 // ------------------------------------------------------------------------ 75 // The "atf_tp_writer" class. 76 // ------------------------------------------------------------------------ 77 78 detail::atf_tp_writer::atf_tp_writer(std::ostream& os) : 79 m_os(os), 80 m_is_first(true) 81 { 82 atf::parser::headers_map hm; 83 atf::parser::attrs_map ct_attrs; 84 ct_attrs["version"] = "1"; 85 hm["Content-Type"] = atf::parser::header_entry("Content-Type", 86 "application/X-atf-tp", ct_attrs); 87 atf::parser::write_headers(hm, m_os); 88 } 89 90 void 91 detail::atf_tp_writer::start_tc(const std::string& ident) 92 { 93 if (!m_is_first) 94 m_os << "\n"; 95 m_os << "ident: " << ident << "\n"; 96 m_os.flush(); 97 } 98 99 void 100 detail::atf_tp_writer::end_tc(void) 101 { 102 if (m_is_first) 103 m_is_first = false; 104 } 105 106 void 107 detail::atf_tp_writer::tc_meta_data(const std::string& name, 108 const std::string& value) 109 { 110 PRE(name != "ident"); 111 m_os << name << ": " << value << "\n"; 112 m_os.flush(); 113 } 114 115 // ------------------------------------------------------------------------ 116 // Free helper functions. 117 // ------------------------------------------------------------------------ 118 119 bool 120 detail::match(const std::string& regexp, const std::string& str) 121 { 122 return atf::text::match(str, regexp); 123 } 124 125 // ------------------------------------------------------------------------ 126 // The "tc" class. 127 // ------------------------------------------------------------------------ 128 129 static std::map< atf_tc_t*, impl::tc* > wraps; 130 static std::map< const atf_tc_t*, const impl::tc* > cwraps; 131 132 struct impl::tc_impl : atf::noncopyable { 133 std::string m_ident; 134 atf_tc_t m_tc; 135 bool m_has_cleanup; 136 137 tc_impl(const std::string& ident, const bool has_cleanup) : 138 m_ident(ident), 139 m_has_cleanup(has_cleanup) 140 { 141 } 142 143 static void 144 wrap_head(atf_tc_t *tc) 145 { 146 std::map< atf_tc_t*, impl::tc* >::iterator iter = wraps.find(tc); 147 INV(iter != wraps.end()); 148 (*iter).second->head(); 149 } 150 151 static void 152 wrap_body(const atf_tc_t *tc) 153 { 154 std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter = 155 cwraps.find(tc); 156 INV(iter != cwraps.end()); 157 try { 158 (*iter).second->body(); 159 } catch (const std::exception& e) { 160 (*iter).second->fail("Caught unhandled exception: " + std::string( 161 e.what())); 162 } catch (...) { 163 (*iter).second->fail("Caught unknown exception"); 164 } 165 } 166 167 static void 168 wrap_cleanup(const atf_tc_t *tc) 169 { 170 std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter = 171 cwraps.find(tc); 172 INV(iter != cwraps.end()); 173 (*iter).second->cleanup(); 174 } 175 }; 176 177 impl::tc::tc(const std::string& ident, const bool has_cleanup) : 178 pimpl(new tc_impl(ident, has_cleanup)) 179 { 180 } 181 182 impl::tc::~tc(void) 183 { 184 cwraps.erase(&pimpl->m_tc); 185 wraps.erase(&pimpl->m_tc); 186 187 atf_tc_fini(&pimpl->m_tc); 188 } 189 190 void 191 impl::tc::init(const vars_map& config) 192 { 193 atf_error_t err; 194 195 auto_array< const char * > array(new const char*[(config.size() * 2) + 1]); 196 const char **ptr = array.get(); 197 for (vars_map::const_iterator iter = config.begin(); 198 iter != config.end(); iter++) { 199 *ptr = (*iter).first.c_str(); 200 *(ptr + 1) = (*iter).second.c_str(); 201 ptr += 2; 202 } 203 *ptr = NULL; 204 205 wraps[&pimpl->m_tc] = this; 206 cwraps[&pimpl->m_tc] = this; 207 208 err = atf_tc_init(&pimpl->m_tc, pimpl->m_ident.c_str(), pimpl->wrap_head, 209 pimpl->wrap_body, pimpl->m_has_cleanup ? pimpl->wrap_cleanup : NULL, 210 array.get()); 211 if (atf_is_error(err)) 212 throw_atf_error(err); 213 } 214 215 bool 216 impl::tc::has_config_var(const std::string& var) 217 const 218 { 219 return atf_tc_has_config_var(&pimpl->m_tc, var.c_str()); 220 } 221 222 bool 223 impl::tc::has_md_var(const std::string& var) 224 const 225 { 226 return atf_tc_has_md_var(&pimpl->m_tc, var.c_str()); 227 } 228 229 const std::string 230 impl::tc::get_config_var(const std::string& var) 231 const 232 { 233 return atf_tc_get_config_var(&pimpl->m_tc, var.c_str()); 234 } 235 236 const std::string 237 impl::tc::get_config_var(const std::string& var, const std::string& defval) 238 const 239 { 240 return atf_tc_get_config_var_wd(&pimpl->m_tc, var.c_str(), defval.c_str()); 241 } 242 243 const std::string 244 impl::tc::get_md_var(const std::string& var) 245 const 246 { 247 return atf_tc_get_md_var(&pimpl->m_tc, var.c_str()); 248 } 249 250 const impl::vars_map 251 impl::tc::get_md_vars(void) 252 const 253 { 254 vars_map vars; 255 256 char **array = atf_tc_get_md_vars(&pimpl->m_tc); 257 try { 258 char **ptr; 259 for (ptr = array; *ptr != NULL; ptr += 2) 260 vars[*ptr] = *(ptr + 1); 261 } catch (...) { 262 atf_utils_free_charpp(array); 263 throw; 264 } 265 266 return vars; 267 } 268 269 void 270 impl::tc::set_md_var(const std::string& var, const std::string& val) 271 { 272 atf_error_t err = atf_tc_set_md_var(&pimpl->m_tc, var.c_str(), val.c_str()); 273 if (atf_is_error(err)) 274 throw_atf_error(err); 275 } 276 277 void 278 impl::tc::run(const std::string& resfile) 279 const 280 { 281 atf_error_t err = atf_tc_run(&pimpl->m_tc, resfile.c_str()); 282 if (atf_is_error(err)) 283 throw_atf_error(err); 284 } 285 286 void 287 impl::tc::run_cleanup(void) 288 const 289 { 290 atf_error_t err = atf_tc_cleanup(&pimpl->m_tc); 291 if (atf_is_error(err)) 292 throw_atf_error(err); 293 } 294 295 void 296 impl::tc::head(void) 297 { 298 } 299 300 void 301 impl::tc::cleanup(void) 302 const 303 { 304 } 305 306 void 307 impl::tc::require_prog(const std::string& prog) 308 const 309 { 310 atf_tc_require_prog(prog.c_str()); 311 } 312 313 void 314 impl::tc::pass(void) 315 { 316 atf_tc_pass(); 317 } 318 319 void 320 impl::tc::fail(const std::string& reason) 321 { 322 atf_tc_fail("%s", reason.c_str()); 323 } 324 325 void 326 impl::tc::fail_nonfatal(const std::string& reason) 327 { 328 atf_tc_fail_nonfatal("%s", reason.c_str()); 329 } 330 331 void 332 impl::tc::skip(const std::string& reason) 333 { 334 atf_tc_skip("%s", reason.c_str()); 335 } 336 337 void 338 impl::tc::check_errno(const char* file, const int line, const int exp_errno, 339 const char* expr_str, const bool result) 340 { 341 atf_tc_check_errno(file, line, exp_errno, expr_str, result); 342 } 343 344 void 345 impl::tc::require_errno(const char* file, const int line, const int exp_errno, 346 const char* expr_str, const bool result) 347 { 348 atf_tc_require_errno(file, line, exp_errno, expr_str, result); 349 } 350 351 void 352 impl::tc::expect_pass(void) 353 { 354 atf_tc_expect_pass(); 355 } 356 357 void 358 impl::tc::expect_fail(const std::string& reason) 359 { 360 atf_tc_expect_fail("%s", reason.c_str()); 361 } 362 363 void 364 impl::tc::expect_exit(const int exitcode, const std::string& reason) 365 { 366 atf_tc_expect_exit(exitcode, "%s", reason.c_str()); 367 } 368 369 void 370 impl::tc::expect_signal(const int signo, const std::string& reason) 371 { 372 atf_tc_expect_signal(signo, "%s", reason.c_str()); 373 } 374 375 void 376 impl::tc::expect_death(const std::string& reason) 377 { 378 atf_tc_expect_death("%s", reason.c_str()); 379 } 380 381 void 382 impl::tc::expect_timeout(const std::string& reason) 383 { 384 atf_tc_expect_timeout("%s", reason.c_str()); 385 } 386 387 // ------------------------------------------------------------------------ 388 // The "tp" class. 389 // ------------------------------------------------------------------------ 390 391 class tp : public atf::application::app { 392 public: 393 typedef std::vector< impl::tc * > tc_vector; 394 395 private: 396 static const char* m_description; 397 398 bool m_lflag; 399 atf::fs::path m_resfile; 400 std::string m_srcdir_arg; 401 atf::fs::path m_srcdir; 402 403 atf::tests::vars_map m_vars; 404 405 std::string specific_args(void) const; 406 options_set specific_options(void) const; 407 void process_option(int, const char*); 408 409 void (*m_add_tcs)(tc_vector&); 410 tc_vector m_tcs; 411 412 void parse_vflag(const std::string&); 413 void handle_srcdir(void); 414 415 tc_vector init_tcs(void); 416 417 enum tc_part { 418 BODY, 419 CLEANUP, 420 }; 421 422 void list_tcs(void); 423 impl::tc* find_tc(tc_vector, const std::string&); 424 static std::pair< std::string, tc_part > process_tcarg(const std::string&); 425 int run_tc(const std::string&); 426 427 public: 428 tp(void (*)(tc_vector&)); 429 ~tp(void); 430 431 int main(void); 432 }; 433 434 const char* tp::m_description = 435 "This is an independent atf test program."; 436 437 tp::tp(void (*add_tcs)(tc_vector&)) : 438 app(m_description, "atf-test-program(1)", "atf(7)", false), 439 m_lflag(false), 440 m_resfile("/dev/stdout"), 441 m_srcdir("."), 442 m_add_tcs(add_tcs) 443 { 444 } 445 446 tp::~tp(void) 447 { 448 for (tc_vector::iterator iter = m_tcs.begin(); 449 iter != m_tcs.end(); iter++) { 450 impl::tc* tc = *iter; 451 452 delete tc; 453 } 454 } 455 456 std::string 457 tp::specific_args(void) 458 const 459 { 460 return "test_case"; 461 } 462 463 tp::options_set 464 tp::specific_options(void) 465 const 466 { 467 using atf::application::option; 468 options_set opts; 469 opts.insert(option('l', "", "List test cases and their purpose")); 470 opts.insert(option('r', "resfile", "The file to which the test program " 471 "will write the results of the " 472 "executed test case")); 473 opts.insert(option('s', "srcdir", "Directory where the test's data " 474 "files are located")); 475 opts.insert(option('v', "var=value", "Sets the configuration variable " 476 "`var' to `value'")); 477 return opts; 478 } 479 480 void 481 tp::process_option(int ch, const char* arg) 482 { 483 switch (ch) { 484 case 'l': 485 m_lflag = true; 486 break; 487 488 case 'r': 489 m_resfile = atf::fs::path(arg); 490 break; 491 492 case 's': 493 m_srcdir_arg = arg; 494 break; 495 496 case 'v': 497 parse_vflag(arg); 498 break; 499 500 default: 501 UNREACHABLE; 502 } 503 } 504 505 void 506 tp::parse_vflag(const std::string& str) 507 { 508 if (str.empty()) 509 throw std::runtime_error("-v requires a non-empty argument"); 510 511 std::vector< std::string > ws = atf::text::split(str, "="); 512 if (ws.size() == 1 && str[str.length() - 1] == '=') { 513 m_vars[ws[0]] = ""; 514 } else { 515 if (ws.size() != 2) 516 throw std::runtime_error("-v requires an argument of the form " 517 "var=value"); 518 519 m_vars[ws[0]] = ws[1]; 520 } 521 } 522 523 void 524 tp::handle_srcdir(void) 525 { 526 if (m_srcdir_arg.empty()) { 527 m_srcdir = atf::fs::path(m_argv0).branch_path(); 528 if (m_srcdir.leaf_name() == ".libs") 529 m_srcdir = m_srcdir.branch_path(); 530 } else 531 m_srcdir = atf::fs::path(m_srcdir_arg); 532 533 if (!atf::fs::exists(m_srcdir / m_prog_name)) 534 throw std::runtime_error("Cannot find the test program in the " 535 "source directory `" + m_srcdir.str() + "'"); 536 537 if (!m_srcdir.is_absolute()) 538 m_srcdir = m_srcdir.to_absolute(); 539 540 m_vars["srcdir"] = m_srcdir.str(); 541 } 542 543 tp::tc_vector 544 tp::init_tcs(void) 545 { 546 m_add_tcs(m_tcs); 547 for (tc_vector::iterator iter = m_tcs.begin(); 548 iter != m_tcs.end(); iter++) { 549 impl::tc* tc = *iter; 550 551 tc->init(m_vars); 552 } 553 return m_tcs; 554 } 555 556 // 557 // An auxiliary unary predicate that compares the given test case's 558 // identifier to the identifier stored in it. 559 // 560 class tc_equal_to_ident { 561 const std::string& m_ident; 562 563 public: 564 tc_equal_to_ident(const std::string& i) : 565 m_ident(i) 566 { 567 } 568 569 bool operator()(const impl::tc* tc) 570 { 571 return tc->get_md_var("ident") == m_ident; 572 } 573 }; 574 575 void 576 tp::list_tcs(void) 577 { 578 tc_vector tcs = init_tcs(); 579 detail::atf_tp_writer writer(std::cout); 580 581 for (tc_vector::const_iterator iter = tcs.begin(); 582 iter != tcs.end(); iter++) { 583 const impl::vars_map vars = (*iter)->get_md_vars(); 584 585 { 586 impl::vars_map::const_iterator iter2 = vars.find("ident"); 587 INV(iter2 != vars.end()); 588 writer.start_tc((*iter2).second); 589 } 590 591 for (impl::vars_map::const_iterator iter2 = vars.begin(); 592 iter2 != vars.end(); iter2++) { 593 const std::string& key = (*iter2).first; 594 if (key != "ident") 595 writer.tc_meta_data(key, (*iter2).second); 596 } 597 598 writer.end_tc(); 599 } 600 } 601 602 impl::tc* 603 tp::find_tc(tc_vector tcs, const std::string& name) 604 { 605 std::vector< std::string > ids; 606 for (tc_vector::iterator iter = tcs.begin(); 607 iter != tcs.end(); iter++) { 608 impl::tc* tc = *iter; 609 610 if (tc->get_md_var("ident") == name) 611 return tc; 612 } 613 throw atf::application::usage_error("Unknown test case `%s'", 614 name.c_str()); 615 } 616 617 std::pair< std::string, tp::tc_part > 618 tp::process_tcarg(const std::string& tcarg) 619 { 620 const std::string::size_type pos = tcarg.find(':'); 621 if (pos == std::string::npos) { 622 return std::make_pair(tcarg, BODY); 623 } else { 624 const std::string tcname = tcarg.substr(0, pos); 625 626 const std::string partname = tcarg.substr(pos + 1); 627 if (partname == "body") 628 return std::make_pair(tcname, BODY); 629 else if (partname == "cleanup") 630 return std::make_pair(tcname, CLEANUP); 631 else { 632 using atf::application::usage_error; 633 throw usage_error("Invalid test case part `%s'", partname.c_str()); 634 } 635 } 636 } 637 638 int 639 tp::run_tc(const std::string& tcarg) 640 { 641 const std::pair< std::string, tc_part > fields = process_tcarg(tcarg); 642 643 impl::tc* tc = find_tc(init_tcs(), fields.first); 644 645 if (!atf::env::has("__RUNNING_INSIDE_ATF_RUN") || atf::env::get( 646 "__RUNNING_INSIDE_ATF_RUN") != "internal-yes-value") 647 { 648 std::cerr << m_prog_name << ": WARNING: Running test cases without " 649 "atf-run(1) is unsupported\n"; 650 std::cerr << m_prog_name << ": WARNING: No isolation nor timeout " 651 "control is being applied; you may get unexpected failures; see " 652 "atf-test-case(4)\n"; 653 } 654 655 try { 656 switch (fields.second) { 657 case BODY: 658 tc->run(m_resfile.str()); 659 break; 660 case CLEANUP: 661 tc->run_cleanup(); 662 break; 663 default: 664 UNREACHABLE; 665 } 666 return EXIT_SUCCESS; 667 } catch (const std::runtime_error& e) { 668 std::cerr << "ERROR: " << e.what() << "\n"; 669 return EXIT_FAILURE; 670 } 671 } 672 673 int 674 tp::main(void) 675 { 676 using atf::application::usage_error; 677 678 int errcode; 679 680 handle_srcdir(); 681 682 if (m_lflag) { 683 if (m_argc > 0) 684 throw usage_error("Cannot provide test case names with -l"); 685 686 list_tcs(); 687 errcode = EXIT_SUCCESS; 688 } else { 689 if (m_argc == 0) 690 throw usage_error("Must provide a test case name"); 691 else if (m_argc > 1) 692 throw usage_error("Cannot provide more than one test case name"); 693 INV(m_argc == 1); 694 695 errcode = run_tc(m_argv[0]); 696 } 697 698 return errcode; 699 } 700 701 namespace atf { 702 namespace tests { 703 int run_tp(int, char* const*, void (*)(tp::tc_vector&)); 704 } 705 } 706 707 int 708 impl::run_tp(int argc, char* const* argv, void (*add_tcs)(tp::tc_vector&)) 709 { 710 return tp(add_tcs).run(argc, argv); 711 } 712