1 // 2 // Automated Testing Framework (atf) 3 // 4 // Copyright (c) 2007 The NetBSD Foundation, Inc. 5 // All rights reserved. 6 // 7 // Redistribution and use in source and binary forms, with or without 8 // modification, are permitted provided that the following conditions 9 // are met: 10 // 1. Redistributions of source code must retain the above copyright 11 // notice, this list of conditions and the following disclaimer. 12 // 2. Redistributions in binary form must reproduce the above copyright 13 // notice, this list of conditions and the following disclaimer in the 14 // documentation and/or other materials provided with the distribution. 15 // 16 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 // 29 30 extern "C" { 31 #include <sys/time.h> 32 } 33 34 #include <cctype> 35 #include <cstdlib> 36 #include <fstream> 37 #include <iomanip> 38 #include <iostream> 39 #include <memory> 40 #include <sstream> 41 #include <utility> 42 #include <vector> 43 44 #include "atf-c/defs.h" 45 46 #include "atf-c++/detail/application.hpp" 47 #include "atf-c++/detail/fs.hpp" 48 #include "atf-c++/detail/sanity.hpp" 49 #include "atf-c++/detail/text.hpp" 50 #include "atf-c++/detail/ui.hpp" 51 52 #include "reader.hpp" 53 54 typedef std::auto_ptr< std::ostream > ostream_ptr; 55 56 static ostream_ptr 57 open_outfile(const atf::fs::path& path) 58 { 59 ostream_ptr osp; 60 if (path.str() == "-") 61 osp = ostream_ptr(new std::ofstream("/dev/stdout")); 62 else 63 osp = ostream_ptr(new std::ofstream(path.c_str())); 64 if (!(*osp)) 65 throw std::runtime_error("Could not create file " + path.str()); 66 return osp; 67 } 68 69 static std::string 70 format_tv(struct timeval* tv) 71 { 72 std::ostringstream output; 73 output << static_cast< long >(tv->tv_sec) << '.' 74 << std::setfill('0') << std::setw(6) 75 << static_cast< long >(tv->tv_usec); 76 return output.str(); 77 } 78 79 // ------------------------------------------------------------------------ 80 // The "writer" interface. 81 // ------------------------------------------------------------------------ 82 83 //! 84 //! \brief A base class that defines an output format. 85 //! 86 //! The writer base class defines a generic interface to output formats. 87 //! This is meant to be subclassed, and each subclass can redefine any 88 //! method to format the information as it wishes. 89 //! 90 //! This class is not tied to a output stream nor a file because, depending 91 //! on the output format, we will want to write to a single file or to 92 //! multiple ones. 93 //! 94 class writer { 95 public: 96 writer(void) {} 97 virtual ~writer(void) {} 98 99 virtual void write_info(const std::string&, const std::string&) {} 100 virtual void write_ntps(size_t) {} 101 virtual void write_tp_start(const std::string&, size_t) {} 102 virtual void write_tp_end(struct timeval*, const std::string&) {} 103 virtual void write_tc_start(const std::string&) {} 104 virtual void write_tc_stdout_line(const std::string&) {} 105 virtual void write_tc_stderr_line(const std::string&) {} 106 virtual void write_tc_end(const std::string&, struct timeval*, 107 const std::string&) {} 108 virtual void write_eof(void) {} 109 }; 110 111 // ------------------------------------------------------------------------ 112 // The "csv_writer" class. 113 // ------------------------------------------------------------------------ 114 115 //! 116 //! \brief A very simple plain-text output format. 117 //! 118 //! The csv_writer class implements a very simple plain-text output 119 //! format that summarizes the results of each executed test case. The 120 //! results are meant to be easily parseable by third-party tools, hence 121 //! they are formatted as a CSV file. 122 //! 123 class csv_writer : public writer { 124 ostream_ptr m_os; 125 bool m_failed; 126 127 std::string m_tpname; 128 std::string m_tcname; 129 130 public: 131 csv_writer(const atf::fs::path& p) : 132 m_os(open_outfile(p)) 133 { 134 } 135 136 virtual 137 void 138 write_tp_start(const std::string& name, 139 size_t ntcs ATF_DEFS_ATTRIBUTE_UNUSED) 140 { 141 m_tpname = name; 142 m_failed = false; 143 } 144 145 virtual 146 void 147 write_tp_end(struct timeval* tv, const std::string& reason) 148 { 149 const std::string timestamp = format_tv(tv); 150 151 if (!reason.empty()) 152 (*m_os) << "tp, " << timestamp << ", " << m_tpname << ", bogus, " 153 << reason << "\n"; 154 else if (m_failed) 155 (*m_os) << "tp, " << timestamp << ", "<< m_tpname << ", failed\n"; 156 else 157 (*m_os) << "tp, " << timestamp << ", "<< m_tpname << ", passed\n"; 158 } 159 160 virtual 161 void 162 write_tc_start(const std::string& name) 163 { 164 m_tcname = name; 165 } 166 167 virtual 168 void 169 write_tc_end(const std::string& state, struct timeval* tv, 170 const std::string& reason) 171 { 172 std::string str = m_tpname + ", " + m_tcname + ", " + state; 173 if (!reason.empty()) 174 str += ", " + reason; 175 (*m_os) << "tc, " << format_tv(tv) << ", " << str << "\n"; 176 177 if (state == "failed") 178 m_failed = true; 179 } 180 }; 181 182 // ------------------------------------------------------------------------ 183 // The "ticker_writer" class. 184 // ------------------------------------------------------------------------ 185 186 //! 187 //! \brief A console-friendly output format. 188 //! 189 //! The ticker_writer class implements a formatter that is user-friendly 190 //! in the sense that it shows the execution of test cases in an easy to 191 //! read format. It is not meant to be parseable and its format can 192 //! freely change across releases. 193 //! 194 class ticker_writer : public writer { 195 ostream_ptr m_os; 196 197 size_t m_curtp, m_ntps; 198 size_t m_tcs_passed, m_tcs_failed, m_tcs_skipped, m_tcs_expected_failures; 199 std::string m_tcname, m_tpname; 200 std::vector< std::string > m_failed_tcs; 201 std::map< std::string, std::string > m_expected_failures_tcs; 202 std::vector< std::string > m_failed_tps; 203 204 void 205 write_info(const std::string& what, const std::string& val) 206 { 207 if (what == "tests.root") { 208 (*m_os) << "Tests root: " << val << "\n\n"; 209 } 210 } 211 212 void 213 write_ntps(size_t ntps) 214 { 215 m_curtp = 1; 216 m_tcs_passed = 0; 217 m_tcs_failed = 0; 218 m_tcs_skipped = 0; 219 m_tcs_expected_failures = 0; 220 m_ntps = ntps; 221 } 222 223 void 224 write_tp_start(const std::string& tp, size_t ntcs) 225 { 226 using atf::text::to_string; 227 using atf::ui::format_text; 228 229 m_tpname = tp; 230 231 (*m_os) << format_text(tp + " (" + to_string(m_curtp) + 232 "/" + to_string(m_ntps) + "): " + 233 to_string(ntcs) + " test cases") 234 << "\n"; 235 (*m_os).flush(); 236 } 237 238 void 239 write_tp_end(struct timeval* tv, const std::string& reason) 240 { 241 using atf::ui::format_text_with_tag; 242 243 m_curtp++; 244 245 if (!reason.empty()) { 246 (*m_os) << format_text_with_tag("BOGUS TEST PROGRAM: Cannot " 247 "trust its results because " 248 "of `" + reason + "'", 249 m_tpname + ": ", false) 250 << "\n"; 251 m_failed_tps.push_back(m_tpname); 252 } 253 (*m_os) << "[" << format_tv(tv) << "s]\n\n"; 254 (*m_os).flush(); 255 256 m_tpname.clear(); 257 } 258 259 void 260 write_tc_start(const std::string& tcname) 261 { 262 m_tcname = tcname; 263 264 (*m_os) << " " + tcname + ": "; 265 (*m_os).flush(); 266 } 267 268 void 269 write_tc_end(const std::string& state, struct timeval* tv, 270 const std::string& reason) 271 { 272 std::string str; 273 274 (*m_os) << "[" << format_tv(tv) << "s] "; 275 276 if (state == "expected_death" || state == "expected_exit" || 277 state == "expected_failure" || state == "expected_signal" || 278 state == "expected_timeout") { 279 str = "Expected failure: " + reason; 280 m_tcs_expected_failures++; 281 m_expected_failures_tcs[m_tpname + ":" + m_tcname] = reason; 282 } else if (state == "failed") { 283 str = "Failed: " + reason; 284 m_tcs_failed++; 285 m_failed_tcs.push_back(m_tpname + ":" + m_tcname); 286 } else if (state == "passed") { 287 str = "Passed."; 288 m_tcs_passed++; 289 } else if (state == "skipped") { 290 str = "Skipped: " + reason; 291 m_tcs_skipped++; 292 } else 293 UNREACHABLE; 294 295 // XXX Wrap text. format_text_with_tag does not currently allow 296 // to specify the current column, which is needed because we have 297 // already printed the tc's name. 298 (*m_os) << str << '\n'; 299 300 m_tcname = ""; 301 } 302 303 static void 304 write_expected_failures(const std::map< std::string, std::string >& xfails, 305 std::ostream& os) 306 { 307 using atf::ui::format_text; 308 using atf::ui::format_text_with_tag; 309 310 os << format_text("Test cases for known bugs:") << "\n"; 311 312 for (std::map< std::string, std::string >::const_iterator iter = 313 xfails.begin(); iter != xfails.end(); iter++) { 314 const std::string& name = (*iter).first; 315 const std::string& reason = (*iter).second; 316 317 os << format_text_with_tag(reason, " " + name + ": ", false) 318 << "\n"; 319 } 320 } 321 322 void 323 write_eof(void) 324 { 325 using atf::text::join; 326 using atf::text::to_string; 327 using atf::ui::format_text; 328 using atf::ui::format_text_with_tag; 329 330 if (!m_failed_tps.empty()) { 331 (*m_os) << format_text("Failed (bogus) test programs:") 332 << "\n"; 333 (*m_os) << format_text_with_tag(join(m_failed_tps, ", "), 334 " ", false) << "\n\n"; 335 } 336 337 if (!m_expected_failures_tcs.empty()) { 338 write_expected_failures(m_expected_failures_tcs, *m_os); 339 (*m_os) << "\n"; 340 } 341 342 if (!m_failed_tcs.empty()) { 343 (*m_os) << format_text("Failed test cases:") << "\n"; 344 (*m_os) << format_text_with_tag(join(m_failed_tcs, ", "), 345 " ", false) << "\n\n"; 346 } 347 348 (*m_os) << format_text("Summary for " + to_string(m_ntps) + 349 " test programs:") << "\n"; 350 (*m_os) << format_text_with_tag(to_string(m_tcs_passed) + 351 " passed test cases.", 352 " ", false) << "\n"; 353 (*m_os) << format_text_with_tag(to_string(m_tcs_failed) + 354 " failed test cases.", 355 " ", false) << "\n"; 356 (*m_os) << format_text_with_tag(to_string(m_tcs_expected_failures) + 357 " expected failed test cases.", 358 " ", false) << "\n"; 359 (*m_os) << format_text_with_tag(to_string(m_tcs_skipped) + 360 " skipped test cases.", 361 " ", false) << "\n"; 362 } 363 364 public: 365 ticker_writer(const atf::fs::path& p) : 366 m_os(open_outfile(p)) 367 { 368 } 369 }; 370 371 // ------------------------------------------------------------------------ 372 // The "xml" class. 373 // ------------------------------------------------------------------------ 374 375 //! 376 //! \brief A single-file XML output format. 377 //! 378 //! The xml_writer class implements a formatter that prints the results 379 //! of test cases in an XML format easily parseable later on by other 380 //! utilities. 381 //! 382 class xml_writer : public writer { 383 ostream_ptr m_os; 384 385 std::string m_tcname, m_tpname; 386 387 static 388 std::string 389 attrval(const std::string& str) 390 { 391 return str; 392 } 393 394 static 395 std::string 396 elemval(const std::string& str) 397 { 398 std::ostringstream buf; 399 for (std::string::const_iterator iter = str.begin(); 400 iter != str.end(); iter++) { 401 const int character = static_cast< unsigned char >(*iter); 402 if (character == '&') { 403 buf << "&"; 404 } else if (character == '<') { 405 buf << "<"; 406 } else if (character == '>') { 407 buf << ">"; 408 } else if (std::isalnum(character) || std::ispunct(character) || 409 std::isspace(character)) { 410 buf << static_cast< char >(character); 411 } else { 412 buf << "&#" << character << ";"; 413 } 414 } 415 return buf.str(); 416 } 417 418 void 419 write_info(const std::string& what, const std::string& val) 420 { 421 (*m_os) << "<info class=\"" << what << "\">" << val << "</info>\n"; 422 } 423 424 void 425 write_tp_start(const std::string& tp, 426 size_t ntcs ATF_DEFS_ATTRIBUTE_UNUSED) 427 { 428 (*m_os) << "<tp id=\"" << attrval(tp) << "\">\n"; 429 } 430 431 void 432 write_tp_end(struct timeval* tv, const std::string& reason) 433 { 434 if (!reason.empty()) 435 (*m_os) << "<failed>" << elemval(reason) << "</failed>\n"; 436 (*m_os) << "<tp-time>" << format_tv(tv) << "</tp-time>"; 437 (*m_os) << "</tp>\n"; 438 } 439 440 void 441 write_tc_start(const std::string& tcname) 442 { 443 (*m_os) << "<tc id=\"" << attrval(tcname) << "\">\n"; 444 } 445 446 void 447 write_tc_stdout_line(const std::string& line) 448 { 449 (*m_os) << "<so>" << elemval(line) << "</so>\n"; 450 } 451 452 void 453 write_tc_stderr_line(const std::string& line) 454 { 455 (*m_os) << "<se>" << elemval(line) << "</se>\n"; 456 } 457 458 void 459 write_tc_end(const std::string& state, struct timeval* tv, 460 const std::string& reason) 461 { 462 std::string str; 463 464 if (state == "expected_death" || state == "expected_exit" || 465 state == "expected_failure" || state == "expected_signal" || 466 state == "expected_timeout") { 467 (*m_os) << "<" << state << ">" << elemval(reason) 468 << "</" << state << ">\n"; 469 } else if (state == "passed") { 470 (*m_os) << "<passed />\n"; 471 } else if (state == "failed") { 472 (*m_os) << "<failed>" << elemval(reason) << "</failed>\n"; 473 } else if (state == "skipped") { 474 (*m_os) << "<skipped>" << elemval(reason) << "</skipped>\n"; 475 } else 476 UNREACHABLE; 477 (*m_os) << "<tc-time>" << format_tv(tv) << "</tc-time>"; 478 (*m_os) << "</tc>\n"; 479 } 480 481 void 482 write_eof(void) 483 { 484 (*m_os) << "</tests-results>\n"; 485 } 486 487 public: 488 xml_writer(const atf::fs::path& p) : 489 m_os(open_outfile(p)) 490 { 491 (*m_os) << "<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>\n" 492 << "<!DOCTYPE tests-results PUBLIC " 493 "\"-//NetBSD//DTD ATF Tests Results 0.1//EN\" " 494 "\"http://www.NetBSD.org/XML/atf/tests-results.dtd\">\n\n" 495 "<tests-results>\n"; 496 } 497 }; 498 499 // ------------------------------------------------------------------------ 500 // The "converter" class. 501 // ------------------------------------------------------------------------ 502 503 //! 504 //! \brief A reader that redirects events to multiple writers. 505 //! 506 //! The converter class implements an atf_tps_reader that, for each event 507 //! raised by the parser, redirects it to multiple writers so that they 508 //! can reformat it according to their output rules. 509 //! 510 class converter : public atf::atf_report::atf_tps_reader { 511 typedef std::vector< writer* > outs_vector; 512 outs_vector m_outs; 513 514 void 515 got_info(const std::string& what, const std::string& val) 516 { 517 for (outs_vector::iterator iter = m_outs.begin(); 518 iter != m_outs.end(); iter++) 519 (*iter)->write_info(what, val); 520 } 521 522 void 523 got_ntps(size_t ntps) 524 { 525 for (outs_vector::iterator iter = m_outs.begin(); 526 iter != m_outs.end(); iter++) 527 (*iter)->write_ntps(ntps); 528 } 529 530 void 531 got_tp_start(const std::string& tp, size_t ntcs) 532 { 533 for (outs_vector::iterator iter = m_outs.begin(); 534 iter != m_outs.end(); iter++) 535 (*iter)->write_tp_start(tp, ntcs); 536 } 537 538 void 539 got_tp_end(struct timeval* tv, const std::string& reason) 540 { 541 for (outs_vector::iterator iter = m_outs.begin(); 542 iter != m_outs.end(); iter++) 543 (*iter)->write_tp_end(tv, reason); 544 } 545 546 void 547 got_tc_start(const std::string& tcname) 548 { 549 for (outs_vector::iterator iter = m_outs.begin(); 550 iter != m_outs.end(); iter++) 551 (*iter)->write_tc_start(tcname); 552 } 553 554 void 555 got_tc_stdout_line(const std::string& line) 556 { 557 for (outs_vector::iterator iter = m_outs.begin(); 558 iter != m_outs.end(); iter++) 559 (*iter)->write_tc_stdout_line(line); 560 } 561 562 void 563 got_tc_stderr_line(const std::string& line) 564 { 565 for (outs_vector::iterator iter = m_outs.begin(); 566 iter != m_outs.end(); iter++) 567 (*iter)->write_tc_stderr_line(line); 568 } 569 570 void 571 got_tc_end(const std::string& state, struct timeval* tv, 572 const std::string& reason) 573 { 574 for (outs_vector::iterator iter = m_outs.begin(); 575 iter != m_outs.end(); iter++) 576 (*iter)->write_tc_end(state, tv, reason); 577 } 578 579 void 580 got_eof(void) 581 { 582 for (outs_vector::iterator iter = m_outs.begin(); 583 iter != m_outs.end(); iter++) 584 (*iter)->write_eof(); 585 } 586 587 public: 588 converter(std::istream& is) : 589 atf::atf_report::atf_tps_reader(is) 590 { 591 } 592 593 ~converter(void) 594 { 595 for (outs_vector::iterator iter = m_outs.begin(); 596 iter != m_outs.end(); iter++) 597 delete *iter; 598 } 599 600 void 601 add_output(const std::string& fmt, const atf::fs::path& p) 602 { 603 if (fmt == "csv") { 604 m_outs.push_back(new csv_writer(p)); 605 } else if (fmt == "ticker") { 606 m_outs.push_back(new ticker_writer(p)); 607 } else if (fmt == "xml") { 608 m_outs.push_back(new xml_writer(p)); 609 } else 610 throw std::runtime_error("Unknown format `" + fmt + "'"); 611 } 612 }; 613 614 // ------------------------------------------------------------------------ 615 // The "atf_report" class. 616 // ------------------------------------------------------------------------ 617 618 class atf_report : public atf::application::app { 619 static const char* m_description; 620 621 typedef std::pair< std::string, atf::fs::path > fmt_path_pair; 622 std::vector< fmt_path_pair > m_oflags; 623 624 void process_option(int, const char*); 625 options_set specific_options(void) const; 626 627 public: 628 atf_report(void); 629 630 int main(void); 631 }; 632 633 const char* atf_report::m_description = 634 "atf-report is a tool that parses the output of atf-run and " 635 "generates user-friendly reports in multiple different formats."; 636 637 atf_report::atf_report(void) : 638 app(m_description, "atf-report(1)", "atf(7)") 639 { 640 } 641 642 void 643 atf_report::process_option(int ch, const char* arg) 644 { 645 switch (ch) { 646 case 'o': 647 { 648 std::string str(arg); 649 std::string::size_type pos = str.find(':'); 650 if (pos == std::string::npos) 651 throw std::runtime_error("Syntax error in -o option"); 652 else { 653 std::string fmt = str.substr(0, pos); 654 atf::fs::path path = atf::fs::path(str.substr(pos + 1)); 655 m_oflags.push_back(fmt_path_pair(fmt, path)); 656 } 657 } 658 break; 659 660 default: 661 UNREACHABLE; 662 } 663 } 664 665 atf_report::options_set 666 atf_report::specific_options(void) 667 const 668 { 669 using atf::application::option; 670 options_set opts; 671 opts.insert(option('o', "fmt:path", "Adds a new output file; multiple " 672 "ones can be specified, and a - " 673 "path means stdout")); 674 return opts; 675 } 676 677 int 678 atf_report::main(void) 679 { 680 if (m_argc > 0) 681 throw std::runtime_error("No arguments allowed"); 682 683 if (m_oflags.empty()) 684 m_oflags.push_back(fmt_path_pair("ticker", atf::fs::path("-"))); 685 686 // Look for path duplicates. 687 std::set< atf::fs::path > paths; 688 for (std::vector< fmt_path_pair >::const_iterator iter = m_oflags.begin(); 689 iter != m_oflags.end(); iter++) { 690 atf::fs::path p = (*iter).second; 691 if (p == atf::fs::path("/dev/stdout")) 692 p = atf::fs::path("-"); 693 if (paths.find(p) != paths.end()) 694 throw std::runtime_error("The file `" + p.str() + "' was " 695 "specified more than once"); 696 paths.insert((*iter).second); 697 } 698 699 // Generate the output files. 700 converter cnv(std::cin); 701 for (std::vector< fmt_path_pair >::const_iterator iter = m_oflags.begin(); 702 iter != m_oflags.end(); iter++) 703 cnv.add_output((*iter).first, (*iter).second); 704 cnv.read(); 705 706 return EXIT_SUCCESS; 707 } 708 709 int 710 main(int argc, char* const* argv) 711 { 712 return atf_report().run(argc, argv); 713 } 714