1 /* 2 * Copyright (c) 2011 Alex Hornung <alex@alexhornung.com>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in 13 * the documentation and/or other materials provided with the 14 * distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 19 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 20 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 22 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 24 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 25 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 26 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/resource.h> 31 #include <sys/time.h> 32 #include <sys/types.h> 33 #include <sys/wait.h> 34 35 #include <errno.h> 36 #include <signal.h> 37 #include <stdio.h> 38 #include <stdlib.h> 39 #include <stdint.h> 40 #include <string.h> 41 #include <unistd.h> 42 43 #include <err.h> 44 45 #include <libprop/proplib.h> 46 47 #include "parser.h" 48 #include "testcase.h" 49 #include "runlist.h" 50 #include <dfregress.h> 51 52 static int count_by_result[10]; 53 54 static void 55 print_row(char c) 56 { 57 int i; 58 59 for (i = 0; i < 80; i++) 60 printf("%c", c); 61 printf("\n"); 62 } 63 64 static void 65 print_header(void) 66 { 67 int i, j; 68 69 i = printf("Test case"); 70 for (j = 60 - i; j > 0; j--) 71 printf(" "); 72 printf("Result\n"); 73 74 print_row('-'); 75 } 76 77 static char *get_arg_string(prop_dictionary_t testcase) 78 { 79 static char buf[2048]; 80 const char **argv; 81 int i; 82 83 buf[0] = '\0'; 84 85 argv = testcase_get_args(testcase); 86 87 /* Skip the first argument, since it's just the test's name */ 88 for (i = 1; argv[i] != NULL; i++) { 89 strcat(buf, argv[i]); 90 if (argv[i+1] != NULL) 91 strcat(buf, " "); 92 } 93 94 return (i > 1) ? buf : NULL; 95 } 96 97 static void 98 print_summary(prop_array_t runlist) 99 { 100 float total_run; 101 float total_tests; 102 103 total_tests = 0.0 + prop_array_count(runlist); 104 total_run = 0.0 + total_tests - count_by_result[RESULT_BUILDFAIL] - 105 count_by_result[RESULT_PREFAIL] - count_by_result[RESULT_NOTRUN] - 106 count_by_result[RESULT_UNKNOWN]; 107 108 printf("\n\n"); 109 print_row('='); 110 printf("Summary:\n\n"); 111 112 113 printf("Tests not built:\t%d\n", count_by_result[RESULT_BUILDFAIL]); 114 115 printf("Tests not run:\t\t%.0f\n", total_tests - total_run); 116 117 printf("Tests pre-failed:\t%d\n", count_by_result[RESULT_PREFAIL]); 118 119 printf("Tests post-failed:\t%d\n", count_by_result[RESULT_POSTFAIL]); 120 121 printf("Tests passed:\t\t%d\n", count_by_result[RESULT_PASS]); 122 123 printf("Tests failed:\t\t%d\n", count_by_result[RESULT_FAIL]); 124 125 printf("Tests crashed:\t\t%d\n", count_by_result[RESULT_SIGNALLED]); 126 127 printf("Tests timed out:\t%d\n", count_by_result[RESULT_TIMEOUT]); 128 129 130 printf("------\n"); 131 132 printf("Run rate:\t\t%.2f\n", total_run/total_tests); 133 printf("Pass rate:\t\t%.2f\n", count_by_result[RESULT_PASS]/total_run); 134 } 135 136 static int 137 runlist_parse_summary(void *arg __unused, prop_dictionary_t testcase) 138 { 139 char *args; 140 int i, j; 141 142 ++count_by_result[testcase_get_result(testcase)]; 143 args = get_arg_string(testcase); 144 145 i = printf("%s", testcase_get_name(testcase)); 146 if (args != NULL) 147 i+= printf(" (%s)", args); 148 149 for (j = 60 - i; j > 0; j--) 150 printf(" "); 151 152 printf("%s\n", testcase_get_result_desc(testcase)); 153 154 return 0; 155 } 156 157 static int 158 runlist_parse_detail(void *arg __unused, prop_dictionary_t testcase) 159 { 160 char *args; 161 162 args = get_arg_string(testcase); 163 164 printf("\n"); 165 print_row('='); 166 167 printf("Test: %s\n", testcase_get_name(testcase)); 168 if (args != NULL) 169 printf("Command line arguments: %s\n", args); 170 171 printf("Type: %s\n", testcase_get_type_desc(testcase)); 172 printf("Result: %s\n", testcase_get_result_desc(testcase)); 173 174 switch (testcase_get_result(testcase)) { 175 case RESULT_FAIL: 176 printf("Exit code: %d\n", testcase_get_exit_value(testcase)); 177 break; 178 179 case RESULT_SIGNALLED: 180 printf("Signal: %d\n", testcase_get_signal(testcase)); 181 break; 182 } 183 184 print_row('-'); 185 printf("driver sysbuf:\n%s\n", testcase_get_sys_buf(testcase)); 186 print_row('-'); 187 printf("build log:\n%s\n", testcase_get_build_buf(testcase)); 188 print_row('-'); 189 printf("'pre' log:\n%s\n", testcase_get_precmd_buf(testcase)); 190 print_row('-'); 191 printf("testcase stdout:\n%s\n", testcase_get_stdout_buf(testcase)); 192 print_row('-'); 193 printf("testcase stderr:\n%s\n", testcase_get_stderr_buf(testcase)); 194 print_row('-'); 195 printf("'post' log:\n%s\n", testcase_get_postcmd_buf(testcase)); 196 print_row('-'); 197 printf("cleanup log:\n%s\n", testcase_get_cleanup_buf(testcase)); 198 199 return 0; 200 } 201 202 int 203 main(int argc, char *argv[]) 204 { 205 if (argc != 2) { 206 fprintf(stderr, "Usage: dfr2text <foo.plist>\n"); 207 exit(1); 208 } 209 210 prop_array_t runlist = runlist_load(argv[1]); 211 212 memset(count_by_result, 0, sizeof(count_by_result)); 213 print_header(); 214 runlist_iterate(runlist, runlist_parse_summary, runlist); 215 print_summary(runlist); 216 printf("\n\nDETAILED RESULTS:\n"); 217 218 runlist_iterate(runlist, runlist_parse_detail, runlist); 219 return 0; 220 } 221 222