1 /* $NetBSD: tinytest.c,v 1.1.1.1 2013/04/11 16:43:32 christos Exp $ */ 2 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 3. The name of the author may not be used to endorse or promote products 13 * derived from this software without specific prior written permission. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <stdio.h> 28 #include <stdlib.h> 29 #include <string.h> 30 #include <assert.h> 31 32 #ifdef TINYTEST_LOCAL 33 #include "tinytest_local.h" 34 #endif 35 36 #ifdef WIN32 37 #include <windows.h> 38 #else 39 #include <sys/types.h> 40 #include <sys/wait.h> 41 #include <unistd.h> 42 #endif 43 44 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) 45 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \ 46 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070) 47 /* Workaround for a stupid bug in OSX 10.6 */ 48 #define FORK_BREAKS_GCOV 49 #include <vproc.h> 50 #endif 51 #endif 52 53 #ifndef __GNUC__ 54 #define __attribute__(x) 55 #endif 56 57 #include "tinytest.h" 58 #include "tinytest_macros.h" 59 60 #define LONGEST_TEST_NAME 16384 61 62 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/ 63 static int n_ok = 0; /**< Number of tests that have passed */ 64 static int n_bad = 0; /**< Number of tests that have failed. */ 65 static int n_skipped = 0; /**< Number of tests that have been skipped. */ 66 67 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ 68 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ 69 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ 70 const char *verbosity_flag = ""; 71 72 enum outcome { SKIP=2, OK=1, FAIL=0 }; 73 static enum outcome cur_test_outcome = 0; 74 const char *cur_test_prefix = NULL; /**< prefix of the current test group */ 75 /** Name of the current test, if we haven't logged is yet. Used for --quiet */ 76 const char *cur_test_name = NULL; 77 78 #ifdef WIN32 79 /* Copy of argv[0] for win32. */ 80 static char commandname[MAX_PATH+1]; 81 #endif 82 83 static void usage(struct testgroup_t *groups, int list_groups) 84 __attribute__((noreturn)); 85 86 static enum outcome 87 _testcase_run_bare(const struct testcase_t *testcase) 88 { 89 void *env = NULL; 90 int outcome; 91 if (testcase->setup) { 92 env = testcase->setup->setup_fn(testcase); 93 if (!env) 94 return FAIL; 95 else if (env == (void*)TT_SKIP) 96 return SKIP; 97 } 98 99 cur_test_outcome = OK; 100 testcase->fn(env); 101 outcome = cur_test_outcome; 102 103 if (testcase->setup) { 104 if (testcase->setup->cleanup_fn(testcase, env) == 0) 105 outcome = FAIL; 106 } 107 108 return outcome; 109 } 110 111 #define MAGIC_EXITCODE 42 112 113 static enum outcome 114 _testcase_run_forked(const struct testgroup_t *group, 115 const struct testcase_t *testcase) 116 { 117 #ifdef WIN32 118 /* Fork? On Win32? How primitive! We'll do what the smart kids do: 119 we'll invoke our own exe (whose name we recall from the command 120 line) with a command line that tells it to run just the test we 121 want, and this time without forking. 122 123 (No, threads aren't an option. The whole point of forking is to 124 share no state between tests.) 125 */ 126 int ok; 127 char buffer[LONGEST_TEST_NAME+256]; 128 STARTUPINFOA si; 129 PROCESS_INFORMATION info; 130 DWORD exitcode; 131 132 if (!in_tinytest_main) { 133 printf("\nERROR. On Windows, _testcase_run_forked must be" 134 " called from within tinytest_main.\n"); 135 abort(); 136 } 137 if (opt_verbosity>0) 138 printf("[forking] "); 139 140 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s", 141 commandname, verbosity_flag, group->prefix, testcase->name); 142 143 memset(&si, 0, sizeof(si)); 144 memset(&info, 0, sizeof(info)); 145 si.cb = sizeof(si); 146 147 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0, 148 0, NULL, NULL, &si, &info); 149 if (!ok) { 150 printf("CreateProcess failed!\n"); 151 return 0; 152 } 153 WaitForSingleObject(info.hProcess, INFINITE); 154 GetExitCodeProcess(info.hProcess, &exitcode); 155 CloseHandle(info.hProcess); 156 CloseHandle(info.hThread); 157 if (exitcode == 0) 158 return OK; 159 else if (exitcode == MAGIC_EXITCODE) 160 return SKIP; 161 else 162 return FAIL; 163 #else 164 int outcome_pipe[2]; 165 pid_t pid; 166 (void)group; 167 168 if (pipe(outcome_pipe)) 169 perror("opening pipe"); 170 171 if (opt_verbosity>0) 172 printf("[forking] "); 173 pid = fork(); 174 #ifdef FORK_BREAKS_GCOV 175 vproc_transaction_begin(0); 176 #endif 177 if (!pid) { 178 /* child. */ 179 int test_r, write_r; 180 char b[1]; 181 close(outcome_pipe[0]); 182 test_r = _testcase_run_bare(testcase); 183 assert(0<=(int)test_r && (int)test_r<=2); 184 b[0] = "NYS"[test_r]; 185 write_r = (int)write(outcome_pipe[1], b, 1); 186 if (write_r != 1) { 187 perror("write outcome to pipe"); 188 exit(1); 189 } 190 exit(0); 191 return FAIL; /* unreachable */ 192 } else { 193 /* parent */ 194 int status, r; 195 char b[1]; 196 /* Close this now, so that if the other side closes it, 197 * our read fails. */ 198 close(outcome_pipe[1]); 199 r = (int)read(outcome_pipe[0], b, 1); 200 if (r == 0) { 201 printf("[Lost connection!] "); 202 return 0; 203 } else if (r != 1) { 204 perror("read outcome from pipe"); 205 } 206 waitpid(pid, &status, 0); 207 close(outcome_pipe[0]); 208 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL); 209 } 210 #endif 211 } 212 213 int 214 testcase_run_one(const struct testgroup_t *group, 215 const struct testcase_t *testcase) 216 { 217 enum outcome outcome; 218 219 if (testcase->flags & TT_SKIP) { 220 if (opt_verbosity>0) 221 printf("%s%s: SKIPPED\n", 222 group->prefix, testcase->name); 223 ++n_skipped; 224 return SKIP; 225 } 226 227 if (opt_verbosity>0 && !opt_forked) { 228 printf("%s%s: ", group->prefix, testcase->name); 229 } else { 230 if (opt_verbosity==0) printf("."); 231 cur_test_prefix = group->prefix; 232 cur_test_name = testcase->name; 233 } 234 235 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) { 236 outcome = _testcase_run_forked(group, testcase); 237 } else { 238 outcome = _testcase_run_bare(testcase); 239 } 240 241 if (outcome == OK) { 242 ++n_ok; 243 if (opt_verbosity>0 && !opt_forked) 244 puts(opt_verbosity==1?"OK":""); 245 } else if (outcome == SKIP) { 246 ++n_skipped; 247 if (opt_verbosity>0 && !opt_forked) 248 puts("SKIPPED"); 249 } else { 250 ++n_bad; 251 if (!opt_forked) 252 printf("\n [%s FAILED]\n", testcase->name); 253 } 254 255 if (opt_forked) { 256 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1)); 257 return 1; /* unreachable */ 258 } else { 259 return (int)outcome; 260 } 261 } 262 263 int 264 _tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag) 265 { 266 int i, j; 267 size_t length = LONGEST_TEST_NAME; 268 char fullname[LONGEST_TEST_NAME]; 269 int found=0; 270 if (strstr(arg, "..")) 271 length = strstr(arg,"..")-arg; 272 for (i=0; groups[i].prefix; ++i) { 273 for (j=0; groups[i].cases[j].name; ++j) { 274 snprintf(fullname, sizeof(fullname), "%s%s", 275 groups[i].prefix, groups[i].cases[j].name); 276 if (!flag) /* Hack! */ 277 printf(" %s\n", fullname); 278 if (!strncmp(fullname, arg, length)) { 279 groups[i].cases[j].flags |= flag; 280 ++found; 281 } 282 } 283 } 284 return found; 285 } 286 287 static void 288 usage(struct testgroup_t *groups, int list_groups) 289 { 290 puts("Options are: [--verbose|--quiet|--terse] [--no-fork]"); 291 puts(" Specify tests by name, or using a prefix ending with '..'"); 292 puts(" To skip a test, list give its name prefixed with a colon."); 293 puts(" Use --list-tests for a list of tests."); 294 if (list_groups) { 295 puts("Known tests are:"); 296 _tinytest_set_flag(groups, "..", 0); 297 } 298 exit(0); 299 } 300 301 int 302 tinytest_main(int c, const char **v, struct testgroup_t *groups) 303 { 304 int i, j, n=0; 305 306 #ifdef WIN32 307 const char *sp = strrchr(v[0], '.'); 308 const char *extension = ""; 309 if (!sp || stricmp(sp, ".exe")) 310 extension = ".exe"; /* Add an exe so CreateProcess will work */ 311 snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension); 312 commandname[MAX_PATH]='\0'; 313 #endif 314 for (i=1; i<c; ++i) { 315 if (v[i][0] == '-') { 316 if (!strcmp(v[i], "--RUNNING-FORKED")) { 317 opt_forked = 1; 318 } else if (!strcmp(v[i], "--no-fork")) { 319 opt_nofork = 1; 320 } else if (!strcmp(v[i], "--quiet")) { 321 opt_verbosity = -1; 322 verbosity_flag = "--quiet"; 323 } else if (!strcmp(v[i], "--verbose")) { 324 opt_verbosity = 2; 325 verbosity_flag = "--verbose"; 326 } else if (!strcmp(v[i], "--terse")) { 327 opt_verbosity = 0; 328 verbosity_flag = "--terse"; 329 } else if (!strcmp(v[i], "--help")) { 330 usage(groups, 0); 331 } else if (!strcmp(v[i], "--list-tests")) { 332 usage(groups, 1); 333 } else { 334 printf("Unknown option %s. Try --help\n",v[i]); 335 return -1; 336 } 337 } else { 338 const char *test = v[i]; 339 int flag = _TT_ENABLED; 340 if (test[0] == ':') { 341 ++test; 342 flag = TT_SKIP; 343 } else { 344 ++n; 345 } 346 if (!_tinytest_set_flag(groups, test, flag)) { 347 printf("No such test as %s!\n", v[i]); 348 return -1; 349 } 350 } 351 } 352 if (!n) 353 _tinytest_set_flag(groups, "..", _TT_ENABLED); 354 355 setvbuf(stdout, NULL, _IONBF, 0); 356 357 ++in_tinytest_main; 358 for (i=0; groups[i].prefix; ++i) 359 for (j=0; groups[i].cases[j].name; ++j) 360 if (groups[i].cases[j].flags & _TT_ENABLED) 361 testcase_run_one(&groups[i], 362 &groups[i].cases[j]); 363 364 --in_tinytest_main; 365 366 if (opt_verbosity==0) 367 puts(""); 368 369 if (n_bad) 370 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad, 371 n_bad+n_ok,n_skipped); 372 else if (opt_verbosity >= 1) 373 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped); 374 375 return (n_bad == 0) ? 0 : 1; 376 } 377 378 int 379 _tinytest_get_verbosity(void) 380 { 381 return opt_verbosity; 382 } 383 384 void 385 _tinytest_set_test_failed(void) 386 { 387 if (opt_verbosity <= 0 && cur_test_name) { 388 if (opt_verbosity==0) puts(""); 389 printf("%s%s: ", cur_test_prefix, cur_test_name); 390 cur_test_name = NULL; 391 } 392 cur_test_outcome = 0; 393 } 394 395 void 396 _tinytest_set_test_skipped(void) 397 { 398 if (cur_test_outcome==OK) 399 cur_test_outcome = SKIP; 400 } 401 402