1 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
2  *
3  * Redistribution and use in source and binary forms, with or without
4  * modification, are permitted provided that the following conditions
5  * are met:
6  * 1. Redistributions of source code must retain the above copyright
7  *    notice, this list of conditions and the following disclaimer.
8  * 2. Redistributions in binary form must reproduce the above copyright
9  *    notice, this list of conditions and the following disclaimer in the
10  *    documentation and/or other materials provided with the distribution.
11  * 3. The name of the author may not be used to endorse or promote products
12  *    derived from this software without specific prior written permission.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25 #ifdef TINYTEST_LOCAL
26 #include "tinytest_local.h"
27 #endif
28 
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <assert.h>
33 
34 #ifndef NO_FORKING
35 
36 #ifdef _WIN32
37 #include <windows.h>
38 #else
39 #include <sys/types.h>
40 #include <sys/wait.h>
41 #include <unistd.h>
42 #endif
43 
44 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
45 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
46     __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
47 /* Workaround for a stupid bug in OSX 10.6 */
48 #define FORK_BREAKS_GCOV
49 #include <vproc.h>
50 #endif
51 #endif
52 
53 #endif /* !NO_FORKING */
54 
55 #ifndef __GNUC__
56 #define __attribute__(x)
57 #endif
58 
59 #include "tinytest.h"
60 #include "tinytest_macros.h"
61 
62 #define LONGEST_TEST_NAME 16384
63 #define DEFAULT_TESTCASE_TIMEOUT 30U
64 #define MAGIC_EXITCODE 42
65 
66 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
67 static int n_ok = 0; /**< Number of tests that have passed */
68 static int n_bad = 0; /**< Number of tests that have failed. */
69 static int n_skipped = 0; /**< Number of tests that have been skipped. */
70 
71 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
72 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
73 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
74 static unsigned int opt_timeout = DEFAULT_TESTCASE_TIMEOUT; /**< Timeout for every test (using alarm()) */
75 const char *verbosity_flag = "";
76 
77 const struct testlist_alias_t *cfg_aliases=NULL;
78 
79 enum outcome { SKIP=2, OK=1, FAIL=0 };
80 static enum outcome cur_test_outcome = 0;
81 const char *cur_test_prefix = NULL; /**< prefix of the current test group */
82 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
83 const char *cur_test_name = NULL;
84 
85 static void usage(struct testgroup_t *groups, int list_groups)
86 	__attribute__((noreturn));
87 static int process_test_option(struct testgroup_t *groups, const char *test);
88 
89 #ifdef _WIN32
90 /* Copy of argv[0] for win32. */
91 static char commandname[MAX_PATH+1];
92 
93 struct timeout_thread_args {
94 	const testcase_fn *fn;
95 	void *env;
96 };
97 
98 static DWORD WINAPI
99 timeout_thread_proc_(LPVOID arg)
100 {
101 	struct timeout_thread_args *args = arg;
102 	(*(args->fn))(args->env);
103 	ExitThread(cur_test_outcome == FAIL ? 1 : 0);
104 }
105 
106 static enum outcome
107 testcase_run_in_thread_(const struct testcase_t *testcase, void *env)
108 {
109 	/* We will never run testcase in a new thread when the
110 	timeout is set to zero */
111 	assert(opt_timeout);
112 	DWORD ret, tid;
113 	HANDLE handle;
114 	struct timeout_thread_args args = {
115 		&(testcase->fn),
116 		env
117 	};
118 
119 	handle =CreateThread(NULL, 0, timeout_thread_proc_,
120 		(LPVOID)&args, 0, &tid);
121 	ret = WaitForSingleObject(handle, opt_timeout * 1000U);
122 	if (ret == WAIT_OBJECT_0) {
123 		ret = 0;
124 		if (!GetExitCodeThread(handle, &ret)) {
125 			printf("GetExitCodeThread failed\n");
126 			ret = 1;
127 		}
128 	} else if (ret == WAIT_TIMEOUT)	{
129 		printf("timeout\n");
130 	} else {
131 		printf("Wait failed\n");
132 	}
133 	CloseHandle(handle);
134 	if (ret == 0)
135 		return OK;
136 	else if (ret == MAGIC_EXITCODE)
137 		return SKIP;
138 	else
139 		return FAIL;
140 }
141 #else
142 static unsigned int testcase_set_timeout_(void)
143 {
144 	return alarm(opt_timeout);
145 }
146 
147 static unsigned int testcase_reset_timeout_(void)
148 {
149 	return alarm(0);
150 }
151 #endif
152 
153 static enum outcome
154 testcase_run_bare_(const struct testcase_t *testcase)
155 {
156 	void *env = NULL;
157 	int outcome;
158 	if (testcase->setup) {
159 		env = testcase->setup->setup_fn(testcase);
160 		if (!env)
161 			return FAIL;
162 		else if (env == (void*)TT_SKIP)
163 			return SKIP;
164 	}
165 
166 	cur_test_outcome = OK;
167 	{
168 		if (opt_timeout) {
169 #ifdef _WIN32
170 			cur_test_outcome = testcase_run_in_thread_(testcase, env);
171 #else
172 			testcase_set_timeout_();
173 			testcase->fn(env);
174 			testcase_reset_timeout_();
175 #endif
176 		} else {
177 			testcase->fn(env);
178 		}
179 	}
180 	outcome = cur_test_outcome;
181 
182 	if (testcase->setup) {
183 		if (testcase->setup->cleanup_fn(testcase, env) == 0)
184 			outcome = FAIL;
185 	}
186 
187 	return outcome;
188 }
189 
190 
191 #ifndef NO_FORKING
192 
193 static enum outcome
194 testcase_run_forked_(const struct testgroup_t *group,
195 		     const struct testcase_t *testcase)
196 {
197 #ifdef _WIN32
198 	/* Fork? On Win32?  How primitive!  We'll do what the smart kids do:
199 	   we'll invoke our own exe (whose name we recall from the command
200 	   line) with a command line that tells it to run just the test we
201 	   want, and this time without forking.
202 
203 	   (No, threads aren't an option.  The whole point of forking is to
204 	   share no state between tests.)
205 	 */
206 	int ok;
207 	char buffer[LONGEST_TEST_NAME+256];
208 	STARTUPINFOA si;
209 	PROCESS_INFORMATION info;
210 	DWORD ret;
211 
212 	if (!in_tinytest_main) {
213 		printf("\nERROR.  On Windows, testcase_run_forked_ must be"
214 		       " called from within tinytest_main.\n");
215 		abort();
216 	}
217 	if (opt_verbosity>0)
218 		printf("[forking] ");
219 
220 	snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s --timeout 0 %s%s",
221 		 commandname, verbosity_flag, group->prefix, testcase->name);
222 
223 	memset(&si, 0, sizeof(si));
224 	memset(&info, 0, sizeof(info));
225 	si.cb = sizeof(si);
226 
227 	ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
228 			   0, NULL, NULL, &si, &info);
229 	if (!ok) {
230 		printf("CreateProcess failed!\n");
231 		return FAIL;
232 	}
233 	ret = WaitForSingleObject(info.hProcess,
234 		(opt_timeout ? opt_timeout * 1000U : INFINITE));
235 
236 	if (ret == WAIT_OBJECT_0) {
237 		GetExitCodeProcess(info.hProcess, &ret);
238 	} else if (ret == WAIT_TIMEOUT) {
239 		printf("timeout\n");
240 	} else {
241 		printf("Wait failed\n");
242 	}
243 	CloseHandle(info.hProcess);
244 	CloseHandle(info.hThread);
245 	if (ret == 0)
246 		return OK;
247 	else if (ret == MAGIC_EXITCODE)
248 		return SKIP;
249 	else
250 		return FAIL;
251 #else
252 	int outcome_pipe[2];
253 	pid_t pid;
254 	(void)group;
255 
256 	if (pipe(outcome_pipe))
257 		perror("opening pipe");
258 
259 	if (opt_verbosity>0)
260 		printf("[forking] ");
261 	pid = fork();
262 #ifdef FORK_BREAKS_GCOV
263 	vproc_transaction_begin(0);
264 #endif
265 	if (!pid) {
266 		/* child. */
267 		int test_r, write_r;
268 		char b[1];
269 		close(outcome_pipe[0]);
270 		test_r = testcase_run_bare_(testcase);
271 		assert(0<=(int)test_r && (int)test_r<=2);
272 		b[0] = "NYS"[test_r];
273 		write_r = (int)write(outcome_pipe[1], b, 1);
274 		if (write_r != 1) {
275 			perror("write outcome to pipe");
276 			exit(1);
277 		}
278 		exit(0);
279 		return FAIL; /* unreachable */
280 	} else {
281 		/* parent */
282 		int status, r, exitcode;
283 		char b[1];
284 		/* Close this now, so that if the other side closes it,
285 		 * our read fails. */
286 		close(outcome_pipe[1]);
287 		r = (int)read(outcome_pipe[0], b, 1);
288 		if (r == 0) {
289 			printf("[Lost connection!] ");
290 			return FAIL;
291 		} else if (r != 1) {
292 			perror("read outcome from pipe");
293 		}
294 		waitpid(pid, &status, 0);
295 		exitcode = WEXITSTATUS(status);
296 		close(outcome_pipe[0]);
297 		if (opt_verbosity>1)
298 			printf("%s%s: exited with %i (%i)\n", group->prefix, testcase->name, exitcode, status);
299 		if (exitcode != 0)
300 		{
301 			printf("[atexit failure!] ");
302 			return FAIL;
303 		}
304 		return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
305 	}
306 #endif
307 }
308 
309 #endif /* !NO_FORKING */
310 
311 int
312 testcase_run_one(const struct testgroup_t *group,
313 		 const struct testcase_t *testcase)
314 {
315 	enum outcome outcome;
316 
317 	if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
318 		if (opt_verbosity>0)
319 			printf("%s%s: %s\n",
320 			   group->prefix, testcase->name,
321 			   (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
322 		++n_skipped;
323 		return SKIP;
324 	}
325 
326 	if (opt_verbosity>0 && !opt_forked) {
327 		printf("%s%s: ", group->prefix, testcase->name);
328 	} else {
329 		if (opt_verbosity==0) printf(".");
330 		cur_test_prefix = group->prefix;
331 		cur_test_name = testcase->name;
332 	}
333 
334 #ifndef NO_FORKING
335 	if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
336 		outcome = testcase_run_forked_(group, testcase);
337 	} else {
338 #else
339 	{
340 #endif
341 		outcome = testcase_run_bare_(testcase);
342 	}
343 
344 	if (outcome == OK) {
345 		if (opt_verbosity>0 && !opt_forked)
346 			puts(opt_verbosity==1?"OK":"");
347 	} else if (outcome == SKIP) {
348 		if (opt_verbosity>0 && !opt_forked)
349 			puts("SKIPPED");
350 	} else {
351 		if (!opt_forked)
352 			printf("\n  [%s FAILED]\n", testcase->name);
353 	}
354 
355 	if (opt_forked) {
356 		exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
357 		return 1; /* unreachable */
358 	} else {
359 		return (int)outcome;
360 	}
361 }
362 
363 int
364 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
365 {
366 	int i, j;
367 	size_t length = LONGEST_TEST_NAME;
368 	char fullname[LONGEST_TEST_NAME];
369 	int found=0;
370 	if (strstr(arg, ".."))
371 		length = strstr(arg,"..")-arg;
372 	for (i=0; groups[i].prefix; ++i) {
373 		for (j=0; groups[i].cases[j].name; ++j) {
374 			struct testcase_t *testcase = &groups[i].cases[j];
375 			snprintf(fullname, sizeof(fullname), "%s%s",
376 				 groups[i].prefix, testcase->name);
377 			if (!flag) { /* Hack! */
378 				printf("    %s", fullname);
379 				if (testcase->flags & TT_OFF_BY_DEFAULT)
380 					puts("   (Off by default)");
381 				else if (testcase->flags & TT_SKIP)
382 					puts("  (DISABLED)");
383 				else
384 					puts("");
385 			}
386 			if (!strncmp(fullname, arg, length)) {
387 				if (set)
388 					testcase->flags |= flag;
389 				else
390 					testcase->flags &= ~flag;
391 				++found;
392 			}
393 		}
394 	}
395 	return found;
396 }
397 
398 static void
399 usage(struct testgroup_t *groups, int list_groups)
400 {
401 	puts("Options are: [--verbose|--quiet|--terse] [--no-fork] [--timeout <sec>]");
402 	puts("  Specify tests by name, or using a prefix ending with '..'");
403 	puts("  To skip a test, prefix its name with a colon.");
404 	puts("  To enable a disabled test, prefix its name with a plus.");
405 	puts("  Use --list-tests for a list of tests.");
406 	if (list_groups) {
407 		puts("Known tests are:");
408 		tinytest_set_flag_(groups, "..", 1, 0);
409 	}
410 	exit(0);
411 }
412 
413 static int
414 process_test_alias(struct testgroup_t *groups, const char *test)
415 {
416 	int i, j, n, r;
417 	for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
418 		if (!strcmp(cfg_aliases[i].name, test)) {
419 			n = 0;
420 			for (j = 0; cfg_aliases[i].tests[j]; ++j) {
421 				r = process_test_option(groups, cfg_aliases[i].tests[j]);
422 				if (r<0)
423 					return -1;
424 				n += r;
425 			}
426 			return n;
427 		}
428 	}
429 	printf("No such test alias as @%s!",test);
430 	return -1;
431 }
432 
433 static int
434 process_test_option(struct testgroup_t *groups, const char *test)
435 {
436 	int flag = TT_ENABLED_;
437 	int n = 0;
438 	if (test[0] == '@') {
439 		return process_test_alias(groups, test + 1);
440 	} else if (test[0] == ':') {
441 		++test;
442 		flag = TT_SKIP;
443 	} else if (test[0] == '+') {
444 		++test;
445 		++n;
446 		if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
447 			printf("No such test as %s!\n", test);
448 			return -1;
449 		}
450 	} else {
451 		++n;
452 	}
453 	if (!tinytest_set_flag_(groups, test, 1, flag)) {
454 		printf("No such test as %s!\n", test);
455 		return -1;
456 	}
457 	return n;
458 }
459 
460 void
461 tinytest_set_aliases(const struct testlist_alias_t *aliases)
462 {
463 	cfg_aliases = aliases;
464 }
465 
466 int
467 tinytest_main(int c, const char **v, struct testgroup_t *groups)
468 {
469 	int i, j, n=0;
470 
471 #ifdef _WIN32
472 	const char *sp = strrchr(v[0], '.');
473 	const char *extension = "";
474 	if (!sp || stricmp(sp, ".exe"))
475 		extension = ".exe"; /* Add an exe so CreateProcess will work */
476 	snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
477 	commandname[MAX_PATH]='\0';
478 #endif
479 	for (i=1; i<c; ++i) {
480 		if (v[i][0] == '-') {
481 			if (!strcmp(v[i], "--RUNNING-FORKED")) {
482 				opt_forked = 1;
483 			} else if (!strcmp(v[i], "--no-fork")) {
484 				opt_nofork = 1;
485 			} else if (!strcmp(v[i], "--quiet")) {
486 				opt_verbosity = -1;
487 				verbosity_flag = "--quiet";
488 			} else if (!strcmp(v[i], "--verbose")) {
489 				opt_verbosity = 2;
490 				verbosity_flag = "--verbose";
491 			} else if (!strcmp(v[i], "--terse")) {
492 				opt_verbosity = 0;
493 				verbosity_flag = "--terse";
494 			} else if (!strcmp(v[i], "--help")) {
495 				usage(groups, 0);
496 			} else if (!strcmp(v[i], "--list-tests")) {
497 				usage(groups, 1);
498 			} else if (!strcmp(v[i], "--timeout")) {
499 				++i;
500 				if (i >= c) {
501 					fprintf(stderr, "--timeout requires argument\n");
502 					return -1;
503 				}
504 				opt_timeout = (unsigned)atoi(v[i]);
505 			} else {
506 				fprintf(stderr, "Unknown option %s. Try --help\n", v[i]);
507 				return -1;
508 			}
509 		} else {
510 			int r = process_test_option(groups, v[i]);
511 			if (r<0)
512 				return -1;
513 			n += r;
514 		}
515 	}
516 	if (!n)
517 		tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
518 
519 #ifdef _IONBF
520 	setvbuf(stdout, NULL, _IONBF, 0);
521 #endif
522 
523 	++in_tinytest_main;
524 	for (i = 0; groups[i].prefix; ++i) {
525 		struct testgroup_t *group = &groups[i];
526 		for (j = 0; group->cases[j].name; ++j) {
527 			struct testcase_t *testcase = &group->cases[j];
528 			int test_attempts = 3;
529 			int test_ret_err;
530 
531 			if (!(testcase->flags & TT_ENABLED_))
532 				continue;
533 
534 			for (;;) {
535 				test_ret_err = testcase_run_one(group, testcase);
536 
537 				if (test_ret_err == OK)
538 					break;
539 				if (!(testcase->flags & TT_RETRIABLE))
540 					break;
541 				printf("\n  [RETRYING %s (%i)]\n", testcase->name, test_attempts);
542 				if (!test_attempts--)
543 					break;
544 			}
545 
546 			switch (test_ret_err) {
547 				case OK:   ++n_ok;      break;
548 				case SKIP: ++n_skipped; break;
549 				default:   ++n_bad;     break;
550 			}
551 		}
552 	}
553 
554 	--in_tinytest_main;
555 
556 	if (opt_verbosity==0)
557 		puts("");
558 
559 	if (n_bad)
560 		printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
561 		       n_bad+n_ok,n_skipped);
562 	else if (opt_verbosity >= 1)
563 		printf("%d tests ok.  (%d skipped)\n", n_ok, n_skipped);
564 
565 	return (n_bad == 0) ? 0 : 1;
566 }
567 
568 int
569 tinytest_get_verbosity_(void)
570 {
571 	return opt_verbosity;
572 }
573 
574 void
575 tinytest_set_test_failed_(void)
576 {
577 	if (opt_verbosity <= 0 && cur_test_name) {
578 		if (opt_verbosity==0) puts("");
579 		printf("%s%s: ", cur_test_prefix, cur_test_name);
580 		cur_test_name = NULL;
581 	}
582 	cur_test_outcome = FAIL;
583 }
584 
585 void
586 tinytest_set_test_skipped_(void)
587 {
588 	if (cur_test_outcome==OK)
589 		cur_test_outcome = SKIP;
590 }
591 
592 char *
593 tinytest_format_hex_(const void *val_, unsigned long len)
594 {
595 	const unsigned char *val = val_;
596 	char *result, *cp;
597 	size_t i;
598 
599 	if (!val)
600 		return strdup("null");
601 	if (!(result = malloc(len*2+1)))
602 		return strdup("<allocation failure>");
603 	cp = result;
604 	for (i=0;i<len;++i) {
605 		*cp++ = "0123456789ABCDEF"[val[i] >> 4];
606 		*cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
607 	}
608 	*cp = 0;
609 	return result;
610 }
611