1 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
2  *
3  * Redistribution and use in source and binary forms, with or without
4  * modification, are permitted provided that the following conditions
5  * are met:
6  * 1. Redistributions of source code must retain the above copyright
7  *    notice, this list of conditions and the following disclaimer.
8  * 2. Redistributions in binary form must reproduce the above copyright
9  *    notice, this list of conditions and the following disclaimer in the
10  *    documentation and/or other materials provided with the distribution.
11  * 3. The name of the author may not be used to endorse or promote products
12  *    derived from this software without specific prior written permission.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25 #ifdef TINYTEST_LOCAL
26 #include "tinytest_local.h"
27 #endif
28 
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <assert.h>
33 
34 #ifndef NO_FORKING
35 
36 #ifdef _WIN32
37 #include <windows.h>
38 #else
39 #include <sys/types.h>
40 #include <sys/wait.h>
41 #include <unistd.h>
42 #endif
43 
44 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
45 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
46     __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
47 /* Workaround for a stupid bug in OSX 10.6 */
48 #define FORK_BREAKS_GCOV
49 #include <vproc.h>
50 #endif
51 #endif
52 
53 #endif /* !NO_FORKING */
54 
55 #ifndef __GNUC__
56 #define __attribute__(x)
57 #endif
58 
59 #include "tinytest.h"
60 #include "tinytest_macros.h"
61 
62 #define LONGEST_TEST_NAME 16384
63 
64 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
65 static int n_ok = 0; /**< Number of tests that have passed */
66 static int n_bad = 0; /**< Number of tests that have failed. */
67 static int n_skipped = 0; /**< Number of tests that have been skipped. */
68 
69 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
70 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
71 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
72 const char *verbosity_flag = "";
73 
74 const struct testlist_alias_t *cfg_aliases=NULL;
75 
76 enum outcome { SKIP=2, OK=1, FAIL=0 };
77 static enum outcome cur_test_outcome = 0;
78 const char *cur_test_prefix = NULL; /**< prefix of the current test group */
79 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
80 const char *cur_test_name = NULL;
81 
82 #ifdef _WIN32
83 /* Copy of argv[0] for win32. */
84 static char commandname[MAX_PATH+1];
85 #endif
86 
87 static void usage(struct testgroup_t *groups, int list_groups)
88   __attribute__((noreturn));
89 static int process_test_option(struct testgroup_t *groups, const char *test);
90 
91 static enum outcome
testcase_run_bare_(const struct testcase_t * testcase)92 testcase_run_bare_(const struct testcase_t *testcase)
93 {
94 	void *env = NULL;
95 	int outcome;
96 	if (testcase->setup) {
97 		env = testcase->setup->setup_fn(testcase);
98 		if (!env)
99 			return FAIL;
100 		else if (env == (void*)TT_SKIP)
101 			return SKIP;
102 	}
103 
104 	cur_test_outcome = OK;
105 	testcase->fn(env);
106 	outcome = cur_test_outcome;
107 
108 	if (testcase->setup) {
109 		if (testcase->setup->cleanup_fn(testcase, env) == 0)
110 			outcome = FAIL;
111 	}
112 
113 	return outcome;
114 }
115 
116 #define MAGIC_EXITCODE 42
117 
118 #ifndef NO_FORKING
119 
120 static enum outcome
testcase_run_forked_(const struct testgroup_t * group,const struct testcase_t * testcase)121 testcase_run_forked_(const struct testgroup_t *group,
122 		     const struct testcase_t *testcase)
123 {
124 #ifdef _WIN32
125 	/* Fork? On Win32?  How primitive!  We'll do what the smart kids do:
126 	   we'll invoke our own exe (whose name we recall from the command
127 	   line) with a command line that tells it to run just the test we
128 	   want, and this time without forking.
129 
130 	   (No, threads aren't an option.  The whole point of forking is to
131 	   share no state between tests.)
132 	 */
133 	int ok;
134 	char buffer[LONGEST_TEST_NAME+256];
135 	STARTUPINFOA si;
136 	PROCESS_INFORMATION info;
137 	DWORD exitcode;
138 
139 	if (!in_tinytest_main) {
140 		printf("\nERROR.  On Windows, testcase_run_forked_ must be"
141 		       " called from within tinytest_main.\n");
142 		abort();
143 	}
144 	if (opt_verbosity>0)
145 		printf("[forking] ");
146 
147 	snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
148 		 commandname, verbosity_flag, group->prefix, testcase->name);
149 
150 	memset(&si, 0, sizeof(si));
151 	memset(&info, 0, sizeof(info));
152 	si.cb = sizeof(si);
153 
154 	ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
155 			   0, NULL, NULL, &si, &info);
156 	if (!ok) {
157 		printf("CreateProcess failed!\n");
158 		return 0;
159 	}
160 	WaitForSingleObject(info.hProcess, INFINITE);
161 	GetExitCodeProcess(info.hProcess, &exitcode);
162 	CloseHandle(info.hProcess);
163 	CloseHandle(info.hThread);
164 	if (exitcode == 0)
165 		return OK;
166 	else if (exitcode == MAGIC_EXITCODE)
167 		return SKIP;
168 	else
169 		return FAIL;
170 #else
171 	int outcome_pipe[2];
172 	pid_t pid;
173 	(void)group;
174 
175 	if (pipe(outcome_pipe))
176 		perror("opening pipe");
177 
178 	if (opt_verbosity>0)
179 		printf("[forking] ");
180 	pid = fork();
181 #ifdef FORK_BREAKS_GCOV
182 	vproc_transaction_begin(0);
183 #endif
184 	if (!pid) {
185 		/* child. */
186 		int test_r, write_r;
187 		char b[1];
188 		close(outcome_pipe[0]);
189 		test_r = testcase_run_bare_(testcase);
190 		assert(0<=(int)test_r && (int)test_r<=2);
191 		b[0] = "NYS"[test_r];
192 		write_r = (int)write(outcome_pipe[1], b, 1);
193 		if (write_r != 1) {
194 			perror("write outcome to pipe");
195 			exit(1);
196 		}
197 		exit(0);
198 		return FAIL; /* unreachable */
199 	} else {
200 		/* parent */
201 		int status, r;
202 		char b[1];
203 		/* Close this now, so that if the other side closes it,
204 		 * our read fails. */
205 		close(outcome_pipe[1]);
206 		r = (int)read(outcome_pipe[0], b, 1);
207 		if (r == 0) {
208 			printf("[Lost connection!] ");
209 			return 0;
210 		} else if (r != 1) {
211 			perror("read outcome from pipe");
212 		}
213 		waitpid(pid, &status, 0);
214 		close(outcome_pipe[0]);
215 		return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
216 	}
217 #endif
218 }
219 
220 #endif /* !NO_FORKING */
221 
222 int
testcase_run_one(const struct testgroup_t * group,const struct testcase_t * testcase)223 testcase_run_one(const struct testgroup_t *group,
224 		 const struct testcase_t *testcase)
225 {
226 	enum outcome outcome;
227 
228 	if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
229 		if (opt_verbosity>0)
230 			printf("%s%s: %s\n",
231 			   group->prefix, testcase->name,
232 			   (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
233 		++n_skipped;
234 		return SKIP;
235 	}
236 
237 	if (opt_verbosity>0 && !opt_forked) {
238 		printf("%s%s: ", group->prefix, testcase->name);
239 	} else {
240 		if (opt_verbosity==0) printf(".");
241 		cur_test_prefix = group->prefix;
242 		cur_test_name = testcase->name;
243 	}
244 
245 #ifndef NO_FORKING
246 	if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
247 		outcome = testcase_run_forked_(group, testcase);
248 	} else {
249 #else
250 	{
251 #endif
252 		outcome = testcase_run_bare_(testcase);
253 	}
254 
255 	if (outcome == OK) {
256 		++n_ok;
257 		if (opt_verbosity>0 && !opt_forked)
258 			puts(opt_verbosity==1?"OK":"");
259 	} else if (outcome == SKIP) {
260 		++n_skipped;
261 		if (opt_verbosity>0 && !opt_forked)
262 			puts("SKIPPED");
263 	} else {
264 		++n_bad;
265 		if (!opt_forked)
266 			printf("\n  [%s FAILED]\n", testcase->name);
267 	}
268 
269 	if (opt_forked) {
270 		exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
271 		return 1; /* unreachable */
272 	} else {
273 		return (int)outcome;
274 	}
275 }
276 
277 int
278 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
279 {
280 	int i, j;
281 	size_t length = LONGEST_TEST_NAME;
282 	char fullname[LONGEST_TEST_NAME];
283 	int found=0;
284 	if (strstr(arg, ".."))
285 		length = strstr(arg,"..")-arg;
286 	for (i=0; groups[i].prefix; ++i) {
287 		for (j=0; groups[i].cases[j].name; ++j) {
288 			struct testcase_t *testcase = &groups[i].cases[j];
289 			snprintf(fullname, sizeof(fullname), "%s%s",
290 				 groups[i].prefix, testcase->name);
291 			if (!flag) { /* Hack! */
292 				printf("    %s", fullname);
293 				if (testcase->flags & TT_OFF_BY_DEFAULT)
294 					puts("   (Off by default)");
295 				else if (testcase->flags & TT_SKIP)
296 					puts("  (DISABLED)");
297 				else
298 					puts("");
299 			}
300 			if (!strncmp(fullname, arg, length)) {
301 				if (set)
302 					testcase->flags |= flag;
303 				else
304 					testcase->flags &= ~flag;
305 				++found;
306 			}
307 		}
308 	}
309 	return found;
310 }
311 
312 static void
313 usage(struct testgroup_t *groups, int list_groups)
314 {
315 	puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
316 	puts("  Specify tests by name, or using a prefix ending with '..'");
317 	puts("  To skip a test, prefix its name with a colon.");
318 	puts("  To enable a disabled test, prefix its name with a plus.");
319 	puts("  Use --list-tests for a list of tests.");
320 	if (list_groups) {
321 		puts("Known tests are:");
322 		tinytest_set_flag_(groups, "..", 1, 0);
323 	}
324 	exit(0);
325 }
326 
327 static int
328 process_test_alias(struct testgroup_t *groups, const char *test)
329 {
330 	int i, j, n, r;
331 	for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
332 		if (!strcmp(cfg_aliases[i].name, test)) {
333 			n = 0;
334 			for (j = 0; cfg_aliases[i].tests[j]; ++j) {
335 				r = process_test_option(groups, cfg_aliases[i].tests[j]);
336 				if (r<0)
337 					return -1;
338 				n += r;
339 			}
340 			return n;
341 		}
342 	}
343 	printf("No such test alias as @%s!",test);
344 	return -1;
345 }
346 
347 static int
348 process_test_option(struct testgroup_t *groups, const char *test)
349 {
350 	int flag = TT_ENABLED_;
351 	int n = 0;
352 	if (test[0] == '@') {
353 		return process_test_alias(groups, test + 1);
354 	} else if (test[0] == ':') {
355 		++test;
356 		flag = TT_SKIP;
357 	} else if (test[0] == '+') {
358 		++test;
359 		++n;
360 		if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
361 			printf("No such test as %s!\n", test);
362 			return -1;
363 		}
364 	} else {
365 		++n;
366 	}
367 	if (!tinytest_set_flag_(groups, test, 1, flag)) {
368 		printf("No such test as %s!\n", test);
369 		return -1;
370 	}
371 	return n;
372 }
373 
374 void
375 tinytest_set_aliases(const struct testlist_alias_t *aliases)
376 {
377 	cfg_aliases = aliases;
378 }
379 
380 int
381 tinytest_main(int c, const char **v, struct testgroup_t *groups)
382 {
383 	int i, j, n=0;
384 
385 #ifdef _WIN32
386 	const char *sp = strrchr(v[0], '.');
387 	const char *extension = "";
388 	if (!sp || stricmp(sp, ".exe"))
389 		extension = ".exe"; /* Add an exe so CreateProcess will work */
390 	snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
391 	commandname[MAX_PATH]='\0';
392 #endif
393 	for (i=1; i<c; ++i) {
394 		if (v[i][0] == '-') {
395 			if (!strcmp(v[i], "--RUNNING-FORKED")) {
396 				opt_forked = 1;
397 			} else if (!strcmp(v[i], "--no-fork")) {
398 				opt_nofork = 1;
399 			} else if (!strcmp(v[i], "--quiet")) {
400 				opt_verbosity = -1;
401 				verbosity_flag = "--quiet";
402 			} else if (!strcmp(v[i], "--verbose")) {
403 				opt_verbosity = 2;
404 				verbosity_flag = "--verbose";
405 			} else if (!strcmp(v[i], "--terse")) {
406 				opt_verbosity = 0;
407 				verbosity_flag = "--terse";
408 			} else if (!strcmp(v[i], "--help")) {
409 				usage(groups, 0);
410 			} else if (!strcmp(v[i], "--list-tests")) {
411 				usage(groups, 1);
412 			} else {
413 				printf("Unknown option %s.  Try --help\n",v[i]);
414 				return -1;
415 			}
416 		} else {
417 			int r = process_test_option(groups, v[i]);
418 			if (r<0)
419 				return -1;
420 			n += r;
421 		}
422 	}
423 	if (!n)
424 		tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
425 
426 #ifdef _IONBF
427 	setvbuf(stdout, NULL, _IONBF, 0);
428 #endif
429 
430 	++in_tinytest_main;
431 	for (i=0; groups[i].prefix; ++i)
432 		for (j=0; groups[i].cases[j].name; ++j)
433 			if (groups[i].cases[j].flags & TT_ENABLED_)
434 				testcase_run_one(&groups[i],
435 						 &groups[i].cases[j]);
436 
437 	--in_tinytest_main;
438 
439 	if (opt_verbosity==0)
440 		puts("");
441 
442 	if (n_bad)
443 		printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
444 		       n_bad+n_ok,n_skipped);
445 	else if (opt_verbosity >= 1)
446 		printf("%d tests ok.  (%d skipped)\n", n_ok, n_skipped);
447 
448 	return (n_bad == 0) ? 0 : 1;
449 }
450 
451 int
452 tinytest_get_verbosity_(void)
453 {
454 	return opt_verbosity;
455 }
456 
457 void
458 tinytest_set_test_failed_(void)
459 {
460 	if (opt_verbosity <= 0 && cur_test_name) {
461 		if (opt_verbosity==0) puts("");
462 		printf("%s%s: ", cur_test_prefix, cur_test_name);
463 		cur_test_name = NULL;
464 	}
465 	cur_test_outcome = 0;
466 }
467 
468 void
469 tinytest_set_test_skipped_(void)
470 {
471 	if (cur_test_outcome==OK)
472 		cur_test_outcome = SKIP;
473 }
474 
475 char *
476 tinytest_format_hex_(const void *val_, unsigned long len)
477 {
478 	const unsigned char *val = val_;
479 	char *result, *cp;
480 	size_t i;
481 
482 	if (!val)
483 		return strdup("null");
484 	if (!(result = malloc(len*2+1)))
485 		return strdup("<allocation failure>");
486 	cp = result;
487 	for (i=0;i<len;++i) {
488 		*cp++ = "0123456789ABCDEF"[val[i] >> 4];
489 		*cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
490 	}
491 	*cp = 0;
492 	return result;
493 }
494