1 #ifndef TEST_RUNNER_HPP
2 #define TEST_RUNNER_HPP
3 
4 #include "TestUtil.hpp"
5 
6 
7 
8 /***************************************************************************************
9  * Begin test runner implementation.
10  * This is a higher-level API that can be used to register tests,
11  * test dependencies, and to run-time select a subset of tests to
12  * run.
13  ***************************************************************************************/
14 
15 /* Register a test to be run */
16 #define REGISTER_TEST( TEST_FUNC ) \
17   runner_register_test( __FILE__, __LINE__, #TEST_FUNC, (TEST_FUNC), NULL )
18 
19 /* Mark a dependency between tests.  The second argument must be
20  * an already-registered test.  The first argument will be registered
21  * as a test if it has not already been registered.  The test specified
22  * by the first argument will be run only if the test specified by
23  * the second argument is run and succeeds.
24  */
25 #define REGISTER_DEP_TEST( TEST_FUNC, REQUIRED_FUNC ) \
26   runner_register_test( __FILE__, __LINE__, #TEST_FUNC, (TEST_FUNC), (REQUIRED_FUNC) )
27 
28 /* Run registered tests.
29  * Arguments should be argc and argv passed to main.
30  * If ARGC is less than or equal to 1 then all tests are run.
31  * Otherwise only tests specified in the argument list are run.
32  * Returns number of failed tests.
33  */
34 #define RUN_TESTS( ARGC, ARGV ) \
35   runner_run_tests( (ARGC), (ARGV) )
36 
37 
38 
39 /***************************************************************************************
40  * NOTE: The remainder of this file contains the implementation of the above macros.
41  *       The above macros constitute the entire intended API.
42  ***************************************************************************************/
43 
44 
45 
46 static void runner_register_test( const char* filename, int line_number,
47                                   const char* name, test_func function,
48                                   test_func requisite = 0 );
49 static int runner_run_tests( int argc, char* argv[] );
50 
51 static void runner_usage( FILE* str, int argc, char* argv[] );
52 static void runner_list_tests(int long_format);
53 static void runner_help( int argc, char* argv[] );
54 
55 enum RunnerStatus { PASSED, FAILED, DESELECTED, SELECTED };
56 struct RunnerTest {
57   test_func testFunc;
58   char* testName;
59   enum RunnerStatus testStatus;
60   int* testRequisites;
61   size_t numRequisites;
62 };
63 
64 
65 struct RunnerTest* RunnerTestList = 0;
66 size_t RunnerTestCount = 0;
67 const size_t RUNNER_NOT_FOUND = ~(size_t)0;
68 static size_t runner_find_test_func( test_func f );
69 static size_t runner_find_test_name( const char* name );
70 static size_t runner_add_test( test_func f, const char* name );
71 static void runner_add_requisite( size_t idx, size_t req );
72 static void free_test_list();
73 
runner_find_test_func(test_func f)74 static size_t runner_find_test_func( test_func f ) {
75   for (size_t i = 0; i < RunnerTestCount; ++i)
76     if (RunnerTestList[i].testFunc == f)
77       return i;
78   return RUNNER_NOT_FOUND;
79 }
runner_find_test_name(const char * name)80 static size_t runner_find_test_name( const char* name ) {
81   for (size_t i = 0; i < RunnerTestCount; ++i)
82     if (!strcmp(RunnerTestList[i].testName, name))
83       return i;
84   return RUNNER_NOT_FOUND;
85 }
runner_add_test(test_func f,const char * name)86 static size_t runner_add_test( test_func f, const char* name ) {
87   size_t idx = runner_find_test_func( f );
88   if (idx == RUNNER_NOT_FOUND) {
89     if (!RunnerTestCount)
90       atexit( &free_test_list );
91     idx = RunnerTestCount++;
92     RunnerTest* new_RunnerTestList = (RunnerTest*)realloc( RunnerTestList, RunnerTestCount * sizeof(RunnerTest) );
93     if (!new_RunnerTestList) {
94       fprintf(stderr, "TestRunner::runner_add_test(): reallocation of RunnerTestList failed\n");
95       atexit( &free_test_list );
96     }
97     else
98       RunnerTestList = new_RunnerTestList;
99     RunnerTestList[idx].testFunc = f;
100     RunnerTestList[idx].testName = strdup(name);
101     RunnerTestList[idx].testStatus = SELECTED;
102     RunnerTestList[idx].testRequisites = 0;
103     RunnerTestList[idx].numRequisites = 0;
104   }
105   return idx;
106 }
runner_add_requisite(size_t idx,size_t req)107 static void runner_add_requisite( size_t idx, size_t req )
108 {
109   size_t i;
110   for (i = 0; i < RunnerTestList[idx].numRequisites; ++i)
111     if (RunnerTestList[idx].testRequisites[i] == (int)req)
112       return;
113   ++RunnerTestList[idx].numRequisites;
114   RunnerTestList[idx].testRequisites = (int*)realloc( RunnerTestList[idx].testRequisites,
115                             RunnerTestList[idx].numRequisites * sizeof(*RunnerTestList[idx].testRequisites) );
116   RunnerTestList[idx].testRequisites[RunnerTestList[idx].numRequisites-1] = req;
117 }
free_test_list()118 static void free_test_list()
119 {
120   for (size_t i = 0; i < RunnerTestCount; ++i) {
121     free( RunnerTestList[i].testName );
122     free( RunnerTestList[i].testRequisites );
123   }
124   free( RunnerTestList );
125 }
126 
runner_register_test(const char * filename,int line_number,const char * name,test_func test,test_func req)127 void runner_register_test( const char* filename,
128                            int line_number,
129                            const char* name,
130                            test_func test,
131                            test_func req )
132 {
133   size_t i = runner_add_test( test, name );
134   size_t req_idx;
135   if (req) {
136     req_idx = runner_find_test_func( req );
137     if (RUNNER_NOT_FOUND == req_idx) {
138       fprintf( stderr, "Error registering requisite for test: \"%s\"\n"
139                        "\tat %s:%d\n"
140                        "\tRequisite test function not registered.\n",
141                        name, filename, line_number );
142       abort();
143     }
144     runner_add_requisite( i, req_idx );
145   }
146 }
147 
runner_usage(FILE * str,int,char * argv[])148 void runner_usage( FILE* str, int /*argc*/, char* argv[] )
149 {
150   fprintf( str, "%s [-l|-L] [-h] [-r] [<test_name> [<test_name> ...]]\n", argv[0] );
151 }
152 
runner_help(int argc,char * argv[])153 void runner_help( int argc, char* argv[] )
154 {
155   runner_usage( stdout, argc, argv );
156   fprintf( stdout, "-l : List test names and exit\n"
157                    "-L : List test names and requisites and exit\n"
158                    "-h : This help text\n"
159                    "-r : Recursively run requisite tests for any specified test\n"
160                    "\n");
161 }
162 
runner_list_tests(int long_format)163 void runner_list_tests( int long_format )
164 {
165   size_t i, j;
166   printf("Test List:\n");
167   for (i = 0; i < RunnerTestCount; ++i) {
168     if (RunnerTestList[i].testStatus == DESELECTED)
169       continue;
170     printf( " o %s\n", RunnerTestList[i].testName );
171     if (!long_format || ! RunnerTestList[i].numRequisites)
172       continue;
173     if (RunnerTestList[i].numRequisites == 1)
174       printf( "  Requires : %s\n", RunnerTestList[RunnerTestList[i].testRequisites[0]].testName );
175     else {
176       printf( "  Requires : \n" );
177       for (j = 0; j < RunnerTestList[i].numRequisites; ++j)
178         printf( "    - %s\n", RunnerTestList[ RunnerTestList[i].testRequisites[j] ].testName );
179     }
180   }
181 }
182 
runner_run_tests(int argc,char * argv[])183 int runner_run_tests( int argc, char* argv[] )
184 {
185     /* Counters */
186   int error_count = 0;
187   int fail_count = 0;
188   int num_selected = 0;
189   int num_run = 0;
190 
191     /* Flags from parsed arguments */
192   int run_requisites = 0;
193   int list_tests = 0;
194   int first_selected = 1;
195 
196     /* Misc iterator vars and such */
197   int changed_some, ran_some, can_run, fail;
198   int k;
199   const char* c;
200   size_t i, j;
201 
202     /* Process command line arguments */
203   for (k = 1; k < argc; ++k) {
204     if (argv[k][0] == '-') {
205       for (c = argv[k] + 1; *c; ++c) {
206         switch (*c) {
207           case 'l': list_tests = 1; break;
208           case 'L': list_tests = 2; break;
209           case 'r': run_requisites = true; break;
210           case 'h': runner_help( argc, argv ); return 0;
211           default:
212             runner_usage( stderr, argc, argv );
213             fprintf( stderr, "Unknown flag: '%c'\n", *c );
214             return 1;
215         }
216       }
217     }
218     else {
219         // If user has specified some tests to run, begin
220         // by marking all tests as de-selected.
221       if (first_selected) {
222         for (i = 0; i < RunnerTestCount; ++i)
223           RunnerTestList[i].testStatus = DESELECTED;
224         first_selected = 0;
225       }
226         // Mark specified test as selected.
227       i = runner_find_test_name( argv[k] );
228       if (RUNNER_NOT_FOUND == i) {
229         fprintf( stderr, "Unknown test name: \"%s\"\n", argv[k] );
230         ++error_count;
231       }
232       else {
233         RunnerTestList[i].testStatus = SELECTED;
234       }
235     }
236   }
237 
238     /* If recursively running requisite tests, select those also. */
239   if (run_requisites) {
240     do {
241       changed_some = 0;
242       for (i = 0; i < RunnerTestCount; ++i) {
243         if (RunnerTestList[i].testStatus == DESELECTED)
244           continue;
245 
246         for (j = 0; j < RunnerTestList[i].numRequisites; ++j) {
247           if (RunnerTestList[ RunnerTestList[i].testRequisites[j] ].testStatus == DESELECTED) {
248             RunnerTestList[ RunnerTestList[i].testRequisites[j] ].testStatus = SELECTED;
249             changed_some = 1;
250           }
251         }
252       }
253     } while(changed_some);
254   }
255 
256     // Count number of selected tests
257   num_selected = 0;
258   for (i = 0; i < RunnerTestCount; ++i)
259     if (RunnerTestList[i].testStatus == SELECTED)
260       ++num_selected;
261 
262   if (list_tests) {
263     runner_list_tests( list_tests - 1 );
264     return error_count;
265   }
266 
267     // Now run the tests
268   num_run = 0;
269   do {
270     ran_some = 0;
271     for (i = 0; i < RunnerTestCount; ++i) {
272       if (RunnerTestList[i].testStatus != SELECTED)
273         continue;
274       can_run = 1;
275       for (j = 0; j < RunnerTestList[i].numRequisites; ++j) {
276         k = RunnerTestList[i].testRequisites[j];
277         if (RunnerTestList[k].testStatus != PASSED &&
278             RunnerTestList[k].testStatus != DESELECTED) {
279           can_run = 0;
280           break;
281         }
282       }
283       if (!can_run)
284         continue;
285 
286       ran_some = 1;
287       ++num_run;
288       fail = run_test( RunnerTestList[i].testFunc, RunnerTestList[i].testName );
289       if (fail) {
290         error_count++;
291         fail_count++;
292         RunnerTestList[i].testStatus = FAILED;
293       }
294       else {
295         RunnerTestList[i].testStatus = PASSED;
296       }
297     }
298   } while (ran_some);
299 
300     // Print brief summary
301   if (num_run == (int)RunnerTestCount && !fail_count) {
302     printf("All %d tests passed.\n", num_run);
303   }
304   else if (num_run == num_selected && !fail_count) {
305     printf("All %d selected tests passed.\n", num_run );
306     printf("Skipped %d non-selected tests\n", (int)(RunnerTestCount - num_selected));
307   }
308   else {
309     printf( "%2d tests registered\n", (int)RunnerTestCount );
310     if (num_selected == num_run)
311       printf( "%2d tests selected\n", num_selected );
312     else
313       printf( "%2d of %2d tests ran\n", num_run, num_selected );
314     if (num_run < (int)RunnerTestCount)
315       printf( "%2d of %2d tests skipped\n",
316         (int)RunnerTestCount - num_run, (int)RunnerTestCount );
317     printf( "%2d of %2d tests passed\n", num_run - fail_count, num_run );
318     if (fail_count)
319       printf( "%2d of %2d tests FAILED\n", fail_count, num_run );
320   }
321 
322   return error_count;
323 }
324 
325 #endif
326