1 /***********************************************************************************************************************************
2 C Test Harness
3 ***********************************************************************************************************************************/
4 #include "build.auto.h"
5
6 #include <fcntl.h>
7 #include <stdio.h>
8 #include <stdlib.h>
9 #include <string.h>
10 #include <sys/time.h>
11 #include <unistd.h>
12
13 #include "common/harnessDebug.h"
14 #include "common/harnessTest.h"
15 #include "common/harnessLog.h"
16
17 #define TEST_LIST_SIZE 64
18
19 typedef struct TestData
20 {
21 bool selected;
22 } TestData;
23
24 static TestData testList[TEST_LIST_SIZE];
25
26 static int testRun = 0;
27 static int testRunSub = 0;
28 static int testTotal = 0;
29 static bool testFirst = true;
30
31 static uint64_t timeMSecBegin;
32
33 static const char *testExeData = NULL;
34 static const char *testProjectExeData = NULL;
35 static bool testContainerData = false;
36 static unsigned int testIdxData = 0;
37 static bool testTiming = true;
38 static const char *testPathData = NULL;
39 static const char *testDataPathData = NULL;
40 static const char *testRepoPathData = NULL;
41
42 static struct HarnessTestLocal
43 {
44 uint64_t logLastBeginTime; // Store the begin time of the last log for deltas
45 int logLastLineNo; // Store the line number to be used in debugging
46
47 struct HarnessTestResult
48 {
49 bool running; // Is the test currently running?
50 const char *statement; // statement that is being tested
51 int lineNo; // Line number the test is on
52 bool result; // Is there a result or is it void?
53 } result;
54 } harnessTestLocal;
55
56 /***********************************************************************************************************************************
57 Extern functions
58 ***********************************************************************************************************************************/
59 #ifdef HRN_FEATURE_LOG
60 void harnessLogInit(void);
61 void harnessLogFinal(void);
62 #endif
63
64 /***********************************************************************************************************************************
65 Initialize harness
66 ***********************************************************************************************************************************/
67 void
hrnInit(const char * testExe,const char * testProjectExe,bool testContainer,unsigned int testIdx,bool timing,const char * testPath,const char * testDataPath,const char * testRepoPath)68 hrnInit(
69 const char *testExe, const char *testProjectExe, bool testContainer, unsigned int testIdx, bool timing, const char *testPath,
70 const char *testDataPath, const char *testRepoPath)
71 {
72 FUNCTION_HARNESS_VOID();
73
74 // Set test configuration
75 testExeData = testExe;
76 testProjectExeData = testProjectExe;
77 testContainerData = testContainer;
78 testIdxData = testIdx;
79 testTiming = timing;
80 testPathData = testPath;
81 testDataPathData = testDataPath;
82 testRepoPathData = testRepoPath;
83
84 FUNCTION_HARNESS_RETURN_VOID();
85 }
86
87 /***********************************************************************************************************************************
88 testAdd - add a new test
89 ***********************************************************************************************************************************/
90 void
hrnAdd(int run,bool selected)91 hrnAdd(int run, bool selected)
92 {
93 FUNCTION_HARNESS_BEGIN();
94 FUNCTION_HARNESS_PARAM(INT, run);
95 FUNCTION_HARNESS_PARAM(BOOL, selected);
96 FUNCTION_HARNESS_END();
97
98 if (run != testTotal + 1)
99 {
100 fprintf(stderr, "ERROR: test run %d is not in order\n", run);
101 fflush(stderr);
102 exit(255);
103 }
104
105 testList[testTotal].selected = selected;
106 testTotal++;
107
108 FUNCTION_HARNESS_RETURN_VOID();
109 }
110
111 /***********************************************************************************************************************************
112 testBegin - should this test run?
113 ***********************************************************************************************************************************/
114 bool
testBegin(const char * name)115 testBegin(const char *name)
116 {
117 FUNCTION_HARNESS_BEGIN();
118 FUNCTION_HARNESS_PARAM(STRINGZ, name);
119
120 FUNCTION_HARNESS_ASSERT(name != NULL);
121 FUNCTION_HARNESS_END();
122
123 bool result = false;
124 testRun++;
125
126 if (testList[testRun - 1].selected)
127 {
128 #ifdef HRN_FEATURE_LOG
129 if (!testFirst)
130 {
131 // Make sure there is nothing untested left in the log
132 harnessLogFinal();
133
134 // Clear out the test directory so the next test starts clean
135 char buffer[2048];
136 snprintf(
137 buffer, sizeof(buffer), "%schmod -R 700 %s/" "* > /dev/null 2>&1;%srm -rf %s/" "*", testContainer() ? "sudo " : "",
138 testPath(), testContainer() ? "sudo " : "", testPath());
139
140 if (system(buffer) != 0)
141 {
142 fprintf(stderr, "ERROR: unable to clear test path '%s'\n", testPath());
143 fflush(stderr);
144 exit(255);
145 }
146
147 // Clear out the data directory so the next test starts clean
148 snprintf(
149 buffer, sizeof(buffer), "%schmod -R 700 %s/" "* > /dev/null 2>&1;%srm -rf %s/" "*", testContainer() ? "sudo " : "",
150 hrnPath(), testContainer() ? "sudo " : "", hrnPath());
151
152 if (system(buffer) != 0)
153 {
154 fprintf(stderr, "ERROR: unable to clear data path '%s'\n", hrnPath());
155 fflush(stderr);
156 exit(255);
157 }
158
159 // Clear any log replacements
160 hrnLogReplaceClear();
161 }
162 #endif
163 // No longer the first test
164 testFirst = false;
165
166 if (testRun != 1)
167 printf("\n");
168
169 printf("run %d - %s\n", testRun, name);
170 fflush(stdout);
171
172 testRunSub = 1;
173 timeMSecBegin = testTimeMSec();
174
175 #ifdef HRN_FEATURE_LOG
176 // Initialize logging
177 harnessLogInit();
178 #endif
179
180 result = true;
181 }
182
183 harnessTestLocal.logLastBeginTime = 0;
184
185 FUNCTION_HARNESS_RETURN(BOOL, result);
186 }
187
188 /***********************************************************************************************************************************
189 testComplete - make sure all expected tests ran
190 ***********************************************************************************************************************************/
191 void
hrnComplete(void)192 hrnComplete(void)
193 {
194 FUNCTION_HARNESS_VOID();
195
196 #ifdef HRN_FEATURE_LOG
197 // Make sure there is nothing untested left in the log
198 harnessLogFinal();
199 #endif
200
201 // Check that all tests ran
202 if (testRun != testTotal)
203 {
204 fprintf(stderr, "ERROR: expected %d tests but %d were run\n", testTotal, testRun);
205 fflush(stderr);
206 exit(255);
207 }
208
209 FUNCTION_HARNESS_RETURN_VOID();
210 }
211
212 /**********************************************************************************************************************************/
213 void
hrnFileRead(const char * fileName,unsigned char * buffer,size_t bufferSize)214 hrnFileRead(const char *fileName, unsigned char *buffer, size_t bufferSize)
215 {
216 int result = open(fileName, O_RDONLY, 0660);
217
218 if (result == -1)
219 {
220 fprintf(stderr, "ERROR: unable to open '%s' for read\n", fileName);
221 fflush(stderr);
222 exit(255);
223 }
224
225 ssize_t bufferRead = read(result, buffer, bufferSize);
226
227 if (bufferRead == -1)
228 {
229 fprintf(stderr, "ERROR: unable to read '%s'\n", fileName);
230 fflush(stderr);
231 exit(255);
232 }
233
234 buffer[bufferRead] = 0;
235
236 close(result);
237 }
238
239 /**********************************************************************************************************************************/
240 void
hrnFileWrite(const char * fileName,const unsigned char * buffer,size_t bufferSize)241 hrnFileWrite(const char *fileName, const unsigned char *buffer, size_t bufferSize)
242 {
243 int result = open(fileName, O_WRONLY | O_CREAT | O_TRUNC, 0660);
244
245 if (result == -1)
246 {
247 fprintf(stderr, "ERROR: unable to open '%s' for write\n", fileName);
248 fflush(stderr);
249 exit(255);
250 }
251
252 if (write(result, buffer, bufferSize) != (int)bufferSize)
253 {
254 fprintf(stderr, "ERROR: unable to write '%s'\n", fileName);
255 fflush(stderr);
256 exit(255);
257 }
258
259 close(result);
260 }
261
262 /**********************************************************************************************************************************/
263 char harnessDiffBuffer[256 * 1024];
264
265 const char *
hrnDiff(const char * expected,const char * actual)266 hrnDiff(const char *expected, const char *actual)
267 {
268 FUNCTION_HARNESS_BEGIN();
269 FUNCTION_HARNESS_PARAM(STRINGZ, expected);
270 FUNCTION_HARNESS_PARAM(STRINGZ, actual);
271 FUNCTION_HARNESS_END();
272
273 ASSERT(actual != NULL);
274
275 // Write expected file
276 char expectedFile[1024];
277 snprintf(expectedFile, sizeof(expectedFile), "%s/diff.expected", hrnPath());
278 hrnFileWrite(expectedFile, (unsigned char *)expected, strlen(expected));
279
280 // Write actual file
281 char actualFile[1024];
282 snprintf(actualFile, sizeof(actualFile), "%s/diff.actual", hrnPath());
283 hrnFileWrite(actualFile, (unsigned char *)actual, strlen(actual));
284
285 // Perform diff
286 char command[2560];
287 snprintf(command, sizeof(command), "diff -u %s %s > %s/diff.result", expectedFile, actualFile, hrnPath());
288
289 if (system(command) == 2)
290 {
291 fprintf(stderr, "ERROR: unable to execute '%s'\n", command);
292 fflush(stderr);
293 exit(255);
294 }
295
296 // Read result
297 char resultFile[1024];
298 snprintf(resultFile, sizeof(resultFile), "%s/diff.result", hrnPath());
299 hrnFileRead(resultFile, (unsigned char *)harnessDiffBuffer, sizeof(harnessDiffBuffer));
300
301 // Remove last linefeed from diff output
302 harnessDiffBuffer[strlen(harnessDiffBuffer) - 1] = 0;
303
304 FUNCTION_HARNESS_RETURN(STRINGZ, harnessDiffBuffer);
305 }
306
307 /**********************************************************************************************************************************/
308 void
hrnTestLogTitle(int lineNo)309 hrnTestLogTitle(int lineNo)
310 {
311 // Output run/test
312 char buffer[32];
313 int bufferSize = snprintf(buffer, sizeof(buffer), "%d/%d", testRun, testRunSub);
314
315 printf("\nrun %s ", buffer);
316
317 // Output dashes
318 for (int dashIdx = 0; dashIdx < 16 - bufferSize; dashIdx++)
319 putchar('-');
320
321 // Output line number
322 printf(" L%04d", lineNo);
323
324 // Increment testSub and reset log time
325 testRunSub++;
326 }
327
328 /**********************************************************************************************************************************/
329 void
hrnTestLogPrefix(const int lineNo)330 hrnTestLogPrefix(const int lineNo)
331 {
332 FUNCTION_HARNESS_BEGIN();
333 FUNCTION_HARNESS_PARAM(INT, lineNo);
334 FUNCTION_HARNESS_END();
335
336 // Always indent at the beginning
337 printf(" ");
338
339 // Add timing if requested
340 if (testTiming)
341 {
342 uint64_t currentTime = testTimeMSec();
343
344 // Print elapsed time size the beginning of the test run
345 printf(
346 "%03" PRIu64 ".%03" PRIu64"s", ((currentTime - testTimeMSecBegin()) / 1000),
347 ((currentTime - testTimeMSecBegin()) % 1000));
348
349 // Print delta time since the last log message
350 if (harnessTestLocal.logLastBeginTime != 0)
351 {
352 printf(
353 " %03" PRIu64 ".%03" PRIu64"s ", ((currentTime - harnessTestLocal.logLastBeginTime) / 1000),
354 ((currentTime - harnessTestLocal.logLastBeginTime) % 1000));
355 }
356 else
357 printf(" ");
358
359 harnessTestLocal.logLastBeginTime = currentTime;
360 }
361
362 // Store line number for
363 harnessTestLocal.logLastLineNo = lineNo;
364
365 // Add line number and padding
366 printf("L%04d ", lineNo);
367 fflush(stdout);
368
369 FUNCTION_HARNESS_RETURN_VOID();
370 }
371
372 /**********************************************************************************************************************************/
373 void
hrnTestResultBegin(const char * const statement,const bool result)374 hrnTestResultBegin(const char *const statement, const bool result)
375 {
376 ASSERT(!harnessTestLocal.result.running);
377 ASSERT(harnessTestLocal.logLastLineNo != 0);
378
379 // Set the line number for the current function in the stack trace
380 FUNCTION_HARNESS_STACK_TRACE_LINE_SET(harnessTestLocal.logLastLineNo);
381
382 // Set info to report if an error is thrown
383 harnessTestLocal.result =
384 (struct HarnessTestResult){
385 .running = true, .statement = statement, .lineNo = harnessTestLocal.logLastLineNo, .result = result};
386
387 // Reset line number so it is not used by another test
388 harnessTestLocal.logLastLineNo = 0;
389 }
390
391 /**********************************************************************************************************************************/
392 void
hrnTestResultComment(const char * const comment)393 hrnTestResultComment(const char *const comment)
394 {
395 if (comment != NULL)
396 printf(" (%s)\n", comment);
397 else
398 puts("");
399
400 fflush(stdout);
401 }
402
403 /**********************************************************************************************************************************/
404 bool
hrnTestResultException(void)405 hrnTestResultException(void)
406 {
407 FUNCTION_HARNESS_VOID();
408
409 if (harnessTestLocal.result.running)
410 {
411 THROW_FMT(
412 #ifdef DEBUG
413 TestError,
414 #else
415 AssertError,
416 #endif
417 "EXPECTED %sRESULT FROM STATEMENT: %s\n\nBUT GOT %s: %s\n\nTHROWN AT:\n%s",
418 harnessTestLocal.result.result ? "" : "VOID ",
419 harnessTestLocal.result.statement, errorName(), errorMessage(), errorStackTrace());
420 }
421
422 FUNCTION_HARNESS_RETURN(BOOL, false);
423 }
424
425 void
hrnTestResultEnd(void)426 hrnTestResultEnd(void)
427 {
428 ASSERT(harnessTestLocal.result.running);
429
430 // Set the line number for the current function back to unknown
431 FUNCTION_HARNESS_STACK_TRACE_LINE_SET(0);
432
433 harnessTestLocal.result.running = false;
434 }
435
436 /**********************************************************************************************************************************/
hrnTestResultDiff(const char * actual,const char * expected)437 static void hrnTestResultDiff(const char *actual, const char *expected)
438 {
439 if (actual != NULL && expected != NULL && (strstr(actual, "\n") != NULL || strstr(expected, "\n") != NULL))
440 {
441 THROW_FMT(
442 #ifdef DEBUG
443 TestError,
444 #else
445 AssertError,
446 #endif
447 "STATEMENT: %s\n\nRESULT IS:\n%s\n\nBUT DIFF FROM EXPECTED IS (- remove from expected, + add to expected):\n%s\n\n",
448 harnessTestLocal.result.statement, actual, hrnDiff(expected, actual));
449 }
450 else
451 {
452 THROW_FMT(
453 #ifdef DEBUG
454 TestError,
455 #else
456 AssertError,
457 #endif
458 "STATEMENT: %s\n\nRESULT IS:\n%s\n\nBUT EXPECTED:\n%s",
459 harnessTestLocal.result.statement, actual == NULL ? "NULL" : actual, expected == NULL ? "NULL" : expected); \
460 }
461 }
462
463 void
hrnTestResultBool(int actual,int expected)464 hrnTestResultBool(int actual, int expected)
465 {
466 ASSERT(harnessTestLocal.result.running);
467
468 if (actual < 0 || actual > 1 || expected < 0 || expected > 1 || actual != expected)
469 {
470 char actualZ[256];
471 char expectedZ[256];
472
473 if (actual < 0 || actual > 1)
474 snprintf(actualZ, sizeof(actualZ), "INVALID(%d)", actual);
475 else
476 actual ? strcpy(actualZ, "true") : strcpy(actualZ, "false");
477
478 if (expected < 0 || expected > 1)
479 snprintf(expectedZ, sizeof(expectedZ), "INVALID(%d)", expected);
480 else
481 expected ? strcpy(expectedZ, "true") : strcpy(expectedZ, "false");
482
483 hrnTestResultDiff(actualZ, expectedZ);
484 }
485
486 hrnTestResultEnd();
487 }
488
489 void
hrnTestResultDouble(double actual,double expected)490 hrnTestResultDouble(double actual, double expected)
491 {
492 ASSERT(harnessTestLocal.result.running);
493
494 if (actual != expected)
495 {
496 char actualZ[256];
497 char expectedZ[256];
498
499 snprintf(actualZ, sizeof(actualZ), "%f", actual);
500 snprintf(expectedZ, sizeof(expectedZ), "%f", expected);
501
502 hrnTestResultDiff(actualZ, expectedZ);
503 }
504
505 hrnTestResultEnd();
506 }
507
508 void
hrnTestResultInt64(int64_t actual,int64_t expected,HarnessTestResultOperation operation)509 hrnTestResultInt64(int64_t actual, int64_t expected, HarnessTestResultOperation operation)
510 {
511 ASSERT(harnessTestLocal.result.running);
512
513 bool result = false;
514
515 switch (operation)
516 {
517 case harnessTestResultOperationEq:
518 result = actual == expected;
519 break;
520
521 case harnessTestResultOperationNe:
522 result = actual != expected;
523 break;
524 }
525
526 if (!result)
527 {
528 char actualZ[256];
529 char expectedZ[256];
530
531 snprintf(actualZ, sizeof(actualZ), "%" PRId64, actual);
532 snprintf(expectedZ, sizeof(expectedZ), "%" PRId64, expected);
533
534 hrnTestResultDiff(actualZ, expectedZ);
535 }
536
537 hrnTestResultEnd();
538 }
539
540 void
hrnTestResultPtr(const void * actual,const void * expected,HarnessTestResultOperation operation)541 hrnTestResultPtr(const void *actual, const void *expected, HarnessTestResultOperation operation)
542 {
543 ASSERT(harnessTestLocal.result.running);
544
545 bool result = false;
546
547 switch (operation)
548 {
549 case harnessTestResultOperationEq:
550 result = actual == expected;
551 break;
552
553 case harnessTestResultOperationNe:
554 result = actual != expected;
555 break;
556 }
557
558 if (!result)
559 {
560 char actualZ[256];
561 char expectedZ[256];
562
563 snprintf(actualZ, sizeof(actualZ), "%p", actual);
564 snprintf(expectedZ, sizeof(expectedZ), "%p", expected);
565
566 hrnTestResultDiff(actualZ, expectedZ);
567 }
568
569 hrnTestResultEnd();
570 }
571
572 #ifdef HRN_FEATURE_STRING
573
574 void
hrnTestResultStringList(const StringList * actual,const char * expected,HarnessTestResultOperation operation)575 hrnTestResultStringList(const StringList *actual, const char *expected, HarnessTestResultOperation operation)
576 {
577 // Return NULL if list is empty
578 if (strLstEmpty(actual))
579 {
580 hrnTestResultZ(NULL, expected, operation);
581 return;
582 }
583
584 hrnTestResultZ(strZ(strCatZ(strLstJoin(actual, "\n"), "\n")), expected, operation);
585 }
586
587 #endif
588
589 void
hrnTestResultUInt64(uint64_t actual,uint64_t expected,HarnessTestResultOperation operation)590 hrnTestResultUInt64(uint64_t actual, uint64_t expected, HarnessTestResultOperation operation)
591 {
592 ASSERT(harnessTestLocal.result.running);
593
594 bool result = false;
595
596 switch (operation)
597 {
598 case harnessTestResultOperationEq:
599 result = actual == expected;
600 break;
601
602 case harnessTestResultOperationNe:
603 result = actual != expected;
604 break;
605 }
606
607 if (!result)
608 {
609 char actualZ[256];
610 char expectedZ[256];
611
612 snprintf(actualZ, sizeof(actualZ), "%" PRIu64, actual);
613 snprintf(expectedZ, sizeof(expectedZ), "%" PRIu64, expected);
614
615 hrnTestResultDiff(actualZ, expectedZ);
616 }
617
618 hrnTestResultEnd();
619 }
620
621 void
hrnTestResultUInt64Int64(uint64_t actual,int64_t expected,HarnessTestResultOperation operation)622 hrnTestResultUInt64Int64(uint64_t actual, int64_t expected, HarnessTestResultOperation operation)
623 {
624 ASSERT(harnessTestLocal.result.running);
625
626 if (actual <= INT64_MAX && expected >= 0)
627 hrnTestResultUInt64(actual, (uint64_t)expected, operation);
628 else
629 {
630 char actualZ[256];
631 char expectedZ[256];
632
633 snprintf(actualZ, sizeof(actualZ), "%" PRIu64, actual);
634 snprintf(expectedZ, sizeof(expectedZ), "%" PRId64, expected);
635
636 hrnTestResultDiff(actualZ, expectedZ);
637 }
638 }
639
640 void
hrnTestResultZ(const char * actual,const char * expected,HarnessTestResultOperation operation)641 hrnTestResultZ(const char *actual, const char *expected, HarnessTestResultOperation operation)
642 {
643 ASSERT(harnessTestLocal.result.running);
644
645 bool result = false;
646
647 switch (operation)
648 {
649 case harnessTestResultOperationEq:
650 result = (actual == NULL && expected == NULL) || (actual != NULL && expected != NULL && strcmp(actual, expected) == 0);
651 break;
652
653 case harnessTestResultOperationNe:
654 result =
655 (actual == NULL && expected != NULL) || (actual != NULL && expected == NULL) ||
656 (actual != NULL && expected != NULL && strcmp(actual, expected) == 0);
657 break;
658 }
659
660 if (!result)
661 hrnTestResultDiff(actual, expected);
662
663 hrnTestResultEnd();
664 }
665
666 /***********************************************************************************************************************************
667 Getters/Setters
668 ***********************************************************************************************************************************/
669 const char *
testExe(void)670 testExe(void)
671 {
672 FUNCTION_HARNESS_VOID();
673 FUNCTION_HARNESS_RETURN(STRINGZ, testExeData);
674 }
675
676 /**********************************************************************************************************************************/
677 const char *
testProjectExe(void)678 testProjectExe(void)
679 {
680 FUNCTION_HARNESS_VOID();
681 FUNCTION_HARNESS_RETURN(STRINGZ, testProjectExeData);
682 }
683
684 /**********************************************************************************************************************************/
685 bool
testContainer(void)686 testContainer(void)
687 {
688 FUNCTION_HARNESS_VOID();
689 FUNCTION_HARNESS_RETURN(BOOL, testContainerData);
690 }
691
692 /**********************************************************************************************************************************/
693 unsigned int
testIdx(void)694 testIdx(void)
695 {
696 FUNCTION_HARNESS_VOID();
697 FUNCTION_HARNESS_RETURN(UINT, testIdxData);
698 }
699
700 /**********************************************************************************************************************************/
701 const char *
testPath(void)702 testPath(void)
703 {
704 FUNCTION_HARNESS_VOID();
705 FUNCTION_HARNESS_RETURN(STRINGZ, testPathData);
706 }
707
708 /**********************************************************************************************************************************/
709 const char *
hrnPath(void)710 hrnPath(void)
711 {
712 FUNCTION_HARNESS_VOID();
713 FUNCTION_HARNESS_RETURN(STRINGZ, testDataPathData);
714 }
715
716 /**********************************************************************************************************************************/
717 const char *
hrnPathRepo(void)718 hrnPathRepo(void)
719 {
720 FUNCTION_HARNESS_VOID();
721 FUNCTION_HARNESS_RETURN(STRINGZ, testRepoPathData);
722 }
723
724 /**********************************************************************************************************************************/
725 uint64_t
testTimeMSec(void)726 testTimeMSec(void)
727 {
728 FUNCTION_HARNESS_VOID();
729
730 struct timeval currentTime;
731 gettimeofday(¤tTime, NULL);
732
733 FUNCTION_HARNESS_RETURN(UINT64, ((uint64_t)currentTime.tv_sec * 1000) + (uint64_t)currentTime.tv_usec / 1000);
734 }
735
736 /**********************************************************************************************************************************/
737 uint64_t
testTimeMSecBegin(void)738 testTimeMSecBegin(void)
739 {
740 FUNCTION_HARNESS_VOID();
741
742 FUNCTION_HARNESS_RETURN(UINT64, timeMSecBegin);
743 }
744