1 /* Testing - Include basic tests macros for the GNUstep Testsuite
2
3 Copyright (C) 2005-2011 Free Software Foundation, Inc.
4
5 Written by: Alexander Malmberg <alexander@malmberg.org>
6 Updated by: Richard Frith-Macdonald <rfm@gnu.org>
7
8 This package is free software; you can redistribute it and/or
9 modify it under the terms of the GNU General Public
10 License as published by the Free Software Foundation; either
11 version 3 of the License, or (at your option) any later version.
12
13 This library is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 General Public License for more details.
17
18 */
19
20 #ifndef Testing_h
21 #define Testing_h
22
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <stdarg.h>
26 #include <string.h>
27
28 #import <Foundation/NSAutoreleasePool.h>
29 #import <Foundation/NSDate.h>
30 #import <Foundation/NSException.h>
31 #import <Foundation/NSObjCRuntime.h>
32 #import <Foundation/NSObject.h>
33 #import <Foundation/NSRegularExpression.h>
34 #import <Foundation/NSString.h>
35
36 /* A flag indicating that the testsuite is currently processing tests
37 * which are actually not expected to pass, but where we hope someone
38 * might have committed a bugfix.
39 * You should set this to YES at the start of any set of tests which
40 * are actually unlikely to pass on all systems.
41 * The state of this flag is preserved by sets ... on exit from a set
42 * it is restored to the state it had on entry.
43 * This flag is ignored if the tests are performed in 'developer' mode
44 * (ie run with the gnustep-tests --developer option and therefore
45 * compiled with the TESTDEV preprocessor macro defined).
46 */
47 static BOOL testHopeful __attribute__((unused)) = NO;
48
49 /* A flag indicating whether the most recently executed test passed.
50 * This is set by the pass() function (and therefore by any test macro).
51 * Do not modify this directly.
52 */
53 static BOOL testPassed __attribute__((unused)) = NO;
54
55 /* A variable storing the line number of the test currently being run.
56 * Do not modify this directly.
57 */
58 static unsigned testLineNumber __attribute__((unused)) = 0;
59
60 /* A variable storing the indentation of the set currently being run.
61 * Do not modify this directly.
62 */
63 static unsigned testIndentation __attribute__((unused)) = 0;
64 static inline void testIndent(void) __attribute__((unused));
testIndent(void)65 static inline void testIndent(void)
66 {
67 unsigned i = testIndentation;
68 while (i-- > 0)
69 {
70 fprintf(stderr, " ");
71 }
72 }
73
74 /* A variable set whenever a test macro is executed. This contains
75 * the exception which terminated the test macro, or nil if no exception
76 * was raised.
77 */
78 static NSException *testRaised __attribute__((unused)) = nil;
79
80 /* The setEnded function pointer may be set to a function which is to be
81 * executed when the set ends, with its three parameters being the name
82 * of the set, a flag to say whether the set completed successfully, and
83 * the duration of the set.
84 * The SET_TIMER() macro turns on/off timing and, if timeing was
85 * already on, adds the time of the current period to the duration.
86 */
87 static void (*setEnded)(const char *name, BOOL completed, double duration)
88 __attribute__((unused)) = 0;
89 #define SET_TIMER(active) \
90 ({ \
91 double started = _setTiming; \
92 _setTiming = [NSDate timeIntervalSinceReferenceDate]; \
93 if (started > 0.0) setDuration += _setTiming - started; \
94 if (NO == active) _setTiming = 0.0; \
95 })\
96
97
98 /* The pass() function is the low-level core of the testsuite.
99 *
100 * You call this with two arguments ... an integer expression indicating the
101 * success or failure of the testcase (0 is a failure) and a string which
102 * describes the testcase.
103 *
104 * The global variable 'testHopeful' can be set to a non-zero value before
105 * calling this function in order to specify that if the condition is
106 * not true it should be treated as a dashed hope rather than a failure
107 * (unless the tests are bing performed in 'developer' mode).
108 *
109 * If there is a better higher-level test macro available, please use
110 * that instead. In particular, please use the PASS_EQUAL() macro wherever
111 * you wish to test the equality of a pair of objective-c objects.
112 *
113 * If you are calling the function directly, please use a format string
114 * beginning "%s:%d" and pass __FILE__ and __LINE__ as the first two arguments
115 * so that this function will print out the location it was called from.
116 *
117 * This function is the most efficient option for general use, but
118 * please don't use it if there is any change that the evaluation of
119 * the expression used as its first argument might cause an exception
120 * in any context where that might be a problem.
121 */
122 static void pass(int passed, const char *format, ...) __attribute__((unused)) __attribute__ ((format(printf, 2, 3)));
pass(int passed,const char * format,...)123 static void pass(int passed, const char *format, ...)
124 {
125 va_list args;
126 va_start(args, format);
127
128 if (passed)
129 {
130 fprintf(stderr, "Passed test: ");
131 testPassed = YES;
132 }
133 #if !defined(TESTDEV)
134 else if (YES == testHopeful)
135 {
136 fprintf(stderr, "Dashed hope: ");
137 testPassed = NO;
138 }
139 #endif
140 else
141 {
142 fprintf(stderr, "Failed test: ");
143 testPassed = NO;
144 }
145 testIndent();
146 vfprintf(stderr, format, args);
147 fprintf(stderr, "\n");
148 va_end(args);
149 #if defined(FAILFAST)
150 if (NO == testPassed && NO == testHopeful)
151 {
152 exit(1); // Abandon testing now.
153 }
154 #endif
155 }
156
157 /* The testStart() function is used by the PASS macros to provide a break
158 * point in the source code after the current test line has been stored in
159 * testLineNumber.
160 * This is provided for when debugging ... you can set a breakpoint in the
161 * testStart() function for the line number reported in a test failure and
162 * have the debugger stop in just the right place.
163 */
164 static void testStart() __attribute__((unused));
testStart()165 static void testStart()
166 {
167 return;
168 }
169
170 /* Tests a code expression which evaluates to an integer value.
171 * The expression may not contain commas unless it is bracketed.
172 * The format must be a literal string printf style format.
173 * If the expression evaluates to zero the test does not pass.
174 * If the expression causes an exception to be raised, the exception
175 * is caught and logged but the test does not pass.
176 * Otherwise, the test passes.
177 * Basically equivalent to pass() but with exception handling.
178 */
179 #define PASS(testExpression__, testFormat__, ...) \
180 NS_DURING \
181 { \
182 int _cond; \
183 id _tmp = testRaised; testRaised = nil; [_tmp release]; \
184 testLineNumber = __LINE__; \
185 testStart(); \
186 _cond = (int)(testExpression__); \
187 pass(_cond, "%s:%d ... " testFormat__, __FILE__, \
188 __LINE__, ## __VA_ARGS__); \
189 } \
190 NS_HANDLER \
191 testRaised = [localException retain]; \
192 pass(0, "%s:%d ... " testFormat__, __FILE__, __LINE__, ## __VA_ARGS__); \
193 printf("%s: %s", [[testRaised name] UTF8String], \
194 [[testRaised description] UTF8String]); \
195 NS_ENDHANDLER
196
197 /* This category declaration should keep the compiler happy ...
198 * it defines an informal protocol specifying methods your test
199 * classes may implement to aid with testing.
200 */
201 @interface NSObject(TestFramework)
202 /* The -isEqualForTestcase: method may be implemented in order to have
203 * the PASS_EQUAL macro perform special equality testing rather than
204 * using the normal equality test method (-isEqual).
205 */
206 - (BOOL) isEqualForTestcase: (id)otherObject;
207 @end
208
209 /* Tests a code expression which evaluates to an object value.
210 * The expression may not contain commas unless it is bracketed.
211 * The expected value may not contain commas unless it is bracketed.
212 * The format must be a literal string printf style format.
213 *
214 * Where the expression evaluates to an object which is identical to
215 * the expect value, or where the expect value responds to -isEqualForTestcase:
216 * and calling [expact -isEqualForTestcase: object] return YES,
217 * or where the expect value does not respond to -isEqualForTestcase: and
218 * calling [expect isEqual: object] returns YES, then the test has passed.
219 *
220 * The particularly useful thing about this macro is that, if the
221 * results of the expression and the expected object are not equal,
222 * the string representation of both values is logged so that you
223 * can get a better idea of what went wrong.
224 */
225 #define PASS_EQUAL(testExpression__, testExpect__, testFormat__, ...) \
226 NS_DURING \
227 { \
228 int _cond; \
229 id _obj; \
230 id _exp; \
231 id _tmp = testRaised; testRaised = nil; [_tmp release]; \
232 testLineNumber = __LINE__; \
233 testStart(); \
234 _obj = (id)(testExpression__);\
235 _exp = (id)(testExpect__);\
236 if (_obj == _exp) \
237 { \
238 _cond = YES; \
239 } \
240 else if ([_obj respondsToSelector: @selector(isEqualForTestcase:)]) \
241 { \
242 _cond = (BOOL)[(id)_exp isEqualForTestcase: _obj]; \
243 } \
244 else \
245 { \
246 _cond = [_exp isEqual: _obj]; \
247 } \
248 pass(_cond, "%s:%d ... " testFormat__, __FILE__, \
249 __LINE__, ## __VA_ARGS__); \
250 if (0 == _cond) \
251 { \
252 NSString *s = [_obj description]; \
253 if ([s length] == 1) \
254 { \
255 fprintf(stderr, \
256 "Expected '%s' and got '%s' (unicode codepoint %d)\n", \
257 [[_exp description] UTF8String], [s UTF8String], \
258 [s characterAtIndex: 0]); \
259 } \
260 else if (nil == s) \
261 { \
262 fprintf(stderr, "Expected '%s' and got (nil)\n", \
263 [[_exp description] UTF8String]); \
264 } \
265 else \
266 { \
267 fprintf(stderr, "Expected '%s' and got '%s'\n", \
268 [[_exp description] UTF8String], [s UTF8String]); \
269 } \
270 } \
271 } \
272 NS_HANDLER \
273 testRaised = [localException retain]; \
274 pass(0, "%s:%d ... " testFormat__, __FILE__, __LINE__, ## __VA_ARGS__); \
275 printf("%s: %s", [[testRaised name] UTF8String], \
276 [[testRaised description] UTF8String]); \
277 NS_ENDHANDLER
278
279 /* Tests a code expression which evaluates to an object value.
280 * The expression may not contain commas unless it is bracketed.
281 * The expected pattern may not contain commas unless it is bracketed.
282 * The format must be a literal string printf style format.
283 *
284 * Where the expression evaluates to an object whose description matches
285 * the expect value reguilar expression, the test has passed.
286 *
287 * If the results of the expression and the expected pattern o not match,
288 * the string representation of both values is logged so that you
289 * can get a better idea of what went wrong.
290 */
291 #define PASS_MATCH(testExpression__, testExpect__, testFormat__, ...) \
292 NS_DURING \
293 { \
294 int _cond; \
295 id _obj; \
296 id _dsc; \
297 id _exp; \
298 id _pat; \
299 id _tmp = testRaised; testRaised = nil; [_tmp release]; \
300 testLineNumber = __LINE__; \
301 testStart(); \
302 _obj = (id)(testExpression__);\
303 _dsc = [_obj description];\
304 _pat = (id)(testExpect__);\
305 _exp = [[[NSRegularExpression alloc] initWithPattern: _pat \
306 options: 0 error: 0] autorelease];\
307 if (nil != _dsc && nil != _exp) \
308 { \
309 NSRange r = NSMakeRange(0, [_dsc length]);\
310 r = [_exp rangeOfFirstMatchInString: _dsc options: 0 range: r];\
311 if (r.length > 0)\
312 { \
313 _cond = YES; \
314 } \
315 } \
316 else \
317 { \
318 _cond = NO; \
319 } \
320 pass(_cond, "%s:%d ... " testFormat__, __FILE__, \
321 __LINE__, ## __VA_ARGS__); \
322 if (0 == _cond) \
323 { \
324 if ([_dsc length] == 1) \
325 { \
326 fprintf(stderr, \
327 "Expected '%s' and got '%s' (unicode codepoint %d)\n", \
328 [[_pat description] UTF8String], [_dsc UTF8String], \
329 [_dsc characterAtIndex: 0]); \
330 } \
331 else if (nil == _dsc) \
332 { \
333 fprintf(stderr, "Expected '%s' and got (nil)\n", \
334 [[_pat description] UTF8String]); \
335 } \
336 else \
337 { \
338 fprintf(stderr, "Expected '%s' and got '%s'\n", \
339 [[_pat description] UTF8String], [_dsc UTF8String]); \
340 } \
341 } \
342 } \
343 NS_HANDLER \
344 testRaised = [localException retain]; \
345 pass(0, "%s:%d ... " testFormat__, __FILE__, __LINE__, ## __VA_ARGS__); \
346 printf("%s: %s", [[testRaised name] UTF8String], \
347 [[testRaised description] UTF8String]); \
348 NS_ENDHANDLER
349
350 /* Please use the PASS_EXCEPTION() macro to handle any code where you
351 * want an exception to be thrown. The macro checks that the supplied
352 * code throws an expection with the specified name. If the code fails
353 * to throw, or throws the wrong exception, then the code does not pass.
354 * You can supply nil for expected exception name if you don't care about
355 * the exact type of exception thrown.
356 * The code fragment may not contain commas unless it is surrounded by
357 * brackets. eg. PASS_EXCEPTION(({code here}), name, "hello")
358 * The format must be a literal string printf style format.
359 */
360 #define PASS_EXCEPTION(testCode__, testExpect__, testFormat__, ...) \
361 NS_DURING \
362 id _tmp = testRaised; testRaised = nil; [_tmp release]; \
363 { \
364 testLineNumber = __LINE__; \
365 testStart(); \
366 testCode__; \
367 } \
368 pass(0, "%s:%d ... " testFormat__, __FILE__, __LINE__, ## __VA_ARGS__); \
369 NS_HANDLER \
370 testRaised = [localException retain]; \
371 pass((nil == (testExpect__) \
372 || [[testRaised name] isEqual: (testExpect__)]), \
373 "%s:%d ... " testFormat__, __FILE__, __LINE__, ## __VA_ARGS__); \
374 if (nil != (testExpect__) \
375 && NO == [(testExpect__) isEqual: [testRaised name]]) \
376 fprintf(stderr, "Expected '%s' and got '%s'\n", \
377 [(testExpect__) UTF8String], \
378 [[testRaised name] UTF8String]); \
379 NS_ENDHANDLER
380
381 /* Please use the PASS_RUNS() macro to handle any code where you want the
382 * code to run to completion without an exception being thrown, but you don't
383 * have a particular expression to be checked.
384 * The code fragment may not contain commas unless it is surrounded by
385 * brackets. eg. PASS_EXCEPTION(({code here}), name, "hello")
386 * The format must be a literal string printf style format.
387 */
388 #define PASS_RUNS(testCode__, testFormat__, ...) \
389 NS_DURING \
390 id _tmp = testRaised; testRaised = nil; [_tmp release]; \
391 { \
392 testLineNumber = __LINE__; \
393 testStart(); \
394 testCode__; \
395 } \
396 pass(1, "%s:%d ... " testFormat__, __FILE__, __LINE__, ## __VA_ARGS__); \
397 NS_HANDLER \
398 testRaised = [localException retain]; \
399 pass(0, "%s:%d ... " testFormat__, __FILE__, __LINE__, ## __VA_ARGS__); \
400 printf("%s: %s", [[testRaised name] UTF8String], \
401 [[testRaised description] UTF8String]); \
402 NS_ENDHANDLER
403
404
405 /* SETs are used to group multiple testcases or code which is outside of
406 * the scope of the current test but could raise exceptions that should
407 * be caught to allow further tests to run.
408 *
409 * You must pass a short description to identify the set at both its
410 * start and its end. This allows the seat to be easily identified in the
411 * log, and also allows for checking to be sure that each start if a set
412 * is matched by a corresponding end.
413 *
414 * The state of the 'testHopeful' flag is saved at the start of the set and
415 * restored at the end of the set, so you can start your code by setting
416 * 'testHopeful=YES;' to mark any tests within the set as being part of a
417 * group of tests we don't expect to pass.
418 *
419 * Importantly, you may skip some or all of the tests in a set if those
420 * tests are not supported in the package being tested (eg. testing of
421 * functionality which depends on some external library which was not
422 * available when the package being tested was buit).
423 *
424 * Any uncaught exception occurring inside a set will abort the entire set
425 * so that remaining tests in the set will not be executed, but you may
426 * also abandon remaining tests upon any test failure.
427 *
428 * The tests within the set are enclosed in an autorelease pool, and any
429 * temporary objects are cleaned up at the end of the set.
430 */
431
432 /* The START_SET() macro starts a set of grouped tests. It must be matched
433 * by a corresponding END_SET() with the same string as an argument.
434 * The argument is a short description to be printed in the log on entry.
435 * The duration of each set is automatically timed (you can suspend/resume
436 * timing using the SET_TIMER macro). Each timed period is added to the
437 * setDuration local variable while a set is executing (you can of course
438 * modify this variable using code inside the set).
439 */
440 #define START_SET(setName) \
441 { \
442 double setDuration = 0.0; \
443 BOOL _setSuccess = YES; \
444 double _setTiming = [NSDate timeIntervalSinceReferenceDate]; \
445 BOOL _save_hopeful = testHopeful; \
446 unsigned _save_indentation = testIndentation; \
447 int _save_line = __LINE__; \
448 char *_save_set = (char*)malloc(strlen(setName) + 1); \
449 strncpy(_save_set, setName, strlen(setName) + 1); \
450 fprintf(stderr, "Start set: "); \
451 testIndent(); \
452 fprintf(stderr, "%s:%d ... %s\n", __FILE__, __LINE__, _save_set); \
453 testIndentation++; \
454 NS_DURING \
455 NSAutoreleasePool *_setPool = [NSAutoreleasePool new]; \
456 {
457
458 /* Helper macro for END_SET() ... do not use directly.
459 */
460 #if defined(TESTDEV)
461 # define OMITTED \
462 { \
463 fprintf(stderr, "Skipped set: "); \
464 testIndent(); \
465 fprintf(stderr, "%s\n", [[localException reason] UTF8String]); \
466 }
467 #else
468 # define OMITTED ;
469 #endif
470
471 /* The END_SET() macro terminates a set of grouped tests. It must be matched
472 * by a corresponding START_SET() with the same string as an argument.
473 * The argument is a short description to be printed in the log on entry.
474 * When a set ends, the function pointed to by the setEnded function is
475 * called with three arguments which allow you to perform extra reporting
476 * or cleanup etc. The three arguments are the set name, a flag to say
477 * whether the set completed successfully, and the duration of the set.
478 */
479 #define END_SET(setName) \
480 } \
481 [_setPool release]; \
482 NS_HANDLER \
483 _setSuccess = NO; \
484 if (YES == [[localException name] isEqualToString: @"SkipSet"]) \
485 { \
486 fprintf(stderr, "Skipped set: "); \
487 testIndent(); \
488 fprintf(stderr, "%s\n", [[localException reason] UTF8String]); \
489 } \
490 else if (YES == [[localException name] isEqualToString: @"OmitSet"]) \
491 OMITTED \
492 else \
493 { \
494 if (YES == [[localException name] isEqualToString: @"FailSet"]) \
495 { \
496 fprintf(stderr, "Failed set: "); \
497 testIndent(); \
498 fprintf(stderr, "%s:%d ... need not met in %s.\n", \
499 __FILE__, _save_line, _save_set); \
500 } \
501 else \
502 { \
503 fprintf(stderr, "EXCEPTION: %s %s %s\n", \
504 [[localException name] UTF8String], \
505 [[localException reason] UTF8String], \
506 [[[localException userInfo] description] UTF8String]); \
507 fprintf(stderr, "Failed set: "); \
508 testIndent(); \
509 fprintf(stderr, "%s:%d ... problem in %s.\n", \
510 __FILE__, _save_line, _save_set); \
511 } \
512 } \
513 NS_ENDHANDLER \
514 SET_TIMER(NO); \
515 if (0 != setEnded) (*setEnded)(setName, _setSuccess, setDuration); \
516 if (strcmp(_save_set, setName) != 0) \
517 fprintf(stderr, "Error: %s:%d ... END(%s) with START(%s).\n", \
518 __FILE__, __LINE__, setName, _save_set); \
519 testIndentation = _save_indentation; \
520 fprintf(stderr, "End set: "); \
521 testIndent(); \
522 fprintf(stderr, "%s:%d ... %s\n", __FILE__, __LINE__, _save_set); \
523 free(_save_set); \
524 testHopeful = _save_hopeful; \
525 }
526
527 /* The NEED macro takes a test macro as an argument and breaks out of a set
528 * and reports it as failed if the test does not pass.
529 */
530 #define NEED(testToTry) \
531 {testToTry;} \
532 if (NO == testPassed) \
533 { \
534 if (nil != testRaised) \
535 { \
536 [testRaised raise]; \
537 } \
538 else \
539 { \
540 [NSException raise: @"FailSet" format: @"Test did not pass"]; \
541 } \
542 }
543
544 /* The SKIP() macro skips the remainder of a set of grouped tests.
545 * Its argument is a literal printf style format string and variable
546 * arguments to print a message giving the reason for skipping the set.
547 * This should be a short one line message (for immediate display),
548 * preferably with a more detailed explanation on subsequent lines.
549 */
550 #define SKIP(testFormat__, ...) \
551 [NSException raise: @"SkipSet" format: @"%s %d ... " testFormat__, \
552 __FILE__, __LINE__, ## __VA_ARGS__];
553
554 /* The OMIT() macro acts just like SKIP() except that it only reports the
555 * set if running in developer mode. The idea is that it should be used for
556 * groups of tests which are not expected to be available on most platforms
557 * yet, so only developers should see them reported.
558 */
559 #define OMIT(testFormat__, ...) \
560 [NSException raise: @"OmitSet" format: @"%s %d ... " testFormat__, \
561 __FILE__, __LINE__, ## __VA_ARGS__];
562
563
564 /* some good macros to compare floating point numbers */
565 #import <math.h>
566 #import <float.h>
567 #define EQ(x, y) (fabs((x) - (y)) <= fabs((x) + (y)) * (FLT_EPSILON * 100))
568 #define LE(x, y) ((x)<(y) || EQ(x, y))
569 #define GE(x, y) ((y)<(x) || EQ(x, y))
570 #define LT(x, y) (!GE(x, y))
571 #define GT(x, y) (!LE(x, y))
572
573 /* A convenience macro to pass an object as a string to a print function.
574 */
575 #define POBJECT(obj) [[(obj) description] UTF8String]
576
577 #endif
578
579 #ifndef CREATE_AUTORELEASE_POOL
580 #define RETAIN(object) [object retain]
581 #define RELEASE(object) [object release]
582 #define AUTORELEASE(object) [object autorelease]
583 #define TEST_RETAIN(object) ({\
584 id __object = (id)(object); (__object != nil) ? [__object retain] : nil; })
585 #define TEST_RELEASE(object) ({\
586 id __object = (id)(object); if (__object != nil) [__object release]; })
587 #define TEST_AUTORELEASE(object) ({\
588 id __object = (id)(object); (__object != nil) ? [__object autorelease] : nil; })
589 #define ASSIGN(object,value) ({\
590 id __value = (id)(value); \
591 id __object = (id)(object); \
592 if (__value != __object) \
593 { \
594 if (__value != nil) \
595 { \
596 [__value retain]; \
597 } \
598 object = __value; \
599 if (__object != nil) \
600 { \
601 [__object release]; \
602 } \
603 } \
604 })
605 #define ASSIGNCOPY(object,value) ({\
606 id __value = (id)(value); \
607 id __object = (id)(object); \
608 if (__value != __object) \
609 { \
610 if (__value != nil) \
611 { \
612 __value = [__value copy]; \
613 } \
614 object = __value; \
615 if (__object != nil) \
616 { \
617 [__object release]; \
618 } \
619 } \
620 })
621 #define DESTROY(object) ({ \
622 if (object) \
623 { \
624 id __o = object; \
625 object = nil; \
626 [__o release]; \
627 } \
628 })
629 #define CREATE_AUTORELEASE_POOL(X) \
630 NSAutoreleasePool *(X) = [NSAutoreleasePool new]
631 #define RECREATE_AUTORELEASE_POOL(X) \
632 if (X == nil) \
633 (X) = [NSAutoreleasePool new]
634 #endif
635
636