1 /*
2 * Copyright © 2004 Red Hat, Inc.
3 * Copyright © 2008 Chris Wilson
4 *
5 * Permission to use, copy, modify, distribute, and sell this software
6 * and its documentation for any purpose is hereby granted without
7 * fee, provided that the above copyright notice appear in all copies
8 * and that both that copyright notice and this permission notice
9 * appear in supporting documentation, and that the name of
10 * Red Hat, Inc. not be used in advertising or publicity pertaining to
11 * distribution of the software without specific, written prior
12 * permission. Red Hat, Inc. makes no representations about the
13 * suitability of this software for any purpose. It is provided "as
14 * is" without express or implied warranty.
15 *
16 * RED HAT, INC. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
17 * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
18 * FITNESS, IN NO EVENT SHALL RED HAT, INC. BE LIABLE FOR ANY SPECIAL,
19 * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
20 * RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
21 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
22 * IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
23 *
24 * Author: Carl D. Worth <cworth@cworth.org>
25 * Chris Wilson <chris@chris-wilson.co.uk>
26 */
27
28 #define _GNU_SOURCE 1 /* for feenableexcept() et al */
29
30 #if HAVE_CONFIG_H
31 #include "config.h"
32 #endif
33
34 #include <stdio.h>
35 #include <stdlib.h>
36 #include <stdarg.h>
37 #include <ctype.h>
38 #include <assert.h>
39 #if HAVE_UNISTD_H
40 #include <unistd.h>
41 #endif
42 #include <errno.h>
43 #include <string.h>
44 #if HAVE_FCFINI
45 #include <fontconfig/fontconfig.h>
46 #endif
47 #if CAIRO_HAS_REAL_PTHREAD
48 #include <pthread.h>
49 #endif
50 #if HAVE_SYS_STAT_H
51 #include <sys/stat.h>
52 #endif
53
54 #if HAVE_VALGRIND
55 #include <valgrind.h>
56 #else
57 #define RUNNING_ON_VALGRIND 0
58 #endif
59
60 #if HAVE_MEMFAULT
61 #include <memfault.h>
62 #define MF(x) x
63 #else
64 #define MF(x)
65 #endif
66
67 #include "cairo-test-private.h"
68
69 #include "buffer-diff.h"
70
71 #ifdef _MSC_VER
72 #include <crtdbg.h>
73 #include <direct.h>
74 #define F_OK 0
75 #define HAVE_MKDIR 1
76 #define mkdir _mkdir
77 #endif
78
79 #ifndef FALSE
80 #define FALSE 0
81 #endif
82 #ifndef TRUE
83 #define TRUE !FALSE
84 #endif
85
86 #if ! HAVE_ALARM || ! defined(SIGALRM)
87 #define alarm(X);
88 #endif
89
90 static const cairo_user_data_key_t _cairo_test_context_key;
91
92 static void
93 _xunlink (const cairo_test_context_t *ctx, const char *pathname);
94
95 static const char *fail_face = "", *xfail_face="", *normal_face = "";
96 static cairo_bool_t print_fail_on_stdout;
97 static int cairo_test_timeout = 60;
98
99 #define NUM_DEVICE_OFFSETS 2
100 #define NUM_DEVICE_SCALE 2
101
102 cairo_bool_t
cairo_test_mkdir(const char * path)103 cairo_test_mkdir (const char *path)
104 {
105 #if ! HAVE_MKDIR
106 return FALSE;
107 #elif HAVE_MKDIR == 1
108 if (mkdir (path) == 0)
109 return TRUE;
110 #elif HAVE_MKDIR == 2
111 if (mkdir (path, 0770) == 0)
112 return TRUE;
113 #else
114 #error Bad value for HAVE_MKDIR
115 #endif
116
117 return errno == EEXIST;
118 }
119
120 static char *
_cairo_test_fixup_name(const char * original)121 _cairo_test_fixup_name (const char *original)
122 {
123 char *name, *s;
124
125 s = name = xstrdup (original);
126 while ((s = strchr (s, '_')) != NULL)
127 *s++ = '-';
128
129 return name;
130 }
131
132 char *
cairo_test_get_name(const cairo_test_t * test)133 cairo_test_get_name (const cairo_test_t *test)
134 {
135 return _cairo_test_fixup_name (test->name);
136 }
137
138 static void
_cairo_test_init(cairo_test_context_t * ctx,const cairo_test_context_t * parent,const cairo_test_t * test,const char * test_name,const char * output)139 _cairo_test_init (cairo_test_context_t *ctx,
140 const cairo_test_context_t *parent,
141 const cairo_test_t *test,
142 const char *test_name,
143 const char *output)
144 {
145 char *log_name;
146
147 MF (MEMFAULT_DISABLE_FAULTS ());
148
149 #if HAVE_FEENABLEEXCEPT
150 feenableexcept (FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW);
151 #endif
152
153 ctx->test = test;
154 ctx->test_name = _cairo_test_fixup_name (test_name);
155 ctx->output = output;
156
157 cairo_test_mkdir (ctx->output);
158
159 ctx->malloc_failure = 0;
160 #if HAVE_MEMFAULT
161 if (getenv ("CAIRO_TEST_MALLOC_FAILURE"))
162 ctx->malloc_failure = atoi (getenv ("CAIRO_TEST_MALLOC_FAILURE"));
163 if (ctx->malloc_failure && ! RUNNING_ON_MEMFAULT ())
164 ctx->malloc_failure = 0;
165 #endif
166
167 ctx->timeout = cairo_test_timeout;
168 if (getenv ("CAIRO_TEST_TIMEOUT"))
169 ctx->timeout = atoi (getenv ("CAIRO_TEST_TIMEOUT"));
170
171 xasprintf (&log_name, "%s/%s%s", ctx->output, ctx->test_name, CAIRO_TEST_LOG_SUFFIX);
172 _xunlink (NULL, log_name);
173
174 ctx->log_file = fopen (log_name, "a");
175 if (ctx->log_file == NULL) {
176 fprintf (stderr, "Error opening log file: %s\n", log_name);
177 ctx->log_file = stderr;
178 }
179 free (log_name);
180
181 ctx->ref_name = NULL;
182 ctx->ref_image = NULL;
183 ctx->ref_image_flattened = NULL;
184
185 if (parent != NULL) {
186 ctx->targets_to_test = parent->targets_to_test;
187 ctx->num_targets = parent->num_targets;
188 ctx->limited_targets = parent->limited_targets;
189 ctx->own_targets = FALSE;
190
191 ctx->srcdir = parent->srcdir;
192 ctx->refdir = parent->refdir;
193 } else {
194 int tmp_num_targets;
195 cairo_bool_t tmp_limited_targets;
196
197 ctx->targets_to_test = cairo_boilerplate_get_targets (&tmp_num_targets, &tmp_limited_targets);
198 ctx->num_targets = tmp_num_targets;
199 ctx->limited_targets = tmp_limited_targets;
200 ctx->own_targets = TRUE;
201
202 ctx->srcdir = getenv ("srcdir");
203 if (ctx->srcdir == NULL)
204 ctx->srcdir = ".";
205
206 ctx->refdir = getenv ("CAIRO_REF_DIR");
207 }
208
209 #ifdef HAVE_UNISTD_H
210 if (*fail_face == '\0' && isatty (2)) {
211 fail_face = "\033[41;37;1m";
212 xfail_face = "\033[43;37;1m";
213 normal_face = "\033[m";
214 if (isatty (1))
215 print_fail_on_stdout = FALSE;
216 }
217 #endif
218
219 printf ("\nTESTING %s\n", ctx->test_name);
220 }
221
222 void
_cairo_test_context_init_for_test(cairo_test_context_t * ctx,const cairo_test_context_t * parent,const cairo_test_t * test)223 _cairo_test_context_init_for_test (cairo_test_context_t *ctx,
224 const cairo_test_context_t *parent,
225 const cairo_test_t *test)
226 {
227 _cairo_test_init (ctx, parent, test, test->name, CAIRO_TEST_OUTPUT_DIR);
228 }
229
230 void
cairo_test_init(cairo_test_context_t * ctx,const char * test_name,const char * output)231 cairo_test_init (cairo_test_context_t *ctx,
232 const char *test_name,
233 const char *output)
234 {
235 _cairo_test_init (ctx, NULL, NULL, test_name, output);
236 }
237
238 void
cairo_test_fini(cairo_test_context_t * ctx)239 cairo_test_fini (cairo_test_context_t *ctx)
240 {
241 if (ctx->log_file == NULL)
242 return;
243
244 if (ctx->log_file != stderr)
245 fclose (ctx->log_file);
246 ctx->log_file = NULL;
247
248 free (ctx->ref_name);
249 cairo_surface_destroy (ctx->ref_image);
250 cairo_surface_destroy (ctx->ref_image_flattened);
251
252 if (ctx->test_name != NULL)
253 free ((char *) ctx->test_name);
254
255 if (ctx->own_targets)
256 cairo_boilerplate_free_targets (ctx->targets_to_test);
257
258 cairo_boilerplate_fini ();
259
260 cairo_debug_reset_static_data ();
261 #if HAVE_FCFINI
262 FcFini ();
263 #endif
264 }
265
266 void
cairo_test_logv(const cairo_test_context_t * ctx,const char * fmt,va_list va)267 cairo_test_logv (const cairo_test_context_t *ctx,
268 const char *fmt, va_list va)
269 {
270 FILE *file = ctx && ctx->log_file ? ctx->log_file : stderr;
271 vfprintf (file, fmt, va);
272 }
273
274 void
cairo_test_log(const cairo_test_context_t * ctx,const char * fmt,...)275 cairo_test_log (const cairo_test_context_t *ctx, const char *fmt, ...)
276 {
277 va_list va;
278
279 va_start (va, fmt);
280 cairo_test_logv (ctx, fmt, va);
281 va_end (va);
282 }
283
284 static void
_xunlink(const cairo_test_context_t * ctx,const char * pathname)285 _xunlink (const cairo_test_context_t *ctx, const char *pathname)
286 {
287 if (unlink (pathname) < 0 && errno != ENOENT) {
288 cairo_test_log (ctx, "Error: Cannot remove %s: %s\n",
289 pathname, strerror (errno));
290 exit (1);
291 }
292 }
293
294 char *
cairo_test_reference_filename(const cairo_test_context_t * ctx,const char * base_name,const char * test_name,const char * target_name,const char * base_target_name,const char * format,const char * suffix,const char * extension)295 cairo_test_reference_filename (const cairo_test_context_t *ctx,
296 const char *base_name,
297 const char *test_name,
298 const char *target_name,
299 const char *base_target_name,
300 const char *format,
301 const char *suffix,
302 const char *extension)
303 {
304 char *ref_name = NULL;
305
306 /* First look for a previous build for comparison. */
307 if (ctx->refdir != NULL && strcmp(suffix, CAIRO_TEST_REF_SUFFIX) == 0) {
308 xasprintf (&ref_name, "%s/%s" CAIRO_TEST_OUT_SUFFIX "%s",
309 ctx->refdir,
310 base_name,
311 extension);
312 if (access (ref_name, F_OK) != 0)
313 free (ref_name);
314 else
315 goto done;
316 }
317
318 if (target_name != NULL) {
319 /* Next look for a target/format-specific reference image. */
320 xasprintf (&ref_name, "%s/reference/%s.%s.%s%s%s",
321 ctx->srcdir,
322 test_name,
323 target_name,
324 format,
325 suffix,
326 extension);
327 if (access (ref_name, F_OK) != 0)
328 free (ref_name);
329 else
330 goto done;
331
332 /* Next, look for target-specific reference image. */
333 xasprintf (&ref_name, "%s/reference/%s.%s%s%s",
334 ctx->srcdir,
335 test_name,
336 target_name,
337 suffix,
338 extension);
339 if (access (ref_name, F_OK) != 0)
340 free (ref_name);
341 else
342 goto done;
343 }
344
345 if (base_target_name != NULL) {
346 /* Next look for a base/format-specific reference image. */
347 xasprintf (&ref_name, "%s/reference/%s.%s.%s%s%s",
348 ctx->srcdir,
349 test_name,
350 base_target_name,
351 format,
352 suffix,
353 extension);
354 if (access (ref_name, F_OK) != 0)
355 free (ref_name);
356 else
357 goto done;
358
359 /* Next, look for base-specific reference image. */
360 xasprintf (&ref_name, "%s/reference/%s.%s%s%s",
361 ctx->srcdir,
362 test_name,
363 base_target_name,
364 suffix,
365 extension);
366 if (access (ref_name, F_OK) != 0)
367 free (ref_name);
368 else
369 goto done;
370 }
371
372 /* Next, look for format-specific reference image. */
373 xasprintf (&ref_name, "%s/reference/%s.%s%s%s",
374 ctx->srcdir,
375 test_name,
376 format,
377 suffix,
378 extension);
379 if (access (ref_name, F_OK) != 0)
380 free (ref_name);
381 else
382 goto done;
383
384 /* Finally, look for the standard reference image. */
385 xasprintf (&ref_name, "%s/reference/%s%s%s", ctx->srcdir,
386 test_name,
387 suffix,
388 extension);
389 if (access (ref_name, F_OK) != 0)
390 free (ref_name);
391 else
392 goto done;
393
394 ref_name = NULL;
395
396 done:
397 return ref_name;
398 }
399
400 cairo_test_similar_t
cairo_test_target_has_similar(const cairo_test_context_t * ctx,const cairo_boilerplate_target_t * target)401 cairo_test_target_has_similar (const cairo_test_context_t *ctx,
402 const cairo_boilerplate_target_t *target)
403 {
404 cairo_surface_t *surface;
405 cairo_test_similar_t has_similar;
406 cairo_t * cr;
407 cairo_surface_t *similar;
408 cairo_status_t status;
409 void *closure;
410 char *path;
411
412 /* ignore image intermediate targets */
413 if (target->expected_type == CAIRO_SURFACE_TYPE_IMAGE)
414 return DIRECT;
415
416 if (getenv ("CAIRO_TEST_IGNORE_SIMILAR"))
417 return DIRECT;
418
419 xasprintf (&path, "%s/%s",
420 cairo_test_mkdir (ctx->output) ? ctx->output : ".",
421 ctx->test_name);
422
423 has_similar = DIRECT;
424 do {
425 do {
426 surface = (target->create_surface) (path,
427 target->content,
428 ctx->test->width,
429 ctx->test->height,
430 ctx->test->width* NUM_DEVICE_SCALE + 25 * NUM_DEVICE_OFFSETS,
431 ctx->test->height* NUM_DEVICE_SCALE + 25 * NUM_DEVICE_OFFSETS,
432 CAIRO_BOILERPLATE_MODE_TEST,
433 &closure);
434 if (surface == NULL)
435 goto out;
436 } while (cairo_test_malloc_failure (ctx, cairo_surface_status (surface)));
437
438 if (cairo_surface_status (surface))
439 goto out;
440
441 cr = cairo_create (surface);
442 cairo_push_group_with_content (cr,
443 cairo_boilerplate_content (target->content));
444 similar = cairo_get_group_target (cr);
445 status = cairo_surface_status (similar);
446
447 if (cairo_surface_get_type (similar) == cairo_surface_get_type (surface))
448 has_similar = SIMILAR;
449 else
450 has_similar = DIRECT;
451
452 cairo_destroy (cr);
453 cairo_surface_destroy (surface);
454
455 if (target->cleanup)
456 target->cleanup (closure);
457 } while (! has_similar && cairo_test_malloc_failure (ctx, status));
458 out:
459 free (path);
460
461 return has_similar;
462 }
463
464 static cairo_surface_t *
_cairo_test_flatten_reference_image(cairo_test_context_t * ctx,cairo_bool_t flatten)465 _cairo_test_flatten_reference_image (cairo_test_context_t *ctx,
466 cairo_bool_t flatten)
467 {
468 cairo_surface_t *surface;
469 cairo_t *cr;
470
471 if (! flatten)
472 return ctx->ref_image;
473
474 if (ctx->ref_image_flattened != NULL)
475 return ctx->ref_image_flattened;
476
477 surface = cairo_image_surface_create (CAIRO_FORMAT_ARGB32,
478 cairo_image_surface_get_width (ctx->ref_image),
479 cairo_image_surface_get_height (ctx->ref_image));
480 cr = cairo_create (surface);
481 cairo_surface_destroy (surface);
482
483 cairo_set_source_rgb (cr, 1, 1, 1);
484 cairo_paint (cr);
485
486 cairo_set_source_surface (cr, ctx->ref_image, 0, 0);
487 cairo_paint (cr);
488
489 surface = cairo_surface_reference (cairo_get_target (cr));
490 cairo_destroy (cr);
491
492 if (cairo_surface_status (surface) == CAIRO_STATUS_SUCCESS)
493 ctx->ref_image_flattened = surface;
494 return surface;
495 }
496
497 cairo_surface_t *
cairo_test_get_reference_image(cairo_test_context_t * ctx,const char * filename,cairo_bool_t flatten)498 cairo_test_get_reference_image (cairo_test_context_t *ctx,
499 const char *filename,
500 cairo_bool_t flatten)
501 {
502 cairo_surface_t *surface;
503
504 if (ctx->ref_name != NULL) {
505 if (strcmp (ctx->ref_name, filename) == 0)
506 return _cairo_test_flatten_reference_image (ctx, flatten);
507
508 cairo_surface_destroy (ctx->ref_image);
509 ctx->ref_image = NULL;
510
511 cairo_surface_destroy (ctx->ref_image_flattened);
512 ctx->ref_image_flattened = NULL;
513
514 free (ctx->ref_name);
515 ctx->ref_name = NULL;
516 }
517
518 surface = cairo_image_surface_create_from_png (filename);
519 if (cairo_surface_status (surface))
520 return surface;
521
522 ctx->ref_name = xstrdup (filename);
523 ctx->ref_image = surface;
524 return _cairo_test_flatten_reference_image (ctx, flatten);
525 }
526
527 static cairo_bool_t
cairo_test_file_is_older(const char * filename,char ** ref_filenames,int num_ref_filenames)528 cairo_test_file_is_older (const char *filename,
529 char **ref_filenames,
530 int num_ref_filenames)
531 {
532 #if HAVE_SYS_STAT_H
533 struct stat st;
534
535 if (stat (filename, &st) < 0)
536 return FALSE;
537
538 while (num_ref_filenames--) {
539 struct stat ref;
540 char *ref_filename = *ref_filenames++;
541
542 if (ref_filename == NULL)
543 continue;
544
545 if (stat (ref_filename++, &ref) < 0)
546 continue;
547
548 if (st.st_mtime <= ref.st_mtime)
549 return TRUE;
550 }
551 #endif
552
553 return FALSE;
554 }
555
556 static cairo_bool_t
cairo_test_files_equal(const char * test_filename,const char * pass_filename)557 cairo_test_files_equal (const char *test_filename,
558 const char *pass_filename)
559 {
560 FILE *test, *pass;
561 int t, p;
562
563 if (test_filename == NULL || pass_filename == NULL)
564 return FALSE;
565
566 test = fopen (test_filename, "rb");
567 if (test == NULL)
568 return FALSE;
569
570 pass = fopen (pass_filename, "rb");
571 if (pass == NULL) {
572 fclose (test);
573 return FALSE;
574 }
575
576 /* as simple as it gets */
577 do {
578 t = getc (test);
579 p = getc (pass);
580 if (t != p)
581 break;
582 } while (t != EOF && p != EOF);
583
584 fclose (pass);
585 fclose (test);
586
587 return t == p; /* both EOF */
588 }
589
590 static cairo_bool_t
cairo_test_copy_file(const char * src_filename,const char * dst_filename)591 cairo_test_copy_file (const char *src_filename,
592 const char *dst_filename)
593 {
594 FILE *src, *dst;
595 int c;
596
597 #if HAVE_LINK
598 if (link (src_filename, dst_filename) == 0)
599 return TRUE;
600
601 unlink (dst_filename);
602 #endif
603
604 src = fopen (src_filename, "rb");
605 if (src == NULL)
606 return FALSE;
607
608 dst = fopen (dst_filename, "wb");
609 if (dst == NULL) {
610 fclose (src);
611 return FALSE;
612 }
613
614 /* as simple as it gets */
615 while ((c = getc (src)) != EOF)
616 putc (c, dst);
617
618 fclose (src);
619 fclose (dst);
620
621 return TRUE;
622 }
623
624 static cairo_test_status_t
cairo_test_for_target(cairo_test_context_t * ctx,const cairo_boilerplate_target_t * target,int dev_offset,int dev_scale,cairo_bool_t similar)625 cairo_test_for_target (cairo_test_context_t *ctx,
626 const cairo_boilerplate_target_t *target,
627 int dev_offset,
628 int dev_scale,
629 cairo_bool_t similar)
630 {
631 cairo_test_status_t status;
632 cairo_surface_t *surface = NULL;
633 cairo_t *cr;
634 const char *empty_str = "";
635 char *offset_str;
636 char *scale_str;
637 char *base_name, *base_path;
638 char *out_png_path;
639 char *ref_path = NULL, *ref_png_path, *cmp_png_path = NULL;
640 char *new_path = NULL, *new_png_path;
641 char *xfail_path = NULL, *xfail_png_path;
642 char *base_ref_png_path;
643 char *base_new_png_path;
644 char *base_xfail_png_path;
645 char *diff_png_path;
646 char *test_filename = NULL, *pass_filename = NULL, *fail_filename = NULL;
647 cairo_test_status_t ret;
648 cairo_content_t expected_content;
649 cairo_font_options_t *font_options;
650 const char *format;
651 cairo_bool_t have_output = FALSE;
652 cairo_bool_t have_result = FALSE;
653 void *closure;
654 double width, height;
655 cairo_bool_t have_output_dir;
656 #if HAVE_MEMFAULT
657 int malloc_failure_iterations = ctx->malloc_failure;
658 int last_fault_count = 0;
659 #endif
660
661 /* Get the strings ready that we'll need. */
662 format = cairo_boilerplate_content_name (target->content);
663 if (dev_offset)
664 xasprintf (&offset_str, ".%d", dev_offset);
665 else
666 offset_str = (char *) empty_str;
667
668 if (dev_scale != 1)
669 xasprintf (&scale_str, ".x%d", dev_scale);
670 else
671 scale_str = (char *) empty_str;
672
673 xasprintf (&base_name, "%s.%s.%s%s%s%s",
674 ctx->test_name,
675 target->name,
676 format,
677 similar ? ".similar" : "",
678 offset_str,
679 scale_str);
680
681 if (offset_str != empty_str)
682 free (offset_str);
683 if (scale_str != empty_str)
684 free (scale_str);
685
686 ref_png_path = cairo_test_reference_filename (ctx,
687 base_name,
688 ctx->test_name,
689 target->name,
690 target->basename,
691 format,
692 CAIRO_TEST_REF_SUFFIX,
693 CAIRO_TEST_PNG_EXTENSION);
694 new_png_path = cairo_test_reference_filename (ctx,
695 base_name,
696 ctx->test_name,
697 target->name,
698 target->basename,
699 format,
700 CAIRO_TEST_NEW_SUFFIX,
701 CAIRO_TEST_PNG_EXTENSION);
702 xfail_png_path = cairo_test_reference_filename (ctx,
703 base_name,
704 ctx->test_name,
705 target->name,
706 target->basename,
707 format,
708 CAIRO_TEST_XFAIL_SUFFIX,
709 CAIRO_TEST_PNG_EXTENSION);
710
711 base_ref_png_path = cairo_test_reference_filename (ctx,
712 base_name,
713 ctx->test_name,
714 NULL, NULL,
715 format,
716 CAIRO_TEST_REF_SUFFIX,
717 CAIRO_TEST_PNG_EXTENSION);
718 base_new_png_path = cairo_test_reference_filename (ctx,
719 base_name,
720 ctx->test_name,
721 NULL, NULL,
722 format,
723 CAIRO_TEST_NEW_SUFFIX,
724 CAIRO_TEST_PNG_EXTENSION);
725 base_xfail_png_path = cairo_test_reference_filename (ctx,
726 base_name,
727 ctx->test_name,
728 NULL, NULL,
729 format,
730 CAIRO_TEST_XFAIL_SUFFIX,
731 CAIRO_TEST_PNG_EXTENSION);
732
733 if (target->file_extension != NULL) {
734 ref_path = cairo_test_reference_filename (ctx,
735 base_name,
736 ctx->test_name,
737 target->name,
738 target->basename,
739 format,
740 CAIRO_TEST_REF_SUFFIX,
741 target->file_extension);
742 new_path = cairo_test_reference_filename (ctx,
743 base_name,
744 ctx->test_name,
745 target->name,
746 target->basename,
747 format,
748 CAIRO_TEST_NEW_SUFFIX,
749 target->file_extension);
750 xfail_path = cairo_test_reference_filename (ctx,
751 base_name,
752 ctx->test_name,
753 target->name,
754 target->basename,
755 format,
756 CAIRO_TEST_XFAIL_SUFFIX,
757 target->file_extension);
758 }
759
760 have_output_dir = cairo_test_mkdir (ctx->output);
761 xasprintf (&base_path, "%s/%s",
762 have_output_dir ? ctx->output : ".",
763 base_name);
764 xasprintf (&out_png_path, "%s" CAIRO_TEST_OUT_PNG, base_path);
765 xasprintf (&diff_png_path, "%s" CAIRO_TEST_DIFF_PNG, base_path);
766
767 if (ctx->test->requirements != NULL) {
768 const char *required;
769
770 required = target->is_vector ? "target=raster" : "target=vector";
771 if (strstr (ctx->test->requirements, required) != NULL) {
772 cairo_test_log (ctx, "Error: Skipping for %s target %s\n",
773 target->is_vector ? "vector" : "raster",
774 target->name);
775 ret = CAIRO_TEST_UNTESTED;
776 goto UNWIND_STRINGS;
777 }
778
779 required = target->is_recording ? "target=!recording" : "target=recording";
780 if (strstr (ctx->test->requirements, required) != NULL) {
781 cairo_test_log (ctx, "Error: Skipping for %s target %s\n",
782 target->is_recording ? "recording" : "non-recording",
783 target->name);
784 ret = CAIRO_TEST_UNTESTED;
785 goto UNWIND_STRINGS;
786 }
787 }
788
789 width = ctx->test->width;
790 height = ctx->test->height;
791 if (width && height) {
792 width *= dev_scale;
793 height *= dev_scale;
794 width += dev_offset;
795 height += dev_offset;
796 }
797
798 #if HAVE_MEMFAULT
799 REPEAT:
800 MEMFAULT_CLEAR_FAULTS ();
801 MEMFAULT_RESET_LEAKS ();
802 ctx->last_fault_count = 0;
803 last_fault_count = MEMFAULT_COUNT_FAULTS ();
804
805 /* Pre-initialise fontconfig so that the configuration is loaded without
806 * malloc failures (our primary goal is to test cairo fault tolerance).
807 */
808 #if HAVE_FCINIT
809 FcInit ();
810 #endif
811
812 MEMFAULT_ENABLE_FAULTS ();
813 #endif
814 have_output = FALSE;
815 have_result = FALSE;
816
817 /* Run the actual drawing code. */
818 ret = CAIRO_TEST_SUCCESS;
819 surface = (target->create_surface) (base_path,
820 target->content,
821 width, height,
822 ctx->test->width * NUM_DEVICE_SCALE + 25 * NUM_DEVICE_OFFSETS,
823 ctx->test->height * NUM_DEVICE_SCALE + 25 * NUM_DEVICE_OFFSETS,
824 CAIRO_BOILERPLATE_MODE_TEST,
825 &closure);
826 if (surface == NULL) {
827 cairo_test_log (ctx, "Error: Failed to set %s target\n", target->name);
828 ret = CAIRO_TEST_UNTESTED;
829 goto UNWIND_STRINGS;
830 }
831
832 #if HAVE_MEMFAULT
833 if (ctx->malloc_failure &&
834 MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
835 cairo_surface_status (surface) == CAIRO_STATUS_NO_MEMORY)
836 {
837 goto REPEAT;
838 }
839 #endif
840
841 if (cairo_surface_status (surface)) {
842 MF (MEMFAULT_PRINT_FAULTS ());
843 cairo_test_log (ctx, "Error: Created an error surface: %s\n",
844 cairo_status_to_string (cairo_surface_status (surface)));
845 ret = CAIRO_TEST_FAILURE;
846 goto UNWIND_STRINGS;
847 }
848
849 /* Check that we created a surface of the expected type. */
850 if (cairo_surface_get_type (surface) != target->expected_type) {
851 MF (MEMFAULT_PRINT_FAULTS ());
852 cairo_test_log (ctx, "Error: Created surface is of type %d (expected %d)\n",
853 cairo_surface_get_type (surface), target->expected_type);
854 ret = CAIRO_TEST_UNTESTED;
855 goto UNWIND_SURFACE;
856 }
857
858 /* Check that we created a surface of the expected content,
859 * (ignore the artificial CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED value).
860 */
861 expected_content = cairo_boilerplate_content (target->content);
862
863 if (cairo_surface_get_content (surface) != expected_content) {
864 MF (MEMFAULT_PRINT_FAULTS ());
865 cairo_test_log (ctx, "Error: Created surface has content %d (expected %d)\n",
866 cairo_surface_get_content (surface), expected_content);
867 ret = CAIRO_TEST_FAILURE;
868 goto UNWIND_SURFACE;
869 }
870
871 if (cairo_surface_set_user_data (surface,
872 &cairo_boilerplate_output_basename_key,
873 base_path,
874 NULL))
875 {
876 #if HAVE_MEMFAULT
877 cairo_surface_destroy (surface);
878
879 if (target->cleanup)
880 target->cleanup (closure);
881
882 goto REPEAT;
883 #else
884 ret = CAIRO_TEST_FAILURE;
885 goto UNWIND_SURFACE;
886 #endif
887 }
888
889 cairo_surface_set_device_offset (surface, dev_offset, dev_offset);
890 cairo_surface_set_device_scale (surface, dev_scale, dev_scale);
891
892 cr = cairo_create (surface);
893 if (cairo_set_user_data (cr, &_cairo_test_context_key, (void*) ctx, NULL)) {
894 #if HAVE_MEMFAULT
895 cairo_destroy (cr);
896 cairo_surface_destroy (surface);
897
898 if (target->cleanup)
899 target->cleanup (closure);
900
901 goto REPEAT;
902 #else
903 ret = CAIRO_TEST_FAILURE;
904 goto UNWIND_CAIRO;
905 #endif
906 }
907
908 if (similar)
909 cairo_push_group_with_content (cr, expected_content);
910
911 /* Clear to transparent (or black) depending on whether the target
912 * surface supports alpha. */
913 cairo_save (cr);
914 cairo_set_operator (cr, CAIRO_OPERATOR_CLEAR);
915 cairo_paint (cr);
916 cairo_restore (cr);
917
918 cairo_select_font_face (cr, CAIRO_TEST_FONT_FAMILY " Sans",
919 CAIRO_FONT_SLANT_NORMAL,
920 CAIRO_FONT_WEIGHT_NORMAL);
921
922 /* Set all components of font_options to avoid backend differences
923 * and reduce number of needed reference images. */
924 font_options = cairo_font_options_create ();
925 cairo_font_options_set_hint_style (font_options, CAIRO_HINT_STYLE_NONE);
926 cairo_font_options_set_hint_metrics (font_options, CAIRO_HINT_METRICS_ON);
927 cairo_font_options_set_antialias (font_options, CAIRO_ANTIALIAS_GRAY);
928 cairo_set_font_options (cr, font_options);
929 cairo_font_options_destroy (font_options);
930
931 cairo_save (cr);
932 alarm (ctx->timeout);
933 status = (ctx->test->draw) (cr, ctx->test->width, ctx->test->height);
934 alarm (0);
935 cairo_restore (cr);
936
937 if (similar) {
938 cairo_pop_group_to_source (cr);
939 cairo_set_operator (cr, CAIRO_OPERATOR_SOURCE);
940 cairo_paint (cr);
941 }
942
943 #if HAVE_MEMFAULT
944 MEMFAULT_DISABLE_FAULTS ();
945
946 /* repeat test after malloc failure injection */
947 if (ctx->malloc_failure &&
948 MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
949 (status == CAIRO_TEST_NO_MEMORY ||
950 cairo_status (cr) == CAIRO_STATUS_NO_MEMORY ||
951 cairo_surface_status (surface) == CAIRO_STATUS_NO_MEMORY))
952 {
953 cairo_destroy (cr);
954 cairo_surface_destroy (surface);
955 if (target->cleanup)
956 target->cleanup (closure);
957 cairo_debug_reset_static_data ();
958 #if HAVE_FCFINI
959 FcFini ();
960 #endif
961 if (MEMFAULT_COUNT_LEAKS () > 0) {
962 MEMFAULT_PRINT_FAULTS ();
963 MEMFAULT_PRINT_LEAKS ();
964 }
965
966 goto REPEAT;
967 }
968 #endif
969
970 /* Then, check all the different ways it could fail. */
971 if (status) {
972 cairo_test_log (ctx, "Error: Function under test failed\n");
973 ret = status;
974 goto UNWIND_CAIRO;
975 }
976
977 #if HAVE_MEMFAULT
978 if (MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
979 MEMFAULT_HAS_FAULTS ())
980 {
981 VALGRIND_PRINTF ("Unreported memfaults...");
982 MEMFAULT_PRINT_FAULTS ();
983 }
984 #endif
985
986 if (target->finish_surface != NULL) {
987 #if HAVE_MEMFAULT
988 /* We need to re-enable faults as most recording-surface processing
989 * is done during cairo_surface_finish().
990 */
991 MEMFAULT_CLEAR_FAULTS ();
992 last_fault_count = MEMFAULT_COUNT_FAULTS ();
993 MEMFAULT_ENABLE_FAULTS ();
994 #endif
995
996 /* also check for infinite loops whilst replaying */
997 alarm (ctx->timeout);
998 status = target->finish_surface (surface);
999 alarm (0);
1000
1001 #if HAVE_MEMFAULT
1002 MEMFAULT_DISABLE_FAULTS ();
1003
1004 if (ctx->malloc_failure &&
1005 MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
1006 status == CAIRO_STATUS_NO_MEMORY)
1007 {
1008 cairo_destroy (cr);
1009 cairo_surface_destroy (surface);
1010 if (target->cleanup)
1011 target->cleanup (closure);
1012 cairo_debug_reset_static_data ();
1013 #if HAVE_FCFINI
1014 FcFini ();
1015 #endif
1016 if (MEMFAULT_COUNT_LEAKS () > 0) {
1017 MEMFAULT_PRINT_FAULTS ();
1018 MEMFAULT_PRINT_LEAKS ();
1019 }
1020
1021 goto REPEAT;
1022 }
1023 #endif
1024 if (status) {
1025 cairo_test_log (ctx, "Error: Failed to finish surface: %s\n",
1026 cairo_status_to_string (status));
1027 ret = CAIRO_TEST_FAILURE;
1028 goto UNWIND_CAIRO;
1029 }
1030 }
1031
1032 /* Skip image check for tests with no image (width,height == 0,0) */
1033 if (ctx->test->width != 0 && ctx->test->height != 0) {
1034 cairo_surface_t *ref_image;
1035 cairo_surface_t *test_image;
1036 cairo_surface_t *diff_image;
1037 buffer_diff_result_t result;
1038 cairo_status_t diff_status;
1039
1040 if (ref_png_path == NULL) {
1041 cairo_test_log (ctx, "Error: Cannot find reference image for %s\n",
1042 base_name);
1043
1044 /* we may be running this test to generate reference images */
1045 _xunlink (ctx, out_png_path);
1046 /* be more generous as we may need to use external renderers */
1047 alarm (4 * ctx->timeout);
1048 test_image = target->get_image_surface (surface, 0,
1049 ctx->test->width,
1050 ctx->test->height);
1051 alarm (0);
1052 diff_status = cairo_surface_write_to_png (test_image, out_png_path);
1053 cairo_surface_destroy (test_image);
1054 if (diff_status) {
1055 if (cairo_surface_status (test_image) == CAIRO_STATUS_INVALID_STATUS)
1056 ret = CAIRO_TEST_CRASHED;
1057 else
1058 ret = CAIRO_TEST_FAILURE;
1059 cairo_test_log (ctx,
1060 "Error: Failed to write output image: %s\n",
1061 cairo_status_to_string (diff_status));
1062 }
1063 have_output = TRUE;
1064
1065 ret = CAIRO_TEST_XFAILURE;
1066 goto UNWIND_CAIRO;
1067 }
1068
1069 if (target->file_extension != NULL) { /* compare vector surfaces */
1070 char *filenames[] = {
1071 ref_png_path,
1072 ref_path,
1073 new_png_path,
1074 new_path,
1075 xfail_png_path,
1076 xfail_path,
1077 base_ref_png_path,
1078 base_new_png_path,
1079 base_xfail_png_path,
1080 };
1081
1082 xasprintf (&test_filename, "%s.out%s",
1083 base_path, target->file_extension);
1084 xasprintf (&pass_filename, "%s.pass%s",
1085 base_path, target->file_extension);
1086 xasprintf (&fail_filename, "%s.fail%s",
1087 base_path, target->file_extension);
1088
1089 if (cairo_test_file_is_older (pass_filename,
1090 filenames,
1091 ARRAY_LENGTH (filenames)))
1092 {
1093 _xunlink (ctx, pass_filename);
1094 }
1095 if (cairo_test_file_is_older (fail_filename,
1096 filenames,
1097 ARRAY_LENGTH (filenames)))
1098 {
1099 _xunlink (ctx, fail_filename);
1100 }
1101
1102 if (cairo_test_files_equal (out_png_path, ref_path)) {
1103 cairo_test_log (ctx, "Vector surface matches reference.\n");
1104 have_output = FALSE;
1105 ret = CAIRO_TEST_SUCCESS;
1106 goto UNWIND_CAIRO;
1107 }
1108 if (cairo_test_files_equal (out_png_path, new_path)) {
1109 cairo_test_log (ctx, "Vector surface matches current failure.\n");
1110 have_output = FALSE;
1111 ret = CAIRO_TEST_NEW;
1112 goto UNWIND_CAIRO;
1113 }
1114 if (cairo_test_files_equal (out_png_path, xfail_path)) {
1115 cairo_test_log (ctx, "Vector surface matches known failure.\n");
1116 have_output = FALSE;
1117 ret = CAIRO_TEST_XFAILURE;
1118 goto UNWIND_CAIRO;
1119 }
1120
1121 if (cairo_test_files_equal (test_filename, pass_filename)) {
1122 /* identical output as last known PASS */
1123 cairo_test_log (ctx, "Vector surface matches last pass.\n");
1124 have_output = TRUE;
1125 ret = CAIRO_TEST_SUCCESS;
1126 goto UNWIND_CAIRO;
1127 }
1128 if (cairo_test_files_equal (test_filename, fail_filename)) {
1129 /* identical output as last known FAIL, fail */
1130 cairo_test_log (ctx, "Vector surface matches last fail.\n");
1131 have_result = TRUE; /* presume these were kept around as well */
1132 have_output = TRUE;
1133 ret = CAIRO_TEST_FAILURE;
1134 goto UNWIND_CAIRO;
1135 }
1136 }
1137
1138 /* be more generous as we may need to use external renderers */
1139 alarm (4 * ctx->timeout);
1140 test_image = target->get_image_surface (surface, 0,
1141 ctx->test->width,
1142 ctx->test->height);
1143 alarm (0);
1144 if (cairo_surface_status (test_image)) {
1145 cairo_test_log (ctx, "Error: Failed to extract image: %s\n",
1146 cairo_status_to_string (cairo_surface_status (test_image)));
1147 if (cairo_surface_status (test_image) == CAIRO_STATUS_INVALID_STATUS)
1148 ret = CAIRO_TEST_CRASHED;
1149 else
1150 ret = CAIRO_TEST_FAILURE;
1151 cairo_surface_destroy (test_image);
1152 goto UNWIND_CAIRO;
1153 }
1154
1155 _xunlink (ctx, out_png_path);
1156 diff_status = cairo_surface_write_to_png (test_image, out_png_path);
1157 if (diff_status) {
1158 cairo_test_log (ctx, "Error: Failed to write output image: %s\n",
1159 cairo_status_to_string (diff_status));
1160 cairo_surface_destroy (test_image);
1161 ret = CAIRO_TEST_FAILURE;
1162 goto UNWIND_CAIRO;
1163 }
1164 have_output = TRUE;
1165
1166 /* binary compare png files (no decompression) */
1167 if (target->file_extension == NULL) {
1168 char *filenames[] = {
1169 ref_png_path,
1170 new_png_path,
1171 xfail_png_path,
1172 base_ref_png_path,
1173 base_new_png_path,
1174 base_xfail_png_path,
1175 };
1176
1177 xasprintf (&test_filename, "%s", out_png_path);
1178 xasprintf (&pass_filename, "%s.pass.png", base_path);
1179 xasprintf (&fail_filename, "%s.fail.png", base_path);
1180
1181 if (cairo_test_file_is_older (pass_filename,
1182 filenames,
1183 ARRAY_LENGTH (filenames)))
1184 {
1185 _xunlink (ctx, pass_filename);
1186 }
1187 if (cairo_test_file_is_older (fail_filename,
1188 filenames,
1189 ARRAY_LENGTH (filenames)))
1190 {
1191 _xunlink (ctx, fail_filename);
1192 }
1193
1194 if (cairo_test_files_equal (test_filename, pass_filename)) {
1195 cairo_test_log (ctx, "PNG file exactly matches last pass.\n");
1196 have_result = TRUE;
1197 cairo_surface_destroy (test_image);
1198 ret = CAIRO_TEST_SUCCESS;
1199 goto UNWIND_CAIRO;
1200 }
1201 if (cairo_test_files_equal (out_png_path, ref_png_path)) {
1202 cairo_test_log (ctx, "PNG file exactly matches reference image.\n");
1203 have_result = TRUE;
1204 cairo_surface_destroy (test_image);
1205 ret = CAIRO_TEST_SUCCESS;
1206 goto UNWIND_CAIRO;
1207 }
1208 if (cairo_test_files_equal (out_png_path, new_png_path)) {
1209 cairo_test_log (ctx, "PNG file exactly matches current failure image.\n");
1210 have_result = TRUE;
1211 cairo_surface_destroy (test_image);
1212 ret = CAIRO_TEST_NEW;
1213 goto UNWIND_CAIRO;
1214 }
1215 if (cairo_test_files_equal (out_png_path, xfail_png_path)) {
1216 cairo_test_log (ctx, "PNG file exactly matches known failure image.\n");
1217 have_result = TRUE;
1218 cairo_surface_destroy (test_image);
1219 ret = CAIRO_TEST_XFAILURE;
1220 goto UNWIND_CAIRO;
1221 }
1222 if (cairo_test_files_equal (test_filename, fail_filename)) {
1223 cairo_test_log (ctx, "PNG file exactly matches last fail.\n");
1224 have_result = TRUE; /* presume these were kept around as well */
1225 cairo_surface_destroy (test_image);
1226 ret = CAIRO_TEST_FAILURE;
1227 goto UNWIND_CAIRO;
1228 }
1229 } else {
1230 if (cairo_test_files_equal (out_png_path, ref_png_path)) {
1231 cairo_test_log (ctx, "PNG file exactly matches reference image.\n");
1232 have_result = TRUE;
1233 cairo_surface_destroy (test_image);
1234 ret = CAIRO_TEST_SUCCESS;
1235 goto UNWIND_CAIRO;
1236 }
1237 if (cairo_test_files_equal (out_png_path, new_png_path)) {
1238 cairo_test_log (ctx, "PNG file exactly matches current failure image.\n");
1239 have_result = TRUE;
1240 cairo_surface_destroy (test_image);
1241 ret = CAIRO_TEST_NEW;
1242 goto UNWIND_CAIRO;
1243 }
1244 if (cairo_test_files_equal (out_png_path, xfail_png_path)) {
1245 cairo_test_log (ctx, "PNG file exactly matches known failure image.\n");
1246 have_result = TRUE;
1247 cairo_surface_destroy (test_image);
1248 ret = CAIRO_TEST_XFAILURE;
1249 goto UNWIND_CAIRO;
1250 }
1251 }
1252
1253 if (cairo_test_files_equal (out_png_path, base_ref_png_path)) {
1254 cairo_test_log (ctx, "PNG file exactly reference image.\n");
1255 have_result = TRUE;
1256 cairo_surface_destroy (test_image);
1257 ret = CAIRO_TEST_SUCCESS;
1258 goto UNWIND_CAIRO;
1259 }
1260 if (cairo_test_files_equal (out_png_path, base_new_png_path)) {
1261 cairo_test_log (ctx, "PNG file exactly current failure image.\n");
1262 have_result = TRUE;
1263 cairo_surface_destroy (test_image);
1264 ret = CAIRO_TEST_NEW;
1265 goto UNWIND_CAIRO;
1266 }
1267 if (cairo_test_files_equal (out_png_path, base_xfail_png_path)) {
1268 cairo_test_log (ctx, "PNG file exactly known failure image.\n");
1269 have_result = TRUE;
1270 cairo_surface_destroy (test_image);
1271 ret = CAIRO_TEST_XFAILURE;
1272 goto UNWIND_CAIRO;
1273 }
1274
1275 /* first compare against the ideal reference */
1276 ref_image = cairo_test_get_reference_image (ctx, base_ref_png_path,
1277 target->content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED);
1278 if (cairo_surface_status (ref_image)) {
1279 cairo_test_log (ctx, "Error: Cannot open reference image for %s: %s\n",
1280 base_ref_png_path,
1281 cairo_status_to_string (cairo_surface_status (ref_image)));
1282 cairo_surface_destroy (test_image);
1283 ret = CAIRO_TEST_FAILURE;
1284 goto UNWIND_CAIRO;
1285 }
1286
1287 diff_image = cairo_image_surface_create (CAIRO_FORMAT_ARGB32,
1288 ctx->test->width,
1289 ctx->test->height);
1290
1291 cmp_png_path = base_ref_png_path;
1292 diff_status = image_diff (ctx,
1293 test_image, ref_image, diff_image,
1294 &result);
1295 _xunlink (ctx, diff_png_path);
1296 if (diff_status ||
1297 image_diff_is_failure (&result, target->error_tolerance))
1298 {
1299 /* that failed, so check against the specific backend */
1300 ref_image = cairo_test_get_reference_image (ctx, ref_png_path,
1301 target->content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED);
1302 if (cairo_surface_status (ref_image)) {
1303 cairo_test_log (ctx, "Error: Cannot open reference image for %s: %s\n",
1304 ref_png_path,
1305 cairo_status_to_string (cairo_surface_status (ref_image)));
1306 cairo_surface_destroy (test_image);
1307 ret = CAIRO_TEST_FAILURE;
1308 goto UNWIND_CAIRO;
1309 }
1310
1311 cmp_png_path = ref_png_path;
1312 diff_status = image_diff (ctx,
1313 test_image, ref_image,
1314 diff_image,
1315 &result);
1316 if (diff_status)
1317 {
1318 cairo_test_log (ctx, "Error: Failed to compare images: %s\n",
1319 cairo_status_to_string (diff_status));
1320 ret = CAIRO_TEST_FAILURE;
1321 }
1322 else if (image_diff_is_failure (&result, target->error_tolerance))
1323 {
1324 ret = CAIRO_TEST_FAILURE;
1325
1326 diff_status = cairo_surface_write_to_png (diff_image,
1327 diff_png_path);
1328 if (diff_status) {
1329 cairo_test_log (ctx, "Error: Failed to write differences image: %s\n",
1330 cairo_status_to_string (diff_status));
1331 } else {
1332 have_result = TRUE;
1333 }
1334
1335 cairo_test_copy_file (test_filename, fail_filename);
1336 }
1337 else
1338 { /* success */
1339 cairo_test_copy_file (test_filename, pass_filename);
1340 }
1341 }
1342 else
1343 { /* success */
1344 cairo_test_copy_file (test_filename, pass_filename);
1345 }
1346
1347 /* If failed, compare against the current image output,
1348 * and attempt to detect systematic failures.
1349 */
1350 if (ret == CAIRO_TEST_FAILURE) {
1351 char *image_out_path;
1352
1353 image_out_path =
1354 cairo_test_reference_filename (ctx,
1355 base_name,
1356 ctx->test_name,
1357 "image",
1358 "image",
1359 format,
1360 CAIRO_TEST_OUT_SUFFIX,
1361 CAIRO_TEST_PNG_EXTENSION);
1362 if (image_out_path != NULL) {
1363 if (cairo_test_files_equal (out_png_path,
1364 image_out_path))
1365 {
1366 ret = CAIRO_TEST_XFAILURE;
1367 }
1368 else
1369 {
1370 ref_image =
1371 cairo_image_surface_create_from_png (image_out_path);
1372 if (cairo_surface_status (ref_image) == CAIRO_STATUS_SUCCESS)
1373 {
1374 diff_status = image_diff (ctx,
1375 test_image, ref_image,
1376 diff_image,
1377 &result);
1378 if (diff_status == CAIRO_STATUS_SUCCESS &&
1379 !image_diff_is_failure (&result, target->error_tolerance))
1380 {
1381 ret = CAIRO_TEST_XFAILURE;
1382 }
1383
1384 cairo_surface_destroy (ref_image);
1385 }
1386 }
1387
1388 free (image_out_path);
1389 }
1390 }
1391
1392 cairo_surface_destroy (test_image);
1393 cairo_surface_destroy (diff_image);
1394 }
1395
1396 if (cairo_status (cr) != CAIRO_STATUS_SUCCESS) {
1397 cairo_test_log (ctx, "Error: Function under test left cairo status in an error state: %s\n",
1398 cairo_status_to_string (cairo_status (cr)));
1399 ret = CAIRO_TEST_ERROR;
1400 goto UNWIND_CAIRO;
1401 }
1402
1403 UNWIND_CAIRO:
1404 free (test_filename);
1405 free (fail_filename);
1406 free (pass_filename);
1407
1408 test_filename = fail_filename = pass_filename = NULL;
1409
1410 #if HAVE_MEMFAULT
1411 if (ret == CAIRO_TEST_FAILURE)
1412 MEMFAULT_PRINT_FAULTS ();
1413 #endif
1414 cairo_destroy (cr);
1415 UNWIND_SURFACE:
1416 cairo_surface_destroy (surface);
1417
1418 if (target->cleanup)
1419 target->cleanup (closure);
1420
1421 #if HAVE_MEMFAULT
1422 cairo_debug_reset_static_data ();
1423
1424 #if HAVE_FCFINI
1425 FcFini ();
1426 #endif
1427
1428 if (MEMFAULT_COUNT_LEAKS () > 0) {
1429 if (ret != CAIRO_TEST_FAILURE)
1430 MEMFAULT_PRINT_FAULTS ();
1431 MEMFAULT_PRINT_LEAKS ();
1432 }
1433
1434 if (ret == CAIRO_TEST_SUCCESS && --malloc_failure_iterations > 0)
1435 goto REPEAT;
1436 #endif
1437
1438 if (have_output)
1439 cairo_test_log (ctx, "OUTPUT: %s\n", out_png_path);
1440
1441 if (have_result) {
1442 if (cmp_png_path == NULL) {
1443 /* XXX presume we matched the normal ref last time */
1444 cmp_png_path = ref_png_path;
1445 }
1446 cairo_test_log (ctx,
1447 "REFERENCE: %s\nDIFFERENCE: %s\n",
1448 cmp_png_path, diff_png_path);
1449 }
1450
1451 UNWIND_STRINGS:
1452 free (out_png_path);
1453 free (ref_png_path);
1454 free (base_ref_png_path);
1455 free (ref_path);
1456 free (new_png_path);
1457 free (base_new_png_path);
1458 free (new_path);
1459 free (xfail_png_path);
1460 free (base_xfail_png_path);
1461 free (xfail_path);
1462 free (diff_png_path);
1463 free (base_path);
1464 free (base_name);
1465
1466 return ret;
1467 }
1468
1469 #if defined(HAVE_SIGNAL_H) && defined(HAVE_SETJMP_H)
1470 #include <signal.h>
1471 #include <setjmp.h>
1472 /* Used to catch crashes in a test, so that we report it as such and
1473 * continue testing, although one crash may already have corrupted memory in
1474 * an nonrecoverable fashion. */
1475 static jmp_buf jmpbuf;
1476
1477 static void
segfault_handler(int signal)1478 segfault_handler (int signal)
1479 {
1480 longjmp (jmpbuf, signal);
1481 }
1482 #endif
1483
1484 cairo_test_status_t
_cairo_test_context_run_for_target(cairo_test_context_t * ctx,const cairo_boilerplate_target_t * target,cairo_bool_t similar,int dev_offset,int dev_scale)1485 _cairo_test_context_run_for_target (cairo_test_context_t *ctx,
1486 const cairo_boilerplate_target_t *target,
1487 cairo_bool_t similar,
1488 int dev_offset, int dev_scale)
1489 {
1490 cairo_test_status_t status;
1491
1492 if (target->get_image_surface == NULL)
1493 return CAIRO_TEST_UNTESTED;
1494
1495 if (similar && ! cairo_test_target_has_similar (ctx, target))
1496 return CAIRO_TEST_UNTESTED;
1497
1498 cairo_test_log (ctx,
1499 "Testing %s with %s%s target (dev offset %d scale: %d)\n",
1500 ctx->test_name,
1501 similar ? " (similar) " : "",
1502 target->name,
1503 dev_offset, dev_scale);
1504
1505 printf ("%s.%s.%s [%dx%d]%s:\t", ctx->test_name, target->name,
1506 cairo_boilerplate_content_name (target->content),
1507 dev_offset, dev_scale,
1508 similar ? " (similar)": "");
1509 fflush (stdout);
1510
1511 #if defined(HAVE_SIGNAL_H) && defined(HAVE_SETJMP_H)
1512 if (! RUNNING_ON_VALGRIND) {
1513 void (* volatile old_segfault_handler)(int);
1514 void (* volatile old_segfpe_handler)(int);
1515 void (* volatile old_sigpipe_handler)(int);
1516 void (* volatile old_sigabrt_handler)(int);
1517 void (* volatile old_sigalrm_handler)(int);
1518
1519 /* Set up a checkpoint to get back to in case of segfaults. */
1520 #ifdef SIGSEGV
1521 old_segfault_handler = signal (SIGSEGV, segfault_handler);
1522 #endif
1523 #ifdef SIGFPE
1524 old_segfpe_handler = signal (SIGFPE, segfault_handler);
1525 #endif
1526 #ifdef SIGPIPE
1527 old_sigpipe_handler = signal (SIGPIPE, segfault_handler);
1528 #endif
1529 #ifdef SIGABRT
1530 old_sigabrt_handler = signal (SIGABRT, segfault_handler);
1531 #endif
1532 #ifdef SIGALRM
1533 old_sigalrm_handler = signal (SIGALRM, segfault_handler);
1534 #endif
1535 if (0 == setjmp (jmpbuf))
1536 status = cairo_test_for_target (ctx, target, dev_offset, dev_scale, similar);
1537 else
1538 status = CAIRO_TEST_CRASHED;
1539 #ifdef SIGSEGV
1540 signal (SIGSEGV, old_segfault_handler);
1541 #endif
1542 #ifdef SIGFPE
1543 signal (SIGFPE, old_segfpe_handler);
1544 #endif
1545 #ifdef SIGPIPE
1546 signal (SIGPIPE, old_sigpipe_handler);
1547 #endif
1548 #ifdef SIGABRT
1549 signal (SIGABRT, old_sigabrt_handler);
1550 #endif
1551 #ifdef SIGALRM
1552 signal (SIGALRM, old_sigalrm_handler);
1553 #endif
1554 } else {
1555 status = cairo_test_for_target (ctx, target, dev_offset, dev_scale, similar);
1556 }
1557 #else
1558 status = cairo_test_for_target (ctx, target, dev_offset, dev_scale, similar);
1559 #endif
1560
1561 cairo_test_log (ctx,
1562 "TEST: %s TARGET: %s FORMAT: %s OFFSET: %d SCALE: %d SIMILAR: %d RESULT: ",
1563 ctx->test_name, target->name,
1564 cairo_boilerplate_content_name (target->content),
1565 dev_offset, dev_scale, similar);
1566 switch (status) {
1567 case CAIRO_TEST_SUCCESS:
1568 printf ("PASS\n");
1569 cairo_test_log (ctx, "PASS\n");
1570 break;
1571
1572 case CAIRO_TEST_UNTESTED:
1573 printf ("UNTESTED\n");
1574 cairo_test_log (ctx, "UNTESTED\n");
1575 break;
1576
1577 default:
1578 case CAIRO_TEST_CRASHED:
1579 if (print_fail_on_stdout) {
1580 printf ("!!!CRASHED!!!\n");
1581 } else {
1582 /* eat the test name */
1583 printf ("\r");
1584 fflush (stdout);
1585 }
1586 cairo_test_log (ctx, "CRASHED\n");
1587 fprintf (stderr, "%s.%s.%s [%dx%d]%s:\t%s!!!CRASHED!!!%s\n",
1588 ctx->test_name, target->name,
1589 cairo_boilerplate_content_name (target->content), dev_offset, dev_scale, similar ? " (similar)" : "",
1590 fail_face, normal_face);
1591 break;
1592
1593 case CAIRO_TEST_ERROR:
1594 if (print_fail_on_stdout) {
1595 printf ("!!!ERROR!!!\n");
1596 } else {
1597 /* eat the test name */
1598 printf ("\r");
1599 fflush (stdout);
1600 }
1601 cairo_test_log (ctx, "ERROR\n");
1602 fprintf (stderr, "%s.%s.%s [%dx%d]%s:\t%s!!!ERROR!!!%s\n",
1603 ctx->test_name, target->name,
1604 cairo_boilerplate_content_name (target->content), dev_offset, dev_scale, similar ? " (similar)" : "",
1605 fail_face, normal_face);
1606 break;
1607
1608 case CAIRO_TEST_XFAILURE:
1609 if (print_fail_on_stdout) {
1610 printf ("XFAIL\n");
1611 } else {
1612 /* eat the test name */
1613 printf ("\r");
1614 fflush (stdout);
1615 }
1616 fprintf (stderr, "%s.%s.%s [%dx%d]%s:\t%sXFAIL%s\n",
1617 ctx->test_name, target->name,
1618 cairo_boilerplate_content_name (target->content), dev_offset, dev_scale, similar ? " (similar)" : "",
1619 xfail_face, normal_face);
1620 cairo_test_log (ctx, "XFAIL\n");
1621 break;
1622
1623 case CAIRO_TEST_NEW:
1624 if (print_fail_on_stdout) {
1625 printf ("NEW\n");
1626 } else {
1627 /* eat the test name */
1628 printf ("\r");
1629 fflush (stdout);
1630 }
1631 fprintf (stderr, "%s.%s.%s [%dx%d]%s:\t%sNEW%s\n",
1632 ctx->test_name, target->name,
1633 cairo_boilerplate_content_name (target->content), dev_offset, dev_scale, similar ? " (similar)" : "",
1634 fail_face, normal_face);
1635 cairo_test_log (ctx, "NEW\n");
1636 break;
1637
1638 case CAIRO_TEST_NO_MEMORY:
1639 case CAIRO_TEST_FAILURE:
1640 if (print_fail_on_stdout) {
1641 printf ("FAIL\n");
1642 } else {
1643 /* eat the test name */
1644 printf ("\r");
1645 fflush (stdout);
1646 }
1647 fprintf (stderr, "%s.%s.%s [%dx%d]%s:\t%sFAIL%s\n",
1648 ctx->test_name, target->name,
1649 cairo_boilerplate_content_name (target->content), dev_offset, dev_scale, similar ? " (similar)" : "",
1650 fail_face, normal_face);
1651 cairo_test_log (ctx, "FAIL\n");
1652 break;
1653 }
1654 fflush (stdout);
1655
1656 return status;
1657 }
1658
1659 const cairo_test_context_t *
cairo_test_get_context(cairo_t * cr)1660 cairo_test_get_context (cairo_t *cr)
1661 {
1662 return cairo_get_user_data (cr, &_cairo_test_context_key);
1663 }
1664
1665 cairo_t *
cairo_test_create(cairo_surface_t * surface,const cairo_test_context_t * ctx)1666 cairo_test_create (cairo_surface_t *surface,
1667 const cairo_test_context_t *ctx)
1668 {
1669 cairo_t *cr = cairo_create (surface);
1670 cairo_set_user_data (cr, &_cairo_test_context_key,
1671 (void*) ctx, NULL);
1672 return cr;
1673 }
1674
1675 cairo_surface_t *
cairo_test_create_surface_from_png(const cairo_test_context_t * ctx,const char * filename)1676 cairo_test_create_surface_from_png (const cairo_test_context_t *ctx,
1677 const char *filename)
1678 {
1679 cairo_surface_t *image;
1680 cairo_status_t status;
1681 char *unique_id;
1682
1683 image = cairo_image_surface_create_from_png (filename);
1684 status = cairo_surface_status (image);
1685 if (status == CAIRO_STATUS_FILE_NOT_FOUND) {
1686 /* expect not found when running with srcdir != builddir
1687 * such as when 'make distcheck' is run
1688 */
1689 if (ctx->srcdir) {
1690 char *srcdir_filename;
1691 xasprintf (&srcdir_filename, "%s/%s", ctx->srcdir, filename);
1692 cairo_surface_destroy (image);
1693 image = cairo_image_surface_create_from_png (srcdir_filename);
1694 free (srcdir_filename);
1695 }
1696 }
1697 unique_id = strdup(filename);
1698 cairo_surface_set_mime_data (image, CAIRO_MIME_TYPE_UNIQUE_ID,
1699 (unsigned char *)unique_id, strlen(unique_id),
1700 free, unique_id);
1701
1702 return image;
1703 }
1704
1705 cairo_pattern_t *
cairo_test_create_pattern_from_png(const cairo_test_context_t * ctx,const char * filename)1706 cairo_test_create_pattern_from_png (const cairo_test_context_t *ctx,
1707 const char *filename)
1708 {
1709 cairo_surface_t *image;
1710 cairo_pattern_t *pattern;
1711
1712 image = cairo_test_create_surface_from_png (ctx, filename);
1713
1714 pattern = cairo_pattern_create_for_surface (image);
1715
1716 cairo_pattern_set_extend (pattern, CAIRO_EXTEND_REPEAT);
1717
1718 cairo_surface_destroy (image);
1719
1720 return pattern;
1721 }
1722
1723 static cairo_surface_t *
_draw_check(int width,int height)1724 _draw_check (int width, int height)
1725 {
1726 cairo_surface_t *surface;
1727 cairo_t *cr;
1728
1729 surface = cairo_image_surface_create (CAIRO_FORMAT_RGB24, 12, 12);
1730 cr = cairo_create (surface);
1731 cairo_surface_destroy (surface);
1732
1733 cairo_set_source_rgb (cr, 0.75, 0.75, 0.75); /* light gray */
1734 cairo_paint (cr);
1735
1736 cairo_set_source_rgb (cr, 0.25, 0.25, 0.25); /* dark gray */
1737 cairo_rectangle (cr, width / 2, 0, width / 2, height / 2);
1738 cairo_rectangle (cr, 0, height / 2, width / 2, height / 2);
1739 cairo_fill (cr);
1740
1741 surface = cairo_surface_reference (cairo_get_target (cr));
1742 cairo_destroy (cr);
1743
1744 return surface;
1745 }
1746
1747 void
cairo_test_paint_checkered(cairo_t * cr)1748 cairo_test_paint_checkered (cairo_t *cr)
1749 {
1750 cairo_surface_t *check;
1751
1752 check = _draw_check (12, 12);
1753
1754 cairo_save (cr);
1755 cairo_set_source_surface (cr, check, 0, 0);
1756 cairo_surface_destroy (check);
1757
1758 cairo_pattern_set_filter (cairo_get_source (cr), CAIRO_FILTER_NEAREST);
1759 cairo_pattern_set_extend (cairo_get_source (cr), CAIRO_EXTEND_REPEAT);
1760 cairo_paint (cr);
1761
1762 cairo_restore (cr);
1763 }
1764
1765 cairo_bool_t
cairo_test_is_target_enabled(const cairo_test_context_t * ctx,const char * target)1766 cairo_test_is_target_enabled (const cairo_test_context_t *ctx,
1767 const char *target)
1768 {
1769 size_t i;
1770
1771 for (i = 0; i < ctx->num_targets; i++) {
1772 const cairo_boilerplate_target_t *t = ctx->targets_to_test[i];
1773 if (strcmp (t->name, target) == 0) {
1774 /* XXX ask the target whether is it possible to run?
1775 * e.g. the xlib backend could check whether it is able to connect
1776 * to the Display.
1777 */
1778 return t->get_image_surface != NULL;
1779 }
1780 }
1781
1782 return FALSE;
1783 }
1784
1785 cairo_bool_t
cairo_test_malloc_failure(const cairo_test_context_t * ctx,cairo_status_t status)1786 cairo_test_malloc_failure (const cairo_test_context_t *ctx,
1787 cairo_status_t status)
1788 {
1789 if (! ctx->malloc_failure)
1790 return FALSE;
1791
1792 if (status != CAIRO_STATUS_NO_MEMORY)
1793 return FALSE;
1794
1795 #if HAVE_MEMFAULT
1796 {
1797 int n_faults;
1798
1799 /* prevent infinite loops... */
1800 n_faults = MEMFAULT_COUNT_FAULTS ();
1801 if (n_faults == ctx->last_fault_count)
1802 return FALSE;
1803
1804 ((cairo_test_context_t *) ctx)->last_fault_count = n_faults;
1805 }
1806 #endif
1807
1808 return TRUE;
1809 }
1810
1811 cairo_test_status_t
cairo_test_status_from_status(const cairo_test_context_t * ctx,cairo_status_t status)1812 cairo_test_status_from_status (const cairo_test_context_t *ctx,
1813 cairo_status_t status)
1814 {
1815 if (status == CAIRO_STATUS_SUCCESS)
1816 return CAIRO_TEST_SUCCESS;
1817
1818 if (cairo_test_malloc_failure (ctx, status))
1819 return CAIRO_TEST_NO_MEMORY;
1820
1821 return CAIRO_TEST_FAILURE;
1822 }
1823