1 /*
2 * Copyright © 2006 Red Hat, Inc.
3 *
4 * Permission to use, copy, modify, distribute, and sell this software
5 * and its documentation for any purpose is hereby granted without
6 * fee, provided that the above copyright notice appear in all copies
7 * and that both that copyright notice and this permission notice
8 * appear in supporting documentation, and that the name of the
9 * copyright holders not be used in advertising or publicity
10 * pertaining to distribution of the software without specific,
11 * written prior permission. The copyright holders make no
12 * representations about the suitability of this software for any
13 * purpose. It is provided "as is" without express or implied
14 * warranty.
15 *
16 * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
17 * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
18 * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
19 * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
20 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
21 * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
22 * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
23 * SOFTWARE.
24 *
25 * Authors: Carl Worth <cworth@cworth.org>
26 */
27
28 #include "cairo-perf.h"
29
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <string.h>
33 #include <errno.h>
34 #include <ctype.h>
35 #include <math.h>
36 #include <assert.h>
37
38 typedef struct _cairo_perf_report_options {
39 double min_change;
40 int use_utf;
41 int print_change_bars;
42 int use_ticks;
43 } cairo_perf_report_options_t;
44
45 typedef struct _cairo_perf_diff_files_args {
46 const char **filenames;
47 int num_filenames;
48 cairo_perf_report_options_t options;
49 } cairo_perf_diff_files_args_t;
50
51 static int
test_diff_cmp_speedup_before_slowdown(const void * a,const void * b)52 test_diff_cmp_speedup_before_slowdown (const void *a,
53 const void *b)
54 {
55 const test_diff_t *a_diff = a;
56 const test_diff_t *b_diff = b;
57
58 /* First make all speedups come before all slowdowns. */
59 if (a_diff->change > 0 && b_diff->change < 0)
60 return -1;
61 if (a_diff->change < 0 && b_diff->change > 0)
62 return 1;
63
64 if (a_diff->change == b_diff->change)
65 return 0;
66
67 /* Large speedups come first. */
68 if (a_diff->change > 0) {
69 if (a_diff->change > b_diff->change)
70 return -1;
71 else
72 return 1;
73 }
74
75 /* Large slowdowns come last. */
76 if (a_diff->change < 0) {
77 if (a_diff->change < b_diff->change)
78 return 1;
79 else
80 return -1;
81 }
82
83 return 0;
84 }
85
86 static int
test_diff_cmp(const void * a,const void * b)87 test_diff_cmp (const void *a,
88 const void *b)
89 {
90 const test_diff_t *a_diff = a;
91 const test_diff_t *b_diff = b;
92
93 /* Reverse sort by magnitude of change so larger changes come
94 * first */
95 if (a_diff->change > b_diff->change)
96 return -1;
97
98 if (a_diff->change < b_diff->change)
99 return 1;
100
101 return 0;
102 }
103
104 #define CHANGE_BAR_WIDTH 70
105 static void
print_change_bar(double change,double max_change,int use_utf)106 print_change_bar (double change,
107 double max_change,
108 int use_utf)
109 {
110 int units_per_cell = ceil (max_change / CHANGE_BAR_WIDTH);
111 static char const *ascii_boxes[8] = {
112 "****","***" ,"***", "**",
113 "**", "*", "*", ""
114 };
115 static char const *utf_boxes[8] = {
116 "█", "▉", "▊", "▋",
117 "▌", "▍", "▎", "▏"
118 };
119 char const **boxes = use_utf ? utf_boxes : ascii_boxes;
120
121 /* For a 1.0x speedup we want a zero-size bar to show "no
122 * change". */
123 change -= 1.0;
124
125 while (change > units_per_cell) {
126 printf ("%s", boxes[0]);
127 change -= units_per_cell;
128 }
129
130 change /= units_per_cell;
131
132 if (change > 7.5/8.0)
133 printf ("%s", boxes[0]);
134 else if (change > 6.5/8.0)
135 printf ("%s", boxes[1]);
136 else if (change > 5.5/8.0)
137 printf ("%s", boxes[2]);
138 else if (change > 4.5/8.0)
139 printf ("%s", boxes[3]);
140 else if (change > 3.5/8.0)
141 printf ("%s", boxes[4]);
142 else if (change > 2.5/8.0)
143 printf ("%s", boxes[5]);
144 else if (change > 1.5/8.0)
145 printf ("%s", boxes[6]);
146 else if (change > 0.5/8.0)
147 printf ("%s", boxes[7]);
148
149 printf ("\n");
150 }
151
152 static void
test_diff_print_binary(test_diff_t * diff,double max_change,cairo_perf_report_options_t * options)153 test_diff_print_binary (test_diff_t *diff,
154 double max_change,
155 cairo_perf_report_options_t *options)
156 {
157 if (diff->tests[0]->size)
158 printf ("%5s-%-4s %26s-%-3d",
159 diff->tests[0]->backend, diff->tests[0]->content,
160 diff->tests[0]->name, diff->tests[0]->size);
161 else
162 printf ("%5s %26s", diff->tests[0]->backend, diff->tests[0]->name);
163
164 printf (" %6.2f (%.2f %4.2f%%) -> %6.2f (%.2f %4.2f%%): %5.2fx ",
165 diff->tests[0]->stats.min_ticks / diff->tests[0]->stats.ticks_per_ms,
166 diff->tests[0]->stats.median_ticks / diff->tests[0]->stats.ticks_per_ms,
167 diff->tests[0]->stats.std_dev * 100,
168 diff->tests[1]->stats.min_ticks / diff->tests[1]->stats.ticks_per_ms,
169 diff->tests[1]->stats.median_ticks / diff->tests[1]->stats.ticks_per_ms,
170 diff->tests[1]->stats.std_dev * 100,
171 fabs (diff->change));
172
173 if (diff->change > 1.0)
174 printf ("speedup\n");
175 else
176 printf ("slowdown\n");
177
178 if (options->print_change_bars)
179 print_change_bar (fabs (diff->change), max_change,
180 options->use_utf);
181 }
182
183 static void
test_diff_print_multi(test_diff_t * diff,double max_change,cairo_perf_report_options_t * options)184 test_diff_print_multi (test_diff_t *diff,
185 double max_change,
186 cairo_perf_report_options_t *options)
187 {
188 int i;
189 double test_time;
190 double change;
191
192 if (diff->tests[0]->size) {
193 printf ("%s (backend: %s-%s, size: %d)\n",
194 diff->tests[0]->name,
195 diff->tests[0]->backend,
196 diff->tests[0]->content,
197 diff->tests[0]->size);
198 } else {
199 printf ("%s (backend: %s)\n",
200 diff->tests[0]->name,
201 diff->tests[0]->backend);
202 }
203
204 for (i = 0; i < diff->num_tests; i++) {
205 test_time = diff->tests[i]->stats.min_ticks;
206 if (! options->use_ticks)
207 test_time /= diff->tests[i]->stats.ticks_per_ms;
208 change = diff->max / test_time;
209 printf ("[%d] %6.2f: %5.2fx ",
210 diff->tests[i]->fileno,
211 diff->tests[i]->stats.min_ticks / diff->tests[i]->stats.ticks_per_ms,
212 change);
213
214 if (options->print_change_bars)
215 print_change_bar (change, max_change, options->use_utf);
216 else
217 printf("\n");
218 }
219
220 printf("\n");
221 }
222
223 static void
cairo_perf_reports_compare(cairo_perf_report_t * reports,int num_reports,cairo_perf_report_options_t * options)224 cairo_perf_reports_compare (cairo_perf_report_t *reports,
225 int num_reports,
226 cairo_perf_report_options_t *options)
227 {
228 int i;
229 test_report_t **tests, *min_test;
230 test_diff_t *diff, *diffs;
231 int num_diffs, max_diffs;
232 double max_change;
233 double test_time;
234 int seen_non_null;
235 cairo_bool_t printed_speedup = FALSE;
236 cairo_bool_t printed_slowdown = FALSE;
237
238 assert (num_reports >= 2);
239
240 tests = xmalloc (num_reports * sizeof (test_report_t *));
241
242 max_diffs = reports[0].tests_count;
243 for (i = 0; i < num_reports; i++) {
244 tests[i] = reports[i].tests;
245 if (reports[i].tests_count > max_diffs)
246 max_diffs = reports[i].tests_count;
247 }
248
249 diff = diffs = xmalloc (max_diffs * sizeof (test_diff_t));
250
251 num_diffs = 0;
252 while (1) {
253 /* We expect iterations values of 0 when multiple raw reports
254 * for the same test have been condensed into the stats of the
255 * first. So we just skip these later reports that have no
256 * stats. */
257 seen_non_null = 0;
258 for (i = 0; i < num_reports; i++) {
259 while (tests[i]->name && tests[i]->stats.iterations == 0)
260 tests[i]++;
261 if (tests[i]->name)
262 seen_non_null++;
263 }
264
265 if (seen_non_null < 2)
266 break;
267
268 /* Find the minimum of all current tests, (we have to do this
269 * in case some reports don't have a particular test). */
270 for (i = 0; i < num_reports; i++) {
271 if (tests[i]->name) {
272 min_test = tests[i];
273 break;
274 }
275 }
276 for (++i; i < num_reports; i++) {
277 if (tests[i]->name &&
278 test_report_cmp_backend_then_name (tests[i], min_test) < 0)
279 {
280 min_test = tests[i];
281 }
282 }
283
284 /* For each report that has the current test, record it into
285 * the diff structure. */
286 diff->num_tests = 0;
287 diff->tests = xmalloc (num_reports * sizeof (test_diff_t));
288 for (i = 0; i < num_reports; i++) {
289 if (tests[i]->name &&
290 test_report_cmp_backend_then_name (tests[i], min_test) == 0)
291 {
292 test_time = tests[i]->stats.min_ticks;
293 if (! options->use_ticks)
294 test_time /= tests[i]->stats.ticks_per_ms;
295 if (diff->num_tests == 0) {
296 diff->min = test_time;
297 diff->max = test_time;
298 } else {
299 if (test_time < diff->min)
300 diff->min = test_time;
301 if (test_time > diff->max)
302 diff->max = test_time;
303 }
304 diff->tests[diff->num_tests++] = tests[i];
305 tests[i]++;
306 }
307 }
308 diff->change = diff->max / diff->min;
309
310 if (num_reports == 2) {
311 double old_time, new_time;
312 if (diff->num_tests == 1) {
313 printf ("Only in %s: %s %s\n",
314 diff->tests[0]->configuration,
315 diff->tests[0]->backend,
316 diff->tests[0]->name);
317 continue;
318 }
319 old_time = diff->tests[0]->stats.min_ticks;
320 new_time = diff->tests[1]->stats.min_ticks;
321 if (! options->use_ticks) {
322 old_time /= diff->tests[0]->stats.ticks_per_ms;
323 new_time /= diff->tests[1]->stats.ticks_per_ms;
324 }
325 diff->change = old_time / new_time;
326 if (diff->change < 1.0)
327 diff->change = - 1.0 / diff->change;
328 }
329
330 diff++;
331 num_diffs++;
332 }
333 if (num_diffs == 0)
334 goto DONE;
335
336 if (num_reports == 2)
337 qsort (diffs, num_diffs, sizeof (test_diff_t),
338 test_diff_cmp_speedup_before_slowdown);
339 else
340 qsort (diffs, num_diffs, sizeof (test_diff_t), test_diff_cmp);
341
342 max_change = 1.0;
343 for (i = 0; i < num_diffs; i++) {
344 if (fabs (diffs[i].change) > max_change)
345 max_change = fabs (diffs[i].change);
346 }
347
348 if (num_reports == 2)
349 printf ("old: %s\n"
350 "new: %s\n",
351 diffs->tests[0]->configuration,
352 diffs->tests[1]->configuration);
353
354 for (i = 0; i < num_diffs; i++) {
355 diff = &diffs[i];
356
357 /* Discard as uninteresting a change which is less than the
358 * minimum change required, (default may be overridden on
359 * command-line). */
360 if (fabs (diff->change) - 1.0 < options->min_change)
361 continue;
362
363 if (num_reports == 2) {
364 if (diff->change > 1.0 && ! printed_speedup) {
365 printf ("Speedups\n"
366 "========\n");
367 printed_speedup = TRUE;
368 }
369 if (diff->change < 1.0 && ! printed_slowdown) {
370 printf ("Slowdowns\n"
371 "=========\n");
372 printed_slowdown = TRUE;
373 }
374 test_diff_print_binary (diff, max_change, options);
375 } else {
376 test_diff_print_multi (diff, max_change, options);
377 }
378 }
379
380 DONE:
381 for (i = 0; i < num_diffs; i++)
382 free (diffs[i].tests);
383 free (diffs);
384 free (tests);
385 }
386
387 static void
usage(const char * argv0)388 usage (const char *argv0)
389 {
390 char const *basename = strrchr(argv0, '/');
391 basename = basename ? basename+1 : argv0;
392 fprintf (stderr,
393 "Usage: %s [options] file1 file2 [...]\n\n",
394 basename);
395 fprintf (stderr,
396 "Computes significant performance differences for cairo performance reports.\n"
397 "Each file should be the output of the cairo-perf program (or \"make perf\").\n"
398 "The following options are available:\n"
399 "\n"
400 "--no-utf Use ascii stars instead of utf-8 change bars.\n"
401 " Four stars are printed per factor of speedup.\n"
402 "\n"
403 "--no-bars Don't display change bars at all.\n\n"
404 "\n"
405 "--use-ms Use milliseconds to calculate differences.\n"
406 " (instead of ticks which are hardware dependent)\n"
407 "\n"
408 "--min-change threshold[%%]\n"
409 " Suppress all changes below the given threshold.\n"
410 " The default threshold of 0.05 or 5%% ignores any\n"
411 " speedup or slowdown of 1.05 or less. A threshold\n"
412 " of 0 will cause all output to be reported.\n"
413 );
414 exit(1);
415 }
416
417 static void
parse_args(int argc,char const ** argv,cairo_perf_diff_files_args_t * args)418 parse_args (int argc,
419 char const **argv,
420 cairo_perf_diff_files_args_t *args)
421 {
422 int i;
423
424 for (i = 1; i < argc; i++) {
425 if (strcmp (argv[i], "--no-utf") == 0) {
426 args->options.use_utf = 0;
427 }
428 else if (strcmp (argv[i], "--no-bars") == 0) {
429 args->options.print_change_bars = 0;
430 }
431 else if (strcmp (argv[i], "--use-ms") == 0) {
432 /* default */
433 }
434 else if (strcmp (argv[i], "--use-ticks") == 0) {
435 args->options.use_ticks = 1;
436 }
437 else if (strcmp (argv[i], "--min-change") == 0) {
438 char *end = NULL;
439 i++;
440 if (i >= argc)
441 usage (argv[0]);
442 args->options.min_change = strtod (argv[i], &end);
443 if (*end) {
444 if (*end == '%') {
445 args->options.min_change /= 100;
446 } else {
447 usage (argv[0]);
448 }
449 }
450 }
451 else {
452 args->num_filenames++;
453 args->filenames = xrealloc (args->filenames,
454 args->num_filenames * sizeof (char *));
455 args->filenames[args->num_filenames - 1] = argv[i];
456 }
457 }
458 }
459
460 int
main(int argc,const char * argv[])461 main (int argc,
462 const char *argv[])
463 {
464 cairo_perf_diff_files_args_t args = {
465 NULL, /* filenames */
466 0, /* num_filenames */
467 {
468 0.05, /* min change */
469 1, /* use UTF-8? */
470 1, /* display change bars? */
471 }
472 };
473 cairo_perf_report_t *reports;
474 test_report_t *t;
475 int i;
476
477 parse_args (argc, argv, &args);
478
479 if (args.num_filenames < 2)
480 usage (argv[0]);
481
482 reports = xmalloc (args.num_filenames * sizeof (cairo_perf_report_t));
483
484 for (i = 0; i < args.num_filenames; i++ ) {
485 cairo_perf_report_load (&reports[i], args.filenames[i], i, NULL);
486 printf ("[%d] %s\n", i, args.filenames[i]);
487 }
488 printf ("\n");
489
490 cairo_perf_reports_compare (reports, args.num_filenames, &args.options);
491
492 /* Pointless memory cleanup, (would be a great place for talloc) */
493 free (args.filenames);
494 for (i = 0; i < args.num_filenames; i++) {
495 for (t = reports[i].tests; t->name; t++) {
496 free (t->samples);
497 free (t->backend);
498 free (t->name);
499 }
500 free (reports[i].tests);
501 free (reports[i].configuration);
502 }
503 free (reports);
504
505 return 0;
506 }
507