1 /* radare - LGPL - Copyright 2020-2021 - thestr4ng3r */
2
3 #include "r2r.h"
4 #include <assert.h>
5
6 #define WORKERS_DEFAULT 8
7 #define RADARE2_CMD_DEFAULT "radare2"
8 #define RASM2_CMD_DEFAULT "rasm2"
9 #define JSON_TEST_FILE_DEFAULT "bins/elf/crackme0x00b"
10 #define TIMEOUT_DEFAULT 960
11
12 #define STRV(x) #x
13 #define STR(x) STRV(x)
14 #define WORKERS_DEFAULT_STR STR(WORKERS_DEFAULT)
15 #define TIMEOUT_DEFAULT_STR STR(TIMEOUT_DEFAULT)
16
17 typedef struct r2r_state_t {
18 R2RRunConfig run_config;
19 bool verbose;
20 R2RTestDatabase *db;
21 PJ *test_results;
22
23 RThreadCond *cond; // signaled from workers to main thread to update status
24 RThreadLock *lock; // protects everything below
25 HtPP *path_left; // char * (path to test file) => ut64 * (count of remaining tests)
26 RPVector completed_paths;
27 ut64 ok_count;
28 ut64 xx_count;
29 ut64 br_count;
30 ut64 fx_count;
31 RPVector queue;
32 RPVector results;
33 } R2RState;
34
35 static RThreadFunctionRet worker_th(RThread *th);
36 static void print_state(R2RState *state, ut64 prev_completed);
37 static void print_log(R2RState *state, ut64 prev_completed, ut64 prev_paths_completed);
38 static void interact(R2RState *state);
39 static void interact_fix(R2RTestResultInfo *result, RPVector *fixup_results);
40 static void interact_break(R2RTestResultInfo *result, RPVector *fixup_results);
41 static void interact_commands(R2RTestResultInfo *result, RPVector *fixup_results);
42 static void interact_diffchar(R2RTestResultInfo *result);
43
help(bool verbose)44 static int help(bool verbose) {
45 printf ("Usage: r2r [-qvVnL] [-j threads] [test file/dir | @test-type]\n");
46 if (verbose) {
47 printf (
48 " -h print this help\n"
49 " -v show version\n"
50 " -q quiet\n"
51 " -V verbose\n"
52 " -i interactive mode\n"
53 " -n do nothing (don't run any test, just load/parse them)\n"
54 " -L log mode (better printing for CI, logfiles, etc.)\n"
55 " -F [dir] run fuzz tests (open and default analysis) on all files in the given dir\n"
56 " -j [threads] how many threads to use for running tests concurrently (default is "WORKERS_DEFAULT_STR")\n"
57 " -r [radare2] path to radare2 executable (default is "RADARE2_CMD_DEFAULT")\n"
58 " -m [rasm2] path to rasm2 executable (default is "RASM2_CMD_DEFAULT")\n"
59 " -f [file] file to use for json tests (default is "JSON_TEST_FILE_DEFAULT")\n"
60 " -C [dir] chdir before running r2r (default follows executable symlink + test/new\n"
61 " -t [seconds] timeout per test (default is "TIMEOUT_DEFAULT_STR")\n"
62 " -o [file] output test run information in JSON format to file"
63 "\n"
64 "R2R_SKIP_ARCHOS=1 # do not run the arch-os-specific tests\n"
65 "Supported test types: @json @unit @fuzz @arch @cmds\n"
66 "OS/Arch for archos tests: "R2R_ARCH_OS"\n");
67 }
68 return 1;
69 }
70
path_left_free_kv(HtPPKv * kv)71 static void path_left_free_kv(HtPPKv *kv) {
72 free (kv->value);
73 }
74
r2r_chdir(const char * argv0)75 static bool r2r_chdir(const char *argv0) {
76 #if __UNIX__
77 if (r_file_is_directory ("db")) {
78 return true;
79 }
80 char src_path[PATH_MAX];
81 char *r2r_path = r_file_path (argv0);
82 bool found = false;
83 if (readlink (r2r_path, src_path, sizeof (src_path)) != -1) {
84 src_path[sizeof (src_path) - 1] = 0;
85 char *p = strstr (src_path, R_SYS_DIR "binr"R_SYS_DIR"r2r"R_SYS_DIR"r2r");
86 if (p) {
87 *p = 0;
88 strcat (src_path, R_SYS_DIR"test"R_SYS_DIR);
89 if (r_file_is_directory (src_path)) {
90 if (chdir (src_path) != -1) {
91 eprintf ("Running from %s\n", src_path);
92 found = true;
93 } else {
94 eprintf ("Cannot find '%s' directory\n", src_path);
95 }
96 }
97 }
98 }
99 free (r2r_path);
100 return found;
101 #else
102 return false;
103 #endif
104 }
105
r2r_test_run_unit(void)106 static bool r2r_test_run_unit(void) {
107 return r_sandbox_system ("make -C unit all run", 1) == 0;
108 }
109
r2r_chdir_fromtest(const char * test_path)110 static bool r2r_chdir_fromtest(const char *test_path) {
111 if (*test_path == '@') {
112 test_path = "";
113 }
114 char *abs_test_path = r_file_abspath (test_path);
115 if (!r_file_is_directory (abs_test_path)) {
116 char *last_slash = (char *)r_str_lchr (abs_test_path, R_SYS_DIR[0]);
117 if (last_slash) {
118 *last_slash = 0;
119 }
120 }
121 if (chdir (abs_test_path) == -1) {
122 free (abs_test_path);
123 return false;
124 }
125 free (abs_test_path);
126 bool found = false;
127 char *cwd = NULL;
128 char *old_cwd = NULL;
129 while (true) {
130 cwd = r_sys_getdir ();
131 if (old_cwd && !strcmp (old_cwd, cwd)) {
132 break;
133 }
134 if (r_file_is_directory ("test")) {
135 r_sys_chdir ("test");
136 if (r_file_is_directory ("db")) {
137 found = true;
138 eprintf ("Running from %s\n", cwd);
139 break;
140 }
141 r_sys_chdir ("..");
142 }
143 if (r_file_is_directory ("db")) {
144 found = true;
145 eprintf ("Running from %s\n", cwd);
146 break;
147 }
148 free (old_cwd);
149 old_cwd = cwd;
150 cwd = NULL;
151 if (chdir ("..") == -1) {
152 break;
153 }
154 }
155 free (old_cwd);
156 free (cwd);
157 return found;
158 }
159
main(int argc,char ** argv)160 int main(int argc, char **argv) {
161 int workers_count = WORKERS_DEFAULT;
162 bool verbose = false;
163 bool nothing = false;
164 bool quiet = false;
165 bool log_mode = false;
166 bool interactive = false;
167 char *radare2_cmd = NULL;
168 char *rasm2_cmd = NULL;
169 char *json_test_file = NULL;
170 char *output_file = NULL;
171 char *fuzz_dir = NULL;
172 const char *r2r_dir = NULL;
173 ut64 timeout_sec = TIMEOUT_DEFAULT;
174 int ret = 0;
175
176 #if __WINDOWS__
177 UINT old_cp = GetConsoleOutputCP ();
178 {
179 HANDLE streams[] = { GetStdHandle (STD_OUTPUT_HANDLE), GetStdHandle (STD_ERROR_HANDLE) };
180 DWORD mode;
181 int i;
182 for (i = 0; i < R_ARRAY_SIZE (streams); i++) {
183 GetConsoleMode (streams[i], &mode);
184 SetConsoleMode (streams[i],
185 mode | ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING);
186 }
187 }
188 #endif
189
190 RGetopt opt;
191 r_getopt_init (&opt, argc, (const char **)argv, "hqvj:r:m:f:C:LnVt:F:io:");
192
193 int c;
194 while ((c = r_getopt_next (&opt)) != -1) {
195 switch (c) {
196 case 'h':
197 ret = help (true);
198 goto beach;
199 case 'q':
200 quiet = true;
201 break;
202 case 'v':
203 if (quiet) {
204 printf (R2_VERSION "\n");
205 } else {
206 char *s = r_str_version ("r2r");
207 printf ("%s\n", s);
208 free (s);
209 }
210 return 0;
211 case 'V':
212 verbose = true;
213 break;
214 case 'i':
215 interactive = true;
216 break;
217 case 'L':
218 log_mode = true;
219 break;
220 case 'F':
221 free (fuzz_dir);
222 fuzz_dir = strdup (opt.arg);
223 break;
224 case 'j':
225 workers_count = atoi (opt.arg);
226 if (workers_count <= 0) {
227 eprintf ("Invalid thread count\n");
228 ret = help (false);
229 goto beach;
230 }
231 break;
232 case 'r':
233 free (radare2_cmd);
234 radare2_cmd = strdup (opt.arg);
235 break;
236 case 'C':
237 r2r_dir = opt.arg;
238 break;
239 case 'n':
240 nothing = true;
241 break;
242 case 'm':
243 free (rasm2_cmd);
244 rasm2_cmd = strdup (opt.arg);
245 break;
246 case 'f':
247 free (json_test_file);
248 json_test_file = strdup (opt.arg);
249 break;
250 case 't':
251 timeout_sec = strtoull (opt.arg, NULL, 0);
252 if (!timeout_sec) {
253 timeout_sec = UT64_MAX;
254 }
255 break;
256 case 'o':
257 free (output_file);
258 output_file = strdup (opt.arg);
259 break;
260 default:
261 ret = help (false);
262 goto beach;
263 }
264 }
265
266 char *cwd = r_sys_getdir ();
267 if (r2r_dir) {
268 if (chdir (r2r_dir) == -1) {
269 eprintf ("Cannot find %s directory.\n", r2r_dir);
270 return -1;
271 }
272 } else {
273 bool dir_found = (opt.ind < argc && argv[opt.ind][0] != '.')
274 ? r2r_chdir_fromtest (argv[opt.ind])
275 : r2r_chdir (argv[0]);
276 if (!dir_found) {
277 eprintf ("Cannot find db/ directory related to the given test.\n");
278 return -1;
279 }
280 }
281
282 if (fuzz_dir) {
283 char *tmp = fuzz_dir;
284 fuzz_dir = r_file_abspath_rel (cwd, fuzz_dir);
285 free (tmp);
286 }
287
288 if (!r2r_subprocess_init ()) {
289 eprintf ("Subprocess init failed\n");
290 return -1;
291 }
292 atexit (r2r_subprocess_fini);
293
294 ut64 time_start = r_time_now_mono ();
295 R2RState state = {{0}};
296 state.run_config.r2_cmd = radare2_cmd ? radare2_cmd : RADARE2_CMD_DEFAULT;
297 state.run_config.rasm2_cmd = rasm2_cmd ? rasm2_cmd : RASM2_CMD_DEFAULT;
298 state.run_config.json_test_file = json_test_file ? json_test_file : JSON_TEST_FILE_DEFAULT;
299 state.run_config.timeout_ms = timeout_sec > UT64_MAX / 1000 ? UT64_MAX : timeout_sec * 1000;
300 state.verbose = verbose;
301 state.db = r2r_test_database_new ();
302 if (!state.db) {
303 return -1;
304 }
305 r_pvector_init (&state.queue, NULL);
306 r_pvector_init (&state.results, (RPVectorFree)r2r_test_result_info_free);
307 r_pvector_init (&state.completed_paths, NULL);
308 if (output_file) {
309 state.test_results = pj_new ();
310 pj_a (state.test_results);
311 }
312 state.lock = r_th_lock_new (false);
313 if (!state.lock) {
314 return -1;
315 }
316 state.cond = r_th_cond_new ();
317 if (!state.cond) {
318 return -1;
319 }
320
321 if (opt.ind < argc) {
322 // Manually specified path(s)
323 int i;
324 for (i = opt.ind; i < argc; i++) {
325 const char *arg = argv[i];
326 if (*arg == '@') {
327 arg++;
328 eprintf ("Category: %s\n", arg);
329 if (!strcmp (arg, "unit")) {
330 if (!r2r_test_run_unit ()) {
331 return -1;
332 }
333 continue;
334 } else if (!strcmp (arg, "fuzz")) {
335 if (!fuzz_dir) {
336 eprintf ("No fuzz dir given. Use -F [dir]\n");
337 return -1;
338 }
339 if (!r2r_test_database_load_fuzz (state.db, fuzz_dir)) {
340 eprintf ("Failed to load fuzz tests from \"%s\"\n", fuzz_dir);
341 }
342 continue;
343 } else if (!strcmp (arg, "json")) {
344 arg = "db/json";
345 } else if (!strcmp (arg, "dasm")) {
346 arg = "db/asm";
347 } else if (!strcmp (arg, "cmds")) {
348 arg = "db";
349 } else {
350 arg = r_str_newf ("db/%s", arg + 1);
351 }
352 }
353 char *tf = r_file_abspath_rel (cwd, arg);
354 if (!tf || !r2r_test_database_load (state.db, tf)) {
355 eprintf ("Failed to load tests from \"%s\"\n", tf);
356 r2r_test_database_free (state.db);
357 free (tf);
358 return -1;
359 }
360 free (tf);
361 }
362 } else {
363 // Default db path
364 if (!r2r_test_database_load (state.db, "db")) {
365 eprintf ("Failed to load tests from ./db\n");
366 r2r_test_database_free (state.db);
367 return -1;
368 }
369 if (fuzz_dir && !r2r_test_database_load_fuzz (state.db, fuzz_dir)) {
370 eprintf ("Failed to load fuzz tests from \"%s\"\n", fuzz_dir);
371 }
372 }
373
374 R_FREE (cwd);
375 uint32_t loaded_tests = r_pvector_len (&state.db->tests);
376 printf ("Loaded %u tests.\n", loaded_tests);
377 if (nothing) {
378 goto coast;
379 }
380
381 bool jq_available = r2r_check_jq_available ();
382 if (!jq_available) {
383 eprintf ("Skipping json tests because jq is not available.\n");
384 size_t i;
385 for (i = 0; i < r_pvector_len (&state.db->tests);) {
386 R2RTest *test = r_pvector_at (&state.db->tests, i);
387 if (test->type == R2R_TEST_TYPE_JSON) {
388 r2r_test_free (test);
389 r_pvector_remove_at (&state.db->tests, i);
390 continue;
391 }
392 i++;
393 }
394 }
395
396 r_pvector_insert_range (&state.queue, 0, state.db->tests.v.a, r_pvector_len (&state.db->tests));
397
398 if (log_mode) {
399 // Log mode prints the state after every completed file.
400 // The count of tests left per file is stored in a ht.
401 state.path_left = ht_pp_new (NULL, path_left_free_kv, NULL);
402 if (state.path_left) {
403 void **it;
404 r_pvector_foreach (&state.queue, it) {
405 R2RTest *test = *it;
406 ut64 *count = ht_pp_find (state.path_left, test->path, NULL);
407 if (!count) {
408 count = malloc (sizeof (ut64));
409 *count = 0;
410 ht_pp_insert (state.path_left, test->path, count);
411 }
412 (*count)++;
413 }
414 }
415 }
416
417 r_th_lock_enter (state.lock);
418
419 RPVector workers;
420 r_pvector_init (&workers, NULL);
421 int i;
422 for (i = 0; i < workers_count; i++) {
423 RThread *th = r_th_new (worker_th, &state, 0);
424 if (!th) {
425 eprintf ("Failed to start thread.\n");
426 exit (-1);
427 }
428 r_pvector_push (&workers, th);
429 }
430
431 ut64 prev_completed = UT64_MAX;
432 ut64 prev_paths_completed = 0;
433 while (true) {
434 ut64 completed = (ut64)r_pvector_len (&state.results);
435 if (log_mode) {
436 print_log (&state, prev_completed, prev_paths_completed);
437 } else if (completed != prev_completed) {
438 print_state (&state, prev_completed);
439 }
440 prev_completed = completed;
441 prev_paths_completed = (ut64)r_pvector_len (&state.completed_paths);
442 if (completed == r_pvector_len (&state.db->tests)) {
443 break;
444 }
445 r_th_cond_wait (state.cond, state.lock);
446 }
447
448 r_th_lock_leave (state.lock);
449
450 printf ("\n");
451
452 void **it;
453 r_pvector_foreach (&workers, it) {
454 RThread *th = *it;
455 r_th_wait (th);
456 r_th_free (th);
457 }
458 r_pvector_clear (&workers);
459
460 ut64 seconds = (r_time_now_mono () - time_start) / 1000000;
461 printf ("Finished in");
462 if (seconds > 60) {
463 ut64 minutes = seconds / 60;
464 printf (" %"PFMT64d" minutes and", seconds / 60);
465 seconds -= (minutes * 60);
466 }
467 printf (" %"PFMT64d" seconds.\n", seconds % 60);
468
469 if (output_file) {
470 pj_end (state.test_results);
471 char *results = pj_drain (state.test_results);
472 r_file_dump (output_file, (ut8 *)results, strlen (results), false);
473 free (results);
474 }
475
476 if (interactive) {
477 interact (&state);
478 }
479
480 if (state.xx_count) {
481 ret = 1;
482 }
483
484 coast:
485 r_pvector_clear (&state.queue);
486 r_pvector_clear (&state.results);
487 r_pvector_clear (&state.completed_paths);
488 r2r_test_database_free (state.db);
489 r_th_lock_free (state.lock);
490 r_th_cond_free (state.cond);
491 beach:
492 free (radare2_cmd);
493 free (rasm2_cmd);
494 free (json_test_file);
495 free (fuzz_dir);
496 #if __WINDOWS__
497 if (old_cp) {
498 (void)SetConsoleOutputCP (old_cp);
499 // chcp doesn't pick up the code page switch for some reason
500 (void)r_sys_cmdf ("chcp %u > NUL", old_cp);
501 }
502 #endif
503 return ret;
504 }
505
test_result_to_json(PJ * pj,R2RTestResultInfo * result)506 static void test_result_to_json(PJ *pj, R2RTestResultInfo *result) {
507 r_return_if_fail (pj && result);
508 pj_o (pj);
509 pj_k (pj, "type");
510 R2RTest *test = result->test;
511 switch (test->type) {
512 case R2R_TEST_TYPE_CMD:
513 pj_s (pj, "cmd");
514 pj_ks (pj, "name", test->cmd_test->name.value);
515 break;
516 case R2R_TEST_TYPE_ASM:
517 pj_s (pj, "asm");
518 pj_ks (pj, "arch", test->asm_test->arch);
519 pj_ki (pj, "bits", test->asm_test->bits);
520 pj_kn (pj, "line", test->asm_test->line);
521 break;
522 case R2R_TEST_TYPE_JSON:
523 pj_s (pj, "json");
524 pj_ks (pj, "cmd", test->json_test->cmd);
525 break;
526 case R2R_TEST_TYPE_FUZZ:
527 pj_s (pj, "fuzz");
528 pj_ks (pj, "file", test->fuzz_test->file);
529 break;
530 }
531 pj_k (pj, "result");
532 switch (result->result) {
533 case R2R_TEST_RESULT_OK:
534 pj_s (pj, "ok");
535 break;
536 case R2R_TEST_RESULT_FAILED:
537 pj_s (pj, "failed");
538 break;
539 case R2R_TEST_RESULT_BROKEN:
540 pj_s (pj, "broken");
541 break;
542 case R2R_TEST_RESULT_FIXED:
543 pj_s (pj, "fixed");
544 break;
545 }
546 pj_kb (pj, "run_failed", result->run_failed);
547 pj_kn (pj, "time_elapsed", result->time_elapsed);
548 pj_kb (pj, "timeout", result->timeout);
549 pj_end (pj);
550 }
551
worker_th(RThread * th)552 static RThreadFunctionRet worker_th(RThread *th) {
553 R2RState *state = th->user;
554 r_th_lock_enter (state->lock);
555 while (true) {
556 if (r_pvector_empty (&state->queue)) {
557 break;
558 }
559 R2RTest *test = r_pvector_pop (&state->queue);
560 r_th_lock_leave (state->lock);
561
562 R2RTestResultInfo *result = r2r_run_test (&state->run_config, test);
563
564 r_th_lock_enter (state->lock);
565 r_pvector_push (&state->results, result);
566 switch (result->result) {
567 case R2R_TEST_RESULT_OK:
568 state->ok_count++;
569 break;
570 case R2R_TEST_RESULT_FAILED:
571 state->xx_count++;
572 break;
573 case R2R_TEST_RESULT_BROKEN:
574 state->br_count++;
575 break;
576 case R2R_TEST_RESULT_FIXED:
577 state->fx_count++;
578 break;
579 }
580 if (state->path_left) {
581 ut64 *count = ht_pp_find (state->path_left, test->path, NULL);
582 if (count) {
583 (*count)--;
584 if (!*count) {
585 r_pvector_push (&state->completed_paths, (void *)test->path);
586 }
587 }
588 }
589 r_th_cond_signal (state->cond);
590 }
591 r_th_lock_leave (state->lock);
592 return R_TH_STOP;
593 }
594
print_diff(const char * actual,const char * expected,bool diffchar,const char * regexp)595 static void print_diff(const char *actual, const char *expected, bool diffchar, const char *regexp) {
596 RDiff *d = r_diff_new ();
597 #ifdef __WINDOWS__
598 d->diff_cmd = "git diff --no-index";
599 #endif
600 char *output = (char *)actual;
601 if (regexp) {
602 RRegex *rx = r_regex_new (regexp, "en");
603 RList *matches = r_regex_match_list (rx, actual);
604 output = r_list_to_str (matches, '\0');
605 r_list_free (matches);
606 r_regex_free (rx);
607 }
608 if (diffchar) {
609 RDiffChar *diff = r_diffchar_new ((const ut8 *)expected, (const ut8 *)actual);
610 if (diff) {
611 r_diffchar_print (diff);
612 r_diffchar_free (diff);
613 goto cleanup;
614 }
615 d->diff_cmd = "git diff --no-index --word-diff=porcelain --word-diff-regex=.";
616 }
617 char *uni = r_diff_buffers_to_string (d, (const ut8 *)expected, (int)strlen (expected),
618 (const ut8 *)output, (int)strlen (output));
619 r_diff_free (d);
620
621 RList *lines = r_str_split_duplist (uni, "\n", false);
622 RListIter *it;
623 char *line;
624 bool header_found = false;
625 r_list_foreach (lines, it, line) {
626 if (!header_found) {
627 if (r_str_startswith (line, "+++ ")) {
628 header_found = true;
629 }
630 continue;
631 }
632 if (r_str_startswith (line, "@@ ") && r_str_endswith (line, " @@")) {
633 printf ("%s%s%s\n", Color_CYAN, line, Color_RESET);
634 continue;
635 }
636 bool color = true;
637 char c = *line;
638 switch (c) {
639 case '+':
640 printf ("%s"Color_INSERT, diffchar ? Color_BGINSERT : "");
641 break;
642 case '-':
643 printf ("%s"Color_DELETE, diffchar ? Color_BGDELETE : "");
644 break;
645 case '~': // can't happen if !diffchar
646 printf ("\n");
647 continue;
648 default:
649 color = false;
650 break;
651 }
652 if (diffchar) {
653 printf ("%s", *line ? line + 1 : "");
654 } else {
655 printf ("%s\n", line);
656 }
657 if (color) {
658 printf ("%s", Color_RESET);
659 }
660 }
661 r_list_free (lines);
662 free (uni);
663 printf ("\n");
664 cleanup:
665 if (regexp) {
666 free (output);
667 }
668 }
669
print_runner(const char * file,const char * args[],size_t args_size,const char * envvars[],const char * envvals[],size_t env_size,ut64 timeout_ms,void * user)670 static R2RProcessOutput *print_runner(const char *file, const char *args[], size_t args_size,
671 const char *envvars[], const char *envvals[], size_t env_size, ut64 timeout_ms, void *user) {
672 size_t i;
673 for (i = 0; i < env_size; i++) {
674 printf ("%s=%s ", envvars[i], envvals[i]);
675 }
676 printf ("%s", file);
677 for (i = 0; i < args_size; i++) {
678 const char *str = args[i];
679 if (strpbrk (str, "\n \'\"")) {
680 printf (" '%s'", str); // TODO: escape
681 } else {
682 printf (" %s", str);
683 }
684 }
685 printf ("\n");
686 return NULL;
687 }
688
r_test_cmp_cmd_output(const char * output,const char * expect,const char * regexp)689 R_API bool r_test_cmp_cmd_output(const char *output, const char *expect, const char *regexp) {
690 if (regexp) {
691 if (!r_regex_match (regexp, "e", output)) {
692 return true;
693 }
694 return false;
695 }
696 return !strcmp (expect, output);
697 }
698
print_result_diff(R2RRunConfig * config,R2RTestResultInfo * result)699 static void print_result_diff(R2RRunConfig *config, R2RTestResultInfo *result) {
700 if (result->run_failed) {
701 printf (Color_RED "RUN FAILED (e.g. wrong radare2 path)" Color_RESET "\n");
702 return;
703 }
704 switch (result->test->type) {
705 case R2R_TEST_TYPE_CMD: {
706 r2r_run_cmd_test (config, result->test->cmd_test, print_runner, NULL);
707 const char *expect = result->test->cmd_test->expect.value;
708 const char *out = result->proc_out->out;
709 const char *regexp_out = result->test->cmd_test->regexp_out.value;
710 if ((expect || regexp_out) && !r_test_cmp_cmd_output (out, expect, regexp_out)) {
711 printf ("-- stdout\n");
712 print_diff (out, expect, false, regexp_out);
713 }
714 expect = result->test->cmd_test->expect_err.value;
715 const char *err = result->proc_out->err;
716 const char *regexp_err = result->test->cmd_test->regexp_err.value;
717 if ((expect || regexp_err) && !r_test_cmp_cmd_output (err, expect, regexp_err)) {
718 printf ("-- stderr\n");
719 print_diff (err, expect, false, regexp_err);
720 } else if (*err) {
721 printf ("-- stderr\n%s\n", err);
722 }
723 if (result->proc_out->ret != 0) {
724 printf ("-- exit status: "Color_RED"%d"Color_RESET"\n", result->proc_out->ret);
725 }
726 break;
727 }
728 case R2R_TEST_TYPE_ASM:
729 // TODO
730 break;
731 case R2R_TEST_TYPE_JSON:
732 break;
733 case R2R_TEST_TYPE_FUZZ:
734 r2r_run_fuzz_test (config, result->test->fuzz_test, print_runner, NULL);
735 printf ("-- stdout\n%s\n", result->proc_out->out);
736 printf ("-- stderr\n%s\n", result->proc_out->err);
737 printf ("-- exit status: "Color_RED"%d"Color_RESET"\n", result->proc_out->ret);
738 break;
739 }
740 }
741
print_new_results(R2RState * state,ut64 prev_completed)742 static void print_new_results(R2RState *state, ut64 prev_completed) {
743 // Detailed test result (with diff if necessary)
744 ut64 completed = (ut64)r_pvector_len (&state->results);
745 ut64 i;
746 for (i = prev_completed; i < completed; i++) {
747 R2RTestResultInfo *result = r_pvector_at (&state->results, (size_t)i);
748 if (state->test_results) {
749 test_result_to_json (state->test_results, result);
750 }
751 if (!state->verbose && (result->result == R2R_TEST_RESULT_OK || result->result == R2R_TEST_RESULT_FIXED || result->result == R2R_TEST_RESULT_BROKEN)) {
752 continue;
753 }
754 char *name = r2r_test_name (result->test);
755 if (!name) {
756 continue;
757 }
758 printf ("\n"R_CONS_CURSOR_UP R_CONS_CLEAR_LINE);
759 switch (result->result) {
760 case R2R_TEST_RESULT_OK:
761 printf (Color_GREEN"[OK]"Color_RESET);
762 break;
763 case R2R_TEST_RESULT_FAILED:
764 printf (Color_RED"[XX]"Color_RESET);
765 break;
766 case R2R_TEST_RESULT_BROKEN:
767 printf (Color_BLUE"[BR]"Color_RESET);
768 break;
769 case R2R_TEST_RESULT_FIXED:
770 printf (Color_CYAN"[FX]"Color_RESET);
771 break;
772 }
773 if (result->timeout) {
774 printf (Color_CYAN" TIMEOUT"Color_RESET);
775 }
776 printf (" %s "Color_YELLOW"%s"Color_RESET"\n", result->test->path, name);
777 if (result->result == R2R_TEST_RESULT_FAILED || (state->verbose && result->result == R2R_TEST_RESULT_BROKEN)) {
778 print_result_diff (&state->run_config, result);
779 }
780 free (name);
781 }
782 }
783
print_state_counts(R2RState * state)784 static void print_state_counts(R2RState *state) {
785 printf ("%8"PFMT64u" OK %8"PFMT64u" BR %8"PFMT64u" XX %8"PFMT64u" FX",
786 state->ok_count, state->br_count, state->xx_count, state->fx_count);
787 }
788
print_state(R2RState * state,ut64 prev_completed)789 static void print_state(R2RState *state, ut64 prev_completed) {
790 #if __WINDOWS__
791 setvbuf (stdout, NULL, _IOFBF, 8192);
792 #endif
793 print_new_results (state, prev_completed);
794
795 // [x/x] OK 42 BR 0 ...
796 printf (R_CONS_CLEAR_LINE);
797 ut64 a = (ut64)r_pvector_len (&state->results);
798 ut64 b = (ut64)r_pvector_len (&state->db->tests);
799 int w = printf ("[%"PFMT64u"/%"PFMT64u"]", a, b);
800 while (w >= 0 && w < 20) {
801 printf (" ");
802 w++;
803 }
804 printf (" ");
805 print_state_counts (state);
806 fflush (stdout);
807 #if __WINDOWS__
808 setvbuf (stdout, NULL, _IONBF, 0);
809 #endif
810 }
811
print_log(R2RState * state,ut64 prev_completed,ut64 prev_paths_completed)812 static void print_log(R2RState *state, ut64 prev_completed, ut64 prev_paths_completed) {
813 print_new_results (state, prev_completed);
814 ut64 paths_completed = r_pvector_len (&state->completed_paths);
815 for (; prev_paths_completed < paths_completed; prev_paths_completed++) {
816 printf ("[**] %50s ", (const char *)r_pvector_at (&state->completed_paths, prev_paths_completed));
817 print_state_counts (state);
818 printf ("\n");
819 }
820 }
821
interact(R2RState * state)822 static void interact(R2RState *state) {
823 void **it;
824 RPVector failed_results;
825 r_pvector_init (&failed_results, NULL);
826 r_pvector_foreach (&state->results, it) {
827 R2RTestResultInfo *result = *it;
828 if (result->result == R2R_TEST_RESULT_FAILED) {
829 r_pvector_push (&failed_results, result);
830 }
831 }
832 if (r_pvector_empty (&failed_results)) {
833 goto beach;
834 }
835
836 #if __WINDOWS__
837 (void)SetConsoleOutputCP (65001); // UTF-8
838 #endif
839 printf ("\n");
840 printf ("#####################\n");
841 printf (" %"PFMT64u" failed test(s) "UTF8_POLICE_CARS_REVOLVING_LIGHT"\n",
842 (ut64)r_pvector_len (&failed_results));
843
844 r_pvector_foreach (&failed_results, it) {
845 R2RTestResultInfo *result = *it;
846 if (result->test->type != R2R_TEST_TYPE_CMD) {
847 // TODO: other types of tests
848 continue;
849 }
850
851 printf ("#####################\n\n");
852 print_result_diff (&state->run_config, result);
853 menu:
854 printf ("Wat do? "
855 "(f)ix "UTF8_WHITE_HEAVY_CHECK_MARK UTF8_VS16 UTF8_VS16 UTF8_VS16" "
856 "(i)gnore "UTF8_SEE_NO_EVIL_MONKEY" "
857 "(b)roken "UTF8_SKULL_AND_CROSSBONES UTF8_VS16 UTF8_VS16 UTF8_VS16" "
858 "(c)ommands "UTF8_KEYBOARD UTF8_VS16" "
859 "(d)iffchar "UTF8_LEFT_POINTING_MAGNIFYING_GLASS" "
860 "(q)uit "UTF8_DOOR"\n");
861 printf ("> ");
862 char buf[0x30];
863 if (!fgets (buf, sizeof (buf), stdin)) {
864 break;
865 }
866 if (strlen (buf) != 2) {
867 goto menu;
868 }
869 switch (buf[0]) {
870 case 'f':
871 if (result->run_failed || result->proc_out->ret != 0) {
872 printf ("This test has failed too hard to be fixed.\n");
873 goto menu;
874 }
875 interact_fix (result, &failed_results);
876 break;
877 case 'i':
878 break;
879 case 'b':
880 interact_break (result, &failed_results);
881 break;
882 case 'c':
883 interact_commands (result, &failed_results);
884 break;
885 case 'd':
886 interact_diffchar (result);
887 goto menu;
888 case 'q':
889 goto beach;
890 default:
891 goto menu;
892 }
893 }
894
895 beach:
896 r_pvector_clear (&failed_results);
897 }
898
format_cmd_kv(const char * key,const char * val)899 static char *format_cmd_kv(const char *key, const char *val) {
900 RStrBuf buf;
901 r_strbuf_init (&buf);
902 r_strbuf_appendf (&buf, "%s=", key);
903 if (strchr (val, '\n')) {
904 r_strbuf_appendf (&buf, "<<EOF\n%sEOF", val);
905 } else {
906 r_strbuf_append (&buf, val);
907 }
908 return r_strbuf_drain_nofree (&buf);
909 }
910
replace_lines(const char * src,size_t from,size_t to,const char * news)911 static char *replace_lines(const char *src, size_t from, size_t to, const char *news) {
912 const char *begin = src;
913 size_t line = 1;
914 while (line < from) {
915 begin = strchr (begin, '\n');
916 if (!begin) {
917 break;
918 }
919 begin++;
920 line++;
921 }
922 if (!begin) {
923 return NULL;
924 }
925
926 const char *end = begin;
927 while (line < to) {
928 end = strchr (end, '\n');
929 if (!end) {
930 break;
931 }
932 end++;
933 line++;
934 }
935
936 RStrBuf buf;
937 r_strbuf_init (&buf);
938 r_strbuf_append_n (&buf, src, begin - src);
939 r_strbuf_append (&buf, news);
940 r_strbuf_append (&buf, "\n");
941 if (end) {
942 r_strbuf_append (&buf, end);
943 }
944 return r_strbuf_drain_nofree (&buf);
945 }
946
947 // After editing a test, fix the line numbers previously saved for all the other tests
fixup_tests(RPVector * results,const char * edited_file,ut64 start_line,st64 delta)948 static void fixup_tests(RPVector *results, const char *edited_file, ut64 start_line, st64 delta) {
949 void **it;
950 r_pvector_foreach (results, it) {
951 R2RTestResultInfo *result = *it;
952 if (result->test->type != R2R_TEST_TYPE_CMD) {
953 continue;
954 }
955 if (result->test->path != edited_file) { // this works because all the paths come from the string pool
956 continue;
957 }
958 R2RCmdTest *test = result->test->cmd_test;
959 test->run_line += delta;
960
961 #define DO_KEY_STR(key, field) \
962 if (test->field.value) { \
963 if (test->field.line_begin >= start_line) { \
964 test->field.line_begin += delta; \
965 } \
966 if (test->field.line_end >= start_line) { \
967 test->field.line_end += delta; \
968 } \
969 }
970
971 #define DO_KEY_BOOL(key, field) \
972 if (test->field.set && test->field.line >= start_line) { \
973 test->field.line += delta; \
974 }
975
976 #define DO_KEY_NUM(key, field) \
977 if (test->field.set && test->field.line >= start_line) { \
978 test->field.line += delta; \
979 }
980
981 R2R_CMD_TEST_FOREACH_RECORD(DO_KEY_STR, DO_KEY_BOOL, DO_KEY_NUM)
982 #undef DO_KEY_STR
983 #undef DO_KEY_BOOL
984 #undef DO_KEY_NUM
985 }
986 }
987
replace_cmd_kv(const char * path,const char * content,size_t line_begin,size_t line_end,const char * key,const char * value,RPVector * fixup_results)988 static char *replace_cmd_kv(const char *path, const char *content, size_t line_begin, size_t line_end, const char *key, const char *value, RPVector *fixup_results) {
989 char *kv = format_cmd_kv (key, value);
990 if (!kv) {
991 return NULL;
992 }
993 size_t kv_lines = r_str_char_count (kv, '\n') + 1;
994 char *newc = replace_lines (content, line_begin, line_end, kv);
995 free (kv);
996 if (!newc) {
997 return NULL;
998 }
999 size_t lines_before = line_end - line_begin;
1000 st64 delta = (st64)kv_lines - (st64)lines_before;
1001 if (line_end == line_begin) {
1002 delta++;
1003 }
1004 fixup_tests (fixup_results, path, line_end, delta);
1005 return newc;
1006 }
1007
replace_cmd_kv_file(const char * path,ut64 line_begin,ut64 line_end,const char * key,const char * value,RPVector * fixup_results)1008 static void replace_cmd_kv_file(const char *path, ut64 line_begin, ut64 line_end, const char *key, const char *value, RPVector *fixup_results) {
1009 char *content = r_file_slurp (path, NULL);
1010 if (!content) {
1011 eprintf ("Failed to read file \"%s\"\n", path);
1012 return;
1013 }
1014 char *newc = replace_cmd_kv (path, content, line_begin, line_end, key, value, fixup_results);
1015 free (content);
1016 if (!newc) {
1017 return;
1018 }
1019 if (r_file_dump (path, (const ut8 *)newc, -1, false)) {
1020 #if __UNIX__
1021 sync ();
1022 #endif
1023 } else {
1024 eprintf ("Failed to write file \"%s\"\n", path);
1025 }
1026 free (newc);
1027 }
1028
interact_fix(R2RTestResultInfo * result,RPVector * fixup_results)1029 static void interact_fix(R2RTestResultInfo *result, RPVector *fixup_results) {
1030 assert (result->test->type == R2R_TEST_TYPE_CMD);
1031 R2RCmdTest *test = result->test->cmd_test;
1032 R2RProcessOutput *out = result->proc_out;
1033 if (test->expect.value && out->out) {
1034 replace_cmd_kv_file (result->test->path, test->expect.line_begin, test->expect.line_end, "EXPECT", out->out, fixup_results);
1035 }
1036 if (test->expect_err.value && out->err) {
1037 replace_cmd_kv_file (result->test->path, test->expect_err.line_begin, test->expect_err.line_end, "EXPECT_ERR", out->err, fixup_results);
1038 }
1039 }
1040
interact_break(R2RTestResultInfo * result,RPVector * fixup_results)1041 static void interact_break(R2RTestResultInfo *result, RPVector *fixup_results) {
1042 assert (result->test->type == R2R_TEST_TYPE_CMD);
1043 R2RCmdTest *test = result->test->cmd_test;
1044 ut64 line_begin;
1045 ut64 line_end;
1046 if (test->broken.set) {
1047 line_begin = test->broken.set;
1048 line_end = line_begin + 1;
1049 } else {
1050 line_begin = line_end = test->run_line;
1051 }
1052 replace_cmd_kv_file (result->test->path, line_begin, line_end, "BROKEN", "1", fixup_results);
1053 }
1054
interact_commands(R2RTestResultInfo * result,RPVector * fixup_results)1055 static void interact_commands(R2RTestResultInfo *result, RPVector *fixup_results) {
1056 assert (result->test->type == R2R_TEST_TYPE_CMD);
1057 R2RCmdTest *test = result->test->cmd_test;
1058 if (!test->cmds.value) {
1059 return;
1060 }
1061 char *name = NULL;
1062 int fd = r_file_mkstemp ("r2r-cmds", &name);
1063 if (fd == -1) {
1064 free (name);
1065 eprintf ("Failed to open tmp file\n");
1066 return;
1067 }
1068 size_t cmds_sz = strlen (test->cmds.value);
1069 if (write (fd, test->cmds.value, cmds_sz) != cmds_sz) {
1070 eprintf ("Failed to write to tmp file\n");
1071 free (name);
1072 close (fd);
1073 return;
1074 }
1075 close (fd);
1076
1077 char *editor = r_sys_getenv ("EDITOR");
1078 if (!editor || !*editor) {
1079 free (editor);
1080 editor = strdup ("vim");
1081 if (!editor) {
1082 free (name);
1083 return;
1084 }
1085 }
1086 r_sys_cmdf ("%s '%s'", editor, name);
1087 free (editor);
1088
1089 char *newcmds = r_file_slurp (name, NULL);
1090 if (!newcmds) {
1091 eprintf ("Failed to read edited command file\n");
1092 free (name);
1093 return;
1094 }
1095 r_str_trim (newcmds);
1096
1097 // if it's multiline we want exactly one trailing newline
1098 if (strchr (newcmds, '\n')) {
1099 char *tmp = newcmds;
1100 newcmds = r_str_newf ("%s\n", newcmds);
1101 free (tmp);
1102 if (!newcmds) {
1103 free (name);
1104 return;
1105 }
1106 }
1107
1108 replace_cmd_kv_file (result->test->path, test->cmds.line_begin, test->cmds.line_end, "CMDS", newcmds, fixup_results);
1109 free (name);
1110 free (newcmds);
1111 }
1112
interact_diffchar(R2RTestResultInfo * result)1113 static void interact_diffchar(R2RTestResultInfo *result) {
1114 const char *actual = result->proc_out->out;
1115 const char *expected = result->test->cmd_test->expect.value;
1116 const char *regexp_out = result->test->cmd_test->regexp_out.value;
1117 printf ("-- stdout\n");
1118 print_diff (actual, expected, true, regexp_out);
1119 }
1120