1#!/usr/bin/env perl 2## 3## Copyright (c) 2012, 2013, 2016 The University of Utah 4## All rights reserved. 5## 6## This file is distributed under the University of Illinois Open Source 7## License. See the file COPYING for details. 8## 9 10use strict; 11use warnings; 12 13my $keep_temp = 0; 14my $invoke_compiler = 0; 15my $run_output = 0; 16my $do_replacement = 0; 17my $replacement = ""; 18my $check_reference = 0; 19my $reference_value = ""; 20my $golden_output = ""; 21my $incremental = 0; 22my $verbose = 0; 23my @all_test_files = (); 24my $SRC_FILE = ""; 25my $FILE_EXT = "c"; 26my @transformations = (); 27my %transformation_results = (); 28my %verified_results = (); 29 30my $CLANG_DELTA = "./clang_delta"; 31my $WORKING_DIR = "./working_dir"; 32 33my $COMPILER = "clang"; 34my $CSMITH_BIN = ""; 35 36my $CSMITH_HOME = $ENV{"CSMITH_HOME"}; 37my $test_iterations = 100; 38my $with_csmith = 0; 39my $tests_dir = ""; 40 41sub print_msg($) { 42 my ($msg) = @_; 43 44 print "$msg" if ($verbose); 45} 46 47sub runit($) { 48 my ($cmd) = @_; 49 print_msg("run: $cmd\n"); 50 my $res = system "$cmd"; 51 return (($? >> 8) || $res); 52} 53 54sub die_on_fail($) { 55 my ($cmd) = @_; 56 57 my $res = runit($cmd); 58 die "Failed to execute: $cmd!\n" if ($res); 59} 60 61sub get_instance_num($$) { 62 my ($trans, $input) = @_; 63 64 my $clang_delta_cmd = "$CLANG_DELTA --query-instances=$trans $input"; 65 66 print_msg("Query the number of available instances for $trans\n"); 67 print_msg("$clang_delta_cmd\n"); 68 my @out = `$clang_delta_cmd`; 69 die "Cannot query the number of instances for $trans!" 70 if ($? >> 8); 71 72 die "Failed to query the number of instances for $trans on $input" 73 if (!defined($out[0])); 74 my $num; 75 if ($out[0] =~ m/Available transformation instances:[\s\t]*([0-9]+)/) { 76 $num = $1; 77 } 78 else { 79 die "Bad output from clang_delta: $out[0]!"; 80 } 81 82 print("Available instances[$num]\n"); 83 return $num; 84} 85 86sub do_one_transformation($$$$) { 87 my ($trans, $counter, $input, $output) = @_; 88 89 my $extra_opt = ""; 90 if ($do_replacement) { 91 $extra_opt = "--replacement=\"$replacement\""; 92 } 93 if ($check_reference) { 94 $extra_opt .= " --check-reference=\"$reference_value\""; 95 } 96 my $clang_delta_cmd = "$CLANG_DELTA --transformation=$trans --counter=$counter $extra_opt --output=$output $input"; 97 print_msg("$clang_delta_cmd\n"); 98 99 print(" increasing counter[$counter] ..."); 100 my $res = runit($clang_delta_cmd); 101 if ($res) { 102 print("[FAIL]\n"); 103 } 104 else { 105 print("[SUCCESS]\n"); 106 } 107 108 return $res; 109} 110 111sub verify_one_output($$$) { 112 my ($trans, $cfile, $counter) = @_; 113 114 #my $cfile = "$WORKING_DIR/$trans/$trans" . "_" . "$counter.$FILE_EXT"; 115 my $out = "$WORKING_DIR/$trans/$trans" . "_$counter.compiler_out"; 116 my $ofile; 117 my $flags = ""; 118 if ($run_output) { 119 $ofile = "$WORKING_DIR/$trans/$trans" . "_" . "$counter.exe"; 120 } 121 else { 122 $flags = "-c"; 123 $ofile = "$WORKING_DIR/$trans/$trans" . "_" . "$counter.o"; 124 } 125 my $compiler_cmd = "$COMPILER $flags $cfile -o $ofile > $out 2>&1"; 126 127 print_msg("Invoking $COMPILER on $cfile ...\n"); 128 print_msg("$compiler_cmd\n"); 129 130 print "Compiling $cfile ...\n"; 131 my $res = runit($compiler_cmd); 132 133 if ($res) { 134 print("[FAIL]\n"); 135 } 136 elsif ($run_output) { 137 open INF1, "$out" or die "can't open $out"; 138 while (my $line = <INF1>) { 139 chomp $line; 140 if ($line =~ m/unknown conversion type character/) { 141 die "bad printf!"; 142 } 143 } 144 close INF1; 145 146 my $exec_out = "$WORKING_DIR/$trans/$trans" . "_$counter.exec_out"; 147 print "Running $ofile...\n"; 148 $res = runit("$ofile > $exec_out 2>&1"); 149 if ($res) { 150 print("[FAIL to run]\n"); 151 } 152 else { 153 if ($golden_output ne "") { 154 my $good = 0; 155 open INF, "$exec_out" or die "can't open $exec_out"; 156 while (my $line = <INF>) { 157 chomp $line; 158 if ($line =~ m/$golden_output/) { 159 $good = 1; 160 last; 161 } 162 elsif ($line =~ m/creduce_value\(%\)/) { 163 print("[FAIL: invalid creduce_value!\n"); 164 die; 165 } 166 } 167 close INF; 168 if ($good) { 169 print("[SUCCESS]\n"); 170 } 171 else { 172 print("[FAIL: can't find golden output!\n"); 173 $res = -1; 174 } 175 } 176 else { 177 print("[SUCCESS]\n"); 178 } 179 } 180 } 181 else { 182 print("[SUCCESS]\n"); 183 } 184 185 return $res; 186} 187 188sub prepare_source_file($) { 189 my ($test_path) = @_; 190 191 my $srcfile = "$test_path/csmith_orig_file.c"; 192 system "rm -rf $srcfile"; 193 194 my $csmith_cmd = "$CSMITH_HOME/src/csmith --output $srcfile"; 195 print_msg("Generating C file...\n"); 196 print_msg("$csmith_cmd\n"); 197 die_on_fail($csmith_cmd); 198 199 my $processed_file = "$test_path/preprocessed.c"; 200 system "rm -rf $processed_file"; 201 202 my $include_path = "$CSMITH_HOME/runtime"; 203 my $preprocessor = "$COMPILER -I$include_path -E $srcfile > $processed_file"; 204 print_msg("Preprocessing the generated C file..\n"); 205 print_msg("$preprocessor\n"); 206 die_on_fail($preprocessor); 207 $SRC_FILE = $processed_file; 208} 209 210sub do_one_test($) { 211 my ($trans) = @_; 212 213 if (!(grep { $trans eq $_ } @transformations)) { 214 die "Unknown transformation: $trans!"; 215 } 216 217 my $input = $SRC_FILE; 218 print "\nTesting $trans on $input ...\n"; 219 my $test_dir = "$WORKING_DIR/$trans"; 220 print_msg("Making dir $test_dir ...\n"); 221 mkdir $test_dir or die; 222 223 my $instance_num = get_instance_num($trans, $input); 224 225 my @results = (); 226 my @veri_results = (); 227 228 print("Running transformation[$trans] on $input ...\n"); 229 if ($instance_num >= 1) { 230 my $i = 1; 231 while (1) { 232 my $orig_backup = "$WORKING_DIR/$trans/$trans" . "_0.$FILE_EXT"; 233 my $output = "$WORKING_DIR/$trans/$trans" . "_" . "$i.$FILE_EXT"; 234 235 print_msg("Copying original file...\n"); 236 die_on_fail("cp $input $orig_backup"); 237 238 my $result = do_one_transformation($trans, $i, $input, $output); 239 push @results, $result; 240 241 if ($invoke_compiler) { 242 if (!$result) { 243 my $verified_result = verify_one_output($trans, $output, $i); 244 push @veri_results, $verified_result; 245 } 246 else { 247 push @veri_results, 1; 248 } 249 } 250 if ($incremental) { 251 my $new_instance_num = get_instance_num($trans, $output); 252 if ($new_instance_num >= $instance_num) { 253 print "[FAIL: generated output has more instances [$new_instance_num] than old one [$instance_num]\n"; 254 die; 255 } 256 $instance_num = $new_instance_num; 257 $input = $output; 258 last if ($new_instance_num == 0); 259 } 260 else { 261 $i++; 262 last if ($i > $instance_num); 263 } 264 } 265 } 266 $transformation_results{$trans} = \@results; 267 $verified_results{$trans} = \@veri_results; 268} 269 270sub doit() { 271 foreach my $trans (@transformations) { 272 do_one_test($trans); 273 } 274} 275 276my @knowns_failures = 277( 278 "nonnull argument with out-of-range operand number", 279); 280 281sub ignore_failures($$) { 282 my ($trans, $failed_counters) = @_; 283 284 my $ignore = 1; 285 foreach my $i (@$failed_counters) { 286 my $out = "$WORKING_DIR/$trans/$trans" . "_$i.compiler_out"; 287 open INF, "$out" or die "Can't open $out!"; 288 while (my $line = <INF>) { 289 chomp $line; 290 if ($line =~ m/error:(.+)/) { 291 if (!(grep { index($1, $_) > -1 } @knowns_failures)) { 292 $ignore = 0; 293 last; 294 } 295 } 296 } 297 } 298 299 close INF; 300 return $ignore; 301} 302 303sub dump_one_results($) { 304 my ($results_hash) = @_; 305 306 my $failure_flag = 0; 307 308 foreach my $trans (keys %$results_hash) { 309 my $trans_failed = 0; 310 my @failed_counters = (); 311 312 my $results = $results_hash->{$trans}; 313 for(my $i = 0; $i < scalar(@$results); $i++) { 314 my $result = @$results[$i]; 315 if ($result) { 316 $trans_failed++; 317 push @failed_counters, ($i+1); 318 } 319 } 320 print " transformation[$trans]: "; 321 if (!$trans_failed) { 322 print "All instances suceeded!\n"; 323 } 324 else { 325 if (($trans ne "return-void") || ignore_failures($trans, \@failed_counters)) { 326 $failure_flag = -1; 327 } 328 print "$trans_failed instances failed [" . join(",", @failed_counters) . "]\n"; 329 } 330 } 331 return $failure_flag; 332} 333 334sub dump_results() { 335 my $failure_flag; 336 print("\nTest Results:\n"); 337 338 print("\nTransformation results:\n"); 339 $failure_flag = dump_one_results(\%transformation_results); 340 341 return $failure_flag if (!$invoke_compiler || ($failure_flag == -1)); 342 343 print("\nCompilation results:\n"); 344 $failure_flag = dump_one_results(\%verified_results); 345 print("\n"); 346 return $failure_flag; 347} 348 349sub finish() { 350 return if ($keep_temp); 351 352 print_msg("deleting $WORKING_DIR\n"); 353 system "rm -rf $WORKING_DIR\n"; 354} 355 356sub do_tests_with_csmith($) { 357 my ($trans) = @_; 358 359 for (my $i = 0; $i < $test_iterations; $i++) { 360 print "Test iteration [$i]...\n"; 361 prepare_source_file($WORKING_DIR); 362 %transformation_results = (); 363 %verified_results = (); 364 365 if (defined($trans)) { 366 do_one_test($trans); 367 } 368 else { 369 doit(); 370 } 371 372 my $failure = dump_results(); 373 if ($failure == -1) { 374 return; 375 } 376 377 system "rm -rf $WORKING_DIR/*"; 378 } 379} 380 381sub prepare() { 382 print_msg("Preparing testing dir ...\n"); 383 print_msg("rm -rf $WORKING_DIR\n"); 384 system "rm -rf $WORKING_DIR"; 385 386 print_msg("Creating $WORKING_DIR ...\n"); 387 mkdir $WORKING_DIR or die; 388 389 print_msg("Querying available transformations ...\n"); 390 my @outs = `$CLANG_DELTA --transformations`; 391 392 die "Failed to get transformations!" if (@outs < 1); 393 394 print("There are " . scalar(@outs) . " transformations available:\n"); 395 foreach (@outs) { 396 print(" $_"); 397 chomp $_; 398 push @transformations, $_; 399 } 400 401 print "\nStart testing ...\n"; 402} 403 404my $help_msg = 'This script is for testing clang_delta. 405If -transformation=<name> option is not provided(see below), the script does the following work: 406 (1) collect all available transformations 407 (2) for each transformation, get the number of transformation instances for each transformation 408 (3) run clang_delta with counter values from 1 to the number of the instances 409 (4) [optional] if -verify-output is given, invoke gcc to compile the transformed output for each output 410 (5) collect and dump test statistics 411 412If -transformation=<name> is given, the script will only test the designated transformation through step (2) to (5) 413 414Options: 415 416test_transformation.pl [-transformation=<name>] [-keep] [-verify-output] [-verbose] source.c 417 -transformation=<name>: specify a transformation to test [By default, the script will test all transformations] 418 -keep: keep all intermediate transformed results 419 -verify-output: invoke gcc on each transformed output 420 -with-csmith: invoke Csmith to generate a testing file. Please set CSMITH_HOME to make Csmith work correctly 421 (we are not allowed to pass source.c if this option is given) 422 -iterations: testing iterations with Csmith 423 -dir=[dir_name]: test all files in the dir_name 424 -run-output: generate and run the executable for the transformed output 425 -incremental: incrementally transform the output from the last transformation 426 -golden-output=<str>: check if the output of running the executable contains str 427 (only useful if -run-output is passed) 428 -replacement=<str>: replace the candidate with the str (only works with 429 --expression-detector) 430 -check-reference=<str>: add reference-checking code for the candidate (only 431 works with --expression-detector) 432 -verbose: output detailed testing process 433 source.c: file under test 434 435'; 436 437sub print_help() { 438 print $help_msg; 439} 440 441sub main() { 442 my $opt; 443 my @unused = (); 444 my $transformation; 445 while(defined ($opt = shift @ARGV)) { 446 if ($opt =~ m/^-(.+)=(.+)$/) { 447 if ($1 eq "transformation") { 448 $transformation = $2; 449 } 450 elsif ($1 eq "iterations") { 451 $test_iterations = $2; 452 } 453 elsif ($1 eq "dir") { 454 $tests_dir = $2; 455 } 456 elsif ($1 eq "golden-output") { 457 $golden_output = $2; 458 } 459 elsif ($1 eq "replacement") { 460 $replacement = $2; 461 $do_replacement = 1; 462 } 463 elsif ($1 eq "check-reference") { 464 $reference_value = $2; 465 $check_reference = 1; 466 } 467 else { 468 die "unknown option: $opt"; 469 } 470 } 471 elsif ($opt =~ m/^-(.+)$/) { 472 if ($1 eq "keep") { 473 $keep_temp = 1; 474 } 475 elsif ($1 eq "verify-output") { 476 $invoke_compiler = 1; 477 } 478 elsif ($1 eq "run-output") { 479 $run_output = 1; 480 } 481 elsif ($1 eq "incremental") { 482 $incremental = 1; 483 } 484 elsif ($1 eq "verbose") { 485 $verbose = 1; 486 } 487 elsif ($1 eq "with-csmith") { 488 $with_csmith = 1; 489 } 490 elsif ($1 eq "help") { 491 print_help(); 492 return; 493 } 494 else { 495 print "Invalid options: $opt\n"; 496 print_help(); 497 die; 498 } 499 } 500 else { 501 push @unused, $opt; 502 } 503 } 504 505 if (@unused == 1) { 506 die "Invalid use of -with-csmith option!" if ($with_csmith); 507 die "Invalid use of -dir=[dir] option!" if ($tests_dir ne ""); 508 push @all_test_files, $unused[0]; 509 } 510 elsif ($with_csmith) { 511 die "Please set CSMITH_HOME env!" if (!defined($CSMITH_HOME)); 512 die "Invalid use of -with-csmith option!" if ($tests_dir ne ""); 513 prepare(); 514 do_tests_with_csmith($transformation); 515 return; 516 } 517 elsif ($tests_dir ne "") { 518 die "$tests_dir doesn't exist!" if (!(-d $tests_dir)); 519 @all_test_files = glob ("$tests_dir/*.*"); 520 } 521 else { 522 die "Please give a test file!"; 523 } 524 525 prepare(); 526 527 my $count = 0; 528 foreach (@all_test_files) { 529 $count++; 530 $SRC_FILE = $_; 531 my @a = split('\.', $SRC_FILE); 532 if (@a > 1) { 533 $FILE_EXT = $a[-1]; 534 } 535 if (defined($transformation)) { 536 do_one_test($transformation); 537 } 538 else { 539 doit(); 540 } 541 542 my $failure = dump_results(); 543 if ($failure != -1) { 544 if ($keep_temp) { 545 system "mv $WORKING_DIR/$transformation $WORKING_DIR/${transformation}_$count"; 546 } 547 else { 548 system "rm -rf $WORKING_DIR/*"; 549 } 550 } 551 else { 552 last; 553 } 554 } 555 556 # dump_results(); 557 finish(); 558} 559 560main(); 561