1#!/usr/bin/env perl
2##
3## Copyright (c) 2012, 2013, 2016 The University of Utah
4## All rights reserved.
5##
6## This file is distributed under the University of Illinois Open Source
7## License.  See the file COPYING for details.
8##
9
10use strict;
11use warnings;
12
13my $keep_temp = 0;
14my $invoke_compiler = 0;
15my $run_output = 0;
16my $do_replacement = 0;
17my $replacement = "";
18my $check_reference = 0;
19my $reference_value = "";
20my $golden_output = "";
21my $incremental = 0;
22my $verbose = 0;
23my @all_test_files = ();
24my $SRC_FILE = "";
25my $FILE_EXT = "c";
26my @transformations = ();
27my %transformation_results = ();
28my %verified_results = ();
29
30my $CLANG_DELTA = "./clang_delta";
31my $WORKING_DIR = "./working_dir";
32
33my $COMPILER = "clang";
34my $CSMITH_BIN = "";
35
36my $CSMITH_HOME = $ENV{"CSMITH_HOME"};
37my $test_iterations = 100;
38my $with_csmith = 0;
39my $tests_dir = "";
40
41sub print_msg($) {
42  my ($msg) = @_;
43
44  print "$msg" if ($verbose);
45}
46
47sub runit($) {
48    my ($cmd) = @_;
49    print_msg("run: $cmd\n");
50    my $res = system "$cmd";
51    return (($? >> 8) || $res);
52}
53
54sub die_on_fail($) {
55    my ($cmd) = @_;
56
57    my $res = runit($cmd);
58    die "Failed to execute: $cmd!\n" if ($res);
59}
60
61sub get_instance_num($$) {
62    my ($trans, $input) = @_;
63
64    my $clang_delta_cmd = "$CLANG_DELTA --query-instances=$trans $input";
65
66    print_msg("Query the number of available instances for $trans\n");
67    print_msg("$clang_delta_cmd\n");
68    my @out = `$clang_delta_cmd`;
69    die "Cannot query the number of instances for $trans!"
70        if ($? >> 8);
71
72    die "Failed to query the number of instances for $trans on $input"
73        if (!defined($out[0]));
74    my $num;
75    if ($out[0] =~ m/Available transformation instances:[\s\t]*([0-9]+)/) {
76         $num = $1;
77    }
78    else {
79        die "Bad output from clang_delta: $out[0]!";
80    }
81
82    print("Available instances[$num]\n");
83    return $num;
84}
85
86sub do_one_transformation($$$$) {
87    my ($trans, $counter, $input, $output) = @_;
88
89    my $extra_opt = "";
90    if ($do_replacement) {
91        $extra_opt = "--replacement=\"$replacement\"";
92    }
93    if ($check_reference) {
94        $extra_opt .= " --check-reference=\"$reference_value\"";
95    }
96    my $clang_delta_cmd = "$CLANG_DELTA --transformation=$trans --counter=$counter $extra_opt --output=$output $input";
97    print_msg("$clang_delta_cmd\n");
98
99    print("  increasing counter[$counter] ...");
100    my $res = runit($clang_delta_cmd);
101    if ($res) {
102        print("[FAIL]\n");
103    }
104    else {
105        print("[SUCCESS]\n");
106    }
107
108    return $res;
109}
110
111sub verify_one_output($$$) {
112    my ($trans, $cfile, $counter) = @_;
113
114    #my $cfile = "$WORKING_DIR/$trans/$trans" . "_" . "$counter.$FILE_EXT";
115    my $out = "$WORKING_DIR/$trans/$trans" . "_$counter.compiler_out";
116    my $ofile;
117    my $flags = "";
118    if ($run_output) {
119       $ofile = "$WORKING_DIR/$trans/$trans" . "_" . "$counter.exe";
120    }
121    else {
122       $flags = "-c";
123       $ofile = "$WORKING_DIR/$trans/$trans" . "_" . "$counter.o";
124    }
125    my $compiler_cmd = "$COMPILER $flags $cfile -o $ofile > $out 2>&1";
126
127    print_msg("Invoking $COMPILER on $cfile ...\n");
128    print_msg("$compiler_cmd\n");
129
130    print "Compiling $cfile ...\n";
131    my $res = runit($compiler_cmd);
132
133    if ($res) {
134        print("[FAIL]\n");
135    }
136    elsif ($run_output) {
137        open INF1, "$out" or die "can't open $out";
138        while (my $line = <INF1>) {
139          chomp $line;
140          if ($line =~ m/unknown conversion type character/) {
141            die "bad printf!";
142          }
143        }
144        close INF1;
145
146        my $exec_out = "$WORKING_DIR/$trans/$trans" . "_$counter.exec_out";
147        print "Running $ofile...\n";
148        $res = runit("$ofile > $exec_out 2>&1");
149        if ($res) {
150          print("[FAIL to run]\n");
151        }
152        else {
153          if ($golden_output ne "") {
154            my $good = 0;
155            open INF, "$exec_out" or die "can't open $exec_out";
156            while (my $line = <INF>) {
157              chomp $line;
158              if ($line =~ m/$golden_output/) {
159                $good = 1;
160                last;
161              }
162              elsif ($line =~ m/creduce_value\(%\)/) {
163                print("[FAIL: invalid creduce_value!\n");
164                die;
165              }
166            }
167            close INF;
168            if ($good) {
169              print("[SUCCESS]\n");
170            }
171            else {
172              print("[FAIL: can't find golden output!\n");
173              $res = -1;
174            }
175          }
176          else {
177            print("[SUCCESS]\n");
178          }
179        }
180    }
181    else {
182        print("[SUCCESS]\n");
183    }
184
185    return $res;
186}
187
188sub prepare_source_file($) {
189    my ($test_path) = @_;
190
191    my $srcfile = "$test_path/csmith_orig_file.c";
192    system "rm -rf $srcfile";
193
194    my $csmith_cmd = "$CSMITH_HOME/src/csmith --output $srcfile";
195    print_msg("Generating C file...\n");
196    print_msg("$csmith_cmd\n");
197    die_on_fail($csmith_cmd);
198
199    my $processed_file = "$test_path/preprocessed.c";
200    system "rm -rf $processed_file";
201
202    my $include_path = "$CSMITH_HOME/runtime";
203    my $preprocessor = "$COMPILER -I$include_path -E $srcfile > $processed_file";
204    print_msg("Preprocessing the generated C file..\n");
205    print_msg("$preprocessor\n");
206    die_on_fail($preprocessor);
207    $SRC_FILE = $processed_file;
208}
209
210sub do_one_test($) {
211    my ($trans) = @_;
212
213    if (!(grep { $trans eq $_ } @transformations)) {
214      die "Unknown transformation: $trans!";
215    }
216
217    my $input = $SRC_FILE;
218    print "\nTesting $trans on $input ...\n";
219    my $test_dir = "$WORKING_DIR/$trans";
220    print_msg("Making dir $test_dir ...\n");
221    mkdir $test_dir or die;
222
223    my $instance_num = get_instance_num($trans, $input);
224
225    my @results = ();
226    my @veri_results = ();
227
228    print("Running transformation[$trans] on $input ...\n");
229    if ($instance_num >= 1) {
230      my $i = 1;
231      while (1) {
232        my $orig_backup = "$WORKING_DIR/$trans/$trans" . "_0.$FILE_EXT";
233        my $output = "$WORKING_DIR/$trans/$trans" . "_" . "$i.$FILE_EXT";
234
235        print_msg("Copying original file...\n");
236        die_on_fail("cp $input $orig_backup");
237
238        my $result = do_one_transformation($trans, $i, $input, $output);
239        push @results, $result;
240
241        if ($invoke_compiler) {
242            if (!$result) {
243                my $verified_result = verify_one_output($trans, $output, $i);
244                push @veri_results, $verified_result;
245            }
246            else {
247                push @veri_results, 1;
248            }
249        }
250        if ($incremental) {
251            my $new_instance_num = get_instance_num($trans, $output);
252            if ($new_instance_num >= $instance_num) {
253                print "[FAIL: generated output has more instances [$new_instance_num] than old one [$instance_num]\n";
254                die;
255            }
256            $instance_num = $new_instance_num;
257            $input = $output;
258            last if ($new_instance_num == 0);
259        }
260        else {
261            $i++;
262            last if ($i > $instance_num);
263        }
264      }
265    }
266    $transformation_results{$trans} = \@results;
267    $verified_results{$trans} = \@veri_results;
268}
269
270sub doit() {
271    foreach my $trans (@transformations) {
272        do_one_test($trans);
273    }
274}
275
276my @knowns_failures =
277(
278    "nonnull argument with out-of-range operand number",
279);
280
281sub ignore_failures($$) {
282    my ($trans, $failed_counters) = @_;
283
284    my $ignore = 1;
285    foreach my $i (@$failed_counters) {
286        my $out = "$WORKING_DIR/$trans/$trans" . "_$i.compiler_out";
287        open INF, "$out" or die "Can't open $out!";
288        while (my $line = <INF>) {
289            chomp $line;
290            if ($line =~ m/error:(.+)/) {
291                 if (!(grep { index($1, $_) > -1 } @knowns_failures)) {
292                     $ignore = 0;
293                     last;
294                 }
295            }
296        }
297    }
298
299    close INF;
300    return $ignore;
301}
302
303sub dump_one_results($) {
304    my ($results_hash) = @_;
305
306    my $failure_flag = 0;
307
308    foreach my $trans (keys %$results_hash) {
309        my $trans_failed = 0;
310        my @failed_counters = ();
311
312        my $results = $results_hash->{$trans};
313        for(my $i = 0; $i < scalar(@$results); $i++) {
314            my $result = @$results[$i];
315            if ($result) {
316                $trans_failed++;
317                push @failed_counters, ($i+1);
318            }
319        }
320        print "  transformation[$trans]: ";
321        if (!$trans_failed) {
322            print "All instances suceeded!\n";
323        }
324        else {
325            if (($trans ne "return-void") || ignore_failures($trans, \@failed_counters)) {
326                $failure_flag = -1;
327            }
328            print "$trans_failed instances failed [" . join(",", @failed_counters) . "]\n";
329        }
330    }
331    return $failure_flag;
332}
333
334sub dump_results() {
335    my $failure_flag;
336    print("\nTest Results:\n");
337
338    print("\nTransformation results:\n");
339    $failure_flag = dump_one_results(\%transformation_results);
340
341    return $failure_flag if (!$invoke_compiler || ($failure_flag == -1));
342
343    print("\nCompilation results:\n");
344    $failure_flag = dump_one_results(\%verified_results);
345    print("\n");
346    return $failure_flag;
347}
348
349sub finish() {
350    return if ($keep_temp);
351
352    print_msg("deleting $WORKING_DIR\n");
353    system "rm -rf $WORKING_DIR\n";
354}
355
356sub do_tests_with_csmith($) {
357    my ($trans) = @_;
358
359    for (my $i = 0; $i < $test_iterations; $i++) {
360        print "Test iteration [$i]...\n";
361        prepare_source_file($WORKING_DIR);
362        %transformation_results = ();
363        %verified_results = ();
364
365        if (defined($trans)) {
366            do_one_test($trans);
367        }
368        else {
369            doit();
370        }
371
372        my $failure = dump_results();
373        if ($failure == -1) {
374            return;
375        }
376
377        system "rm -rf $WORKING_DIR/*";
378    }
379}
380
381sub prepare() {
382    print_msg("Preparing testing dir ...\n");
383    print_msg("rm -rf $WORKING_DIR\n");
384    system "rm -rf $WORKING_DIR";
385
386    print_msg("Creating $WORKING_DIR ...\n");
387    mkdir $WORKING_DIR or die;
388
389    print_msg("Querying available transformations ...\n");
390    my @outs = `$CLANG_DELTA --transformations`;
391
392    die "Failed to get transformations!" if (@outs < 1);
393
394    print("There are " . scalar(@outs) . " transformations available:\n");
395    foreach (@outs) {
396        print("  $_");
397        chomp $_;
398        push @transformations, $_;
399    }
400
401    print "\nStart testing ...\n";
402}
403
404my $help_msg = 'This script is for testing clang_delta.
405If -transformation=<name> option is not provided(see below), the script does the following work:
406  (1) collect all available transformations
407  (2) for each transformation, get the number of transformation instances for each transformation
408  (3) run clang_delta with counter values from 1 to the number of the instances
409  (4) [optional] if -verify-output is given, invoke gcc to compile the transformed output for each output
410  (5) collect and dump test statistics
411
412If -transformation=<name> is given, the script will only test the designated transformation through step (2) to (5)
413
414Options:
415
416test_transformation.pl [-transformation=<name>] [-keep] [-verify-output] [-verbose] source.c
417  -transformation=<name>: specify a transformation to test [By default, the script will test all transformations]
418  -keep: keep all intermediate transformed results
419  -verify-output: invoke gcc on each transformed output
420  -with-csmith: invoke Csmith to generate a testing file. Please set CSMITH_HOME to make Csmith work correctly
421                (we are not allowed to pass source.c if this option is given)
422  -iterations: testing iterations with Csmith
423  -dir=[dir_name]: test all files in the dir_name
424  -run-output: generate and run the executable for the transformed output
425  -incremental: incrementally transform the output from the last transformation
426  -golden-output=<str>: check if the output of running the executable contains str
427                        (only useful if -run-output is passed)
428  -replacement=<str>: replace the candidate with the str (only works with
429                      --expression-detector)
430  -check-reference=<str>: add reference-checking code for the candidate (only
431                          works with --expression-detector)
432  -verbose: output detailed testing process
433  source.c: file under test
434
435';
436
437sub print_help() {
438  print $help_msg;
439}
440
441sub main() {
442    my $opt;
443    my @unused = ();
444    my $transformation;
445    while(defined ($opt = shift @ARGV)) {
446        if ($opt =~ m/^-(.+)=(.+)$/) {
447            if ($1 eq "transformation") {
448                $transformation = $2;
449            }
450            elsif ($1 eq "iterations") {
451                $test_iterations = $2;
452            }
453            elsif ($1 eq "dir") {
454                $tests_dir = $2;
455            }
456            elsif ($1 eq "golden-output") {
457                $golden_output = $2;
458            }
459            elsif ($1 eq "replacement") {
460                $replacement = $2;
461                $do_replacement = 1;
462            }
463            elsif ($1 eq "check-reference") {
464                $reference_value = $2;
465                $check_reference = 1;
466            }
467            else {
468                die "unknown option: $opt";
469            }
470        }
471        elsif ($opt =~ m/^-(.+)$/) {
472            if ($1 eq "keep") {
473                $keep_temp = 1;
474            }
475            elsif ($1 eq "verify-output") {
476                $invoke_compiler = 1;
477            }
478            elsif ($1 eq "run-output") {
479                $run_output = 1;
480            }
481            elsif ($1 eq "incremental") {
482                $incremental = 1;
483            }
484            elsif ($1 eq "verbose") {
485                $verbose = 1;
486            }
487            elsif ($1 eq "with-csmith") {
488                $with_csmith = 1;
489            }
490            elsif ($1 eq "help") {
491                print_help();
492                return;
493            }
494            else {
495                print "Invalid options: $opt\n";
496                print_help();
497                die;
498            }
499        }
500        else {
501            push @unused, $opt;
502        }
503    }
504
505    if (@unused == 1) {
506        die "Invalid use of -with-csmith option!" if ($with_csmith);
507        die "Invalid use of -dir=[dir] option!" if ($tests_dir ne "");
508        push @all_test_files, $unused[0];
509    }
510    elsif ($with_csmith) {
511        die "Please set CSMITH_HOME env!" if (!defined($CSMITH_HOME));
512        die "Invalid use of -with-csmith option!" if ($tests_dir ne "");
513        prepare();
514        do_tests_with_csmith($transformation);
515        return;
516    }
517    elsif ($tests_dir ne "") {
518      die "$tests_dir doesn't exist!" if (!(-d $tests_dir));
519      @all_test_files = glob ("$tests_dir/*.*");
520    }
521    else {
522        die "Please give a test file!";
523    }
524
525    prepare();
526
527    my $count = 0;
528    foreach (@all_test_files) {
529        $count++;
530        $SRC_FILE = $_;
531        my @a = split('\.', $SRC_FILE);
532        if (@a > 1) {
533            $FILE_EXT = $a[-1];
534        }
535        if (defined($transformation)) {
536            do_one_test($transformation);
537        }
538        else {
539            doit();
540        }
541
542        my $failure = dump_results();
543        if ($failure != -1) {
544            if ($keep_temp) {
545                system "mv $WORKING_DIR/$transformation $WORKING_DIR/${transformation}_$count";
546            }
547            else {
548                system "rm -rf $WORKING_DIR/*";
549            }
550        }
551        else {
552            last;
553        }
554    }
555
556    # dump_results();
557    finish();
558}
559
560main();
561