#!/usr/bin/env perl
##
## Copyright (c) 2012, 2013, 2016 The University of Utah
## All rights reserved.
##
## This file is distributed under the University of Illinois Open Source
## License.  See the file COPYING for details.
##

use strict;
use warnings;

my $keep_temp = 0;
my $invoke_compiler = 0;
my $run_output = 0;
my $do_replacement = 0;
my $replacement = "";
my $check_reference = 0;
my $reference_value = "";
my $golden_output = "";
my $incremental = 0;
my $verbose = 0;
my @all_test_files = ();
my $SRC_FILE = "";
my $FILE_EXT = "c";
my @transformations = ();
my %transformation_results = ();
my %verified_results = ();

my $CLANG_DELTA = "./clang_delta";
my $WORKING_DIR = "./working_dir";

my $COMPILER = "clang";
my $CSMITH_BIN = "";

my $CSMITH_HOME = $ENV{"CSMITH_HOME"};
my $test_iterations = 100;
my $with_csmith = 0;
my $tests_dir = "";

sub print_msg($) {
  my ($msg) = @_;

  print "$msg" if ($verbose);
}

sub runit($) {
    my ($cmd) = @_;
    print_msg("run: $cmd\n");
    my $res = system "$cmd";
    return (($? >> 8) || $res);
}

sub die_on_fail($) {
    my ($cmd) = @_;

    my $res = runit($cmd);
    die "Failed to execute: $cmd!\n" if ($res);
}

sub get_instance_num($$) {
    my ($trans, $input) = @_;

    my $clang_delta_cmd = "$CLANG_DELTA --query-instances=$trans $input";

    print_msg("Query the number of available instances for $trans\n");
    print_msg("$clang_delta_cmd\n");
    my @out = `$clang_delta_cmd`;
    die "Cannot query the number of instances for $trans!" 
        if ($? >> 8);

    die "Failed to query the number of instances for $trans on $input" 
        if (!defined($out[0]));
    my $num;
    if ($out[0] =~ m/Available transformation instances:[\s\t]*([0-9]+)/) {
         $num = $1; 
    }
    else {
        die "Bad output from clang_delta: $out[0]!";
    }
    
    print("Available instances[$num]\n");
    return $num;
}

sub do_one_transformation($$$$) {
    my ($trans, $counter, $input, $output) = @_;

    my $extra_opt = "";
    if ($do_replacement) {
        $extra_opt = "--replacement=\"$replacement\"";
    }
    if ($check_reference) {
        $extra_opt .= " --check-reference=\"$reference_value\"";
    }
    my $clang_delta_cmd = "$CLANG_DELTA --transformation=$trans --counter=$counter $extra_opt --output=$output $input";
    print_msg("$clang_delta_cmd\n");

    print("  increasing counter[$counter] ...");
    my $res = runit($clang_delta_cmd);
    if ($res) {
        print("[FAIL]\n");
    }
    else {
        print("[SUCCESS]\n");
    }

    return $res;
}

sub verify_one_output($$$) {
    my ($trans, $cfile, $counter) = @_;

    #my $cfile = "$WORKING_DIR/$trans/$trans" . "_" . "$counter.$FILE_EXT";
    my $out = "$WORKING_DIR/$trans/$trans" . "_$counter.compiler_out";
    my $ofile;
    my $flags = "";
    if ($run_output) {
       $ofile = "$WORKING_DIR/$trans/$trans" . "_" . "$counter.exe";
    }
    else {
       $flags = "-c";
       $ofile = "$WORKING_DIR/$trans/$trans" . "_" . "$counter.o";
    }
    my $compiler_cmd = "$COMPILER $flags $cfile -o $ofile > $out 2>&1";

    print_msg("Invoking $COMPILER on $cfile ...\n");
    print_msg("$compiler_cmd\n");

    print "Compiling $cfile ...\n";
    my $res = runit($compiler_cmd);

    if ($res) {
        print("[FAIL]\n");
    }
    elsif ($run_output) {
        open INF1, "$out" or die "can't open $out";
        while (my $line = <INF1>) {
          chomp $line;
          if ($line =~ m/unknown conversion type character/) {
            die "bad printf!";
          }
        }
        close INF1;

        my $exec_out = "$WORKING_DIR/$trans/$trans" . "_$counter.exec_out";
        print "Running $ofile...\n";
        $res = runit("$ofile > $exec_out 2>&1");
        if ($res) {
          print("[FAIL to run]\n");
        }
        else {
          if ($golden_output ne "") {
            my $good = 0;
            open INF, "$exec_out" or die "can't open $exec_out";
            while (my $line = <INF>) {
              chomp $line;
              if ($line =~ m/$golden_output/) {
                $good = 1;
                last;
              }
              elsif ($line =~ m/creduce_value\(%\)/) {
                print("[FAIL: invalid creduce_value!\n");
                die;
              }
            }
            close INF;
            if ($good) {
              print("[SUCCESS]\n");
            }
            else {
              print("[FAIL: can't find golden output!\n");
              $res = -1;
            }
          }
          else {
            print("[SUCCESS]\n");
          }
        }
    }
    else {
        print("[SUCCESS]\n");
    }

    return $res;
}

sub prepare_source_file($) {
    my ($test_path) = @_;

    my $srcfile = "$test_path/csmith_orig_file.c";
    system "rm -rf $srcfile";

    my $csmith_cmd = "$CSMITH_HOME/src/csmith --output $srcfile";
    print_msg("Generating C file...\n");
    print_msg("$csmith_cmd\n");
    die_on_fail($csmith_cmd);

    my $processed_file = "$test_path/preprocessed.c";
    system "rm -rf $processed_file";

    my $include_path = "$CSMITH_HOME/runtime";
    my $preprocessor = "$COMPILER -I$include_path -E $srcfile > $processed_file";
    print_msg("Preprocessing the generated C file..\n");
    print_msg("$preprocessor\n");
    die_on_fail($preprocessor);
    $SRC_FILE = $processed_file;
}

sub do_one_test($) {
    my ($trans) = @_;

    if (!(grep { $trans eq $_ } @transformations)) {
      die "Unknown transformation: $trans!";
    }

    my $input = $SRC_FILE;
    print "\nTesting $trans on $input ...\n";
    my $test_dir = "$WORKING_DIR/$trans";
    print_msg("Making dir $test_dir ...\n");
    mkdir $test_dir or die;

    my $instance_num = get_instance_num($trans, $input);

    my @results = ();
    my @veri_results = ();

    print("Running transformation[$trans] on $input ...\n");
    if ($instance_num >= 1) {
      my $i = 1;
      while (1) {
        my $orig_backup = "$WORKING_DIR/$trans/$trans" . "_0.$FILE_EXT";
        my $output = "$WORKING_DIR/$trans/$trans" . "_" . "$i.$FILE_EXT";
  
        print_msg("Copying original file...\n");
        die_on_fail("cp $input $orig_backup");

        my $result = do_one_transformation($trans, $i, $input, $output);
        push @results, $result;

        if ($invoke_compiler) {
            if (!$result) {
                my $verified_result = verify_one_output($trans, $output, $i);
                push @veri_results, $verified_result;
            }
            else {
                push @veri_results, 1;
            }
        }
        if ($incremental) {
            my $new_instance_num = get_instance_num($trans, $output);
            if ($new_instance_num >= $instance_num) {
                print "[FAIL: generated output has more instances [$new_instance_num] than old one [$instance_num]\n";
                die;
            }
            $instance_num = $new_instance_num;
            $input = $output;
            last if ($new_instance_num == 0);
        }
        else {
            $i++;
            last if ($i > $instance_num);
        }
      }
    }
    $transformation_results{$trans} = \@results;
    $verified_results{$trans} = \@veri_results;
}

sub doit() {
    foreach my $trans (@transformations) {
        do_one_test($trans);
    }
}

my @knowns_failures = 
(
    "nonnull argument with out-of-range operand number",
);

sub ignore_failures($$) {
    my ($trans, $failed_counters) = @_;

    my $ignore = 1;
    foreach my $i (@$failed_counters) {
        my $out = "$WORKING_DIR/$trans/$trans" . "_$i.compiler_out";
        open INF, "$out" or die "Can't open $out!";
        while (my $line = <INF>) {
            chomp $line;
            if ($line =~ m/error:(.+)/) {
                 if (!(grep { index($1, $_) > -1 } @knowns_failures)) {
                     $ignore = 0;
                     last;
                 }
            }
        }
    }

    close INF;
    return $ignore;
}

sub dump_one_results($) {
    my ($results_hash) = @_;

    my $failure_flag = 0;

    foreach my $trans (keys %$results_hash) {
        my $trans_failed = 0;
        my @failed_counters = ();

        my $results = $results_hash->{$trans};
        for(my $i = 0; $i < scalar(@$results); $i++) {
            my $result = @$results[$i];
            if ($result) {
                $trans_failed++;
                push @failed_counters, ($i+1);
            }
        }
        print "  transformation[$trans]: ";
        if (!$trans_failed) {
            print "All instances suceeded!\n";
        }
        else {
            if (($trans ne "return-void") || ignore_failures($trans, \@failed_counters)) {
                $failure_flag = -1;
            }
            print "$trans_failed instances failed [" . join(",", @failed_counters) . "]\n";
        }
    }
    return $failure_flag;
}

sub dump_results() {
    my $failure_flag;
    print("\nTest Results:\n");

    print("\nTransformation results:\n");
    $failure_flag = dump_one_results(\%transformation_results);
    
    return $failure_flag if (!$invoke_compiler || ($failure_flag == -1));

    print("\nCompilation results:\n");
    $failure_flag = dump_one_results(\%verified_results);
    print("\n");
    return $failure_flag;
}

sub finish() {
    return if ($keep_temp);

    print_msg("deleting $WORKING_DIR\n");
    system "rm -rf $WORKING_DIR\n";
}

sub do_tests_with_csmith($) {
    my ($trans) = @_;

    for (my $i = 0; $i < $test_iterations; $i++) {
        print "Test iteration [$i]...\n";
        prepare_source_file($WORKING_DIR);
        %transformation_results = ();
        %verified_results = ();
      
        if (defined($trans)) {
            do_one_test($trans);
        }
        else {
            doit();
        }

        my $failure = dump_results();
        if ($failure == -1) {
            return;
        }

        system "rm -rf $WORKING_DIR/*";
    }
}

sub prepare() {
    print_msg("Preparing testing dir ...\n");
    print_msg("rm -rf $WORKING_DIR\n");
    system "rm -rf $WORKING_DIR";

    print_msg("Creating $WORKING_DIR ...\n");
    mkdir $WORKING_DIR or die;

    print_msg("Querying available transformations ...\n");
    my @outs = `$CLANG_DELTA --transformations`;
    
    die "Failed to get transformations!" if (@outs < 1);
  
    print("There are " . scalar(@outs) . " transformations available:\n");
    foreach (@outs) {
        print("  $_");
        chomp $_;
        push @transformations, $_;
    }

    print "\nStart testing ...\n";
}

my $help_msg = 'This script is for testing clang_delta.
If -transformation=<name> option is not provided(see below), the script does the following work:
  (1) collect all available transformations
  (2) for each transformation, get the number of transformation instances for each transformation
  (3) run clang_delta with counter values from 1 to the number of the instances
  (4) [optional] if -verify-output is given, invoke gcc to compile the transformed output for each output
  (5) collect and dump test statistics

If -transformation=<name> is given, the script will only test the designated transformation through step (2) to (5)

Options:

test_transformation.pl [-transformation=<name>] [-keep] [-verify-output] [-verbose] source.c
  -transformation=<name>: specify a transformation to test [By default, the script will test all transformations]
  -keep: keep all intermediate transformed results
  -verify-output: invoke gcc on each transformed output
  -with-csmith: invoke Csmith to generate a testing file. Please set CSMITH_HOME to make Csmith work correctly 
                (we are not allowed to pass source.c if this option is given)
  -iterations: testing iterations with Csmith
  -dir=[dir_name]: test all files in the dir_name
  -run-output: generate and run the executable for the transformed output
  -incremental: incrementally transform the output from the last transformation
  -golden-output=<str>: check if the output of running the executable contains str
                        (only useful if -run-output is passed)
  -replacement=<str>: replace the candidate with the str (only works with
                      --expression-detector)
  -check-reference=<str>: add reference-checking code for the candidate (only
                          works with --expression-detector)
  -verbose: output detailed testing process
  source.c: file under test

';

sub print_help() {
  print $help_msg;  
}

sub main() {
    my $opt;
    my @unused = ();
    my $transformation;
    while(defined ($opt = shift @ARGV)) {
        if ($opt =~ m/^-(.+)=(.+)$/) {
            if ($1 eq "transformation") {
                $transformation = $2;
            }
            elsif ($1 eq "iterations") {
                $test_iterations = $2;
            }
            elsif ($1 eq "dir") {
                $tests_dir = $2;
            }
            elsif ($1 eq "golden-output") {
                $golden_output = $2;
            }
            elsif ($1 eq "replacement") {
                $replacement = $2;
                $do_replacement = 1;
            }
            elsif ($1 eq "check-reference") {
                $reference_value = $2;
                $check_reference = 1;
            }
            else {
                die "unknown option: $opt";
            }
        }
        elsif ($opt =~ m/^-(.+)$/) {
            if ($1 eq "keep") {
                $keep_temp = 1;
            }
            elsif ($1 eq "verify-output") {
                $invoke_compiler = 1;
            }
            elsif ($1 eq "run-output") {
                $run_output = 1;
            }
            elsif ($1 eq "incremental") {
                $incremental = 1;
            }
            elsif ($1 eq "verbose") {
                $verbose = 1;
            }
            elsif ($1 eq "with-csmith") {
                $with_csmith = 1;
            }
            elsif ($1 eq "help") {
                print_help();
                return;
            }
            else {
                print "Invalid options: $opt\n";
                print_help();
                die;
            }
        }
        else {
            push @unused, $opt;
        }
    }

    if (@unused == 1) {
        die "Invalid use of -with-csmith option!" if ($with_csmith);
        die "Invalid use of -dir=[dir] option!" if ($tests_dir ne "");
        push @all_test_files, $unused[0];
    }
    elsif ($with_csmith) {
        die "Please set CSMITH_HOME env!" if (!defined($CSMITH_HOME));
        die "Invalid use of -with-csmith option!" if ($tests_dir ne "");
        prepare();
        do_tests_with_csmith($transformation);
        return;
    }
    elsif ($tests_dir ne "") {
      die "$tests_dir doesn't exist!" if (!(-d $tests_dir));
      @all_test_files = glob ("$tests_dir/*.*");
    }
    else {
        die "Please give a test file!";
    }

    prepare();

    my $count = 0;
    foreach (@all_test_files) {
        $count++;
        $SRC_FILE = $_;
        my @a = split('\.', $SRC_FILE);
        if (@a > 1) {
            $FILE_EXT = $a[-1];
        }
        if (defined($transformation)) {
            do_one_test($transformation);
        }
        else {
            doit();
        }

        my $failure = dump_results();
        if ($failure != -1) {
            if ($keep_temp) {
                system "mv $WORKING_DIR/$transformation $WORKING_DIR/${transformation}_$count";
            }
            else {
                system "rm -rf $WORKING_DIR/*";
            }
        }
        else {
            last;
        }
    }

    # dump_results();
    finish();
}

main();
