#!/usr/bin/env perl
#
# creates diagrams from jobstate log file, see --help for usage info.
#
use 5.006;
use strict;
use File::Spec;
use File::Basename;
use File::Temp qw(tempfile);
use Time::Local;
use Data::Dumper;
use Getopt::Long qw(:config bundling no_ignore_case);

$main::version = 'unknown';
$_ = '$Revision: 3291 $';       # don't edit
$main::version = $1 if /Revision:\s+([0-9.]+)/o;

sub usage {
    my $base = basename($0,'.pl');
    print "Usage: $base [options] dot-dag-file\n\n";
    print "Mandatory arguments:\n";
    print " dot-dag-file  path to the DAGMan input file. This path is also\n";
    print "               used to derive the location of the jobstate.log.\n";
    print "\n";
    print "Optional arguments:\n";
    print " -h|--help     print usage information and exit.\n";
    print " -V|--version  print version information and exit.\n";
    print " -k|--keep     don't unlink temporary data files.\n";
    print " -a|--adjust s adjust time information by s seconds.\n";
    print " -c|--color-file <file> the file mapping job transformation names to color.\n";
    print " -t|--title <title name> the title to be assigned to generated graph. Replaces the default title.\n";
    print " -s|--show-jobnames whether to show the jobnames on right y axis \n";
    print "\n";
    exit(1);
}

# global variables
my $adjustment = 0;		# time zone adjustment (@#~! Condor)
my $nounlink = 0;		# remove temporary files later
my $color_fn ; # the path to the file containing color scheme for jobs
my $user_title; # the user provided title to be applied to the graph
my $show_jobnames; #whether to display jobnames on the right y axis
GetOptions( 'help|h' => \&usage, 
	    'version|V' => sub { print "$main::version\n"; exit(0); },
	    "keep|k!" => \$nounlink,
	    "show-jobnames|s!" => \$show_jobnames,
	    "adjust|a=i" => \$adjustment,
	    "color-file|c=s" => \$color_fn,
	    "title|t=s" => \$user_title );

# determine dag filename to find kickstart records
my $dagfn = shift || die "ERROR: Need the name of the DAG file\n";
die "ERROR: No DAGMan file $dagfn\n" unless -r $dagfn;
my $dagdir = File::Spec->rel2abs( dirname($dagfn) );

# determine job log
my $joblog = shift || File::Spec->catfile( $dagdir, 'jobstate.log' );
die "ERROR: No jobstate file $joblog\n" unless -r $joblog;

# globals
my $dpo;			# used later as $diff+1
$main::duration = 0.0;
$main::min = $main::kmin = 1E20;
$main::max = $main::kmax = 0;
$main::failure = 0;
%main::fail = ();
%main::order = ();		# records earliest timestamp for jobid
%main::state = 
    ( UN_READY => -1,
      PRE_SCRIPT_STARTED => 0,
      PRE_SCRIPT_SUCCESS => 1,
      PRE_SCRIPT_FAILURE => 1,
      SUBMIT => 2,
      GLOBUS_SUBMIT => 3,
      GRID_SUBMIT => 3,
      EXECUTE => 4,
      IMAGE_SIZE => 4,
      JOB_TERMINATED => 5,
      JOB_ABORTED => 5,
      JOB_SUCCESS => 5, #success and failure go to JOB_TERMINATED
      JOB_FAILURE => 5,
      POST_SCRIPT_STARTED => 6,
      POST_SCRIPT_SUCCESS => 7,
      POST_SCRIPT_TERMINATED => -1,
      POST_SCRIPT_FAILURE => 7 );

#ASSIGN DIFFERENT COLORS TO DIFFERENT JOB
#THE HASH IS POPULATED AT RUNTIME by --color-file option
%main::color = 
   ( 
     "pegasus::dirmanager" => "lavender",
     "pegasus::pegasus-transfer" => "magenta",
     "pegasus::rc-client" => "powderblue2",
     "unknown" => "gray(0.1)", 
    );


sub load_job_colors( $ ){
    # purpose: load the color scheme file.
    # paramtr: path to file. 
    # fileformat: each line in file is complete_transformation_name color
    #             e.g. genome::solsanger:1.0   blue
    
    my $color_fn = shift;

    if ( -r $color_fn && -s $color_fn && open( CS, "<$color_fn" ) ) {
	my ($job, $color);
	while ( <CS> ) {
	    chomp;
	    #work on non empty strings
	    if( $_ ){
		($job,$color)=split /\s+/, $_ ;	    
		#print("For job $job color coding is $color \n");
		$main::color{$job}=$color;
	    }
	}
    }
    else{
	die "Unable to open color file $color_fn";	
    }
}

sub find_job_out_file($){
    # purpose: determine the latest .out file to pick up for a job
    # paramtr: $program (IN): the job name
    #          
    # returns: name for the .out file to load

    my $job = shift;
    my ($out,$new_out);
    for( my $i = 0; $i < 1000; $i++){
	$new_out = sprintf( "%s.out.%03d",$job,$i);
	#try and open the file to detect
	if( open( KS, "<$new_out" ) ){
	    close KS;
	    $out = $new_out;
	    next;
	}
	else{
	    #break
	    last;
	}
    }
    
    #if no job.out.xxx exists try for .out
    $out = sprintf( "%s.out", $job ) unless defined( $out );
    #print "Correct out file to pick is $out\n";
    return $out;
}

sub find_exec($;$) {
    # purpose: determine location of given binary in $PATH
    # paramtr: $program (IN): executable basename to look for
    #          $curdir (opt. IN): if true, logically also check '.'
    # returns: fully qualified path to binary, undef if not found
    my $program = shift;
    my $curdir = shift;
    foreach my $dir ( File::Spec->path ) {
        my $fs = File::Spec->catfile( $dir, $program );
        return $fs if -x $fs;
    }
    if ( defined $curdir && $curdir ) {
        my $fs = File::Spec->catfile( File::Spec->curdir(), $program );
        return $fs if -x $fs;
    }
    undef;
}

sub unix2iso (;$) {
    my $stamp = shift || time();
    my $offset = int($stamp) - timelocal( (gmtime($stamp))[0..5] );
    my @stamp = localtime($stamp);
    my $result = sprintf( "%04d-%02d-%02dT%02d:%02d:%02d", 
			  $stamp[5]+1900, $stamp[4]+1, $stamp[3],
			  $stamp[2], $stamp[1], $stamp[0] );
    $result .= ( ( $offset >= 0 ) ? '+' : '-' );
    $offset = abs($offset);
    $result .= sprintf( "%02d:%02d", $offset / 3600, ($offset % 3600) / 60 );
}

sub iso2unix ($) {
    local $_ = shift;
    die unless /(\d{4})-?(\d{2})-?(\d{2})T(\d{2}):?(\d{2}):?(\d{2})/;
    my $stamp = timegm($6,$5,$4,$3,$2-1,$1-1900);
    die unless /\.(\d+)([-+])(\d{2}):?(\d{2})$/;
    my ($fraction,$pm,$offset) = ("0.$1",$2,$3*3600+$4*60);
    $stamp += $fraction;
    $stamp += (( $pm eq '-' ) ? $offset : -$offset);
}

sub default_title($) {
    my $dagfn = shift;
    my @stat = stat($dagfn);

    my $result;
    if ( @stat > 0 ) {
	$result = unix2iso($stat[9]) . ' by ' . getpwuid($stat[4]);
    } else {
	$result = unix2iso() . ' by ' . getpwuid($>);
    }
    $result;
}

sub slurp_dagfile($) {
    my $dagfn = shift;
    my $dagdir = File::Spec->rel2abs( dirname($dagfn) );
    my %result = ();
    open( DAG, "<$dagfn" ) || die "ERROR: open $dagfn: $!\n";
    my @x;
    while ( <DAG> ) {
	next unless /^job/i;
	chomp;
	@x = split;
	$result{$x[1]} = ( index($x[2],0,1) eq '/' ) ? $x[2] : 
	    File::Spec->catfile( $dagdir, $x[2] );
    }
    close DAG;
    %result;
}

sub slurp_jobstate($) {
    my $fn = shift;
    open( JOB, "<$fn" ) ||
	die "ERROR: Unable to read jobstate from $fn: $!\n";

    #   stamp jobid state condorid sitehandle walltime
    # * 1113521855 ID000017 UN_READY - - -
    # x 1113521856 ID000017 PRE_SCRIPT_STARTED - - -
    # x 1113521990 ID000017 PRE_SCRIPT_SUCCESS - - -
    #   1113521996 ID000017 SUBMIT 1633.0 - -
    # * 1113522089 ID000017 GLOBUS_SUBMIT 1633.0 griodine 120
    #   1113522114 ID000017 EXECUTE 1633.0 griodine 120
    #   1113522224 ID000017 JOB_TERMINATED 1633.0 griodine 120
    # x 1113522224 ID000017 POST_SCRIPT_STARTED - griodine 120
    # x 1113522426 ID000017 POST_SCRIPT_TERMINATED 1633.0 griodine 120
    # x 1113522426 ID000017 POST_SCRIPT_SUCCESS - griodine 120
    my (%result,@x);
    while ( <JOB> ) {
	my ($stamp,$jobid,$state,$condorid,$site) = split;
	#print( "[DEBUG] $stamp  $jobid $state $condorid $site\n" );
	if ( $stamp !~ /^\d+$/ && $stamp > 0 ) {
	    warn "Warning: Ignoring illegal input line $_";
	    next;
	}
	    
	$stamp += $adjustment;
#	$main::min = $stamp if $main::min > $stamp;
#	$main::max = $stamp if $main::max < $stamp;
	if ( $jobid eq 'INTERNAL' ) {
	    $main::min = $stamp 
		if ( $condorid eq 'DAGMAN_STARTED' && $main::min > $stamp );
	    $main::max = $stamp 
		if ( $condorid eq 'DAGMAN_FINISHED' && $main::max < $stamp );
	} else {
	    $main::min = $stamp if $main::min > $stamp;
	    $main::max = $stamp if $main::max < $stamp;
	    if ( exists $main::state{$state} ) {
		if ( $state eq 'PRE_SCRIPT_FAILURE' ) {
		    push( @{$main::fail{$jobid}}, [ $result{$jobid}[0], $stamp ] );
		    $main::failure++;
		} elsif ( $state eq 'POST_SCRIPT_FAILURE' ) {
		    push( @{$main::fail{$jobid}}, [ $result{$jobid}[6], $stamp ] );
		    $main::failure++;
		} elsif ( $state eq 'JOB_ABORTED' ) {
		    push( @{$main::fail{$jobid}}, [ $result{$jobid}[2], $stamp ] );
		    $main::failure++;
		}
		elsif( $state eq 'JOB_FAILURE' ){
		    #display between EXECUTE and JOB_FAILURE
		    push( @{$main::fail{$jobid}}, [ $result{$jobid}[4], $stamp ] );
                    $main::failure++;
		}

		if ( $main::state{$state} >= 0 ) {
		    $main::order{$jobid} = 1E20
			unless exists $main::order{$jobid};
		    $main::order{$jobid} = $stamp
			if $main::order{$jobid} > $stamp;
		    $result{$jobid}[$main::state{$state}] = $stamp;
		}
	    } else {
		warn "Warning: $jobid: Ignoring state $state\n";
	    }
	}
    }

    close JOB;
    %result;
}

sub gen_data($$$\%) {
    my $dfn = shift;
    my $y = shift || die;
    my $n = shift;		# number of jobs
    my $p = shift;
    die unless ref $p eq 'HASH';

    warn "# Ploticus Data File generated is $dfn\n"  unless !$nounlink;
    my $nn = $n+1; 

    # try auto-guess x scale
    my ($xstubs,$xtics,$width);
    my $diff = $main::max - $main::min;
    if ( $diff <= 60 ) {
	$xstubs = 5;
	$xtics = 1;
	$width = 8.0;
    } elsif ( $diff <= 3600 ) {
	$xstubs = 600;
	$xtics = 60;
	$width = 8.0;
    } elsif ( $diff <= 14400 ) {
	$xstubs = 1800;
	$xtics = 600;
	$width = ( $diff < 7200 ? 8.0 : $diff / 900 );
    } elsif ( $diff <= 43200 ) {
	$xstubs = 3600;
	$xtics = 600;
	$width = $diff / 3600;
    } elsif ( $diff <= 86400 ) {
	$xstubs = 7200;
	$xtics = 1200;
	$width = $diff / 7200;
    } elsif ( $diff <= 345600 ) {
	$xstubs = 28800;
	$xtics = 4800;
	$width = $diff / 28800;
    } else {
	die "ERROR: $diff s workflow is just too long!\n";
    }

    my $title = defined( $user_title )? $user_title : default_title($dagfn);

    my $divisor  = (defined $show_jobnames )? 10 : 20; #for show-jobnames we need names on right y axis to be displayed correctly.
    my $height = ( $n <= 101 ? 5.0 : $n / $divisor );
    warn "# xstubs=$xstubs, xticks=$xtics, width=$width, height=$height\n";

    my $cfn = substr( $dfn, 0, -4 ) . "-$y.pls";
    #print "CFN file is $cfn \n";
    open( OUT, ">$cfn" ) || die "ERROR: open $cfn: $!\n";
    print OUT << "END";
//
// generated: @{[scalar localtime]}
//
#proc getdata
  file: $dfn

#proc areadef
  rectangle: 0 0 $width $height
  xautorange: datafields=3,5,10,12
  yautorange: datafield=$y
  frame: width=0.5 color=gray(0.3)
  title: $title
  titledetails: align=C style=B size=14

#proc xaxis
  ticincrement: $xtics
  grid: color=rgb(1,0.9,0.8) style=1 dashscale=2

#proc xaxis
  label: jobs over time ( in seconds )
  tics: yes
  stubs: incremental $xstubs
  minorticinc: $xtics
  grid: color=gray(0.8)

#proc yaxis
  ticincrement: 1
  grid: color=rgb(1,0.9,0.8) style=1 dashscale=2

#proc yaxis
  tics: yes
  stubs: incremental 5
  minorticinc: 1
  grid: color=gray(0.8)

#proc bars
  select: \@3 != $dpo
  outline: no
  barwidth: 0.03
  horizontalbars: yes
  segmentfields: 3 4
  locfield: $y
  color: darkblue
  tails: 0.03

#proc legendentry
  sampletype: color
  label: pre script
  details: darkblue

#proc bars
  outline: no
  barwidth: 0.03
  horizontalbars: yes
  segmentfields: 5 8
  locfield: $y
  color: yellow
  tails: 0.03

#proc legendentry
  sampletype: color
  label: condor job
  details: yellow

#proc bars
  select: \@9 != $dpo
  outline: no
  barwidth: 0.03
  horizontalbars: yes
  segmentfields: 9 10
  locfield: $y
  color: purple
  tails: 0.03

#proc legendentry
  sampletype: color
  label: post script
  details: purple

#proc bars
  select: \@6 != $dpo
  outline: color=orange
  color: lightorange
  horizontalbars: yes
  barwidth: 0.04
  tails: no
  segmentfields: 6 7
  locfield: $y

#proc legendentry
  sampletype: color
  label: Q delay
  details: lightorange

#proc scatterplot
  select: \@7 != $dpo
  xfield: 7
  yfield: $y
  symbol: shape=square style=spokes linecolor=black

END
    ;

# taken care of in for loop below
#proc bars
#  select: \@11 != $dpo
#  outline: color=gray(0.4)
#  color: gray(0.75)
#  horizontalbars: yes
#  barwidth: 0.06
#  tails: no
#  segmentfields: 11 12
#  locfield: $y
#
##proc legendentry
#  sampletype: color
#  label: job duration
#  details: gray(0.75)
#
#END
#    ;

    #generate legends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::gends for specific type of jobs
    foreach my $job (keys %main::ge