#!/usr/bin/env perl
#
# collect information about workflows and display their states.
#
##
#  Copyright 2007-2010 University Of Southern California
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
##
#
# Author: Jens-S. V??ckler voeckler at isi dot edu
# Revision: $Revision: 4486 $
#
use v5.8.8;			# unbroken unicode requires perl >= 5.8.8
use strict; 

BEGIN {
    # use very early - before loading most modules!
    $main::isutf8 = ( exists $ENV{LANG} && $ENV{LANG} =~ m{utf-?8}i );
    delete $ENV{LANG};
    $ENV{LANG} = 'C';
}

use utf8; 			# tell Perl "this script contains UTF-8"
use Carp; 
use Cwd qw(getcwd abs_path); 
use File::Spec;
use File::Basename qw(basename dirname); 
use Getopt::Long qw(:config bundling no_ignore_case);
use Data::Dumper; 

# Path to load Pegasus Perl modules
BEGIN { 
    my $pegasus_config = File::Spec->catfile( dirname($0), 'pegasus-config' );
    eval `$pegasus_config --perl-dump`;
    die("Unable to eval pegasus-config output. $@") if $@;
}
use Pegasus::Common;
use Pegasus::Properties qw(%initial); # parses -Dprop=val from @ARGV

# set function TIOCGWINSZ to return ioctl() argument
if ( $^O eq 'darwin' ) { 
    # h2ph is broken on Darwin
    *TIOCGWINSZ = sub { 0x40087468; };
} elsif ( $^O eq 'linux' ) {
    # not broken, but may not be installed
    *TIOCGWINSZ = sub { 0x5413; };
} elsif ( lc $^O eq 'sunos' ) {
    # may be easier than getting the headers right
    *TIOCGWINSZ = sub { 0x5468; }; 
} else {
    # Neither linux nor macosx
    eval { require "sys/ioctl.ph" }; 
}

#
# --- globals ----------------------------------------------
#
$main::debug = 0;		# debug output
$main::color = 0; 		# default: no color (black bg terminal)
$main::user = $ENV{USER} || $ENV{LOGNAME} || scalar getpwuid($>);
$_ = '$Revision: 4486 $';     # don't edit, automatically updated by CVS
$main::revision=$1 if /Revision:\s+([0-9.]+)/o;
$main::onatty = -t STDOUT; 	# are we connected to a terminal?
$main::dirsep = File::Spec->catdir( '', '' ); 
$main::space = '  ';		# 2 spaces in basic mode
@main::clong = 			# Condor job states (basic mode)
    ( 'Unsub'			# U
    , 'Idle'			# I
    , 'Run'			# R
    , 'Del'			# X
    , 'Done'			# C
    , 'Held'			# H
    ); 
@main::cstat = qw(U I R X C H);	# Condor job states (expert mode)
@main::ccolor = 
    ( "\033[0;37m"		# unsubmitted (gray)
    , "\033[0;34m"		# idle (blue)
    , "\033[0;32m"		# running (green)
    , "\033[0;35m"		# removing (magenta)
    , "\033[0;36m"		# completed (cyan)
    , "\033[0;31m"		# held (red)
    );
$main::bold  = "\033[1m";	# start real xterm as "xterm -bdc"
$main::reset = "\033[0m";	# reset all color/bold/etc.
%main::ccolor = map { $main::cstat[$_] => $main::ccolor[$_] } 
	0 .. $#main::cstat;
@main::dstat = qw(? I R S F);	# Workflow states
@main::dlong = 
    ( 'Unknown'
    , 'Unknown'			# actually indeterminable 
    , 'Running'
    , 'Success'
    , 'Failure'
    ); 
@main::dcolor =
    ( "\033[0;37m"		# unknown (gray)
    , "\033[0;36m"		# indeterminable (cyan)
    , "\033[0;34m"		# running (blue)
    , "\033[0;30m"		# success (black)
    , "\033[0;31m"		# failure (red)
    );
%main::gstat = 			# Globus job states (expert mode)
    (   0 => '?'		# unknown
    ,   1 => 'P'                # pending
    ,   2 => 'A'                # active
    ,   4 => 'F'                # failed 
    ,   8 => 'D'                # done
    ,  16 => 'S'                # suspend
    ,  32 => 'U'                # unsuspend
    ,  64 => 'I'                # stage-in
    , 128 => 'O'                # stage-out
    );
@main::jobclass = 
    ( 'unknown'			# 0
    , 'compute'			# 1
    , 'stage-in'		# 2
    , 'stage-out'		# 3
    , 'register'		# 4
    , 'xsite-xfer'		# 5
    , 'createdir'		# 6
    , 's-compute'		# 7 -- deprecated
    , 'clean-up'		# 8
    , 'chmod'			# 9
    , 'subdax'			# 10
    , 'subdag'			# 11
    );
@main::jobclass_desc = 
    ( 'unknown (do not use)'
    , 'regular computation job'
    , 'auxilliary stage-in transfer job'
    , 'auxilliary stage-out transfer job'
    , 'auxilliary replica registration job'
    , 'auxilliary inter-site transfer job'
    , 'auxilliary createdir job'
    , 'remote compute job (do not use)'
    , 'auxilliary clean-up job'
    , 'auxilliary chmod job'
    , 'unplanned DAX sub-workflow job'
    , 'planned DAG sub-workflow job'
    );
@main::jobshort =		# short job class (job type)
    (   '-', 'job',  'si',  'so'
    ,  'rr', 'isx',  'cd', 'stc'
    , 'clu', 'chm', 'dax', 'dag' ); 
$main::time = $^T;		# initialization only 
%main::dagman_p =		# predicate to determine variations on DAGMan
    map { $_ => 1 } qw(pegasus-dagman condor_dagman); 
%main::width = 			# width selection (0=unlimited)
    ( dagnodename => 30		# width of concrete dag node identifiers
    , pegasus_wf_name => 24	# width of abstract workflow identifiers
    , cmd => 20			# width of executable that is actually run
    );
$main::cache = undef;		# debug 
my ($rows,$cols) = &initialize_winch;

# %qtitle describes any head we would want to show, 
# indexed by a short internal key:
# {header} is what to put into the title of the output
# {function} is an fptr, being called with current row (q) job classads
# {minwidth} is an minimal width, with negative width meaning left adjustment
# {legend} is what to display for this column in the legend
#
my %qtitle =
    ( 'STAT' => { header => 'STAT'
		, function => \&x_jobstatus
		, minwidth => -4
		, legend => 'Condor job status'
		},
      'S' => { header => 'S'
	     , function => \&x_cstat
	     , minwidth => 1
	     , legend => 'Condor job status'
	     },
      'IN_STATE' => { header => 'IN_STATE'
		    , function => \&x_in_state
		    , minwidth => 8
		    , legend => 'Time job spent in current Condor status'
                    },
      'JPRIO' => { header => 'PRI'
		 , minwidth => 3
		 , function => \&x_jobpriority
		 , legend => 'Condor job priority'
	         },
      'PJC' => { header => 'CLASS'
	       , minwidth => -5
	       , function => \&x_pegasus_jobtype
	       , legend => 'Pegasus job type'
	       },
      'PJCN' => { header => 'C'
	        , minwidth => 1
	        , function => \&x_pegasus_jobtypenum
	        , legend => 'Pegasus job type'
	       },
      'PJCS' => { header => 'PJC'
	        , minwidth => -3
	        , function => \&x_pegasus_jobtypeshort
	        , legend => 'Pegasus job type'
	       },
      'JOB1' => { header => 'JOB'
	        , minwidth => -50
	        , function => \&x_job1
		, legend => 'Workflow- or DAG-Node ID'
                },
      'JOB2' => { header => 'JOB'
	        , minwidth => -32
	        , function => \&x_job2
		, legend => 'DAG-Node ID, command, and workflow-ID'
                },
      'CONDORID' => { header => 'ID'
		    , minwidth => 2
		    , function => \&x_condorid
		    , legend => 'Condor cluster ID'
		    },
      'SITE' => { header => 'SITE'
		, minwidth => -5
		, function => \&x_site
		, legend => 'Job site'
                },
      'C/G' => { header => 'C/G'
	       , function => \&x_cgstatus
	       , minwidth => 3
	       , legend => 'Condor- and Globus job status'
	       }
    );

# @main::qtitle is an array of default outputs, indexed by 'expert
# level'. Each entry is a key into %qtitle. (Eventually, this can
# be overwritten by a CLI option for your own mix-n-match (TBD).)
@main::qtitle = 
    ( 
       # first level is the novice state, leave out distractive information
       [qw(STAT IN_STATE JOB1) ],

       # some more complex information in first expert level 
       [qw(CONDORID S IN_STATE JPRIO JOB2) ],

       # even more complex information in next expert level 
       [qw(CONDORID C/G IN_STATE JPRIO PJCS SITE JOB2) ]
    );


# %dtitle describes any head we would want to show, 
# indexed by a short internal key:
# {header} is what to put into the title of the output
# {function} is an fptr, being called with current row (dag) workflow entry
# {minwidth} is an minimal width, with negative width meaning left adjustment
# {legend} is what to display for this column in the legend
#
my %dtitle =
    ( 'DONE' => { header => 'DONE'
		, function => \&y_dag_done
		, minwidth => 5
		, legend => 'Job completed with success'
	        },
      'PRE' => { header => 'PRE',
	       , function => \&y_dag_pre
	       , minwidth => 5
	       , legend => 'PRE-Scripts running'
	       },
      'QUEUED' => { header => 'IN_Q',
		  , function => \&y_dag_queued
		  , minwidth => 5
		  , legend => 'Submitted jobs'
	          },
      'POST' => { header => 'POST',
	        , function => \&y_dag_post
	        , minwidth => 5
	        , legend => 'POST-Scripts running'
	        },
      'READY' => { header => 'READY',
	         , function => \&y_dag_ready
	         , minwidth => 5
	         , legend => 'Jobs ready for submission'
	         },
      'UNREADY' => { header => 'UNRDY',
		   , function => \&y_dag_unready
		   , minwidth => 5
		   , legend => 'Jobs blocked by dependencies'
	           },
      'FAILED' => { header => 'FAIL',
		  , function => \&y_dag_failed
		  , minwidth => 5
		  , legend => 'Jobs completed with failure'
		  },
      'TOTAL' => { header => 'TOTAL'
		 , function => \&y_dag_total
		 , minwidth => 5
		 , legend => 'Jobs in workflow'
	         },
      'S_DONE' => { header => 'SUCCESS'
		, function => \&y_dag_done
		, minwidth => 7
		, legend => 'Job completed with success'
	        },
      'S_PRE' => { header => 'PRE',
	       , function => \&y_dag_pre
	       , minwidth => 7
	       , legend => 'PRE-Scripts running'
	       },
      'S_QUEUED' => { header => 'QUEUED',
		  , function => \&y_dag_queued
		  , minwidth => 7
		  , legend => 'Submitted jobs'
	          },
      'S_POST' => { header => 'POST',
	        , function => \&y_dag_post
	        , minwidth => 7
	        , legend => 'POST-Scripts running'
	        },
      'S_READY' => { header => 'READY',
	         , function => \&y_dag_ready
	         , minwidth => 7
	         , legend => 'Jobs ready for submission'
	         },
      'S_UNREADY' => { header => 'UNREADY',
		   , function => \&y_dag_unready
		   , minwidth => 7
		   , legend => 'Jobs blocked by dependencies'
	           },
      'S_FAILED' => { header => 'FAILURE',
		  , function => \&y_dag_failed
		  , minwidth => 7
		  , legend => 'Jobs completed with failure'
		  },
      'S_TOTAL' => { header => 'TOTAL'
		 , function => \&y_dag_total
		 , minwidth => 7
		 , legend => 'Total of jobs'
	         },
      'S' => { header => 'S'
	     , function => \&y_dstat
	     , minwidth => 1
	     , legend => 'Workflow state'
	     },
      'DOFT' => { header => 'D/T'
		, function => \&y_done_total
		, minwidth => 3
		, legend => 'Jobs done of total'
                },
      'PERCENT' => { header => '%DONE'
		   , function => \&y_percent
		   , minwidth => 5
		   , legend => 'Success percentage'
		   },
      'STATE' => { header => 'STATE'
		 , function => \&y_dlong
		 , minwidth => -7
		 , legend => 'Workflow state'
		 },
      'EC' => { header => 'EC'
	      , function => \&y_status
	      , minwidth => 1
	      , legend => 'Workflow exit status'
	      },
      'WORKFLOW' => { header => 'DAGNAME'
		    , function => \&y_name
		    , minwidth => -40
		    , legend => 'Name of workflow'
		    }
      );

# @main::dtitle is an array of default outputs, currently fixed. Each
# entry is a key into %qtitle. (Eventually, this can be overwritten by a
# CLI option for your own mix-n-match (TBD).)
@main::dtitle = 
    ( # summary mode
      [ qw(S_UNREADY S_READY S_PRE S_QUEUED S_POST S_DONE S_FAILED PERCENT) ]
      # show subdag mode
    , [ qw(UNREADY READY PRE QUEUED POST DONE FAILED PERCENT STATE WORKFLOW) ],
    );


#
# --- functions --------------------------------------------
#
sub usage(;$) {
    my $msg = shift;
    my $flag = defined $msg && lc($msg) ne 'help';
    if ( $flag ) {
	print $main::bold if $main::color;
	print "ERROR: $msg\n";
	print $main::reset if $main::color; 
    }

    my $app = basename($0); 
    print << "EOF";

Usage: $app [options ] [dagdir]
 $app helps monitor a workflow by querying Condor and directories. 

Optional arguments:
 -h|--help        print this help and exit.
 -V|--version     print version information and exit. 
 -w|--watch [s]   repeatedly print output every 's' seconds, default 60.
 -L|--[no]legend  Enable or disable showing of the legends, default off. 
 -c|--[no]color   Enable or disable ANSI colors, default off.
 -U|--[no]utf8    Enable or disable UTF-8 graphics, default from \$LANG.

Optional arguments affecting Condor Q output:
 -Q|--[no]queue   Disable or enable Condor Q output, default is on. 
 -v|--verbose     increase expert level. 
 -d|--debug       increase debug level (Pegasus debugging only). 
 -u|--user name   monitor jobs for user 'name', default is $main::user.
 -i|--[no]idle    Omit jobs in state 'idle' from output. 
 --[no]held       Disable or enable showing HoldReason, default on.
 --[no]heavy      Disable or enable heavy Unicode lines, default on. 
 -j|--jobtype jt  *Only show jobs of type 'jt', default is all jobs.
                  (run with 'jt' of 'help' to see available job types.)
 -s|--site sid    *Only show jobs running on site 'sid', default is all sites.

Optional arguments affecting DAG output: 
 rundir           directory to monitor, default is CWD.
 -l|--long        Show all DAG states, including sub-DAGs, default only totals.
 -r|--rows        Show summary in rows, not columns. Mutually-exclusive wrt -l. 
 -S|--[no]success Omit 'Success' workflows from --long output, default show. 

(*) denotes a multi-option, which may be given multiple times or comma lists. 

EOF
    exit ($flag ? 1 : 0); 
}

sub myversion() {
    my $version = version();
    print "Pegasus $version, @{[basename($0)]} $main::revision\n"; 
    exit 0; 
}

sub profile_start($) {
    my $fn = shift; 
    if ( CORE::open( $main::profile, ">$fn" ) ) {
	profile_log('start');
    }
}

sub profile_log { 
    if ( defined $main::profile ) { 
	printf { $main::profile } "%s\n", join(' ', isomsdate(), @_ );
    }
}

sub profile_done { 
    profile_log('final');
    close $main::profile;
    undef $main::profile;
}

END { profile_done if defined $main::profile }

sub trim($) { 
    # purpose: remove leading and trailing whitespace, quotes around
    # paramtr: $s (IN): a string
    # returns: possibly shortened string
    #
    my $s = shift;
    $s =~ s/^\s+//;
    $s =~ s/\s+$//;
    $s = substr($s,1,-1) if substr($s,0,1) =~ /[""'']/;
    $s; 
}

sub commas($) {
    # purpose: put commas to separate engineering dimensions
    # paramtr: $x (IN): numerical string
    # returns: string with commas inserted as necessary
    # warning: assumes english locale
    #
    my $text = reverse shift();
    $text =~ s/(\d{3})(?=\d)(?!\d*\.)/$1,/g;
    return scalar reverse $text;
}

sub plural($$) {
    # purpose: print number space item. Add plural-s if number != 1
    # paramtr: $n (IN): count
    #          $s (IN): item string
    # returns: constructed string with proper plural
    #
    my $n = shift;
    my $s = shift;
    return "$n $s" if $n == 1;

    my $last = substr($s,-1);
    if ( $last eq 'y' ) {
        commas($n) . ' ' . substr($s,0,-1) . 'ies';
    } elsif ( $last eq 's' ) {
	commas($n) . " ${s}es"; 
    } else {
        commas($n) . " ${s}s";
    }
}

sub initialize_winch {
    # purpose: determine rows and columns of current window
    # returns: ($rows,$cols)
    # warning: Make sure that this function stays POSIX signal safe!
    #
    my $r = $ENV{LINES} || 25; 
    my $c = $ENV{COLUMNS} || 80; 

    if ( $main::onatty ) { 
	my $ws = pack('S!4',()); 
	if ( defined &TIOCGWINSZ && ioctl( STDOUT, &TIOCGWINSZ, $ws ) ) { 
	    ($r,$c) = unpack("S!4",$ws);
	}
    } else {
	$r = $c = 1E10;	# virtually unlimited
    }

    ($r,$c); 
}

sub sigwinch {
    # purpose: adjust global $rows and $cols when window size changes
    # globals: $rows (OUT): new row count
    #          $cols (OUT): new column count
    # warning: Make sure that this function stays POSIX signal safe!
    # warning: Make sure the handler is only installed for ttys!
    #
    my $ws = pack('S!4',()); 
    if ( defined &TIOCGWINSZ && ioctl( STDOUT, &TIOCGWINSZ, $ws ) ) { 
	($rows,$cols) = unpack("S!4",$ws);
    }
}

sub interval($) {
    # purpose: convert a number of seconds into days, hours, mins, secs
    # paramtr: $s (IN): total number of seconds
    # returns: formatted string with or without days
    #
    use integer;
    my $total = int( shift() );
    my $s = $total % 60;
    my $m = ($total % 3600) / 60; 

    if ( $total < 3600 ) { 
	# no days or hours
	sprintf "%02d:%02d", $m, $s;
    } elsif ( $total < 86400 ) { 
	# no days, don't show days
	sprintf "%02d:%02d:%02d", ($total / 3600), $m, $s;
    } else {
	my $h = ($total % 86400) / 3600;
	my $d = $total / 86400; 
	sprintf "%d+%02d:%02d:%02d", $d, $h, $m, $s;
    }
}

sub fit($$) {
    # purpose: fit a string into a given width, truncate start or end
    # paramtr: $width (IN): maximum space
    #          $s (IN): input string to format
    # returns: formatted string
    # 
    my $width = shift; 
    my $s = shift;
    my $len = length($s); 

    if ( $width != 0 && $len > abs($width) ) {
	if ( $width < 0 ) {
	    # fit from back
	    '..' . substr($s,($len+$width)+2);
	} else {
	    # forward fit
	    substr($s,0,$width-2) . '..';
	}
    } else {
	$s;
    }
}

sub cfit($$) {
    # purpose: fit a string into a given width, truncate center
    # paramtr: $width (IN): maximum space
    #          $s (IN): string to fit
    # returns: fitted string
    #
    my $width = abs( shift() );
    my $s = shift;
    my $len = length($s); 

    if ( $width == 0 || $len <= $width ) {
	# string fits
	$s;
    } else {
	use integer; 
	my $diff = $len - $width + 2; 
	substr( $s, 0, ($len-$diff)/2 ) . 
	    '..' .
	    substr( $s, ($len+$diff)/2 ); 
    }
}

sub headline($$$;$) {
    # purpose: format header from 3 strings
    # paramtr: $left (IN): what to put on left side
    #          $center (IN): what to put into center
    #          $right (IN): what to put on right side
    #          $width (opt. IN): total width (typically terminal)
    # globals: $cols (IN): terminal width default
    # returns: formatted string
    #
    my $l = shift || '';
    my $c = shift || '';
    my $r = shift || '';
    my $width = shift || $cols;

    my $llen = length($l);
    my $clen = length($c);
    my $rlen = length($r); 
    if ( $llen+$clen+$rlen > $width ) {
	# FIXME: fit strings
	$l . $c . $r;
    } else {
	# adjust strings
	use integer; 
	my $room = $width - $llen - $clen - $rlen; 
	my $x = ' ' x ( $room / 2 ); 

	if ( ( $room & 1 ) == 1 ) { 
	    # odd 
	    $l . $x . ' ' . $c . $x . $r;
	} else {
	    # even
	    $l . $x . $c . $x . $r;
	}
    }
}

sub whittle_down(\%\@\@) {
    # purpose: taken a full input set (Q or DAG), and reduce to only 
    #          columns that we'll show
    # paramtr: %title (IN): hash of all available title definitions
    #          @title (IN): current set of columns to show
    #          @input (IN): array of job classads or workflows
    # returns: array of columns to show for the given input
    #
    my $avail = shift;		# %[qd]title
    my $title = shift; 		# @[qd]title
    my $input = shift;		# @q or @dags
    my @result = (); 

    foreach my $row ( @{$input} ) {
	my @y = (); 		# all columns go into @y
	foreach my $k ( @{$title} ) { 
	    confess "FATAL: title \"$k\" does not exist" 
		unless exists $avail->{$k}; 
	    push( @y, &{$avail->{$k}->{function}}( $row ) );
	}
	push( @result, [ @y ] ); # a row goes into @result
    }

    @result; 
}

sub signum($) { 
    # purpose: sign (lat.: signum) function
    # paramtr: $x (IN) number
    # returns: -1 for negative $x, +1 for positive $x and 0 for $x==0
    # warning: comparison with 0 should use |x| < epsilon for floats. 
    #
    my $x = shift; 
    ( $x < 0 ? -1 : ( $x > 0 ? 1 : 0 ) );
}

sub column_widths(\%\@\@) {
    # purpose: compute width of output column from data requirements
    # paramtr: %title (IN): minimum column width comes from this
    #          @title (IN): currently selected set of columns
    #          @input (IN): whittled down input rows (array of arrays)
    # globals: $main::space (IN): current column spacing
    #          $cols (IN): terminal width 
    # returns: array of column widths. 
    # warning: The last column is (attempted to) adjust to the screen width
    #
    my $avail = shift;
    my $title = shift;
    my $input = shift;
    my @max = ();
    my @sgn = (); 

    # start with title minimum width, separating sign and magnitude
    foreach my $k ( @{$title} ) { 
	my $x = $avail->{$k}{minwidth};
	push( @max, abs($x) );
	push( @sgn, signum($x) ); 
    }

    # determine the width of each column, but no smaller than the title width
    foreach my $row ( @{$input} ) {
	for ( my $i=0; $i < @{$row}; ++$i ) { 
	    my $len = length( $row->[$i] ); 
	    $max[$i] = $len if $len > $max[$i]; 
	}
    }

    # fix last column to match maximum terminal width
    my $s = 0; 
    my $l = length( $main::space ); 
    for ( my $i=0; $i < @max; ++$i ) {
	$s += $max[$i] + $l; 
    }
    if ( $s > $cols ) {
	$s -= $max[$#max];
	$max[$#max] = $cols - $s;
	$max[$#max] = 0 if $max[$#max] < 0; 
    }

    # return results
    map { $max[$_] * $sgn[$_] } 0 .. $#max;
}

sub create_legend(\%\@) {
    # purpose: show the legend based on a title (both: Q and DAG)
    # paramtr: %title (IN): hash of all available title defs
    #          @title (IN): select titles
    # globals: $cols (IN): current terminal width
    #          $main::color (IN): whether to use ANSI colors
    #          $main::bold (IN): turn on bold
    #          $main::reset (IN): turn off bold 
    # returns: scalar: string containing the legend
    #          array: [0] string containing legend
    #                 [1] rows required to show legend
    #
    my $avail = shift;		# %[qd]title ref
    my $title = shift;		# @[qd]title ref

    my $result = ''; 
    my $cursor = 0; 

    my $p = 0;
    for ( my $i=0; $i<@{$title}; ++$i ) { 
	my $k = $avail->{ $title->[$i] }{header};
	my $v = $avail->{ $title->[$i] }{legend}; 
	my $l = length($k) + length($v); 
	my $s = '';
	$s .= $main::bold if $main::color;
	$s .= $k;
	$s .= $main::reset if $main::color; 
	$s .= ': ' . $v; 
	if ( $p + $l + 2 > $cols ) { 
	    $result .= "\n$s";
	    ++$cursor;
	    $p = $l + 2;
	} else {
	    $result .= ' ' if $i; 
	    $result .= $s;
	    $p += $l + 3; 
	}
    }
    $result .= "\n\n"; 
    $cursor += 2; 

    wantarray ? ( $result, $cursor ) : $result;
}

sub kickstart($) {
    # purpose: remove kickstart arguments from commandline
    # paramtr: job classad 'Arguments' value
    # returns: remaining commandline, with kickstart removed
    # warning: also applies 'basename' to all absolute filenames
    #
    my @arg = split /\s+/, shift(); # FIXME: deal with quoting properly!
    my @result = ();
    my $state = 0;
    for ( my $i=0; $i<@arg; ++$i ) {
	if ( $state == 0 ) {
	    if ( substr($arg[$i],0,1) eq '-' ) {
		my $opt = substr($arg[$i],1,1);
		if ( index('ioelnNRBLTIwWSs',$opt) >= 0 ) {
		    # skip argument
		    ++$i;
		} elsif ( index('HVX',$opt) >= 0 ) {
		    # do nothing
		} else {
		    warn "Warning: Unknown kickstart argument $arg[$i]\n";
		}
	    } else {
		# this better be the application that we are starting
		$state = 1;
		push( @result, basename($arg[$i]) );
	    }
	} else {
	    # we can only apply basename to absolute filenames, because
	    # those are the only element we can recognize as such. 
	    if ( substr($arg[$i],0,1) eq $main::dirsep ) {
		push( @result, basename($arg[$i]) );
	    } else {
		push( @result, $arg[$i] );
	    }
	}
    }

    wantarray ? @result : join( ' ', @result );
}

sub seqexec(\%) {
    # purpose: count number of jobs in seqexec input file
    # paramtr: %r (IN): job class ad representation
    # returns: number of seqexec sub-jobs
    #
    my $r = shift;
    my $result = 0;
    local(*S); 

    my $fn = File::Spec->rel2abs( $r->{in}, $r->{iwd} );
    profile_log( "open $fn" ) if defined $main::profile;
    if ( open( S, "<$fn" ) ) {
	my @ok = ();
	while ( <S> ) {
	    s/[ \r\n]+$//;
	    s/\#.*//;
	    next if length($_) < 3;
	    push( @ok, $_ );
	}
	close S;
	$result = @ok+0;
    } else {
	warn "Warning: open $fn: $!, skipping\n"
	    if $main::debug > 3; 
    }

    $result;
}

sub cstat($) {
    # purpose: parse condor job state into string.
    # paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  paramtr: $s (IN): job classad 'JobStatus'
    # returns: string  para