#! /usr/bin/perl -w
#
#  This script makes sure that all the standard monitors and infrastructure 
#  processes are running. There are four phases, each controlled by a 
#  separate file. These are:
#    1)  procmgt Initialization - procmgt reads an initialization file
#        containing a list of environment variables to be set
#    2)  System Startup - this phase should be started when the node
#        on which procmgt is running is booted. It is run by specifying
#        "start" as the agrgument to progmgt. During system startup,
#        procmgt starts a series of initialization tasks and optionally 
#        waits for them to complete.
#    3)  Monitoring - Start a list of standard monitors and watch to make 
#        sure that they continue to run.
#    4)  Clean Up - Shut down the listed monitors, and perform cleanup tasks.
#
#  05/17/00 Revert all instances of e.g. " for (my $i..." to 
#           "my $i ; for($i..." for compatibility with old versions of perl.
#  09/14/00 Test process dependent env. vbl generation, directory creation.
#  11/29/01 Remember processes from startphase. Resolve envars in -cd: and
#           in -log:
#  08/14/07 Add '-env=' parameter. Use <filename>_archxxxx.
#
#--- Define Phase subroutines
#
sub Initial; sub Run; sub Start; sub Cleanup; sub ReadDB;
sub getFileName; sub procStart; sub procParse; sub tStamp;
sub rmProc; sub purgeDB; sub getProcName; sub proc_read;
sub find_exe; sub subs_env; sub repair_parts;
#
#--- Process database list
#
#    @Process contains a list of processes that are to be managed.
#    Each process entry contains $EntrySz elements, i.e.
#
#      Offset  Contents
#      ------  --------
#         0    Process name
#         1    Command executed by the process
#         2    PID of executing process (0 if process isn't currently running)
#         3    Restart count (#restarts remaining).
#         4    Log File name
#         5    Execution flags
#         6    When condition.
#         7    Status
#         8    Working directory
#         9    environment "env1=val1;env2=val2;...envn=valn"
#
#     The execution flags (Process[$inx+5]) have the following bit values 
#
#        Bit   Meaning
#       -----  -------
#        0x01  Wait for process to complete (-wait modifier)
#        0x02  Kill process when manager completes (-hup)
#        0x04  No trend output (-no-output:trend)
#        0x08  No report output (-no-output:html)
#        0x10  No data output (-no-output:data)
#        0x20  Disable at start
#
#     The process status has the following bit values:
#
#        Bit   Meaning
#       -----  -------
#        0x01  Evaluation of when clause in progress
#        0x02  When clause satisfied.
#        0x04  Execution disabled
#
$EntrySz = 10;
@Process = ();
$MaxRestart = 100;
#
#--- Do initialization
#
$|=1;  #autoflush

$Phase = (grep(!/-/, @ARGV))[0];
$Phase = "run" if (! $Phase);

$Node = `uname -n`;
chomp $Node;

$Arch = `uname -m`;
chomp $Arch;

print('Running procmgt version: $Id$ \n' );

$Debug = grep(/--debug/, @ARGV);
$RunList = grep(/--runlist/, @ARGV);
$FileBase = (grep(/--file:/, @ARGV))[0];
$prefix="/usr";
$datarootdir="${prefix}/share";
if (defined($FileBase)) {
    $FileBase =~ s/--file://;
} else {
    $FileBase = "${datarootdir}/gds-utilities/procmgt/procmgt";
}
$Site = "X";


$Time = tStamp();
print "Node: ${Node} Phase: ${Phase} Time: $Time\n" if ($Debug);
print (tStamp(), "--------------------------------  initialization phase\n");
Initial;

#--------------------------------------  Set the run-time environment
umask 022;

#--------------------------------------  Clean/update the process database
$ProcDB = "$ENV{HOME}/Process_List_$Node";
print " Check for process database in $ProcDB\n" if ($Debug);
if (-e $ProcDB) {
    purgeDB();
} else {
    mkdir $ProcDB, 0775;
    print($Time, "Created process database directory $ProcDB\n");
}

#---------------------------------------  Initialize control variables.
$ENV{DMTSTORE}  = "$ENV{HOME}"                   if (!defined($ENV{DMTSTORE}));
$ENV{DMTRENDIR} = "$ENV{DMTSTORE}/monitors/data" if (!defined($ENV{DMTRENDIR}));
$ENV{GDSAPACHE} = "$ENV{HOME}/html"              if (!defined($ENV{GDSAPACHE}));

#--------------------------------------  This code is supposed to allow
# use POSIX ':signal_h';
# $term = 0;
# sigaction SIGTERM, new POSIX::SigAction sub { $term = 1 }
#           or die "Error setting SIGTERM handler: $!\n";

$Go = 1;
$SIG{TERM} = \&Cleanup;
$SIG{INT}  = \&Cleanup;
$SIG{HUP}  = \&ReadDB;
$SIG{USR1} = "IGNORE";
#
#--- Execute phases (iterate)
#
while ($Phase) {
    print (tStamp(), "--------------------------------  $Phase phase\n");
    if ($Phase eq "start") {
	Start();
    } elsif ($Phase eq "run") {
	Run();
    } elsif ($Phase eq "stop") {
	Cleanup();
    } else {
	die "Invalid phase name ${Phase},  specified\n";
    }
}

#===========================================================================
#
#    Subroutine:  Initial
#
#    Arguments:   none
#
#    Returns:     The next phase to be executed is returned in $Phase.
#
#    Purpose:     Initialization phase execution. 
#
#===========================================================================
sub Initial {
    my $File;
    return if (!($File = getFileName("Initial")));
    open(InitFile, $File);
    print (tStamp(), "Reading $File\n");
    while (defined($Line=proc_read(\*InitFile))) {
	if ($Line =~ /^\s*(\w+)(\s*)(.*)/) {
	    my $Vbl   = $1;
	    my $Value = $3;
	    $Value =~ s/^"(.*)"$/$1/;
	    print (tStamp(), "Environment variable: $Vbl=$Value\n");
	    if ($Value =~ s/\$(\w*)/$ENV{$1}/g) {
		print(tStamp(), "Substitutions:        $Vbl=$Value\n");
	    }
	    $ENV{$Vbl} = $Value;
	} else {
	    warn "Undecipherable initialization command: $Line\n";
	}
    }
    close(InitFile);

    #-----------------------------------  Add a few standard names
    $ENV{HOST} = $Node;
    $Site = substr($ENV{DMTIFOS}, 0, 1);

    #-----------------------------------  Print process limits
    print(tStamp, "Process limits are as follows: \n");
    system("/bin/sh -c 'ulimit -a'");
}

#===========================================================================
#
#    Subroutine:  Start
#
#    Arguments:   none
#
#    Returns:     The next phase to be executed is returned in $Phase.
#
#    Purpose:     Start phase execution. The Startup process are read
#                 from the Start phase configuration file (procmgt_Start)
#                 and run.
#
#===========================================================================
sub Start {
    #-----------------------------------  Get file name
    my $File;
    if (!($File = getFileName("Start"))) {
	$Phase = "";
	return;
    }    

    #-----------------------------------  Parse the startup commands.
    my $Line;
    open(StartFile, $File);
    print (tStamp(), "Reading $File\n");
    while (defined($Line=proc_read(\*StartFile))) {
       	my @myproc = procParse($Line);
	next if (! @myproc);
	$myproc[3] = 1;
	if ($myproc[6] ne "") {
	    warn "-when not recognized in Run phase... use -wait\n";
	    $myproc[6] = "";
	}
	my $p = @Process;
	push(@Process, @myproc);
	if (! procStart($p)) {
	    pop(@Process) while (@Process > $p);

        #-------------------------------  Wait if requested
	} elsif (($Process[$p+5] & 1) != 0) {
	    my $child = $Process[$p+2];
	    print "Waiting for process $child to complete..." if ($Debug);
	    waitpid($child, 0);
	    print "Done \n" if ($Debug);
	    rmProc($Process[$p]);
	    pop(@Process) while (@Process > $p);

        #-------------------------------  Keep process if -hup specified
	} elsif (($Process[$p+5] & 2) != 0) {
	    $Process[$p+7] |= 4;

        #-------------------------------  Otherwise, we're done
	} else {
	    pop(@Process) while (@Process > $p);
	}
    }
    close(StartFile);
    $Phase = "run";
}

#===========================================================================
#
#    Run Phase
#
#===========================================================================
sub Run {
    
    #------------------------------------  Open the run file
    my $File;
    if (!($File = getFileName("Run"))) {
	$Phase = "";
	return;
    }
    open(RunFile, $File);
    print (tStamp(), "Reading $File\n");
    
    #------------------------------------  Fill the process list.
    my $Line;
    while (defined($Line=proc_read(\*RunFile))) {
	my @myproc = procParse($Line);
	if (@myproc) {
	    if (($myproc[5] & 3) != 0) {
		warn "-wait and -hup not recognized in Run phase. Use -when\n";
		$myproc[5] &= ~3;
	    }
	    push(@Process, @myproc);
	}
    }
    close(RunFile);

    #--------------------------------------  Write the MonMustRun list
    if ($RunList) {
	my $pFile = ">$ENV{GDSAPACHE}/dmt/Monitors/${Node}_MonMustRun.list";
	if (open(ProcFile, "$pFile")) {
	    for ($i=0 ; $i < @Process ; $i+=$EntrySz) {
		my @List  = split(/ /,$Process[$i+1]);
		my $pName = $List[0];
		$pName =~ s|.*/||g;
		print ProcFile "$pName      100   250   1    25\n";
	    }
	    close(ProcFile);
	} else {
	    print "Can't open $pFile\n";
	}
    }

    #--------------------------------------  Iterate over list.
    while ($Go && @Process) {

	#----------------------------------  Add missing processes
	@addlist = ();
	my $i;
	for ($i=0 ; $i < @Process ; $i+=$EntrySz) {
	    $PID = $Process[$i+2];
	    if (! $PID) {
		push(@addlist, ($i));
	    } elsif (! kill(0,$PID)) {
		$Process[$i+2] = 0;
		push(@addlist, ($i));
	    }
	}
	procStart(@addlist);
	
	#----------------------------------  Wait for a process to choke
	print (tStamp(), "Waiting for something to happen.\n");
	my $PID = wait();
	my $PIDStat = $?;
	my $Errno   = $!;

	#----------------------------------  Mark it gone.
	my $Time = tStamp();
	if ($PID != -1 ) {
	    for ($i=0 ; $i < @Process ; $i+=$EntrySz) {
		if ($Process[$i+2] == $PID) {
		    my $r = $Process[$i+3];
		    $Process[$i+2] = 0;
		    if ($Process[$i+7] & 1) {     # Evaluating the when clause?
			$Process[$i+7] &= ~1;     # No more.
			$Process[$i+7] |= 2 if (!$PIDStat);
			print $Time,"[$Process[$i]] condition is $PIDStat\n";
		    } else {
			print $Time,"[$Process[$i]] incarnation died [$r left]\n";
			repair_parts $PID;
			# rmProc("$Process[$i]");
			my $pidl = "$ProcDB/$Process[$i]/pid";
			unlink("$pidl") if (-l "$pidl");
			symlink("NA", "$pidl");
		    }
		    $PID = 0;
		    last;
		}
	    }

	    #----------------------------------  Remove process from fixed list
	    if ($PID != 0) {
	        repair_parts $PID;
		my $rName = getProcName($PID);
		if ($rName) {rmProc($rName);}
	    }
	} else {
	    $Go = 0 if ($Errno eq "No child processes");
	    print("Wait failed, errno=$Errno \n") if ($Debug);
	}

	#--------------------------------------  Count processes alive
	my $nLeft=0;
	for (my $j=0 ; $j<@Process ; $j+=$EntrySz) {
	    $nLeft++ if ($Process[$j+2] != 0 || ($Process[$j+7] & 4) == 0);
	}
	$Go = 0 if ($nLeft == 0);
    }
    $Phase = "stop";
}

#===========================================================================
#
#    Cleanup Phase
#
#===========================================================================
sub Cleanup {
    print (tStamp(), "Cleanup phase\n");
    $Phase = "";
    $Go = 0;
    purgeDB();
    return if (! @Process);

    #-----------------------------------  Kill all current processes in 
    #                                     reverse order of creation
    my $pid; my $pCount = 0;
    for (my $i=@Process-$EntrySz ; $i>=0 ; $i-=$EntrySz) {
	$pid = $Process[$i+2];
	if ($pid != 0) {
	    print (tStamp(), "Killing $Process[$i] (PID $pid)\n");
	    kill(SIGTERM, -$pid);
	    $pCount++;
	} else {
	    print (tStamp(), "Process $Process[$i] already expired\n");
	    rmProc("$Process[$i]");
	}
    }

    #-----------------------------------  Add a sweep process
    if ($pCount != 0) {
	my $p = @Process;
	my @swproc=("_sweep_", "/bin/sleep 30", 0, 1, "", 0x1e, "", 0, 
		    "$ENV{DMTSTORE}/pars", "");
	push(@Process, @swproc);
	procStart($p);
    }

    #-----------------------------------  Wait until all process have finished
    while($pCount != 0 && ($pid = wait()) != -1) {
	for (my $i=0 ; $i<@Process ; $i+=$EntrySz) {
	    if ($Process[$i+2] == $pid) {
	        rmProc("$Process[$i]");
		$Process[$i+2] = 0;
	        if ($Process[$i] eq "_sweep_") {
		    for (my $j=0 ; $j<@Process ; $j+=$EntrySz) {
		    	next if ($Process[$j+2] == 0);
			print(tStamp(), "[_sweep_] Kill process $Process[$j] \n");
			kill(SIGKILL, $Process[$j+2]);
		    }
		} else {
		    $pCount--;
		}
		last;
	    }
	}
    }
    $Phase = "";
}

#===========================================================================
#
#    Get the creation date of the specified file
#
#===========================================================================
sub getFileDate {
    my @pStat=lstat("$_[0]");
    return $pStat[10];
}

#===========================================================================
#
#    Read Database
#
#===========================================================================
sub ReadDB {
    print (tStamp(), "*****  Read DataBase invoked by SigHup  *****\n");
    for (my $i=0 ; $i<@Process ; $i+=$EntrySz) {
	my $restart_proc = 0;
	my $pName=$Process[$i];
	my $subd = "$ProcDB/$pName";

        #-------------------------------  Check restart count of idled process
	if ($Process[$i+2] == 0) {
	    my $cntl = "$subd/restarts";
	    if (-l "$cntl") {
		my $rcnt = readlink($cntl);
		$Process[$i+3] = $rcnt;
		if ($rcnt != 0) {
		    $Process[$i+7] &= ~4;
		    $restart_proc = 1;
		}
		print(tStamp(), "Process $pName restarts set to $rcnt\n");
	    }
	}

        #-------------------------------  Check for reset of arguments.
	if (getFileDate("$subd/args") > getFileDate("$subd/pid")) {
	    print(tStamp(), "Process $pName argument list was reset\n");
	    my @List=split(/ /, $Process[$i+1]);
	    my $argList=`cat $subd/args`;
	    chomp($argList);
	    my $inx = 0;
	    for (my $j=1 ; $j<@List ; $j++) {
		last if ($List[$j] eq ">>");
		$inx = $j;
	    }
	    splice(@List, 1, $inx, $argList);
	    $Process[$i+1]=join(' ', @List);
	    print("New argument list: $Process[$i+1]\n") if ($Debug);
	    $restart_proc = 1;
	}

        #-------------------------------  Restart updated process
	procStart($i) if ($restart_proc);
    }
}
#===========================================================================
#
#   Get a file name
#
#===========================================================================
sub getFileName {
    foreach my $Sufx (".$Node", ".arch_$Arch", "") {
	my $File = "${FileBase}_$_[0]$Sufx";
	print "Try file $File\n" if ($Debug);
	return $File if (-e $File);
    }
    print "No valid file for Phase $_[0]\n";
    return 0;
}

#===========================================================================
#
#    Subroutine:  procStart
#
#    Arguments:   a list of offsets into @Process of processes to (re)start
#
#    Returns:     Number of processes successfully (re)started.
#
#    Purpose:     Start a list of processes. The processes to be started are
#                 defined by a list of offsets into the @Process database.
#                 If an unsatisfied -when clause has been specified, the 
#                 when conditions is tested. Otherwise the process is started.
#                 The DMTOUTPUT, DMTHTMLOUT and DMTRENDOUT environment 
#                 variables are set to $DMTSTORE/monitors/data/<name>, 
#                 $GDSAPACHE/monitor_reports/<name> and $DMTRENDIR/<name>, 
#                 respectively.
#
#===========================================================================
sub procStart {
    my $Success = 0;
    my $Cmd;
    foreach my $i (@_) {

	#-------------------------------  Initialize for this go-round
	my $Pnam = $Process[$i];
	my $sbd = "$ProcDB/$Pnam";

	#-------------------------------  See if this process is still active
	next if (($Process[$i+7] & 4) != 0);
	next if (-e "$sbd/disable");

	#-------------------------------  Check the restart count.
	my $Time = tStamp();
	if ($Process[$i+3] <= 0) {
	    print($Time, "[$Pnam] - too many restarts!\n");
	    $Process[$i+7] |= 4;

	#-------------------------------  Start the command.
	} else {

	    #---------------------------  Get the command or when clause
	    my $When = $Process[$i+6];
	    if ($When eq "" || ($Process[$i+7] & 2)) {
		$Cmd = $Process[$i+1];
		$Process[$i+7] &= ~3;
		print($Time, "[$Pnam] Run: $Cmd\n");
	    } else {
		$Cmd = $When;
		$Process[$i+7] |= 1;
		print($Time, "[$Pnam] Evaluate: $When\n");
	    }

	    #-------------------------------  Set environment variables
	    my $Dout=""; my $Dhtm=""; my $Dtrnd="";
	    my $rFlgs = $Process[$i+5];
	    if ($Phase eq "run") {
		if (! ($rFlgs & 0x10)) {
		    $Dout = "$ENV{DMTSTORE}/monitors/data/$Pnam";
		    if (! -e $Dout) {
			mkdir $Dout, 0775;
			print $Time,"[$Pnam] Created directory $Dout\n";
		    }
		}
		if (! ($rFlgs & 0x08)) {
		    $Dhtm = "$ENV{GDSAPACHE}/monitor_reports/$Pnam";
		    if (! -e $Dhtm) {
			mkdir $Dhtm, 0775;
			print $Time,"[$Pnam] Created directory $Dhtm\n";
		    }
		}
		if (! ($rFlgs & 0x04)) {
		    $Dtrnd = "$ENV{DMTRENDIR}/$Pnam";
		    if (! -e $Dtrnd) {
			mkdir $Dtrnd, 0775;
			print $Time,"[$Pnam] Created directory $Dtrnd\n";
		    }

		    if ( defined($ENV{TEMPLATE_TREND_SUBDIR})) {
		        $Dtrnd = "$Dtrnd/$ENV{TEMPLATE_TREND_SUBDIR}";
		    } else {
		        my $curt  = int(`when 0 %s` / 1000000);
		        my $fortd = "$Dtrnd/$Site-M-$curt";
		        if (! -e "$fortd" ) {
			    mkdir "$fortd", 0775;
			    print "$Time [$Pnam] Created directory $fortd \n";
		        }
                        $Dtrnd = "$Dtrnd/current";
		        if (! -e "$Dtrnd") {
			    symlink "$fortd", "$Dtrnd";
			    print $Time,"[$Pnam] Created symlink $Dtrnd\n";
		    	}
 		    }
		}
	    }
	    $ENV{DMTOUTPUT}  = $Dout;
	    $ENV{DMTHTMLOUT} = $Dhtm;
	    $ENV{DMTRENDOUT} = $Dtrnd;
	    $ENV{DMTPROCESS} = $Pnam;

	    #---------------------------  Translate environment variables
	    my $T = substr($Time,0,16);
	    my $D = substr($Time,0,10);
	    $Cmd =~ s/%t/$T/g;
	    $Cmd =~ s/%d/$D/g;
	    $Cmd =~ s/%h/$Node/g;
	    $Cmd =~ s/\$(\w*)/$ENV{$1}/g;
	    print "Execute command: $Cmd\n" if ($Debug);
	    my $enable = 1;

	    #---------------------------  Set up the log file name
	    my $Log = $Process[$i+4];
            $Log = "" if (($Process[$i+7] & 1) != 0);
            if ($Log ne "") {
                $Log =~ s/%t/$T/g;
                $Log =~ s/%d/$D/g;
                $Log =~ s/%h/$Node/g;
                $Log =~ s/\$(\w*)/$ENV{$1}/g;
            }

	    #---------------------------  Check that the command is valid
	    my $Cwd = subs_env($Process[$i+8]);
	    print "Working directory: $Cwd\n" if ($Debug);
	    my @List = split(/ /,$Cmd);
	    my $child = "NA";
	    my $exeName = find_exe($List[0], "$Cwd");
	    if (!defined($exeName)) {
		$exeName = $List[0];
		print $Time,"[$Pnam] Command not found: $exeName\n";
		# $Process[$i+7] |= 4;
		$enable = 0;
	    }

	    #---------------------------  Check the disable on start flag
	    if ($rFlgs & 0x20) {
		print $Time,"[$Pnam] Disable on start requested\n";
		$enable = 0;
	    }

	    #---------------------------  Run the new process.
	    if ($enable) {
		unlink("$sbd/mgrpid") if (-l "$sbd/mgrpid");
		$child = fork();
		if (!defined($child)) {
		    print $Time,"[$Pnam] Fork failed!\n";
		    $Process[$i+7] &= ~3;
		} elsif (!$child) {
		    my $ppid = getppid();
		    symlink("$ppid", "$sbd/mgrpid");
		    chdir($Cwd);
		    setpgrp();
		    $ENV{PWD} = $Cwd;
                    if ($Log ne "") {
                        open STDOUT, ">>$Log";
                        open STDERR, ">&STDOUT";
			select STDERR; $|=1;
			select STDOUT; $|=1;
                    }
		    if ($Process[$i+9] ne "") {
			foreach my $env (split(/;/, $Process[$i+9])) {
			    if ( $env =~ '^\s*(\w+)\s*=\s*(.*)$' ) {
				my $Vbl = $1;
				my $Val = subs_env($2);
				print $Time,"[$Pnam] Environment: $Vbl=$Val\n"
				    if ($Debug);
				$ENV{$Vbl} = $Val;
			    } else {
				print $Time,"[$Pnam] Invalid env syntax: $env\n";
			    }
			}
		    }
		    exec($Cmd) or print $Time,"[$Pnam] Failed to exec $Cmd\n";
		    exit();
		} else {
		    $Process[$i+2] = $child;
		    $Process[$i+3]--;
		    $Success++;
		    print $Time,"[$Pnam] Process $child created \n";
		}
	    }

            #---------------------------  Add a database entry
	    if (!($Process[$i+7] & 1)) {
		mkdir "$sbd", 0775 if (! -e $sbd);
		unlink("$sbd/exe") if (-l "$sbd/exe");
		symlink("$exeName", "$sbd/exe");
		unlink("$sbd/dmt_output") if (-l "$sbd/dmt_output");
		symlink("$Dout", "$sbd/dmt_output") if (! ($rFlgs & 0x10));
		unlink("$sbd/html_output") if (-l "$sbd/html_output");
		symlink("$Dhtm", "$sbd/html_output") if (! ($rFlgs & 0x08));
		unlink("$sbd/trend_output") if (-l "$sbd/trend_output");
		symlink("$Dtrnd", "$sbd/trend_output") if (! ($rFlgs & 0x04));
		unlink("$sbd/cwd") if (-l "$sbd/cwd");
       	       	symlink("$Cwd",     "$sbd/cwd");
		unlink("$sbd/log") if (-l "$sbd/log");
		my $nArgs = 0;
		for (my $ix=1 ; $ix<@List ; $ix++) {
		    if ($List[$ix] eq ">>") {
			symlink($List[$ix+1], "$sbd/log");
			last;
		    }
		    $nArgs++;
		}
		unlink("$sbd/args") if (-e "$sbd/args");
		open(ARGFILE, ">$sbd/args");
		print(ARGFILE join(' ', splice(@List, 1, $nArgs) ) );
		close(ARGFILE);
		unlink("$sbd/restarts") if (-l "$sbd/restarts");
		symlink("$Process[$i+3]",   "$sbd/restarts");
		unlink("$sbd/pid") if (-l "$sbd/pid");
		symlink("$child",   "$sbd/pid");
		if ($rFlgs & 0x20) {
		    open(DBLFILE, ">$sbd/disable");
		    close(DBLFILE);
		    $Process[$i+5] &= ~0x20;
		}
	    }
	}
    }
    return $Success;
}

#===========================================================================
#
#    Subroutine:  procParse
#
#    Arguments:   A process list command line
#
#    Returns:     A list comprising a process entry.
#
#    Purpose:     Parse a process definition line. The process definition 
#                 parsed and a process entry is produced. The general form
#                 for a process definition is as follows:
#                     [-name:<name>] [-when:<when-condition>] 
#                                    [-cd:<directory>] [-log[:<log-file>]] 
#                                    [-nlives:<n-restart>]
#                                    <command>
#                 where:
#                     <name>           the process name
#                     <when-condition> an sh command which if it evaluates 
#                                      to 0 (no error) will enable the 
#                                      process to be run.
#                     <log-file>       a log file into which the standard
#                                      output will be copied. If -log is
#                                      specified without a file name, the 
#                                      $HOME/logs/<name>-%t.log is used.
#                     <n-restart>      Number of restarts
#                     <command>        the shell command to be executed.
#
#===========================================================================
sub procParse {
    my $Line = $_[0];
    return () if (! length($Line));
    $Line =~ s/\s(\s*)/ /g;
    $Line =~ s/^ //;
    while($Line =~ s/"(.*?) (.*?)"/"$1#$2"/g) {}
    $Line =~ s/"(.*?)"/$1/g;
    my @List = split(/ /,$Line);
    for (my $i=0 ; $i < @List ; $i++) {
	$List[$i] =~ s/#/ /g;
    }
    my $Wait   = 0;
    my $Name   = "";
    my $Log    = "";
    my $When   = "";
    my $envmt  = "";
    my $Resets = $MaxRestart;
    my $Cwd    = "$ENV{DMTSTORE}/pars";
    while ($List[0] =~ "^-") {
	my $Key = shift(@List);
	print "Key: $Key \n" if($Debug);
	if ($Key eq "-wait") {
	    $Wait |= 1;
	} elsif ($Key eq "-hup") {
	    $Wait |= 2;
	} elsif ($Key =~ "^-cd:") {
	    $Cwd = substr($Key, 4);
	    $Cwd =~ s/%h/$Node/g;
	} elsif ($Key eq "-disable") {
	    $Wait |= 0x20;
	} elsif ($Key =~ "^-env=") {
	    $envmt = substr($Key, 5);
	} elsif ($Key =~ "^-name:") {
	    $Name = substr($Key, 6);
	    $Name =~ s/%h/$Node/g;
	} elsif ($Key =~ "^-nlives:") {
	    $Resets = substr($Key, 8);
	} elsif ($Key =~ "^-when:") {
	    $When = substr($Key, 6);
	} elsif ($Key eq "-log") {
	    $Log = "<default>";
	} elsif ($Key =~ "^-log:") {
	    $Log = substr($Key, 5);
	} elsif ($Key =~ "^-no-output:") {
	    my @OutList = split(':', substr($Key, 11));
	    $Wait |= 0x04 if (scalar(grep(/trend/, @OutList)) > 0);
	    $Wait |= 0x08 if (scalar(grep(/html/,  @OutList)) > 0);
	    $Wait |= 0x10 if (scalar(grep(/data/,  @OutList)) > 0);
	} elsif ($Key eq "-no-output") {
	    $Wait |= 0x1c;
	} else {
	    warn "Unidentified option: $Key\n";
	    return ();
	}
    }
    my $Cmd = join(' ', @List);
    if (! $Name) {
	$Name = $List[0];
	$Name =~ s|.*/||g;
	my $inst = 0;
	my $test = $Name;
	for (my $i=0; $i < @Process ; $i+=$EntrySz) {
	    if ($Process[$i] eq $test) {
	       $inst += 1;
	       $test = "${Name}_$inst";
	    }
	}
	$Name = $test;
	print "Process assigned name $Name \n" if ($Debug);
    }
    $Log = "\$DMTSTORE/logs/${Name}-\%t.log" if ($Log eq "<default>");
    return ($Name, $Cmd, 0, $Resets, $Log, $Wait, $When, 0, $Cwd, $envmt);
}

#===========================================================================
#
#    Subroutine:  tStamp
#
#    Arguments:   none
#
#    Returns:     The current time formatted in a standard way
#
#    Purpose:     Get the current time to be used as a time stamp for the
#                 Log file.
#
#=======================================  Generate a time stamp
sub tStamp {
    my ($sec, $min, $hour, $mday, $mon, $year) = localtime();
    $year += 1900;
    $mon  += 1;
    return sprintf("%d.\%02d.\%02d-\%02d.\%02d.\%02d ", 
		   $year, $mon, $mday, $hour, $min, $sec);
}

#===========================================================================
#
#    Subroutine:  getProcName
#
#    Arguments:   A unix process ID
#
#    Returns:     A string containing the process name
#
#    Purpose:     Search the external database for a process with the
#                 specified pid.
#
#=========================================================================
sub getProcName {
    my $rName = "";
    opendir(PROCDBHANDLE, $ProcDB);
    while(defined(my $pName = readdir(PROCDBHANDLE))) {
	my $subdir = "$ProcDB/$pName";
	next if (-d $subdir );
	my $pid = readlink("$subdir/pid");
	if (defined($pid) && $pid == $_[0]) {
	    $rName = $pName;
	    last;
	}
    }
    closedir(PROCDBHANDLE);
    return $rName;
}

#===========================================================================
#
#    Subroutine:  purgeID
#
#    Arguments:   none
#
#    Returns:     none
#
#    Purpose:     Remove expired processes from existing database
#
#===========================================================================
sub purgeDB {
    opendir(PROCDBHANDLE, $ProcDB);
    while(defined(my $pName = readdir(PROCDBHANDLE))) {
	next if ($pName =~ "\..*");
	my $subdir = "$ProcDB/$pName";
	next if (! -d $subdir );
	my $pid = readlink("$subdir/pid");
	if (!defined($pid) || $pid == 0 || $pid =~ "NA") {
	    rmProc($pName);
	    next;
	}
	my $pidok = kill 0, $pid;
	if ($pidok) {
	    print $Time,"Process: $pName (PID $pid) already running\n";
	} else {
	    rmProc($pName);
	}
    }
    closedir(PROCDBHANDLE);
}

#===========================================================================
#
#   Remove a specified process directory.
#
#   p0 - Process name
#
#===========================================================================
sub rmProc {
    my $pDir = "$ProcDB/$_[0]";
    return 0 if (! -d $pDir);
    print tStamp(), "Removing expired process: $_[0]\n";
    unlink "$pDir/args", "$pDir/cwd", "$pDir/dmt_output", "$pDir/exe";
    unlink "$pDir/html_output", "$pDir/pid", "$pDir/log", "$pDir/restarts";
    unlink "$pDir/trend_output", "$pDir/disable";
    unlink "$pDir/mgrpid";
    rmdir($pDir);
}

#===========================================================================
#
#   Read a line from the specified file handle. Comment lines starting
#   with a '#' are ignored and a trailing '\' indicates that the line 
#   is continued.
#
#   p0 - reference to the file handle
#
#===========================================================================
sub proc_read {
    my $fh = $_[0];
    my $Line; my $Cont;
    while (defined($Line=<$fh>) && $Line =~ /^#/) {}
    return undef if (!defined($Line));
    chomp $Line;
    while ($Line =~ s/\\$//) {
	if (defined($Cont=<$fh>)) {
	    chomp $Cont;
	    $Line .= $Cont;
	} else {
	    last;
	}
    }
    return $Line;
}

#===========================================================================
#
#   Read a line from the specified file handle. Comment lines starting
#   with a '#' are ignored and a trailing '\' indicates that the line 
#   is continued.
#
#   p0 - reference to the file handle
#
#===========================================================================
sub find_exe {
    my $exeName = $_[0];
    my $Cwd     = $_[1];
    print "Test for valid command: $exeName in: $Cwd\n" if ($Debug);
    my $svdir  = $ENV{PWD};
    chdir($Cwd);
    if ($exeName =~ /^\// || $exeName =~ /^\./) {
	if (-x "$exeName") {
	    chdir($svdir);
	    return "$exeName";
	}
    } else {
	foreach (split(/:/, $ENV{PATH})) {
	    if (-x "$_/$exeName") {
		chdir($svdir);
		return "$_/$exeName";
	    }
	}
    }
    chdir($svdir);
    return undef;
}
#
#   Substitute environment variable in a string.
sub subs_env {
    my $str = $_[0];
    $str =~ s/\$(\w*)/$ENV{$1}/g; 
    return $str;
}
#
#   Repair partitions in use by argument pid
sub repair_parts {
    my $smrepair_exe = find_exe("smrepair", ".");
    return 0 if (!defined(smrepair_exe));
    my $smlist_exe   = find_exe("smlist", ".");
    my $xpid = $_[0];
    if (!defined($xpid) || $xpid == 0 || !defined($smlist_exe)) {
    	my $repair = `$smrepair_exe --keep $ENV{LIGOSMPART}`;
    } else {
	@parlist = split(/\n/, `$smlist_exe`);
      	foreach my $line (@parlist) {
	    chomp $line;
	    $line =~ s/\s(\s*)/ /g;
	    $line =~ s/^ //g;
	    $line =~ s/(\s)/:/g;
	    my @smlist = split(":", $line);
	    my $part = $smlist[1];	    
	    next if ($part =~ "name");
	    my @conlist = split(/\n/, `smdump -c $part`);
	    my $nfound = grep {/ $xpid /} @conlist;
	    print $Time, "pid: $xpid found: $nfound times in: @conlist \n" if ($Debug);
	    if ($nfound != 0) {
                print $Time,"Repair partition: $part \n";	    
		my $repair = `$smrepair_exe --keep $part`;
	    }
	}
    }
}
