#!/usr/bin/perl -w
#
#    Watch dog script.
#
#    WatchDawg is a perl script that looks for abnormal conditions in the
#    DMT processing nodes. If an abnormal condition is detected a DMT 
#    Alarm is generated. Conditions  to be monitored are listed in a 
#    configuration file that is specified on the command line. 
#    The following conditions are implemented:
#    A) check-space: look for an almost fill or dismounted NFS file system
#       P1: directory path.
#    B) check-insert: Check SeqInsert status
#    C) check-oldata: Check that online partition exists
#    D) check-data-loss: Check for data loss
#    E) check-fpu-traps: Check for fpu traps
#    F) check-stats:  Check iostat data
#    G) no-command:  Ignore this command.
#
#    Define a few subroutines
sub tStamp; sub find_exec;
#
#    Set up global variables
$prefix="/usr";
$datarootdir="${prefix}/share";
$Debug = grep(/--debug/, @ARGV);
$Node = `uname -n`;
chomp $Node;
$OSname = `uname -s`;
chomp $OSname;
$ArchName = `uname -m`;
chomp $ArchName;

#---------------------------------------  Look for a config file
$File = pop(@ARGV);
if (!defined($File) || $File eq "" || $File =~ /^-/) {
    my $defpath="${datarootdir}/gds-utilities/procmgt";
    foreach my $ext ("$Node", "$ArchName", "$ENV{LIGOSITE}") {
	$File = "WatchDawg_$ext.dat";
	last if ( -e $File);
	$File = "$defpath/WatchDawg_$ext.dat";
	last if ( -e $File);
    }
    die("Configuration file not found\n") unless ( -e $File );
}

#---------------------------------------  Get the TCL nodename
if ( find_exec("tclshexe") eq "" ) {
    print(tStamp(), "[WatchDawg] Warning: can't find tclshexe\n");
    $TclNode = $Node;
} else {
    $TclNode = `echo "puts [info hostname]" | tclshexe`;
    chomp $TclNode;
}
$Partition = $ENV{"LIGOSMPART"};
#
$|=1;
#
#   Set limits
#
$WaitTime=30;
$SeqInsertQueueLimit=20;
$MaxDpushTimeOffset=5;
$MaxFpuTrapHz=100;
$MaxBroadcastHz=10;
#
#   Status
#
$SMVirtualStart=0;
$FpuTrapCount=0;
$BroadcastCount=0;
#
#--> Get kernel statistic
sub getKernelStat {
    return 0 if ($OSname ne "SunOS");
    print(tStamp(), "--> getKernelStat ", join(" ", @_)) if ($Debug);
    my $modname   = $_[0];
    my $statname  = $_[1];
    my $trapCount = `kstat -n $modname | grep $statname`;
    $trapCount    =~ s/^(\s)*$statname(\s)*//;
    print(" >>> $trapCount\n") if ($Debug);
    return $trapCount;
}
#
#--> Find the specified executable
sub find_exec {
    my $exename = "$_[0]";
    my @pathlist = split(":", $ENV{PATH});
    foreach my $bindir (@pathlist) {
    	my $path = "$bindir/$exename";
	return $path if ( -x $path );
    }
    return "";
}
#
#  Set parameters
sub SetParam {
    print(tStamp(), "set-parameter ", join(" ", @_), "\n") if ($Debug);
    if (@_ < 2) {
	print("set-parameter: Insufficient arguments\n");
	return;
    }
    my $parm  = "$_[0]";
    my $value = "$_[1]";
    if ($parm eq "Debug") {
	$Debug = $value;
    } elsif ($parm eq "MaxFpuTrapHz") {
	$MaxFpuTrapHz = $value;
    } elsif ($parm eq "WaitTime") {
	$WaitTime = $value;
    } else {
	print("set-parameter: Unrecognized parameter: $parm\n");
	return;
    }
}
#
#  Get the amount of space on the specified disk  
sub CheckSpace {
    print(tStamp(), "check-space ", join(" ", @_), "\n") if ($Debug);
    my $dir   = "$_[0]";
    if (! -d "$dir") {
	print "No directory: $dir\n";
	return;
    }

    #---------------------------------  Parse limits (pct, blocks)
    my $minpct = 5.0;
    my $minblk = 10000;
    $minpct = $_[1] if (@_ >= 2);
    $minblk = $_[2] if (@_ >= 3);
    print "Space: Limits are ${minpct}% or $minblk M-bytes\n" if (!$nLoop && $Debug);

    my @space = split('[ \t\n]+', `df -BM $dir | grep -v Filesystem`);
    if ($space[5] ne $dir) {
	print "Device: $dir is not mounted on node $Node\n" if ($Debug);
	AlarmSet("NotMounted", "$dir $Node");	
	return;
    }

    my $free    = $space[3];
    $free =~ s,M,,;
    my $total   = $space[1];
    $total =~ s,M,,;
    my $pctfree = $free / ($total / 100);
    print "Free space for device $dir: $free/$total (${pctfree}%)\n" if ($Debug);
    if ( $pctfree < $minpct && $free < $minblk ) {
	print "Device $Node:$dir has less than ${minpct}% free\n" if ($Debug);
	AlarmSet("DeviceFull", "$Node:$dir $free $total");
    }
}
#
#--->  Check SeqInsert is running
#
#      Note: `cat $file | wc -l` instead of `wc -l $file` keeps wc from 
#            printing the file name to stdout.
sub CheckInsert {
    if ( find_exec("SeqInsert") eq "" ) {
        print(tStamp(), "[WatchDawg] SeqInsert not installed on node $Node\n");
	return;
    }
    print(tStamp(), "check-insert ", join(" ", @_), "\n") if ($Debug);
    my $response = `SeqInsert check`;
    chomp $response;
    if ($response =~ "SeqInsert server is not being run by you") {
	print "SeqInsert not running on node $Node\n" if ($Debug);
	AlarmSet("NoSeqInsert", "$Node");
	return;
    }
    $response =~ s/State is //;
    print "SeqInsert state is: $response\n" if ($Debug);
    if ("$response" ne "run" && "$response" ne "idle") {
        print "SeqInsert state error: $response\n" if ($Debug);
	AlarmSet("SeqInsertError", "$response $Node");
    }
    my $Qfile = "$ENV{HOME}/.SeqInsertQueue.$TclNode";
    if ( -e "$Qfile" ) {
	$response = `cat $Qfile | wc -l`;
	chomp $response;
	$response =~ s/^\s*//;
	print "SeqInsert backlog is: $response\n" if ($Debug);
	if ($response > $SeqInsertQueueLimit) {
	    print "SeqInsert Backlog error: $response\n" if ($Debug);
	    AlarmSet("SeqInsertBacklog", "$response $Node");
	}
    }
}

#
#--->  Check that data are current in partition.
sub CheckOLData {
    print(tStamp(), "check-oldata ", join(" ", @_), "\n") if ($Debug);
    my $GPSNow = `when 0 %s`;
    chomp $GPSNow;
    my $part = $Partition;
    $part = $_[0] if (defined($_[0]));
    my $LastGPS = `smstat last_ID $part`;
    chomp $LastGPS;
    my $LastDate;
    if ($LastGPS eq "" || $LastGPS =~ "-0") {
	$LastGPS  = 0;
	$LastDate = "no data";
    } else {
	$LastDate = `when $LastGPS %Y-%m-%d_%H:%N:%S`;
	chomp $LastDate;
    }
    print "Last data inserted: $LastGPS, current date: $GPSNow\n" if ($Debug);
    if ($GPSNow - $LastGPS > $MaxDpushTimeOffset) {
        print "Data are not current ($LastDate) on $Node\n" if ($Debug);
	AlarmSet("DataNotCurrent", "$Node $LastDate");
    }
    if ($LastGPS - $GPSNow > $MaxDpushTimeOffset) {
        print "Data have future date ($LastDate) on $Node\n" if ($Debug);
	AlarmSet("DateInFuture", "$Node $LastDate");
    }
}

#
#--->  Check for lost data
sub CheckSMDataLoss {
    print(tStamp(), "check-data-loss ", join(" ", @_), "\n") if ($Debug);
    my $part = $Partition;
    $part = $_[0] if (defined($_[0]));
    my $VirtStart = `smstat vstart $part`;
    chomp $VirtStart;
    if ($VirtStart eq "" || $VirtStart =~ "-0") {
	$VirtStart = 0;
    } elsif ( $SMVirtualStart == 0 ) {
	$SMVirtualStart = $VirtStart;
    } elsif ($SMVirtualStart < $VirtStart) {
	my $tLost = $VirtStart - $SMVirtualStart;
	$SMVirtualStart = $VirtStart;
	AlarmSet("SMDataLost", "$tLost $Node");
    } else {
	$SMVirtualStart = $VirtStart;
    }
}

#
#--->  Check for FPU Traps
$trap_string="fpu_unfinished_traps";
$fpu_stat_name="fpu_traps";
sub CheckFpuTraps {
    return 0 if ($ArchName ne "sun4u");
    print(tStamp(), "check-fpu-traps ", join(" ", @_), "\n") if ($Debug);
    my $trapCount = getKernelStat($fpu_stat_name, $trap_string);
    if ($FpuTrapCount != 0) {
	my $trapHz = ($trapCount-$FpuTrapCount) / $WaitTime;
	print(tStamp(), "[WatchDawg] Node: $Node, Fpu Trap rate: $trapHz\n")
	    if ($Debug);
	if ($trapHz > $MaxFpuTrapHz) {
	    $trapHz = int($trapHz + 0.5);
	    AlarmSet("FPU_Traps", "$trapHz $Node");
	}
    }
    $FpuTrapCount = $trapCount;
}

#
#--->  Check Broadcast count
sub CheckBroadcast {
    print(tStamp(), "check-broadcast ", join(" ", @_), "\n") if ($Debug);
    my $netdev = "skge0";
    $netdev = $_[0] if (@_ >= 1);
    my $bcastCount = getKernelStat($netdev, "brdcstrcv");
    $bcastCount    =~ s/^(\s)*$trap_string(\s)*//;
    if ($BroadcastCount != 0) {
	my $trapHz = ($bcastCount-$BroadcastCount) / $WaitTime;
	print(tStamp(), "[WatchDawg] Node: $Node, Broadcast rate: $trapHz\n")
	    if ($Debug);
	if ($trapHz > $MaxBroadcastHz) {
	    $trapHz = int($trapHz + 0.5);
	    AlarmSet("Broadcast", "$trapHz $Node:$netdev");
	}
    }
    $BroadcastCount = $bcastCount;
}

#
#--->  Check paging statistics
@StatCols = ();
if ($OSname eq "SunOS") {
    @StatCols = ("thr_r", "thr_b", "thr_w", 
		 "mem_swap", "mem_free", 
		 "page_re", "page_mf", "page_pi", "page_po", "page_fr", "page_de", 
		 "disk_sr", "disk_dd", "disk_f0", "disk_s0", "disk_xx", 
		 "fault_in", "fault_sy", "fault_cs", 
		 "cpu_user", "cpu_sys", "cpu_idle");
} elsif ($OSname eq "Linux") {
    @StatCols = ("thr_r", "thr_b", 
		 "mem_swapd", "mem_free", "mem_buff", "mem_cache", 
		 "swap_si", "swap_so",
		 "disk_bi", "disk_bo", 
		 "fault_in", "fault_cs", 
		 "cpu_user", "cpu_sys", "cpu_idle", "cpu_wait");
}

sub CheckStats {
    print ("Enter CheckStats, O/S $OSname\n") if ($Debug);
    return 1 if (! @StatCols );
    my @stats = `vmstat 2 2`;
    chomp(@stats);
    my @statlist = split(' ', $stats[3]);
    my %stathash;
    foreach my $key (@StatCols) {
	my $value = shift(@statlist);
	$stathash{$key} = $value;
    }
    foreach my $cmd (@_) {
	if ( $cmd =~ /([a-zA-Z_0-9]*)([<=>\!]{1,2})([0-9]*)/ ) {
	    my $op    = $2;
	    my $limit = $3;
	    my $vbl   = $1;
	    my $val   = $stathash{$vbl};
	    if (!defined($vbl)) {
		print("Invalid variable name: $cmd\n");
		next;
	    }
	    print("Evaluate: $vbl (=$val) $op $limit\n") if ($Debug); 
	    if ($op eq "=" || $op eq "==") {
		AlarmSet("StatLimit", "$cmd $Node") if ($val  = $limit);
	    } elsif ($op eq "!=") {
		AlarmSet("StatLimit", "$cmd $Node") if ($val != $limit);
	    } elsif ($op eq "<") {
		AlarmSet("StatLimit", "$cmd $Node") if ($val  < $limit);
	    } elsif ($op eq "<=") {
		AlarmSet("StatLimit", "$cmd $Node") if ($val <= $limit);
	    } elsif ($op eq ">") {
		AlarmSet("StatLimit", "$cmd $Node") if ($val  > $limit);
	    } elsif ($op eq ">=") {
		AlarmSet("StatLimit", "$cmd $Node") if ($val >= $limit);
	    } else {
		print("Invalid stat operator: $op\n");
	    }
	}
    }
}

#
#---> Define an alarm
sub AlarmDefine {
    my $desc = $_[1];
    $desc =~ s/\$/\\\$/g;
    my $exptime = $WaitTime + 5;
    my $Rc = `Alarm define WatchDawg "$_[0]" -sever 5 -time $exptime -desc "$desc" -flags 4 -webfile "Monitors/WatchDawg/WatchDawg_Alarms.html#$_[0]"`;
}

#
#---> Define an alarm
sub AlarmSet {
    my $Rc = `Alarm set WatchDawg "$_[0]" -sever 5 -params '$_[1]'`;
    print("Sent WatchDawg alarm: $_[0] with parameters: $_[1]\n") if ($Debug);
}

#===========================================================================
#
#    Subroutine:  tStamp
#
#    Arguments:   none
#
#    Returns:     The current time formatted in a standard way
#
#    Purpose:     Get the current time to be used as a time stamp for the
#                 Log file.
#
#=======================================  Generate a time stamp
sub tStamp {
    (my $sec, my $min, my $hour, my $mday, my $mon, my $year) = localtime();
    $year += 1900;
    $mon  += 1;
    return sprintf("%d.\%02d.\%02d-\%02d.\%02d.\%02d ", 
		   $year, $mon, $mday,$hour, $min, $sec);
}

#===========================================================================
#
#   Read a line from the specified file handle. Comment lines starting
#   with a '#' are ignored and a trailing '\' indicates that the line 
#   is continued.
#
#   p0 - reference to the file handle
#
#===========================================================================
sub proc_read {
    my $fh = $_[0];
    my $Cont;
    my $Line=<$fh>;
    return undef if (!defined($Line));
    chomp $Line;
    $Line =~ s/\#.*$//g;
    while ($Line =~ s/\\$//) {
	if (defined($Cont=<$fh>)) {
	    chomp $Cont;
	    $Line .= $Cont;
	} else {
	    last;
	}
	$Line =~ s/\#.*$//g;
    }
    $Line =~ s/^\s*//g;
    return $Line;
}


#===========================================================================
#
#                            W A T C H D A W G
#
#                              Main  Routine 
#
#===========================================================================
#
#   Define alarms
#
AlarmDefine("NotMounted", "Device: \$1 is not mounted on node \$2.");
AlarmDefine("DeviceFull", "Device: \$1 is >95% full. \$2 kB of \$3 kB left.");
AlarmDefine("SeqInsertError", "SeqInsert is in state: \$1 on node: \$2");
AlarmDefine("NoSeqInsert",    "SeqInsert is not running on node: \$1");
AlarmDefine("SeqInsertBacklog", "SeqInsert has backlog of \$1 on node: \$2");
AlarmDefine("DataNotCurrent", "Data on node \$1 are not current (\$2).");
AlarmDefine("DateInFuture",   "Data on node \$1 have future date (\$2).");
AlarmDefine("SMDataLost",     "SM Data (\$1 seconds) lost on node \$2.");
AlarmDefine("FPU_Traps",     " High FPU trap rate (\$1 Hz) on node \$2.");
AlarmDefine("Broadcast",     " High broadcast rate (\$1 Hz) on node:dev \$2.");
AlarmDefine("StatLimit",     " System stat exceeds limit: \$1 on node \$2.");

#
#    Get list
#
@CheckList="";
$nCheck=0;
open(InitFile, '<', $File) or die(tStamp(), "[WatchDawg] Can't open $File\n");
print (tStamp(), "[WatchDawg] Reading check-list: $File\n");
while (defined($Line=proc_read(\*InitFile))) {
    next if ("$Line" eq "");
    print (tStamp(), "[WatchDawg] Check command: $Line\n");
    my @cmdlist = split(/ /, $Line);
    my $Cmd = shift(@cmdlist);
    if ($Cmd eq "check-insert") {
        if (find_exec("SeqInsert") eq "") {
	   print(tStamp(), "[WatchDawg] SeqInsert not found on node $Node\n");
	   print(tStamp(), "[WatchDawg] check-insert won't be run\n");
	   next;
	}
    }
    $CheckList[$nCheck++] = $Line;
}
close(InitFile);
#
#  Loop over items in list
#
$nLoop=0;
while (1) {
    foreach my $cmdline (@CheckList) {
	my @cmdlist = split(/ /, $cmdline);
	my $Cmd = shift(@cmdlist);
	if ($Cmd eq "check-space") {
	    CheckSpace( @cmdlist );
	} elsif ($Cmd eq "check-insert") {
	    CheckInsert( @cmdlist );
	} elsif ($Cmd eq "check-oldata") {
	    CheckOLData( @cmdlist );
	} elsif ($Cmd eq "check-data-loss") {
	    CheckSMDataLoss( @cmdlist );
	} elsif ($Cmd eq "check-fpu-traps") {
	    CheckFpuTraps( @cmdlist );
	} elsif ($Cmd eq "check-broadcast") {
	    CheckBroadcast( @cmdlist );
	} elsif ($Cmd eq "check-stats") {
	    CheckStats( @cmdlist );
	} elsif ($Cmd eq "no-commmand") {
	    # fail silently
	} elsif ($Cmd eq "set-parameter") {
	    SetParam( @cmdlist );
	} else {
	    die("Unidentified command $Cmd\n");
	}
    }
    $nLoop++;
    sleep($WaitTime);
}
