#!/usr/bin/perl # The LearningOnline Network # Housekeeping program, started by cron # # (TCP networking package # 6/1/99,6/2,6/10,6/11,6/12,6/14,6/26,6/28,6/29,6/30, # 7/1,7/2,7/9,7/10,7/12 Gerd Kortemeyer) # # 7/14,7/15,7/19,7/21,7/22,11/18 Gerd Kortemeyer use IO::File; use IO::Socket; # -------------------------------------------------- Non-critical communication sub reply { my ($cmd,$server)=@_; my $peerfile="$perlvar{'lonSockDir'}/$server"; my $client=IO::Socket::UNIX->new(Peer =>"$peerfile", Type => SOCK_STREAM, Timeout => 10) or return "con_lost"; print $client "$cmd\n"; my $answer=<$client>; chomp($answer); if (!$answer) { $answer="con_lost"; } return $answer; } # --------------------------------------------------------- Output error status sub errout { my $fh=shift; print $fh (<
Notices$notices
Warnings$warnings
Errors$errors

Top

ENDERROUT } # ================================================================ Main Program # ------------------------------------------------------------ Read access.conf { my $config=IO::File->new("/etc/httpd/conf/access.conf"); while (my $configline=<$config>) { if ($configline =~ /PerlSetVar/) { my ($dummy,$varname,$varvalue)=split(/\s+/,$configline); $perlvar{$varname}=$varvalue; } } } # ------------------------------------------------------------- Read hosts file { my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab"); while (my $configline=<$config>) { my ($id,$domain,$role,$name,$ip)=split(/:/,$configline); $hostname{$id}=$name; $hostdom{$id}=$domain; $hostrole{$id}=$role; $hostip{$id}=$ip; if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) { $libserv{$id}=$name; } } } # ------------------------------------------------------ Read spare server file { my $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab"); while (my $configline=<$config>) { chomp($configline); if (($configline) && ($configline ne $perlvar{'lonHostID'})) { $spareid{$configline}=1; } } } # ---------------------------------------------------------------- Start report $statusdir="/home/httpd/html/lon-status"; $errors=0; $warnings=0; $notices=0; $now=time; $date=localtime($now); { my $fh=IO::File->new(">$statusdir/newstatus.html"); print $fh (< LON Status Report $perlvar{'lonHostID'}

LON Status Report $perlvar{'lonHostID'}

$date ($now)

  1. Configuration
  2. Machine Information
  3. httpd
  4. lond
  5. lonc
  6. lonnet
  7. Connections
  8. Delayed Messages
  9. Error Count

Configuration

PerlVars

ENDHEADERS foreach $varname (keys %perlvar) { print $fh "\n"; } print $fh "
$varname$perlvar{$varname}

Hosts

"; foreach $id (keys %hostname) { print $fh ""; print $fh "\n"; } print $fh "
$id$hostdom{$id}$hostrole{$id}$hostname{$id}$hostip{$id}

Spare Hosts

    "; foreach $id (keys %spareid) { print $fh "
  1. $id\n"; } print $fh "
\n"; # --------------------------------------------------------------------- Machine print $fh '

Machine Information

'; print $fh "

loadavg

"; open (LOADAVGH,"/proc/loadavg"); $loadavg=; close (LOADAVGH); print $fh "$loadavg"; @parts=split(/\s+/,$loadavg); if ($parts[1]>3.0) { $errors++; } elsif ($parts[1]>2.0) { $warnings++; } elsif ($parts[1]>1.0) { $notices++; } print $fh "

df

"; print $fh "
";

open (DFH,"df|");
while ($line=) { 
   print $fh "$line"; 
   @parts=split(/\s+/,$line);
   $usage=$parts[4];
   $usage=~s/\W//g;
   if ($usage>90) { 
      $errors++; 
   } elsif ($usage>80) {
      $warnings++;
   } elsif ($usage>60) {
      $notices++;
   }
   if ($usage>95) { $errors++; }
}
close (DFH);
print $fh "
"; &errout($fh); # ----------------------------------------------------------------------- httpd print $fh '

httpd

Access Log

';

open (DFH,"tail -n40 /etc/httpd/logs/access_log|");
while ($line=) { print $fh "$line" };
close (DFH);

print $fh "

Error Log

";

open (DFH,"tail -n50 /etc/httpd/logs/error_log|");
while ($line=) { 
   print $fh "$line";
   if ($line=~/\[error\]/) { $notices++; } 
};
close (DFH);
print $fh "
"; &errout($fh); # ------------------------------------------------------------------------ lond print $fh '

lond

Log

';

if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){
open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lond.log|");
while ($line=) { 
   print $fh "$line";
   if ($line=~/giving up/) { $notices++; }
};
close (DFH);
}
print $fh "
"; my $londfile="$perlvar{'lonDaemons'}/logs/lond.pid"; if (-e $londfile) { my $lfh=IO::File->new("$londfile"); my $londpid=<$lfh>; chomp($londpid); if (kill 0 => $londpid) { print $fh "

lond at pid $londpid responding

"; } else { $errors++; $errors++; print $fh "

lond at pid $londpid not responding

"; } } else { $errors++; print $fh "

lond not running, trying to start

"; system("$perlvar{'lonDaemons'}/lond"); sleep 120; if (-e $londfile) { print $fh "Seems like it started ...

"; my $lfh=IO::File->new("$londfile"); my $londpid=<$lfh>; chomp($londpid); sleep 30; if (kill 0 => $londpid) { print $fh "

lond at pid $londpid responding

"; } else { $errors++; $errors++; print $fh "

lond at pid $londpid not responding

"; print $fh "Give it one more try ...

"; system("$perlvar{'lonDaemons'}/lond"); sleep 120; } } else { print $fh "Seems like that did not work!

"; $errors++; } } $fname="$perlvar{'lonDaemons'}/logs/lond.log"; my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); if ($size>40000) { print $fh "Rotating logs ...

"; rename("$fname.2","$fname.3"); rename("$fname.1","$fname.2"); rename("$fname","$fname.1"); } &errout($fh); # ------------------------------------------------------------------------ lonc print $fh '

lonc

Log

';

if (-e "$perlvar{'lonDaemons'}/logs/lonc.log"){
open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonc.log|");
while ($line=) { 
   print $fh "$line";
   if ($line=~/died/) { $notices++; }
};
close (DFH);
}
print $fh "
"; my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid"; if (-e $loncfile) { my $lfh=IO::File->new("$loncfile"); my $loncpid=<$lfh>; chomp($loncpid); if (kill 0 => $loncpid) { print $fh "

lonc at pid $loncpid responding, sending USR1

"; kill USR1 => $loncpid; } else { $errors++; $errors++; print $fh "

lonc at pid $loncpid not responding

"; } } else { $errors++; print $fh "

lonc not running, trying to start

"; system("$perlvar{'lonDaemons'}/lonc"); sleep 120; if (-e $loncfile) { print $fh "Seems like it started ...

"; my $lfh=IO::File->new("$loncfile"); my $loncpid=<$lfh>; chomp($loncpid); sleep 30; if (kill 0 => $loncpid) { print $fh "

lonc at pid $loncpid responding

"; } else { $errors++; $errors++; print $fh "

lonc at pid $loncpid not responding

"; print $fh "Give it one more try ...

"; system("$perlvar{'lonDaemons'}/lonc"); sleep 120; } } else { print $fh "Seems like that did not work!

"; $errors++; } } $fname="$perlvar{'lonDaemons'}/logs/lonc.log"; my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); if ($size>40000) { print $fh "Rotating logs ...

"; rename("$fname.2","$fname.3"); rename("$fname.1","$fname.2"); rename("$fname","$fname.1"); } &errout($fh); # ---------------------------------------------------------------------- lonnet print $fh '

lonnet

Temp Log

';
if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
while ($line=) { 
    print $fh "$line";
    if ($line=~/Delayed/) { $warnings++; }
    if ($line=~/giving up/) { $warnings++; }
    if ($line=~/FAILED/) { $errors++; }
};
close (DFH);
}
print $fh "

Perm Log

"; if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") { open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|"); while ($line=) { print $fh "$line"; }; close (DFH); } else { print $fh "No perm log\n" } $fname="$perlvar{'lonDaemons'}/logs/lonnet.log"; my ($dev,$ino,$mode,$nlink, $uid,$gid,$rdev,$size, $atime,$mtime,$ctime, $blksize,$blocks)=stat($fname); if ($size>40000) { print $fh "Rotating logs ...

"; rename("$fname.2","$fname.3"); rename("$fname.1","$fname.2"); rename("$fname","$fname.1"); } print $fh ""; &errout($fh); # ----------------------------------------------------------------- Connections print $fh '

Connections

'; print $fh ""; foreach $tryserver (keys %hostname) { $answer=reply("pong",$tryserver); if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { $result="ok"; } else { $result=$answer; $warnings++; if ($answer eq 'con_lost') { $warnings++; } } print $fh "\n"; } print $fh "
$tryserver$result
"; &errout($fh); # ------------------------------------------------------------ Delayed messages print $fh '

Delayed Messages

'; print $fh '

Scanning Permanent Log

'; $unsend=0; { my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log"); while ($line=<$dfh>) { ($time,$sdf,$dserv,$dcmd)=split(/:/,$line); if ($sdf eq 'F') { $local=localtime($time); print "Failed: $time, $dserv, $dcmd
"; $warnings++; } if ($sdf eq 'S') { $unsend--; } if ($sdf eq 'D') { $unsend++; } } } print $fh "Total unsend messages: $unsend

\n"; $warnings=$warnings+5*$unsend; print $fh "

Outgoing Buffer

"; open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|"); while ($line=) { print $fh "$line
"; }; close (DFH); # ------------------------------------------------------------------------- End print $fh "\n"; $totalcount=$notices+4*$warnings+100*$errors; &errout($fh); print $fh "

Total Error Count: $totalcount

"; $now=time; $date=localtime($now); print $fh "
$date ($now)\n"; } rename ("$statusdir/newstatus.html","$statusdir/index.html"); if ($totalcount>200) { $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; system( "metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html"); } 1; 500 Internal Server Error

Internal Server Error

The server encountered an internal error or misconfiguration and was unable to complete your request.

Please contact the server administrator at root@localhost to inform them of the time this error occurred, and the actions you performed just before this error.

More information about this error may be available in the server error log.