--- loncom/loncron 2000/12/08 22:08:57 1.8 +++ loncom/loncron 2004/05/11 19:15:03 1.47 @@ -1,17 +1,45 @@ #!/usr/bin/perl -# The LearningOnline Network -# Housekeeping program, started by cron +# Housekeeping program, started by cron, loncontrol and loncron.pl # -# (TCP networking package -# 6/1/99,6/2,6/10,6/11,6/12,6/14,6/26,6/28,6/29,6/30, -# 7/1,7/2,7/9,7/10,7/12 Gerd Kortemeyer) +# $Id: loncron,v 1.47 2004/05/11 19:15:03 albertel Exp $ # -# 7/14,7/15,7/19,7/21,7/22,11/18, -# 2/8 Gerd Kortemeyer +# Copyright Michigan State University Board of Trustees +# +# This file is part of the LearningOnline Network with CAPA (LON-CAPA). +# +# LON-CAPA is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# LON-CAPA is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with LON-CAPA; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# /home/httpd/html/adm/gpl.txt +# +# http://www.lon-capa.org/ +# + +$|=1; + +use lib '/home/httpd/lib/perl/'; +use LONCAPA::Configuration; use IO::File; use IO::Socket; +use strict; +#globals +use vars qw (%perlvar %simplestatus $errors $warnings $notices $totalcount); + +my $statusdir="/home/httpd/html/lon-status"; + # -------------------------------------------------- Non-critical communication sub reply { @@ -30,9 +58,14 @@ sub reply { # --------------------------------------------------------- Output error status +sub log { + my $fh=shift; + if ($fh) { print $fh @_ } +} + sub errout { my $fh=shift; - print $fh (< @@ -41,64 +74,205 @@ sub errout { ENDERROUT } -# ================================================================ Main Program +sub start_daemon { + my ($fh,$daemon,$pidfile) = @_; + my $progname=$daemon; + if ($daemon eq 'lonc' && $ARGV[0] eq 'new') { + $progname='loncnew'; + print "new "; + } + system("$perlvar{'lonDaemons'}/$progname 2>>$perlvar{'lonDaemons'}/logs/${daemon}_errors"); + sleep 2; + if (-e $pidfile) { + &log($fh,"Seems like it started ...

"); + my $lfh=IO::File->new("$pidfile"); + my $daemonpid=<$lfh>; + chomp($daemonpid); + sleep 2; + if (kill 0 => $daemonpid) { + return 1; + } else { + return 0; + } + } + &log($fh,"Seems like that did not work!

"); + $errors++; + return 0; +} +sub checkon_daemon { + my ($fh,$daemon,$maxsize,$sendusr1)=@_; -# ------------------------------------------------------------ Read access.conf -{ - my $config=IO::File->new("/etc/httpd/conf/access.conf"); + &log($fh,'

'.$daemon.'

Log

');
+    printf("%-10s ",$daemon);
+    if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){
+	open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|");
+	while (my $line=) { 
+	    &log($fh,"$line");
+	    if ($line=~/INFO/) { $notices++; }
+	    if ($line=~/WARNING/) { $notices++; }
+	    if ($line=~/CRITICAL/) { $warnings++; }
+	};
+	close (DFH);
+    }
+    &log($fh,"
"); + + my $pidfile="$perlvar{'lonDaemons'}/logs/$daemon.pid"; + + my $restartflag=1; + my $daemonpid; + if (-e $pidfile) { + my $lfh=IO::File->new("$pidfile"); + $daemonpid=<$lfh>; + chomp($daemonpid); + if (kill 0 => $daemonpid) { + &log($fh,"

$daemon at pid $daemonpid responding"); + if ($sendusr1) { &log($fh,", sending USR1"); } + &log($fh,"

"); + if ($sendusr1) { kill USR1 => $daemonpid; } + $restartflag=0; + print "running\n"; + } else { + $errors++; + &log($fh,"

$daemon at pid $daemonpid not responding

"); + $restartflag=1; + &log($fh,"

Decided to clean up stale .pid file and restart $daemon

"); + } + } + if ($restartflag==1) { + $simplestatus{$daemon}='off'; + $errors++; + &log($fh,'
Killall '.$daemon.': '. + `killall $daemon 2>&1`.' - '); + sleep 2; + &log($fh,unlink($pidfile).' - '. + `killall -9 $daemon 2>&1`. + '
'); + &log($fh,"

$daemon not running, trying to start

"); + + if (&start_daemon($fh,$daemon,$pidfile)) { + &log($fh,"

$daemon at pid $daemonpid responding

"); + $simplestatus{$daemon}='restarted'; + print "started\n"; + } else { + $errors++; + &log($fh,"

$daemon at pid $daemonpid not responding

"); + &log($fh,"Give it one more try ...

"); + print " "; + if (&start_daemon($fh,$daemon,$pidfile)) { + &log($fh,"

$daemon at pid $daemonpid responding

"); + $simplestatus{$daemon}='restarted'; + print "started\n"; + } else { + print " failed\n"; + $simplestatus{$daemon}='failed'; + $errors++; $errors++; + &log($fh,"

$daemon at pid $daemonpid not responding

"); + &log($fh,"Unable to start $daemon

"); + } + } - while (my $configline=<$config>) { - if ($configline =~ /PerlSetVar/) { - my ($dummy,$varname,$varvalue)=split(/\s+/,$configline); - $perlvar{$varname}=$varvalue; - } + if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ + &log($fh,"

");
+	    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/$daemon.log|");
+	    while (my $line=) { 
+		&log($fh,"$line");
+		if ($line=~/WARNING/) { $notices++; }
+		if ($line=~/CRITICAL/) { $notices++; }
+	    };
+	    close (DFH);
+	    &log($fh,"
"); + } + } + + my $fname="$perlvar{'lonDaemons'}/logs/$daemon.log"; + + my ($dev,$ino,$mode,$nlink, + $uid,$gid,$rdev,$size, + $atime,$mtime,$ctime, + $blksize,$blocks)=stat($fname); + + if ($size>$maxsize) { + &log($fh,"Rotating logs ...

"); + rename("$fname.2","$fname.3"); + rename("$fname.1","$fname.2"); + rename("$fname","$fname.1"); } -} -# ------------------------------------------------------------- Read hosts file -{ - my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab"); + &errout($fh); +} - while (my $configline=<$config>) { - my ($id,$domain,$role,$name,$ip)=split(/:/,$configline); - $hostname{$id}=$name; - $hostdom{$id}=$domain; - $hostrole{$id}=$role; - $hostip{$id}=$ip; - if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) { - $libserv{$id}=$name; - } +# --------------------------------------------------------------------- Machine +sub log_machine_info { + my ($fh)=@_; + &log($fh,'

Machine Information

'); + &log($fh,"

loadavg

"); + + open (LOADAVGH,"/proc/loadavg"); + my $loadavg=; + close (LOADAVGH); + + &log($fh,"$loadavg"); + + my @parts=split(/\s+/,$loadavg); + if ($parts[1]>4.0) { + $errors++; + } elsif ($parts[1]>2.0) { + $warnings++; + } elsif ($parts[1]>1.0) { + $notices++; } -} -# ------------------------------------------------------ Read spare server file -{ - my $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab"); + &log($fh,"

df

"); + &log($fh,"
");
 
-    while (my $configline=<$config>) {
-       chomp($configline);
-       if (($configline) && ($configline ne $perlvar{'lonHostID'})) {
-          $spareid{$configline}=1;
-       }
+    open (DFH,"df|");
+    while (my $line=) { 
+	&log($fh,"$line"); 
+	@parts=split(/\s+/,$line);
+	my $usage=$parts[4];
+	$usage=~s/\W//g;
+	if ($usage>90) { 
+	    $warnings++;
+	    $notices++; 
+	} elsif ($usage>80) {
+	    $warnings++;
+	} elsif ($usage>60) {
+	    $notices++;
+	}
+	if ($usage>95) { $warnings++; $warnings++; $simplestatus{'diskfull'}++; }
     }
-}
+    close (DFH);
+    &log($fh,"
"); -# ---------------------------------------------------------------- Start report -$statusdir="/home/httpd/html/lon-status"; + &log($fh,"

ps

"); + &log($fh,"
");
+    my $psproc=0;
 
-$errors=0;
-$warnings=0;
-$notices=0;
+    open (PSH,"ps -aux --cols 140 |");
+    while (my $line=) { 
+	&log($fh,"$line"); 
+	$psproc++;
+    }
+    close (PSH);
+    &log($fh,"
"); -$now=time; -$date=localtime($now); + if ($psproc>200) { $notices++; } + if ($psproc>250) { $notices++; } -{ -my $fh=IO::File->new(">$statusdir/newstatus.html"); + &errout($fh); +} + +sub start_logging { + my ($hostdom,$hostrole,$hostname,$spareid)=@_; + my $fh=IO::File->new(">$statusdir/newstatus.html"); + my %simplestatus=(); + my $now=time; + my $date=localtime($now); + -print $fh (< LON Status Report $perlvar{'lonHostID'} @@ -110,13 +284,17 @@ print $fh (<
  • Configuration
  • Machine Information +
  • Temporary Files +
  • Session Tokens
  • httpd +
  • lonsql
  • lond
  • lonc +
  • lonhttpd
  • lonnet
  • Connections
  • Delayed Messages -
  • Error Count +
  • Error Coindex.html.unt
    @@ -125,151 +303,128 @@ print $fh (< ENDHEADERS -foreach $varname (keys %perlvar) { - print $fh "
    • \n"; -} -print $fh "
    Notices$notices
    Warnings$warnings
    $varname$perlvar{$varname}

    Hosts

    "; -foreach $id (keys %hostname) { -print $fh - ""; -print $fh "\n"; -} -print $fh "
    $id$hostdom{$id}$hostrole{$id}$hostname{$id}$hostip{$id}

    Spare Hosts

      "; -foreach $id (keys %spareid) { - print $fh "
    1. $id\n"; + foreach my $varname (sort(keys(%perlvar))) { + &log($fh,"$varname$perlvar{$varname}\n"); + } + &log($fh,"

      Hosts

      "); + foreach my $id (sort(keys(%{$hostname}))) { + &log($fh, + "\n"); + } + &log($fh,"
      $id".$hostdom->{$id}. + "".$hostrole->{$id}. + "".$hostname->{$id}."

      Spare Hosts

        "); + foreach my $id (sort(keys(%{$spareid}))) { + &log($fh,"
      1. $id\n"); + } + &log($fh,"
      \n"); + return $fh; } -print $fh "
    \n"; - -# --------------------------------------------------------------------- Machine - -print $fh '

    Machine Information

    '; -print $fh "

    loadavg

    "; +# --------------------------------------------------------------- clean out tmp +sub clean_tmp { + my ($fh)=@_; + &log($fh,'

    Temporary Files

    '); + my $cleaned=0; + my $old=0; + while (my $fname=<$perlvar{'lonDaemons'}/tmp/*>) { + my ($dev,$ino,$mode,$nlink, + $uid,$gid,$rdev,$size, + $atime,$mtime,$ctime, + $blksize,$blocks)=stat($fname); + my $now=time; + my $since=$now-$mtime; + if ($since>$perlvar{'lonExpire'}) { + my $line=''; + if (open(PROBE,$fname)) { + $line=; + close(PROBE); + } + unless ($line=~/^CHECKOUTTOKEN\&/) { + $cleaned++; + unlink("$fname"); + } else { + if ($since>365*$perlvar{'lonExpire'}) { + $cleaned++; + unlink("$fname"); + } else { $old++; } + } + } + } + &log($fh,"Cleaned up ".$cleaned." files (".$old." old checkout tokens)."); +} -open (LOADAVGH,"/proc/loadavg"); -$loadavg=; -close (LOADAVGH); +# ------------------------------------------------------------ clean out lonIDs +sub clean_lonIDs { + my ($fh)=@_; + &log($fh,'

    Session Tokens

    '); + my $cleaned=0; + my $active=0; + while (my $fname=<$perlvar{'lonIDsDir'}/*>) { + my ($dev,$ino,$mode,$nlink, + $uid,$gid,$rdev,$size, + $atime,$mtime,$ctime, + $blksize,$blocks)=stat($fname); + my $now=time; + my $since=$now-$mtime; + if ($since>$perlvar{'lonExpire'}) { + $cleaned++; + &log($fh,"Unlinking $fname
    "); + unlink("$fname"); + } else { + $active++; + } + } + &log($fh,"

    Cleaned up ".$cleaned." stale session token(s)."); + &log($fh,"

    $active open session(s)

    "); +} -print $fh "$loadavg"; -@parts=split(/\s+/,$loadavg); -if ($parts[1]>4.0) { - $errors++; -} elsif ($parts[1]>2.0) { - $warnings++; -} elsif ($parts[1]>1.0) { - $notices++; -} - -print $fh "

    df

    "; -print $fh "
    ";
-
-open (DFH,"df|");
-while ($line=) { 
-   print $fh "$line"; 
-   @parts=split(/\s+/,$line);
-   $usage=$parts[4];
-   $usage=~s/\W//g;
-   if ($usage>90) { 
-      $warnings++; 
-   } elsif ($usage>80) {
-      $warnings++;
-   } elsif ($usage>60) {
-      $notices++;
-   }
-   if ($usage>95) { $warnings++; $warnings++ }
-}
-close (DFH);
-print $fh "
    "; -&errout($fh); # ----------------------------------------------------------------------- httpd +sub check_httpd_logs { + my ($fh)=@_; + &log($fh,'

    httpd

    Access Log

    ');
+    
+    open (DFH,"tail -n25 /etc/httpd/logs/access_log|");
+    while (my $line=) { &log($fh,"$line") };
+    close (DFH);
+	
+    &log($fh,"

    Error Log

    ");
+	
+    open (DFH,"tail -n25 /etc/httpd/logs/error_log|");
+    while (my $line=) { 
+	&log($fh,"$line");
+	if ($line=~/\[error\]/) { $notices++; } 
+    }
+    close (DFH);
+    &log($fh,"
    "); + &errout($fh); +} -print $fh '

    httpd

    Access Log

    ';
-
-open (DFH,"tail -n40 /etc/httpd/logs/access_log|");
-while ($line=) { print $fh "$line" };
-close (DFH);
-
-print $fh "

    Error Log

    ";
-
-open (DFH,"tail -n50 /etc/httpd/logs/error_log|");
-while ($line=) { 
-   print $fh "$line";
-   if ($line=~/\[error\]/) { $notices++; } 
-};
-close (DFH);
-print $fh "
    "; -&errout($fh); - - -# ------------------------------------------------------------------------ lonsql - -if ($perlvar{'lonRole'} eq "library" && 1==0) { +# ---------------------------------------------------------------------- lonnet - print $fh '

    lonsql

    Log

    ';
-    
-    if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
-	open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
-	while ($line=) { 
-	    print $fh "$line";
-	    if ($line=~/INFO/) { $notices++; }
-	    if ($line=~/WARNING/) { $notices++; }
-	    if ($line=~/CRITICAL/) { $warnings++; }
-	};
+sub rotate_logs {
+    my ($fh)=@_;
+    &log($fh,'
    lonnet
    Temp Log
    ');
+    print "checking logs\n";
+    if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
+	open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
+	while (my $line=) { 
+	    &log($fh,"$line");
+	}
 	close (DFH);
     }
-    print $fh "
    ";
-    
-    my $lonsqlfile="$perlvar{'lonDaemons'}/logs/lonsql.pid";
+    &log($fh,"

    Perm Log

    ");
     
-    if (-e $lonsqlfile) {
-	my $lfh=IO::File->new("$lonsqlfile");
-	my $lonsqlpid=<$lfh>;
-	chomp($lonsqlpid);
-	if (kill 0 => $lonsqlpid) {
-	    print $fh "
    lonsql at pid $lonsqlpid responding
    ";
-	} else {
-	    $errors++; $errors++;
-	    print $fh "
    lonsql at pid $lonsqlpid not responding
    ";
-	}
-    } else {
-	$errors++;
-	print $fh "
    lonsql not running, trying to start
    ";
-	system("$perlvar{'lonDaemons'}/lonsql");
-	sleep 120;
-	if (-e $lonsqlfile) {
-	    print $fh "Seems like it started ...
    ";
-	    my $lfh=IO::File->new("$lonsqlfile");
-	    my $lonsqlpid=<$lfh>;
-	    chomp($lonsqlpid);
-	    sleep 30;
-	    if (kill 0 => $lonsqlpid) {
-		print $fh "
    lonsql at pid $lonsqlpid responding
    ";
-	    } else {
-		$errors++; $errors++;
-		print $fh "
    lonsql at pid $lonsqlpid not responding
    ";
-		print $fh "Give it one more try ...
    ";
-		system("$perlvar{'lonDaemons'}/lonsql");
-		sleep 120;
-	    }
-	} else {
-	    print $fh "Seems like that did not work!
    ";
-	    $errors++;
+    if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") {
+	open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|");
+	while (my $line=) { 
+	    &log($fh,"$line");
 	}
-	if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){
-	    print $fh "
    ";
-	    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");
-	    while ($line=) { 
-		print $fh "$line";
-		if ($line=~/WARNING/) { $notices++; }
-		if ($line=~/CRITICAL/) { $notices++; }
-	    };
-	    close (DFH);
-	    print $fh "
    ";
-	}
-    }
+	close (DFH);
+    } else { &log($fh,"No perm log\n") }
 
-    $fname="$perlvar{'lonDaemons'}/logs/lonsql.log";
+    my $fname="$perlvar{'lonDaemons'}/logs/lonnet.log";
 
     my ($dev,$ino,$mode,$nlink,
 	$uid,$gid,$rdev,$size,
@@ -277,325 +432,206 @@ if ($perlvar{'lonRole'} eq "library" &&
 	$blksize,$blocks)=stat($fname);
 
     if ($size>40000) {
-	print $fh "Rotating logs ...
    ";
+	&log($fh,"Rotating logs ...
    ");
 	rename("$fname.2","$fname.3");
 	rename("$fname.1","$fname.2");
 	rename("$fname","$fname.1");
     }
 
+    &log($fh,"
    "); &errout($fh); } -# ------------------------------------------------------------------------ lond -print $fh '

    lond

    Log

    ';
+# ----------------------------------------------------------------- Connections
+sub test_connections {
+    my ($fh,$hostname)=@_;
+    &log($fh,'
    Connections
    ');
+    print "testing connections\n";
+    &log($fh,"");
+    foreach my $tryserver (sort(keys(%{$hostname}))) {
+	print(".");
+	my $result;
+	my $answer=reply("pong",$tryserver);
+	if ($answer eq "$tryserver:$perlvar{'lonHostID'}") {
+	    $result="ok";
+	} else {
+	    $result=$answer;
+	    $warnings++;
+	    if ($answer eq 'con_lost') { $warnings++; }
+	}
+	if ($answer =~ /con_lost/) { print(" $tryserver down\n"); }
+	&log($fh,"\n");
+    }
+    &log($fh,"
    $tryserver$result
    ");
 
-if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){
-open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
-while ($line=) { 
-   print $fh "$line";
-   if ($line=~/INFO/) { $notices++; }
-   if ($line=~/WARNING/) { $notices++; }
-   if ($line=~/CRITICAL/) { $warnings++; }
-};
-close (DFH);
-}
-print $fh "
    "; - -my $londfile="$perlvar{'lonDaemons'}/logs/lond.pid"; - -my $restartflag=1; -if (-e $londfile) { - my $lfh=IO::File->new("$londfile"); - my $londpid=<$lfh>; - chomp($londpid); - if (kill 0 => $londpid) { - print $fh "

    lond at pid $londpid responding

    "; - $restartflag=0; - } else { - $errors++; - print $fh "

    lond at pid $londpid not responding

    "; - # Intelligently handle this. - # Possibility #1: there is no process - # Solution: remove .pid file and restart - if (getpgrp($londpid)==-1) { - unlink($londfile); - $restartflag=1; - } - else { - # Possibility #2: there is a zombie process - # Possibility #3: there is a live process that is not responding - # for an unknown reason - # Solution: kill process, remove .pid and restart - kill 2 => $londpid; - unlink($londfile); - $restartflag=1; - } - print $fh - "

    Deciding to clean up stale .pid file and restart lond

    "; - } -} -if ($restartflag==1) { - $errors++; - print $fh "

    lond not running, trying to start

    "; - system("$perlvar{'lonDaemons'}/lond"); - sleep 120; - if (-e $londfile) { - print $fh "Seems like it started ...

    "; - my $lfh=IO::File->new("$londfile"); - my $londpid=<$lfh>; - chomp($londpid); - sleep 30; - if (kill 0 => $londpid) { - print $fh "

    lond at pid $londpid responding

    "; - } else { - $errors++; $errors++; - print $fh "

    lond at pid $londpid not responding

    "; - print $fh "Give it one more try ...

    "; - system("$perlvar{'lonDaemons'}/lond"); - sleep 120; - } - } else { - print $fh "Seems like that did not work!

    "; - $errors++; - } - if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){ - print $fh "

    ";
-    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
-    while ($line=) { 
-      print $fh "$line";
-      if ($line=~/WARNING/) { $notices++; }
-      if ($line=~/CRITICAL/) { $notices++; }
-    };
-    close (DFH);
-    print $fh "
    "; - } + &errout($fh); } -$fname="$perlvar{'lonDaemons'}/logs/lond.log"; - my ($dev,$ino,$mode,$nlink, - $uid,$gid,$rdev,$size, - $atime,$mtime,$ctime, - $blksize,$blocks)=stat($fname); - -if ($size>40000) { - print $fh "Rotating logs ...

    "; - rename("$fname.2","$fname.3"); - rename("$fname.1","$fname.2"); - rename("$fname","$fname.1"); -} - -&errout($fh); -# ------------------------------------------------------------------------ lonc - -print $fh '

    lonc

    Log

    ';
-
-if (-e "$perlvar{'lonDaemons'}/logs/lonc.log"){
-open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
-while ($line=) { 
-   print $fh "$line";
-   if ($line=~/INFO/) { $notices++; }
-   if ($line=~/WARNING/) { $notices++; }
-   if ($line=~/CRITICAL/) { $warnings++; }
-};
-close (DFH);
-}
-print $fh "
    "; - -my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid"; - -$restartflag=1; -if (-e $loncfile) { - my $lfh=IO::File->new("$loncfile"); - my $loncpid=<$lfh>; - chomp($loncpid); - if (kill 0 => $loncpid) { - print $fh "

    lonc at pid $loncpid responding, sending USR1

    "; - kill USR1 => $loncpid; - $restartflag=0; - } else { - $errors++; - print $fh "

    lonc at pid $loncpid not responding

    "; - # Intelligently handle this. - # Possibility #1: there is no process - # Solution: remove .pid file and restart - if (getpgrp($loncpid)==-1) { - unlink($loncfile); - $restartflag=1; - } - else { - # Possibility #2: there is a zombie process - # Possibility #3: there is a live process that is not responding - # for an unknown reason - # Solution: kill process, remove .pid and restart - kill 2 => $loncpid; - unlink($loncfile); - $restartflag=1; - } - print $fh - "

    Deciding to clean up stale .pid file and restart lonc

    "; - } -} -if ($restartflag==1) { - $errors++; - print $fh "

    lonc not running, trying to start

    "; - system("$perlvar{'lonDaemons'}/lonc"); - sleep 120; - if (-e $loncfile) { - print $fh "Seems like it started ...

    "; - my $lfh=IO::File->new("$loncfile"); - my $loncpid=<$lfh>; - chomp($loncpid); - sleep 30; - if (kill 0 => $loncpid) { - print $fh "

    lonc at pid $loncpid responding

    "; - } else { - $errors++; $errors++; - print $fh "

    lonc at pid $loncpid not responding

    "; - print $fh "Give it one more try ...

    "; - system("$perlvar{'lonDaemons'}/lonc"); - sleep 120; - } - } else { - print $fh "Seems like that did not work!

    "; - $errors++; - } - if (-e "$perlvar{'lonDaemons'}/logs/lonc.log") { - print $fh "

    ";
-    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
-    while ($line=) { 
-      print $fh "$line";
-      if ($line=~/WARNING/) { $notices++; }
-      if ($line=~/CRITICAL/) { $notices++; }
-    };
-    close (DFH);
-    print $fh "
    "; - } -} +# ------------------------------------------------------------ Delayed messages +sub check_delayed_msg { + my ($fh)=@_; + &log($fh,'

    Delayed Messages

    '); + print "checking buffers\n"; + + &log($fh,'

    Scanning Permanent Log

    '); -$fname="$perlvar{'lonDaemons'}/logs/lonc.log"; + my $unsend=0; - my ($dev,$ino,$mode,$nlink, - $uid,$gid,$rdev,$size, - $atime,$mtime,$ctime, - $blksize,$blocks)=stat($fname); + my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log"); + while (my $line=<$dfh>) { + my ($time,$sdf,$dserv,$dcmd)=split(/:/,$line); + if ($sdf eq 'F') { + my $local=localtime($time); + &log($fh,"Failed: $time, $dserv, $dcmd
    "); + $warnings++; + } + if ($sdf eq 'S') { $unsend--; } + if ($sdf eq 'D') { $unsend++; } + } -if ($size>40000) { - print $fh "Rotating logs ...

    "; - rename("$fname.2","$fname.3"); - rename("$fname.1","$fname.2"); - rename("$fname","$fname.1"); -} + &log($fh,"Total unsend messages: $unsend

    \n"); + $warnings=$warnings+5*$unsend; - -&errout($fh); -# ---------------------------------------------------------------------- lonnet + if ($unsend) { $simplestatus{'unsend'}=$unsend; } + &log($fh,"

    Outgoing Buffer

    "); -print $fh '

    lonnet

    Temp Log

    ';
-if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
-open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
-while ($line=) { 
-    print $fh "$line";
-};
-close (DFH);
-}
-print $fh "

    Perm Log

    "; - -if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") { - open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|"); -while ($line=) { - print $fh "$line"; -}; -close (DFH); -} else { print $fh "No perm log\n" } - -$fname="$perlvar{'lonDaemons'}/logs/lonnet.log"; - - my ($dev,$ino,$mode,$nlink, - $uid,$gid,$rdev,$size, - $atime,$mtime,$ctime, - $blksize,$blocks)=stat($fname); - -if ($size>40000) { - print $fh "Rotating logs ...

    "; - rename("$fname.2","$fname.3"); - rename("$fname.1","$fname.2"); - rename("$fname","$fname.1"); + open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|"); + while (my $line=) { + &log($fh,"$line
    "); + } + close (DFH); } -print $fh ""; -&errout($fh); -# ----------------------------------------------------------------- Connections +sub finish_logging { + my ($fh)=@_; + &log($fh,"    \n"); + $totalcount=$notices+4*$warnings+100*$errors; + &errout($fh); + &log($fh,"

    Total Error Count: $totalcount

    "); + my $now=time; + my $date=localtime($now); + &log($fh,"
    $date ($now)\n"); + print "lon-status webpage updated\n"; + $fh->close(); + + if ($errors) { $simplestatus{'errors'}=$errors; } + if ($warnings) { $simplestatus{'warnings'}=$warnings; } + if ($notices) { $simplestatus{'notices'}=$notices; } + $simplestatus{'time'}=time; +} -print $fh '

    Connections

    '; +sub log_simplestatus { + rename ("$statusdir/newstatus.html","$statusdir/index.html"); + + my $sfh=IO::File->new(">$statusdir/loncron_simple.txt"); + foreach (keys %simplestatus) { + print $sfh $_.'='.$simplestatus{$_}.'&'; + } + print $sfh "\n"; + $sfh->close(); +} -print $fh ""; -foreach $tryserver (keys %hostname) { +sub send_mail { + print "sending mail\n"; + my $emailto="$perlvar{'lonAdmEMail'}"; + if ($totalcount>1000) { + $emailto.=",$perlvar{'lonSysEMail'}"; + } + my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; + system("metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html"); +} - $answer=reply("pong",$tryserver); - if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { - $result="ok"; - } else { - $result=$answer; - $warnings++; - if ($answer eq 'con_lost') { $warnings++; } +# ================================================================ Main Program +sub main () { +# --------------------------------- Read loncapa_apache.conf and loncapa.conf + my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf'); + %perlvar=%{$perlvarref}; + undef $perlvarref; + delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed + delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed + +# --------------------------------------- Make sure that LON-CAPA is configured +# I only test for one thing here (lonHostID). This is just a safeguard. + if ('{[[[[lonHostID]]]]}' eq $perlvar{'lonHostID'}) { + print("Unconfigured machine.\n"); + my $emailto=$perlvar{'lonSysEMail'}; + my $hostname=`/bin/hostname`; + chop $hostname; + $hostname=~s/[^\w\.]//g; # make sure is safe to pass through shell + my $subj="LON: Unconfigured machine $hostname"; + system("echo 'Unconfigured machine $hostname.' |\ + mailto $emailto -s '$subj' > /dev/null"); + exit 1; } - print $fh "\n"; -} -print $fh "
    $tryserver$result
    "; +# ----------------------------- Make sure this process is running from user=www + my $wwwid=getpwnam('www'); + if ($wwwid!=$<) { + print("User ID mismatch. This program must be run as user 'www'\n"); + my $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; + my $subj="LON: $perlvar{'lonHostID'} User ID mismatch"; + system("echo 'User ID mismatch. loncron must be run as user www.' |\ + mailto $emailto -s '$subj' > /dev/null"); + exit 1; + } -&errout($fh); -# ------------------------------------------------------------ Delayed messages +# ------------------------------------------------------------- Read hosts file + my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab"); + + my (%hostname,%hostdom,%hostrole,%spareid); + while (my $configline=<$config>) { + my ($id,$domain,$role,$name,$ip,$domdescr)=split(/:/,$configline); + if ($id && $domain && $role && $name && $ip) { + $hostname{$id}=$name; + $hostdom{$id}=$domain; + $hostrole{$id}=$role; + } else { + if ($configline) { +# &logthis("Skipping hosts.tab line -$configline-"); + } + } + } + undef $config; -print $fh '

    Delayed Messages

    '; +# ------------------------------------------------------ Read spare server file + $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab"); + + while (my $configline=<$config>) { + chomp($configline); + if (($configline) && ($configline ne $perlvar{'lonHostID'})) { + $spareid{$configline}=1; + } + } + undef $config; -print $fh '

    Scanning Permanent Log

    '; +# ---------------------------------------------------------------- Start report -$unsend=0; -{ - my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log"); - while ($line=<$dfh>) { - ($time,$sdf,$dserv,$dcmd)=split(/:/,$line); - if ($sdf eq 'F') { - $local=localtime($time); - print "Failed: $time, $dserv, $dcmd
    "; - $warnings++; - } - if ($sdf eq 'S') { $unsend--; } - if ($sdf eq 'D') { $unsend++; } - } -} -print $fh "Total unsend messages: $unsend

    \n"; -$warnings=$warnings+5*$unsend; - -print $fh "

    Outgoing Buffer

    "; - -open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|"); -while ($line=) { - print $fh "$line
    "; -}; -close (DFH); - -# ------------------------------------------------------------------------- End -print $fh "    \n"; -$totalcount=$notices+4*$warnings+100*$errors; -&errout($fh); -print $fh "

    Total Error Count: $totalcount

    "; -$now=time; -$date=localtime($now); -print $fh "
    $date ($now)\n"; + $errors=0; + $warnings=0; + $notices=0; + + my $fh=&start_logging(\%hostdom,\%hostrole,\%hostname,\%spareid); + + &log_machine_info($fh); + &clean_tmp($fh); + &clean_lonIDs($fh); + &check_httpd_logs($fh); + &checkon_daemon($fh,'lonsql',200000); + &checkon_daemon($fh,'lond',40000,1); + &checkon_daemon($fh,'lonc',40000,1); + &checkon_daemon($fh,'lonhttpd',40000); + + &test_connections($fh,\%hostname); + &check_delayed_msg($fh); + &finish_logging($fh); + &log_simplestatus(); + + if ($totalcount>200) { &send_mail(); } } -rename ("$statusdir/newstatus.html","$statusdir/index.html"); - -if ($totalcount>200) { - $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; - $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; - system( - "metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html"); -} +&main(); 1; 500 Internal Server Error

    Internal Server Error

    The server encountered an internal error or misconfiguration and was unable to complete your request.

    Please contact the server administrator at root@localhost to inform them of the time this error occurred, and the actions you performed just before this error.

    More information about this error may be available in the server error log.