--- loncom/loncron 2000/02/08 17:34:24 1.3 +++ loncom/loncron 2010/08/17 22:22:00 1.86 @@ -1,476 +1,881 @@ #!/usr/bin/perl -# The LearningOnline Network -# Housekeeping program, started by cron +# Housekeeping program, started by cron, loncontrol and loncron.pl # -# (TCP networking package -# 6/1/99,6/2,6/10,6/11,6/12,6/14,6/26,6/28,6/29,6/30, -# 7/1,7/2,7/9,7/10,7/12 Gerd Kortemeyer) +# $Id: loncron,v 1.86 2010/08/17 22:22:00 raeburn Exp $ +# +# Copyright Michigan State University Board of Trustees +# +# This file is part of the LearningOnline Network with CAPA (LON-CAPA). +# +# LON-CAPA is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# LON-CAPA is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with LON-CAPA; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# /home/httpd/html/adm/gpl.txt +# +# http://www.lon-capa.org/ # -# 7/14,7/15,7/19,7/21,7/22,11/18, -# 2/8 Gerd Kortemeyer + +$|=1; +use strict; + +use lib '/home/httpd/lib/perl/'; +use LONCAPA::Configuration; +use Apache::lonnet; +use Apache::loncommon; use IO::File; use IO::Socket; +use HTML::Entities; +use Getopt::Long; +#globals +use vars qw (%perlvar %simplestatus $errors $warnings $notices $totalcount); + +my $statusdir="/home/httpd/html/lon-status"; -# -------------------------------------------------- Non-critical communication -sub reply { - my ($cmd,$server)=@_; - my $peerfile="$perlvar{'lonSockDir'}/$server"; - my $client=IO::Socket::UNIX->new(Peer =>"$peerfile", - Type => SOCK_STREAM, - Timeout => 10) - or return "con_lost"; - print $client "$cmd\n"; - my $answer=<$client>; - chomp($answer); - if (!$answer) { $answer="con_lost"; } - return $answer; -} # --------------------------------------------------------- Output error status +sub log { + my $fh=shift; + if ($fh) { print $fh @_ } +} + sub errout { my $fh=shift; - print $fh (< + &log($fh,(< -
Notices$notices
Warnings$warnings
Errors$errors

Top

+

Top

ENDERROUT } -# ================================================================ Main Program +sub rotate_logfile { + my ($file,$fh,$description) = @_; + my $size=(stat($file))[7]; + if ($size>40000) { + &log($fh,"

Rotating $description ...

"); + rename("$file.2","$file.3"); + rename("$file.1","$file.2"); + rename("$file","$file.1"); + } +} + +sub start_daemon { + my ($fh,$daemon,$pidfile,$args) = @_; + my $progname=$daemon; + if ($daemon eq 'lonc') { + $progname='loncnew'; + } + my $error_fname="$perlvar{'lonDaemons'}/logs/${daemon}_errors"; + &rotate_logfile($error_fname,$fh,'error logs'); + if ($daemon eq 'lonc') { + &clean_sockets($fh); + } + system("$perlvar{'lonDaemons'}/$progname 2>$perlvar{'lonDaemons'}/logs/${daemon}_errors"); + sleep 1; + if (-e $pidfile) { + &log($fh,"

Seems like it started ...

"); + my $lfh=IO::File->new("$pidfile"); + my $daemonpid=<$lfh>; + chomp($daemonpid); + if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) { + return 1; + } else { + return 0; + } + } + &log($fh,"

Seems like that did not work!

"); + $errors++; + return 0; +} +sub checkon_daemon { + my ($fh,$daemon,$maxsize,$send,$args)=@_; -# ------------------------------------------------------------ Read access.conf -{ - my $config=IO::File->new("/etc/httpd/conf/access.conf"); - - while (my $configline=<$config>) { - if ($configline =~ /PerlSetVar/) { - my ($dummy,$varname,$varvalue)=split(/\s+/,$configline); - $perlvar{$varname}=$varvalue; - } + my $result; + &log($fh,'

'.$daemon.'

Log

'); + printf("%-15s ",$daemon); + if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ + open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|"); + while (my $line=) { + &log($fh,"$line"); + if ($line=~/INFO/) { $notices++; } + if ($line=~/WARNING/) { $notices++; } + if ($line=~/CRITICAL/) { $warnings++; } + }; + close (DFH); + } + &log($fh,"

"); + + my $pidfile="$perlvar{'lonDaemons'}/logs/$daemon.pid"; + + my $restartflag=1; + my $daemonpid; + if (-e $pidfile) { + my $lfh=IO::File->new("$pidfile"); + $daemonpid=<$lfh>; + chomp($daemonpid); + if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) { + &log($fh,"

$daemon at pid $daemonpid responding"); + if ($send) { &log($fh,", sending $send"); } + &log($fh,"

"); + if ($send eq 'USR1') { kill USR1 => $daemonpid; } + if ($send eq 'USR2') { kill USR2 => $daemonpid; } + $restartflag=0; + if ($send eq 'USR2') { + $result = 'reloaded'; + print "reloaded\n"; + } else { + $result = 'running'; + print "running\n"; + } + } else { + $errors++; + &log($fh,"

$daemon at pid $daemonpid not responding

"); + $restartflag=1; + &log($fh,"

Decided to clean up stale .pid file and restart $daemon

"); + } + } + if ($restartflag==1) { + $simplestatus{$daemon}='off'; + $errors++; + my $kadaemon=$daemon; + if ($kadaemon eq 'lonmemcached') { $kadaemon='memcached'; } + &log($fh,'
Killall '.$daemon.': '. + `killall $kadaemon 2>&1`.' - '); + sleep 1; + &log($fh,unlink($pidfile).' - '. + `killall -9 $kadaemon 2>&1`. + '
'); + &log($fh,"

$daemon not running, trying to start

"); + + if (&start_daemon($fh,$daemon,$pidfile,$args)) { + &log($fh,"

$daemon at pid $daemonpid responding

"); + $simplestatus{$daemon}='restarted'; + $result = 'started'; + print "started\n"; + } else { + $errors++; + &log($fh,"

$daemon at pid $daemonpid not responding

"); + &log($fh,"

Give it one more try ...

"); + print " "; + if (&start_daemon($fh,$daemon,$pidfile,$args)) { + &log($fh,"

$daemon at pid $daemonpid responding

"); + $simplestatus{$daemon}='restarted'; + $result = 'started'; + print "started\n"; + } else { + $result = 'failed'; + print " failed\n"; + $simplestatus{$daemon}='failed'; + $errors++; $errors++; + &log($fh,"

$daemon at pid $daemonpid not responding

"); + &log($fh,"

Unable to start $daemon

"); + } + } + + if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){ + &log($fh,"

");
+	    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/$daemon.log|");
+	    while (my $line=) { 
+		&log($fh,"$line");
+		if ($line=~/WARNING/) { $notices++; }
+		if ($line=~/CRITICAL/) { $notices++; }
+	    };
+	    close (DFH);
+	    &log($fh,"

"); + } } + + my $fname="$perlvar{'lonDaemons'}/logs/$daemon.log"; + &rotate_logfile($fname,$fh,'logs'); + + &errout($fh); + return $result; } -# ------------------------------------------------------------- Read hosts file -{ - my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab"); - - while (my $configline=<$config>) { - my ($id,$domain,$role,$name,$ip)=split(/:/,$configline); - $hostname{$id}=$name; - $hostdom{$id}=$domain; - $hostrole{$id}=$role; - $hostip{$id}=$ip; - if (($role eq 'library') && ($id ne $perlvar{'lonHostID'})) { - $libserv{$id}=$name; - } +# --------------------------------------------------------------------- Machine +sub log_machine_info { + my ($fh)=@_; + &log($fh,'

Machine Information

'); + &log($fh,"

loadavg

"); + + open (LOADAVGH,"/proc/loadavg"); + my $loadavg=; + close (LOADAVGH); + + &log($fh,"$loadavg"); + + my @parts=split(/\s+/,$loadavg); + if ($parts[1]>4.0) { + $errors++; + } elsif ($parts[1]>2.0) { + $warnings++; + } elsif ($parts[1]>1.0) { + $notices++; } -} -# ------------------------------------------------------ Read spare server file -{ - my $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab"); - - while (my $configline=<$config>) { - chomp($configline); - if (($configline) && ($configline ne $perlvar{'lonHostID'})) { - $spareid{$configline}=1; - } + &log($fh,"

df

"); + &log($fh,"
");
+
+    open (DFH,"df|");
+    while (my $line=) { 
+	&log($fh,&encode_entities($line,'<>&"')); 
+	@parts=split(/\s+/,$line);
+	my $usage=$parts[4];
+	$usage=~s/\W//g;
+	if ($usage>90) { 
+	    $warnings++;
+	    $notices++; 
+	} elsif ($usage>80) {
+	    $warnings++;
+	} elsif ($usage>60) {
+	    $notices++;
+	}
+	if ($usage>95) { $warnings++; $warnings++; $simplestatus{'diskfull'}++; }
     }
-}
+    close (DFH);
+    &log($fh,"
"); -# ---------------------------------------------------------------- Start report -$statusdir="/home/httpd/html/lon-status"; + &log($fh,"

ps

"); + &log($fh,"
");
+    my $psproc=0;
 
-$errors=0;
-$warnings=0;
-$notices=0;
+    open (PSH,"ps aux --cols 140 |");
+    while (my $line=) { 
+	&log($fh,&encode_entities($line,'<>&"')); 
+	$psproc++;
+    }
+    close (PSH);
+    &log($fh,"
"); -$now=time; -$date=localtime($now); + if ($psproc>200) { $notices++; } + if ($psproc>250) { $notices++; } -{ -my $fh=IO::File->new(">$statusdir/newstatus.html"); + &log($fh,"

distprobe

"); + &log($fh,"
");
+    open(DSH,"$perlvar{'lonDaemons'}/distprobe |");
+    while (my $line=) { 
+	&log($fh,&encode_entities($line,'<>&"')); 
+	$psproc++;
+    }
+    close(DSH);
+    &log($fh,"
"); -print $fh (<new(">$statusdir/newstatus.html"); + my %simplestatus=(); + my $now=time; + my $date=localtime($now); + + + &log($fh,(< LON Status Report $perlvar{'lonHostID'} - +

LON Status Report $perlvar{'lonHostID'}

$date ($now)

    -
  1. Configuration -
  2. Machine Information -
  3. httpd -
  4. lond -
  5. lonc -
  6. lonnet -
  7. Connections -
  8. Delayed Messages -
  9. Error Count +
  10. Configuration
    11. +
  11. Machine Information
    12. +
  12. Temporary Files
    13. +
  13. Session Tokens
    14. +
  14. httpd
    15. +
  15. lonsql
    16. +
  16. lond
    17. +
  17. lonc
    18. +
  18. lonnet
    19. +
  19. Connections
    20. +
  20. Delayed Messages
    21. +
  21. Error Count
-
- +
+

Configuration

PerlVars

- +
ENDHEADERS -foreach $varname (keys %perlvar) { - print $fh "\n"; -} -print $fh "
$varname$perlvar{$varname}

Hosts

"; -foreach $id (keys %hostname) { -print $fh - ""; -print $fh "\n"; -} -print $fh "
$id$hostdom{$id}$hostrole{$id}$hostname{$id}$hostip{$id}

Spare Hosts

    "; -foreach $id (keys %spareid) { - print $fh "
  1. $id\n"; + foreach my $varname (sort(keys(%perlvar))) { + &log($fh,"$varname". + &encode_entities($perlvar{$varname},'<>&"')."\n"); + } + &log($fh,"

    Hosts

    "); + my %hostname = &Apache::lonnet::all_hostnames(); + foreach my $id (sort(keys(%hostname))) { + my $role = (&Apache::lonnet::is_library($id) ? 'library' + : 'access'); + &log($fh, + "\n"); + } + &log($fh,"
    $id".&Apache::lonnet::host_domain($id). + "".$role. + "".&Apache::lonnet::hostname($id)."

    Spare Hosts

      "); + foreach my $type (sort(keys(%Apache::lonnet::spareid))) { + &log($fh,"
    • $type\n
        "); + foreach my $id (@{ $Apache::lonnet::spareid{$type} }) { + &log($fh,"
      1. $id
        2. \n"); + } + &log($fh,"
      \n
      • \n"); + } + &log($fh,"
    \n"); + return $fh; } -print $fh "
\n"; +# --------------------------------------------------------------- clean out tmp +sub clean_tmp { + my ($fh)=@_; + &log($fh,'

Temporary Files

'); + my ($cleaned,$old,$removed) = (0,0,0); + my %errors = ( + dir => [], + file => [], + failopen => [], + ); + my %error_titles = ( + dir => 'failed to remove empty directory:', + file => 'failed to unlike stale file', + failopen => 'failed to open file or directory' + ); + ($cleaned,$old,$removed) = &recursive_clean_tmp('',$cleaned,$old,$removed,\%errors); + &log($fh,"Cleaned up: ".$cleaned." files; removed: $removed empty directories; (found: $old old checkout tokens)"); + foreach my $key (sort(keys(%errors))) { + if (ref($errors{$key}) eq 'ARRAY') { + if (@{$errors{$key}} > 0) { + &log($fh,"Error during cleanup ($error_titles{$key}):
'); + } + } + } +} -# --------------------------------------------------------------------- Machine +sub recursive_clean_tmp { + my ($subdir,$cleaned,$old,$removed,$errors) = @_; + my $base = "$perlvar{'lonDaemons'}/tmp"; + my $path = $base; + next if ($subdir =~ m{\.\./}); + next unless (ref($errors) eq 'HASH'); + unless ($subdir eq '') { + $path .= '/'.$subdir; + } + if (opendir(my $dh,"$path")) { + while (my $file = readdir($dh)) { + next if ($file =~ /^\.\.?$/); + my $fname = "$path/$file"; + if (-d $fname) { + my $innerdir; + if ($subdir eq '') { + $innerdir = $file; + } else { + $innerdir = $subdir.'/'.$file; + } + ($cleaned,$old,$removed) = + &recursive_clean_tmp($innerdir,$cleaned,$old,$removed,$errors); + my @doms = &Apache::lonnet::current_machine_domains(); + + if (open(my $dirhandle,$fname)) { + unless (($innerdir eq 'helprequests') || + (($innerdir =~ /^addcourse/) && ($innerdir !~ m{/\d+$}))) { + my @contents = grep {!/^\.\.?$/} readdir($dirhandle); + join('&&',@contents)."\n"; + if (scalar(grep {!/^\.\.?$/} readdir($dirhandle)) == 0) { + closedir($dirhandle); + if ($fname =~ m{^\Q$perlvar{'lonDaemons'}\E/tmp/}) { + if (rmdir($fname)) { + $removed ++; + } elsif (ref($errors->{dir}) eq 'ARRAY') { + push(@{$errors->{dir}},$fname); + } + } + } + } else { + closedir($dirhandle); + } + } + } else { + my ($dev,$ino,$mode,$nlink, + $uid,$gid,$rdev,$size, + $atime,$mtime,$ctime, + $blksize,$blocks)=stat($fname); + my $now=time; + my $since=$now-$mtime; + if ($since>$perlvar{'lonExpire'}) { + if ($subdir eq '') { + my $line=''; + if ($fname =~ /\.db$/) { + if (unlink($fname)) { + $cleaned++; + } elsif (ref($errors->{file}) eq 'ARRAY') { + push(@{$errors->{file}},$fname); + } + } elsif (open(PROBE,$fname)) { + my $line=''; + $line=; + close(PROBE); + if ($line=~/^CHECKOUTTOKEN\&/) { + if ($since>365*$perlvar{'lonExpire'}) { + if (unlink($fname)) { + $cleaned++; + } elsif (ref($errors->{file}) eq 'ARRAY') { + push(@{$errors->{file}},$fname); + } + } else { + $old++; + } + } else { + if (unlink($fname)) { + $cleaned++; + } elsif (ref($errors->{file}) eq 'ARRAY') { + push(@{$errors->{file}},$fname); + } + } + } elsif (ref($errors->{failopen}) eq 'ARRAY') { + push(@{$errors->{failopen}},$fname); + } + } else { + if (unlink($fname)) { + $cleaned++; + } elsif (ref($errors->{file}) eq 'ARRAY') { + push(@{$errors->{file}},$fname); + } + } + } + } + } + closedir($dh); + } elsif (ref($errors->{failopen}) eq 'ARRAY') { + push(@{$errors->{failopen}},$path); + } + return ($cleaned,$old,$removed); +} -print $fh '

Machine Information

'; -print $fh "

loadavg

"; +# ------------------------------------------------------------ clean out lonIDs +sub clean_lonIDs { + my ($fh)=@_; + &log($fh,'

Session Tokens

'); + my $cleaned=0; + my $active=0; + while (my $fname=<$perlvar{'lonIDsDir'}/*>) { + my ($dev,$ino,$mode,$nlink, + $uid,$gid,$rdev,$size, + $atime,$mtime,$ctime, + $blksize,$blocks)=stat($fname); + my $now=time; + my $since=$now-$mtime; + if ($since>$perlvar{'lonExpire'}) { + $cleaned++; + &log($fh,"Unlinking $fname
"); + unlink("$fname"); + } else { + $active++; + } + } + &log($fh,"

Cleaned up ".$cleaned." stale session token(s).

"); + &log($fh,"

$active open session(s)

"); +} -open (LOADAVGH,"/proc/loadavg"); -$loadavg=; -close (LOADAVGH); +# ----------------------------------------------------------- clean out sockets +sub clean_sockets { + my ($fh)=@_; + my $cleaned=0; + opendir(SOCKETS,$perlvar{'lonSockDir'}); + while (my $fname=readdir(SOCKETS)) { + next if (-d $fname + || $fname=~/(mysqlsock|maximasock|rsock|\Q$perlvar{'lonSockDir'}\E)/); + $cleaned++; + &log($fh,"Unlinking $fname
"); + unlink("/home/httpd/sockets/$fname"); + } + &log($fh,"

Cleaned up ".$cleaned." stale sockets.

"); +} -print $fh "$loadavg"; -@parts=split(/\s+/,$loadavg); -if ($parts[1]>3.0) { - $errors++; -} elsif ($parts[1]>2.0) { - $warnings++; -} elsif ($parts[1]>1.0) { - $notices++; -} - -print $fh "

df

"; -print $fh "
";
-
-open (DFH,"df|");
-while ($line=) { 
-   print $fh "$line"; 
-   @parts=split(/\s+/,$line);
-   $usage=$parts[4];
-   $usage=~s/\W//g;
-   if ($usage>90) { 
-      $errors++; 
-   } elsif ($usage>80) {
-      $warnings++;
-   } elsif ($usage>60) {
-      $notices++;
-   }
-   if ($usage>95) { $errors++; }
-}
-close (DFH);
-print $fh "
"; -&errout($fh); # ----------------------------------------------------------------------- httpd - -print $fh '

httpd

Access Log

';
-
-open (DFH,"tail -n40 /etc/httpd/logs/access_log|");
-while ($line=) { print $fh "$line" };
-close (DFH);
-
-print $fh "

Error Log

";
-
-open (DFH,"tail -n50 /etc/httpd/logs/error_log|");
-while ($line=) { 
-   print $fh "$line";
-   if ($line=~/\[error\]/) { $notices++; } 
-};
-close (DFH);
-print $fh "
"; -&errout($fh); -# ------------------------------------------------------------------------ lond - -print $fh '

lond

Log

';
-
-if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){
-open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
-while ($line=) { 
-   print $fh "$line";
-   if ($line=~/INFO/) { $notices++; }
-   if ($line=~/WARNING/) { $warnings++; }
-   if ($line=~/CRITICAL/) { $errors++; }
-};
-close (DFH);
-}
-print $fh "
"; - -my $londfile="$perlvar{'lonDaemons'}/logs/lond.pid"; - -if (-e $londfile) { - my $lfh=IO::File->new("$londfile"); - my $londpid=<$lfh>; - chomp($londpid); - if (kill 0 => $londpid) { - print $fh "

lond at pid $londpid responding

"; - } else { - $errors++; $errors++; - print $fh "

lond at pid $londpid not responding

"; - } -} else { - $errors++; - print $fh "

lond not running, trying to start

"; - system("$perlvar{'lonDaemons'}/lond"); - sleep 120; - if (-e $londfile) { - print $fh "Seems like it started ...

"; - my $lfh=IO::File->new("$londfile"); - my $londpid=<$lfh>; - chomp($londpid); - sleep 30; - if (kill 0 => $londpid) { - print $fh "

lond at pid $londpid responding

"; - } else { - $errors++; $errors++; - print $fh "

lond at pid $londpid not responding

"; - print $fh "Give it one more try ...

"; - system("$perlvar{'lonDaemons'}/lond"); - sleep 120; - } - } else { - print $fh "Seems like that did not work!

"; - $errors++; - } - if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){ - print $fh "

";
-    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");
-    while ($line=) { 
-      print $fh "$line";
-      if ($line=~/INFO/) { $notices++; }
-      if ($line=~/WARNING/) { $warnings++; }
-      if ($line=~/CRITICAL/) { $errors++; }
-    };
+sub check_httpd_logs {
+    my ($fh)=@_;
+    &log($fh,'
httpd
Access Log
');
+    
+    open (DFH,"tail -n25 /etc/httpd/logs/access_log|");
+    while (my $line=) { &log($fh,&encode_entities($line,'<>&"')) };
     close (DFH);
-    print $fh "
";
-   }
-}
-
-$fname="$perlvar{'lonDaemons'}/logs/lond.log";
-
-                          my ($dev,$ino,$mode,$nlink,
-                              $uid,$gid,$rdev,$size,
-                              $atime,$mtime,$ctime,
-                              $blksize,$blocks)=stat($fname);
-
-if ($size>40000) {
-    print $fh "Rotating logs ...
";
-    rename("$fname.2","$fname.3");
-    rename("$fname.1","$fname.2");
-    rename("$fname","$fname.1");
-}
-
-&errout($fh);
-# ------------------------------------------------------------------------ lonc
-
-print $fh '
lonc
Log
';
-
-if (-e "$perlvar{'lonDaemons'}/logs/lonc.log"){
-open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
-while ($line=) { 
-   print $fh "$line";
-   if ($line=~/INFO/) { $notices++; }
-   if ($line=~/WARNING/) { $warnings++; }
-   if ($line=~/CRITICAL/) { $errors++; }
-};
-close (DFH);
-}
-print $fh "
";
-
-my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid";
-
-if (-e $loncfile) {
-   my $lfh=IO::File->new("$loncfile");
-   my $loncpid=<$lfh>;
-   chomp($loncpid);
-   if (kill 0 => $loncpid) {
-      print $fh "
lonc at pid $loncpid responding, sending USR1
";
-      kill USR1 => $loncpid;
-   } else {
-      $errors++; $errors++;
-      print $fh "
lonc at pid $loncpid not responding
";
-   }
-} else {
-   $errors++;
-   print $fh "
lonc not running, trying to start
";
-   system("$perlvar{'lonDaemons'}/lonc");
-   sleep 120;
-   if (-e $loncfile) {
-       print $fh "Seems like it started ...
";
-       my $lfh=IO::File->new("$loncfile");
-       my $loncpid=<$lfh>;
-       chomp($loncpid);
-       sleep 30;
-       if (kill 0 => $loncpid) {
-          print $fh "
lonc at pid $loncpid responding
";
-       } else {
-          $errors++; $errors++;
-          print $fh "
lonc at pid $loncpid not responding
";
-          print $fh "Give it one more try ...
";
-          system("$perlvar{'lonDaemons'}/lonc");
-          sleep 120;
-       }
-   } else {
-       print $fh "Seems like that did not work!
";
-       $errors++;
-   }
-   if (-e "$perlvar{'lonDaemons'}/logs/lonc.log") {
-    print $fh "
";
-    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");
-    while ($line=) { 
-      print $fh "$line";
-      if ($line=~/INFO/) { $notices++; }
-      if ($line=~/WARNING/) { $warnings++; }
-      if ($line=~/CRITICAL/) { $errors++; }
-    };
+	
+    &log($fh,"
Error Log
");
+	
+    open (DFH,"tail -n25 /etc/httpd/logs/error_log|");
+    while (my $line=) { 
+	&log($fh,"$line");
+	if ($line=~/\[error\]/) { $notices++; } 
+    }
     close (DFH);
-    print $fh "
";
-   }
+    &log($fh,"
"); + &errout($fh); } -$fname="$perlvar{'lonDaemons'}/logs/lonc.log"; +# ---------------------------------------------------------------------- lonnet - my ($dev,$ino,$mode,$nlink, - $uid,$gid,$rdev,$size, - $atime,$mtime,$ctime, - $blksize,$blocks)=stat($fname); +sub rotate_lonnet_logs { + my ($fh)=@_; + &log($fh,'

lonnet

Temp Log

');
+    print "checking logs\n";
+    if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
+	open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
+	while (my $line=) { 
+	    &log($fh,&encode_entities($line,'<>&"'));
+	}
+	close (DFH);
+    }
+    &log($fh,"

Perm Log

");
+    
+    if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") {
+	open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|");
+	while (my $line=) { 
+	    &log($fh,&encode_entities($line,'<>&"'));
+	}
+	close (DFH);
+    } else { &log($fh,"No perm log\n") }
+
+    my $fname="$perlvar{'lonDaemons'}/logs/lonnet.log";
+    &rotate_logfile($fname,$fh,'lonnet log');
+
+    &log($fh,"
"); + &errout($fh); +} + +sub rotate_other_logs { + my ($fh) = @_; + my %logs = ( + autoenroll => 'Auto Enroll log', + autocreate => 'Create Course log', + searchcat => 'Search Cataloguing log', + autoupdate => 'Auto Update log', + refreshcourseids_db => 'Refresh CourseIDs db log', + ); + foreach my $item (keys(%logs)) { + my $fname=$perlvar{'lonDaemons'}.'/logs/'.$item.'.log'; + &rotate_logfile($fname,$fh,$logs{$item}); + } +} -if ($size>40000) { - print $fh "Rotating logs ...

"; - rename("$fname.2","$fname.3"); - rename("$fname.1","$fname.2"); - rename("$fname","$fname.1"); +# ----------------------------------------------------------------- Connections +sub test_connections { + my ($fh)=@_; + &log($fh,'

Connections

'); + print "testing connections\n"; + &log($fh,""); + my ($good,$bad)=(0,0); + my %hostname = &Apache::lonnet::all_hostnames(); + foreach my $tryserver (sort(keys(%hostname))) { + print("."); + my $result; + my $answer=&Apache::lonnet::reply("ping",$tryserver); + if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { + $result="ok"; + $good++; + } else { + $result=$answer; + $warnings++; + if ($answer eq 'con_lost') { + $bad++; + $warnings++; + } else { + $good++; #self connection + } + } + if ($answer =~ /con_lost/) { print(" $tryserver down\n"); } + &log($fh,"\n"); + } + &log($fh,"
$tryserver$result
"); + print "\n$good good, $bad bad connections\n"; + &errout($fh); } - -&errout($fh); -# ---------------------------------------------------------------------- lonnet -print $fh '

lonnet

Temp Log

';
-if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
-open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
-while ($line=) { 
-    print $fh "$line";
-};
-close (DFH);
-}
-print $fh "

Perm Log

"; - -if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") { - open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|"); -while ($line=) { - print $fh "$line"; -}; -close (DFH); -} else { print $fh "No perm log\n" } - -$fname="$perlvar{'lonDaemons'}/logs/lonnet.log"; - - my ($dev,$ino,$mode,$nlink, - $uid,$gid,$rdev,$size, - $atime,$mtime,$ctime, - $blksize,$blocks)=stat($fname); - -if ($size>40000) { - print $fh "Rotating logs ...

"; - rename("$fname.2","$fname.3"); - rename("$fname.1","$fname.2"); - rename("$fname","$fname.1"); -} +# ------------------------------------------------------------ Delayed messages +sub check_delayed_msg { + my ($fh)=@_; + &log($fh,'

Delayed Messages

'); + print "checking buffers\n"; + + &log($fh,'

Scanning Permanent Log

'); -print $fh ""; -&errout($fh); -# ----------------------------------------------------------------- Connections + my $unsend=0; -print $fh '

Connections

'; + my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log"); + while (my $line=<$dfh>) { + my ($time,$sdf,$dserv,$dcmd)=split(/:/,$line); + if ($sdf eq 'F') { + my $local=localtime($time); + &log($fh,"Failed: $time, $dserv, $dcmd
"); + $warnings++; + } + if ($sdf eq 'S') { $unsend--; } + if ($sdf eq 'D') { $unsend++; } + } -print $fh ""; -foreach $tryserver (keys %hostname) { + &log($fh,"

Total unsend messages: $unsend

\n"); + $warnings=$warnings+5*$unsend; - $answer=reply("pong",$tryserver); - if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { - $result="ok"; - } else { - $result=$answer; - $warnings++; - if ($answer eq 'con_lost') { $warnings++; } + if ($unsend) { $simplestatus{'unsend'}=$unsend; } + &log($fh,"

Outgoing Buffer

\n
");
+# list directory with delayed messages and remember offline servers
+    my %servers=();
+    open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|");
+    while (my $line=) {
+        my ($server)=($line=~/\.(\w+)$/);
+        if ($server) { $servers{$server}=1; }
+	&log($fh,&encode_entities($line,'<>&"'));
+    }
+    &log($fh,"
\n"); + close (DFH); +# pong to all servers that have delayed messages +# this will trigger a reverse connection, which should flush the buffers + foreach my $tryserver (keys %servers) { + my $answer=&Apache::lonnet::reply("pong",$tryserver); + &log($fh,"Pong to $tryserver: $answer
"); } - print $fh "\n"; +} +sub finish_logging { + my ($fh)=@_; + &log($fh,"\n"); + $totalcount=$notices+4*$warnings+100*$errors; + &errout($fh); + &log($fh,"

Total Error Count: $totalcount

"); + my $now=time; + my $date=localtime($now); + &log($fh,"
$date ($now)\n"); + print "lon-status webpage updated\n"; + $fh->close(); + + if ($errors) { $simplestatus{'errors'}=$errors; } + if ($warnings) { $simplestatus{'warnings'}=$warnings; } + if ($notices) { $simplestatus{'notices'}=$notices; } + $simplestatus{'time'}=time; +} + +sub log_simplestatus { + rename("$statusdir/newstatus.html","$statusdir/index.html"); + + my $sfh=IO::File->new(">$statusdir/loncron_simple.txt"); + foreach (keys %simplestatus) { + print $sfh $_.'='.$simplestatus{$_}.'&'; + } + print $sfh "\n"; + $sfh->close(); } -print $fh "
$tryserver$result
"; -&errout($fh); -# ------------------------------------------------------------ Delayed messages +sub write_loncaparevs { + if (open(my $fh,">$perlvar{'lonTabDir'}/loncaparevs.tab")) { + my %hostname = &Apache::lonnet::all_hostnames(); + foreach my $id (sort(keys(%hostname))) { + if ($id ne '') { + my $loncaparev = &Apache::lonnet::get_server_loncaparev('',$id,1,'loncron'); + if ($loncaparev =~ /^[\w.\-]+$/) { + print $fh $id.':'.$loncaparev."\n"; + } + } + } + close($fh); + } + return; +} -print $fh '

Delayed Messages

'; +sub write_serverhomeIDs { + if (open(my $fh,">$perlvar{'lonTabDir'}/serverhomeIDs.tab")) { + my %name_to_host = &Apache::lonnet::all_names(); + foreach my $name (sort(keys(%name_to_host))) { + if ($name ne '') { + if (ref($name_to_host{$name}) eq 'ARRAY') { + my $serverhomeID = &Apache::lonnet::get_server_homeID($name,1,'loncron'); + if ($serverhomeID ne '') { + print $fh $name.':'.$serverhomeID."\n"; + } else { + print $fh $name.':'.$name_to_host{$name}->[0]."\n"; + } + } + } + } + close($fh); + } + return; +} -print $fh '

Scanning Permanent Log

'; +sub send_mail { + print "sending mail\n"; + my $defdom = $perlvar{'lonDefDomain'}; + my $origmail = $perlvar{'lonAdmEMail'}; + my $emailto = &Apache::loncommon::build_recipient_list(undef, + 'lonstatusmail',$defdom,$origmail); + if ($totalcount>2500) { + $emailto.=",$perlvar{'lonSysEMail'}"; + } + my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; -$unsend=0; -{ - my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log"); - while ($line=<$dfh>) { - ($time,$sdf,$dserv,$dcmd)=split(/:/,$line); - if ($sdf eq 'F') { - $local=localtime($time); - print "Failed: $time, $dserv, $dcmd
"; - $warnings++; - } - if ($sdf eq 'S') { $unsend--; } - if ($sdf eq 'D') { $unsend++; } + my $result=system("metasend -b -S 4000000 -t $emailto -s '$subj' -f $statusdir/index.html -m text/html >& /dev/null"); + if ($result != 0) { + $result=system("mail -s '$subj' $emailto < $statusdir/index.html"); } } -print $fh "Total unsend messages: $unsend

\n"; -$warnings=$warnings+5*$unsend; -print $fh "

Outgoing Buffer

"; +sub usage { + print(<) { - print $fh "$line
"; -}; -close (DFH); +# ================================================================ Main Program +sub main () { + my ($help,$justcheckdaemons,$noemail,$justcheckconnections, + $justreload); + &GetOptions("help" => \$help, + "justcheckdaemons" => \$justcheckdaemons, + "noemail" => \$noemail, + "justcheckconnections" => \$justcheckconnections, + "justreload" => \$justreload + ); + if ($help) { &usage(); return; } +# --------------------------------- Read loncapa_apache.conf and loncapa.conf + my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf'); + %perlvar=%{$perlvarref}; + undef $perlvarref; + delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed + delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed + chdir($perlvar{'lonDaemons'}); +# --------------------------------------- Make sure that LON-CAPA is configured +# I only test for one thing here (lonHostID). This is just a safeguard. + if ('{[[[[lonHostID]]]]}' eq $perlvar{'lonHostID'}) { + print("Unconfigured machine.\n"); + my $emailto=$perlvar{'lonSysEMail'}; + my $hostname=`/bin/hostname`; + chop $hostname; + $hostname=~s/[^\w\.]//g; # make sure is safe to pass through shell + my $subj="LON: Unconfigured machine $hostname"; + system("echo 'Unconfigured machine $hostname.' |\ + mailto $emailto -s '$subj' > /dev/null"); + exit 1; + } -# ------------------------------------------------------------------------- End -print $fh "\n"; -$totalcount=$notices+4*$warnings+100*$errors; -&errout($fh); -print $fh "

Total Error Count: $totalcount

"; -$now=time; -$date=localtime($now); -print $fh "
$date ($now)\n"; +# ----------------------------- Make sure this process is running from user=www + my $wwwid=getpwnam('www'); + if ($wwwid!=$<) { + print("User ID mismatch. This program must be run as user 'www'\n"); + my $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; + my $subj="LON: $perlvar{'lonHostID'} User ID mismatch"; + system("echo 'User ID mismatch. loncron must be run as user www.' |\ + mailto $emailto -s '$subj' > /dev/null"); + exit 1; + } -} +# -------------------------------------------- Force reload of host information + &Apache::lonnet::load_hosts_tab(1); + &Apache::lonnet::load_domain_tab(1); + &Apache::lonnet::get_iphost(1); + +# ----------------------------------------- Force firewall update for lond port + + if ((!$justcheckdaemons) && (!$justreload)) { + my $now = time; + my $tmpfile = $perlvar{'lonDaemons'}.'/tmp/lciptables_iphost_'. + $now.$$.int(rand(10000)); + if (open(my $fh,">$tmpfile")) { + my %iphosts = &Apache::lonnet::get_iphost(); + foreach my $key (keys(%iphosts)) { + print $fh "$key\n"; + } + close($fh); + my $execpath = $perlvar{'lonDaemons'}.'/lciptables'; + system("$execpath $tmpfile"); + unlink($fh); + } + } -rename ("$statusdir/newstatus.html","$statusdir/index.html"); +# ---------------------------------------------------------------- Start report -if ($totalcount>200) { - $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}"; - $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; - system( - "metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html"); + $errors=0; + $warnings=0; + $notices=0; + + + my $fh; + if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { + $fh=&start_logging(); + + &log_machine_info($fh); + &clean_tmp($fh); + &clean_lonIDs($fh); + &check_httpd_logs($fh); + &rotate_lonnet_logs($fh); + &rotate_other_logs($fh); + } + if (!$justcheckconnections && !$justreload) { + &checkon_daemon($fh,'lonmemcached',40000); + &checkon_daemon($fh,'lonsql',200000); + if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') { + &checkon_daemon($fh,'lond',40000,'USR2'); + } + &checkon_daemon($fh,'lonc',40000,'USR1'); + &checkon_daemon($fh,'lonmaxima',40000); + &checkon_daemon($fh,'lonr',40000); + } + if ($justreload) { + &checkon_daemon($fh,'lond',40000,'USR2'); + &checkon_daemon($fh,'lonc',40000,'USR2'); + } + if ($justcheckconnections) { + &test_connections($fh); + } + if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { + &check_delayed_msg($fh); + &finish_logging($fh); + &log_simplestatus(); + + if ($totalcount>200 && !$noemail) { &send_mail(); } + } + if (!$justcheckconnections && !$justreload) { + &write_loncaparevs(); + &write_serverhomeIDs(); + } } + +&main(); 1; 500 Internal Server Error

Internal Server Error

The server encountered an internal error or misconfiguration and was unable to complete your request.

Please contact the server administrator at root@localhost to inform them of the time this error occurred, and the actions you performed just before this error.

More information about this error may be available in the server error log.