[BACK]Return to loncron CVS log [TXT] [DIR] Up to [LON-CAPA] / loncom

Diff for /loncom/loncron between versions 1.40 and 1.43

version 1.40, 2003/07/29 21:00:21 version 1.43, 2003/09/11 19:49:59
Line 49  sub errout { Line 49  sub errout {
 ENDERROUT  ENDERROUT
 }  }
   
   sub start_daemon {
       my ($fh,$daemon,$pidfile) = @_;
       system("$perlvar{'lonDaemons'}/$daemon 2>>$perlvar{'lonDaemons'}/logs/${daemon}_errors");
       sleep 2;
       if (-e $pidfile) {
    print $fh "Seems like it started ...<p>";
    my $lfh=IO::File->new("$pidfile");
    my $daemonpid=<$lfh>;
    chomp($daemonpid);
    sleep 2;
    if (kill 0 => $daemonpid) {
       return 1;
    } else {
       return 0;
    }
       }
       print $fh "Seems like that did not work!<p>";
       $errors++;
       return 0;
   }
   
   sub checkon_daemon {
       my ($fh,$daemon,$maxsize,$sendusr1)=@_;
   
       print $fh '<hr><a name="'.$daemon.'"><h2>'.$daemon.'</h2><h3>Log</h3><pre>';
       printf("%-10s ",$daemon);
       if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){
    open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|");
    while ($line=<DFH>) { 
       print $fh "$line";
       if ($line=~/INFO/) { $notices++; }
       if ($line=~/WARNING/) { $notices++; }
       if ($line=~/CRITICAL/) { $warnings++; }
    };
    close (DFH);
       }
       print $fh "</pre>";
       
       my $pidfile="$perlvar{'lonDaemons'}/logs/$daemon.pid";
       
       my $restartflag=1;
       
       if (-e $pidfile) {
    my $lfh=IO::File->new("$pidfile");
    my $daemonpid=<$lfh>;
    chomp($daemonpid);
    if (kill 0 => $daemonpid) {
       print $fh "<h3>$daemon at pid $daemonpid responding";
       if ($sendusr1) { print $fh ", sending USR1"; }
       print $fh "</h3>";
       if ($sendusr1) { kill USR1 => $daemonpid; }
       $restartflag=0;
       print "running\n";
    } else {
       $errors++;
       print $fh "<h3>$daemon at pid $daemonpid not responding</h3>";
       $restartflag=1;
       print $fh "<h3>Decided to clean up stale .pid file and restart $daemon</h3>";
    }
       }
       if ($restartflag==1) {
    $simplestatus{$daemon}='off';
    $errors++;
    print $fh '<br><font color="red">Killall '.$daemon.': '.
       `killall $daemon 2>&1`.' - ';
    sleep 2;
    print $fh unlink($pidfile).' - '.
       `killall -9 $daemon 2>&1`.
       '</font><br>';
    print $fh "<h3>$daemon not running, trying to start</h3>";
   
    if (&start_daemon($fh,$daemon,$pidfile)) {
       print $fh "<h3>$daemon at pid $daemonpid responding</h3>";
       $simplestatus{$daemon}='restarted';
       print "started\n";
    } else {
       $errors++;
       print $fh "<h3>$daemon at pid $daemonpid not responding</h3>";
       print $fh "Give it one more try ...<p>";
       print " ";
       if (&start_daemon($fh,$daemon,$pidfile)) {
    print $fh "<h3>$daemon at pid $daemonpid responding</h3>";
    $simplestatus{$daemon}='restarted';
    print "started\n";
       } else {
    print " failed\n";
    $simplestatus{$daemon}='failed';
    $errors++; $errors++;
    print $fh "<h3>$daemon at pid $daemonpid not responding</h3>";
    print $fh "Unable to start $daemon<p>";
       }
    }
   
    if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){
       print $fh "<p><pre>";
       open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/$daemon.log|");
       while ($line=<DFH>) { 
    print $fh "$line";
    if ($line=~/WARNING/) { $notices++; }
    if ($line=~/CRITICAL/) { $notices++; }
       };
       close (DFH);
       print $fh "</pre>";
    }
       }
       
       $fname="$perlvar{'lonDaemons'}/logs/$daemon.log";
       
       my ($dev,$ino,$mode,$nlink,
    $uid,$gid,$rdev,$size,
    $atime,$mtime,$ctime,
    $blksize,$blocks)=stat($fname);
       
       if ($size>$maxsize) {
    print $fh "Rotating logs ...<p>";
    rename("$fname.2","$fname.3");
    rename("$fname.1","$fname.2");
    rename("$fname","$fname.1");
       }
   
       &errout($fh);
   }
 # ================================================================ Main Program  # ================================================================ Main Program
   
 # --------------------------------- Read loncapa_apache.conf and loncapa.conf  # --------------------------------- Read loncapa_apache.conf and loncapa.conf
Line 61  delete $perlvar{'lonSqlAccess'}; # remov Line 183  delete $perlvar{'lonSqlAccess'}; # remov
 # --------------------------------------- Make sure that LON-CAPA is configured  # --------------------------------------- Make sure that LON-CAPA is configured
 # I only test for one thing here (lonHostID).  This is just a safeguard.  # I only test for one thing here (lonHostID).  This is just a safeguard.
 if ('{[[[[lonHostID]]]]}' eq $perlvar{'lonHostID'}) {  if ('{[[[[lonHostID]]]]}' eq $perlvar{'lonHostID'}) {
    print("Unconfigured machine.\n");      print("Unconfigured machine.\n");
    $emailto=$perlvar{'lonSysEMail'};      $emailto=$perlvar{'lonSysEMail'};
    $hostname=`/bin/hostname`;      $hostname=`/bin/hostname`;
    chop $hostname;      chop $hostname;
    $hostname=~s/[^\w\.]//g; # make sure is safe to pass through shell      $hostname=~s/[^\w\.]//g; # make sure is safe to pass through shell
    $subj="LON: Unconfigured machine $hostname";      $subj="LON: Unconfigured machine $hostname";
    system("echo 'Unconfigured machine $hostname.' |\      system("echo 'Unconfigured machine $hostname.' |\
  mailto $emailto -s '$subj' > /dev/null");   mailto $emailto -s '$subj' > /dev/null");
     exit 1;      exit 1;
 }  }
Line 75  if ('{[[[[lonHostID]]]]}' eq $perlvar{'l Line 197  if ('{[[[[lonHostID]]]]}' eq $perlvar{'l
 # ----------------------------- Make sure this process is running from user=www  # ----------------------------- Make sure this process is running from user=www
 my $wwwid=getpwnam('www');  my $wwwid=getpwnam('www');
 if ($wwwid!=$<) {  if ($wwwid!=$<) {
    print("User ID mismatch.  This program must be run as user 'www'\n");      print("User ID mismatch.  This program must be run as user 'www'\n");
    $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}";      $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}";
    $subj="LON: $perlvar{'lonHostID'} User ID mismatch";      $subj="LON: $perlvar{'lonHostID'} User ID mismatch";
    system("echo 'User ID mismatch.  loncron must be run as user www.' |\      system("echo 'User ID mismatch.  loncron must be run as user www.' |\
  mailto $emailto -s '$subj' > /dev/null");   mailto $emailto -s '$subj' > /dev/null");
    exit 1;      exit 1;
 }  }
   
 # ------------------------------------------------------------- Read hosts file  # ------------------------------------------------------------- Read hosts file
 {  {
     my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab");      my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab");
       
     while (my $configline=<$config>) {      while (my $configline=<$config>) {
  my ($id,$domain,$role,$name,$ip,$domdescr)=split(/:/,$configline);   my ($id,$domain,$role,$name,$ip,$domdescr)=split(/:/,$configline);
  if ($id && $domain && $role && $name && $ip) {   if ($id && $domain && $role && $name && $ip) {
Line 109  if ($wwwid!=$<) { Line 231  if ($wwwid!=$<) {
 # ------------------------------------------------------ Read spare server file  # ------------------------------------------------------ Read spare server file
 {  {
     my $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab");      my $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab");
       
     while (my $configline=<$config>) {      while (my $configline=<$config>) {
        chomp($configline);   chomp($configline);
        if (($configline) && ($configline ne $perlvar{'lonHostID'})) {   if (($configline) && ($configline ne $perlvar{'lonHostID'})) {
           $spareid{$configline}=1;      $spareid{$configline}=1;
        }   }
     }      }
 }  }
   
Line 130  $now=time; Line 252  $now=time;
 $date=localtime($now);  $date=localtime($now);
   
 {  {
 my $fh=IO::File->new(">$statusdir/newstatus.html");      my $fh=IO::File->new(">$statusdir/newstatus.html");
       my %simplestatus=();
 print $fh (<<ENDHEADERS);      
       print $fh (<<ENDHEADERS);
 <html>  <html>
 <head>  <head>
 <title>LON Status Report $perlvar{'lonHostID'}</title>  <title>LON Status Report $perlvar{'lonHostID'}</title>
Line 163  print $fh (<<ENDHEADERS); Line 286  print $fh (<<ENDHEADERS);
 <table border=2>  <table border=2>
 ENDHEADERS  ENDHEADERS
   
 foreach $varname (sort(keys(%perlvar))) {      foreach $varname (sort(keys(%perlvar))) {
     print $fh "<tr><td>$varname</td><td>$perlvar{$varname}</td></tr>\n";   print $fh "<tr><td>$varname</td><td>$perlvar{$varname}</td></tr>\n";
 }      }
 print $fh "</table><h3>Hosts</h3><table border=2>";      print $fh "</table><h3>Hosts</h3><table border=2>";
 foreach $id (sort(keys(%hostname))) {      foreach $id (sort(keys(%hostname))) {
     print $fh    print $fh 
  "<tr><td>$id</td><td>$hostdom{$id}</td><td>$hostrole{$id}</td>";      "<tr><td>$id</td><td>$hostdom{$id}</td><td>$hostrole{$id}</td>";
     print $fh "<td>$hostname{$id}</td><td>$hostip{$id}</td></tr>\n";   print $fh "<td>$hostname{$id}</td><td>$hostip{$id}</td></tr>\n";
 }      }
 print $fh "</table><h3>Spare Hosts</h3><ol>";      print $fh "</table><h3>Spare Hosts</h3><ol>";
 foreach $id (sort(keys(%spareid))) {      foreach $id (sort(keys(%spareid))) {
     print $fh "<li>$id\n";   print $fh "<li>$id\n";
 }      }
       
 print $fh "</ol>\n";      print $fh "</ol>\n";
   
 # --------------------------------------------------------------------- Machine  # --------------------------------------------------------------------- Machine
   
 print $fh '<hr><a name="machine"><h2>Machine Information</h2>';  
 print $fh "<h3>loadavg</h3>";  
   
 open (LOADAVGH,"/proc/loadavg");  
 $loadavg=<LOADAVGH>;  
 close (LOADAVGH);  
   
 print $fh "<tt>$loadavg</tt>";  
   
 @parts=split(/\s+/,$loadavg);  
 if ($parts[1]>4.0) {  
     $errors++;  
 } elsif ($parts[1]>2.0) {  
     $warnings++;  
 } elsif ($parts[1]>1.0) {  
     $notices++;  
 }  
   
 print $fh "<h3>df</h3>";  
 print $fh "<pre>";  
   
 open (DFH,"df|");  
 while ($line=<DFH>) {   
    print $fh "$line";   
    @parts=split(/\s+/,$line);  
    $usage=$parts[4];  
    $usage=~s/\W//g;  
    if ($usage>90) {   
       $warnings++;  
       $notices++;   
    } elsif ($usage>80) {  
       $warnings++;  
    } elsif ($usage>60) {  
       $notices++;  
    }  
    if ($usage>95) { $warnings++; $warnings++ }  
 }  
 close (DFH);  
 print $fh "</pre>";  
   
   
 print $fh "<h3>ps</h3>";  
 print $fh "<pre>";  
 $psproc=0;  
   
 open (PSH,"ps -aux|");  
 while ($line=<PSH>) {   
    print $fh "$line";   
    $psproc++;  
 }  
 close (PSH);  
 print $fh "</pre>";  
   
 if ($psproc>200) { $notices++; }  
 if ($psproc>250) { $notices++; }  
   
 &errout($fh);  
   
 # --------------------------------------------------------------- clean out tmp  
 print $fh '<hr><a name="tmp"><h2>Temporary Files</h2>';  
 $cleaned=0;  
 $old=0;  
 while ($fname=<$perlvar{'lonDaemons'}/tmp/*>) {  
                           my ($dev,$ino,$mode,$nlink,  
                               $uid,$gid,$rdev,$size,  
                               $atime,$mtime,$ctime,  
                               $blksize,$blocks)=stat($fname);  
                           $now=time;  
                           $since=$now-$mtime;  
                           if ($since>$perlvar{'lonExpire'}) {  
                               $line='';  
                               if (open(PROBE,$fname)) {  
   $line=<PROBE>;  
                                   close(PROBE);  
       }  
       unless ($line=~/^CHECKOUTTOKEN\&/) {  
                                  $cleaned++;  
                                  unlink("$fname");  
       } else {  
   if ($since>365*$perlvar{'lonExpire'}) {  
                                      $cleaned++;  
                                      unlink("$fname");  
  } else { $old++; }  
                               }  
                           }  
           
 }      print $fh '<hr><a name="machine"><h2>Machine Information</h2>';
 print $fh "Cleaned up ".$cleaned." files (".$old." old checkout tokens).";      print $fh "<h3>loadavg</h3>";
   
 # ------------------------------------------------------------ clean out lonIDs  
 print $fh '<hr><a name="tokens"><h2>Session Tokens</h2>';  
 $cleaned=0;  
 $active=0;  
 while ($fname=<$perlvar{'lonIDsDir'}/*>) {  
                           my ($dev,$ino,$mode,$nlink,  
                               $uid,$gid,$rdev,$size,  
                               $atime,$mtime,$ctime,  
                               $blksize,$blocks)=stat($fname);  
                           $now=time;  
                           $since=$now-$mtime;  
                           if ($since>$perlvar{'lonExpire'}) {  
                               $cleaned++;  
                               print $fh "Unlinking $fname<br>";  
                               unlink("$fname");  
                           } else {  
                               $active++;  
                           }  
           
 }      open (LOADAVGH,"/proc/loadavg");
 print $fh "<p>Cleaned up ".$cleaned." stale session token(s).";      $loadavg=<LOADAVGH>;
 print $fh "<h3>$active open session(s)</h3>";      close (LOADAVGH);
       
 # ----------------------------------------------------------------------- httpd      print $fh "<tt>$loadavg</tt>";
       
 print $fh '<hr><a name="httpd"><h2>httpd</h2><h3>Access Log</h3><pre>';      @parts=split(/\s+/,$loadavg);
       if ($parts[1]>4.0) {
    $errors++;
       } elsif ($parts[1]>2.0) {
    $warnings++;
       } elsif ($parts[1]>1.0) {
    $notices++;
       }
   
 open (DFH,"tail -n25 /etc/httpd/logs/access_log|");      print $fh "<h3>df</h3>";
 while ($line=<DFH>) { print $fh "$line" };      print $fh "<pre>";
 close (DFH);  
   
 print $fh "</pre><h3>Error Log</h3><pre>";  
   
 open (DFH,"tail -n25 /etc/httpd/logs/error_log|");  
 while ($line=<DFH>) {   
    print $fh "$line";  
    if ($line=~/\[error\]/) { $notices++; }   
 };  
 close (DFH);  
 print $fh "</pre>";  
 &errout($fh);  
   
       open (DFH,"df|");
       while ($line=<DFH>) { 
    print $fh "$line"; 
    @parts=split(/\s+/,$line);
    $usage=$parts[4];
    $usage=~s/\W//g;
    if ($usage>90) { 
       $warnings++;
       $notices++; 
    } elsif ($usage>80) {
       $warnings++;
    } elsif ($usage>60) {
       $notices++;
    }
    if ($usage>95) { $warnings++; $warnings++; $simplestatus{'diskfull'}++; }
       }
       close (DFH);
       print $fh "</pre>";
   
 # ---------------------------------------------------------------------- lonsql  
   
 my $restartflag=1;      print $fh "<h3>ps</h3>";
     print $fh '<hr><a name="lonsql"><h2>lonsql</h2><h3>Log</h3><pre>';      print $fh "<pre>";
     print "lonsql\n";      $psproc=0;
     if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){  
  open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");      open (PSH,"ps -aux|");
  while ($line=<DFH>) {       while ($line=<PSH>) { 
     print $fh "$line";   print $fh "$line"; 
     if ($line=~/INFO/) { $notices++; }   $psproc++;
     if ($line=~/WARNING/) { $notices++; }  
     if ($line=~/CRITICAL/) { $warnings++; }  
  };  
  close (DFH);  
     }      }
       close (PSH);
     print $fh "</pre>";      print $fh "</pre>";
       
     my $lonsqlfile="$perlvar{'lonDaemons'}/logs/lonsql.pid";      if ($psproc>200) { $notices++; }
        if ($psproc>250) { $notices++; }
     $restartflag=1;  
          &errout($fh);
     if (-e $lonsqlfile) {  
  my $lfh=IO::File->new("$lonsqlfile");  # --------------------------------------------------------------- clean out tmp
  my $lonsqlpid=<$lfh>;      print $fh '<hr><a name="tmp"><h2>Temporary Files</h2>';
  chomp($lonsqlpid);      $cleaned=0;
  if (kill 0 => $lonsqlpid) {      $old=0;
     print $fh "<h3>lonsql at pid $lonsqlpid responding</h3>";      while ($fname=<$perlvar{'lonDaemons'}/tmp/*>) {
     $restartflag=0;   my ($dev,$ino,$mode,$nlink,
  } else {      $uid,$gid,$rdev,$size,
     $errors++; $errors++;      $atime,$mtime,$ctime,
     print $fh "<h3>lonsql at pid $lonsqlpid not responding</h3>";      $blksize,$blocks)=stat($fname);
  $restartflag=1;   $now=time;
  print $fh    $since=$now-$mtime;
     "<h3>Decided to clean up stale .pid file and restart lonsql</h3>";   if ($since>$perlvar{'lonExpire'}) {
  }      $line='';
     }      if (open(PROBE,$fname)) {
     if ($restartflag==1) {   $line=<PROBE>;
  $errors++;   close(PROBE);
          print $fh '<br><font color="red">Killall lonsql: '.      }
                     system('killall lonsql').' - ';      unless ($line=~/^CHECKOUTTOKEN\&/) {
                     sleep 2;   $cleaned++;
                     print $fh unlink($lonsqlfile).' - '.   unlink("$fname");
                               system('killall -9 lonsql').  
                     '</font><br>';  
  print $fh "<h3>lonsql not running, trying to start</h3>";  
  system(  
  "$perlvar{'lonDaemons'}/lonsql 2>>$perlvar{'lonDaemons'}/logs/lonsql_errors");  
  sleep 2;  
  if (-e $lonsqlfile) {  
     print $fh "Seems like it started ...<p>";  
     my $lfh=IO::File->new("$lonsqlfile");  
     my $lonsqlpid=<$lfh>;  
     chomp($lonsqlpid);  
     sleep 2;  
     if (kill 0 => $lonsqlpid) {  
  print $fh "<h3>lonsql at pid $lonsqlpid responding</h3>";  
     } else {      } else {
  $errors++; $errors++;   if ($since>365*$perlvar{'lonExpire'}) {
  print $fh "<h3>lonsql at pid $lonsqlpid not responding</h3>";      $cleaned++;
  print $fh "Give it one more try ...<p>";      unlink("$fname");
  system(   } else { $old++; }
  "$perlvar{'lonDaemons'}/lonsql 2>>$perlvar{'lonDaemons'}/logs/lonsql_errors");  
  sleep 2;  
     }      }
  } else {  
     print $fh "Seems like that did not work!<p>";  
     $errors++;  
  }  
  if (-e "$perlvar{'lonDaemons'}/logs/lonsql.log"){  
     print $fh "<p><pre>";  
     open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonsql.log|");  
     while ($line=<DFH>) {   
  print $fh "$line";  
  if ($line=~/WARNING/) { $notices++; }  
  if ($line=~/CRITICAL/) { $notices++; }  
     };  
     close (DFH);  
     print $fh "</pre>";  
  }   }
       
     }      }
       print $fh "Cleaned up ".$cleaned." files (".$old." old checkout tokens).";
   
     $fname="$perlvar{'lonDaemons'}/logs/lonsql.log";  # ------------------------------------------------------------ clean out lonIDs
       print $fh '<hr><a name="tokens"><h2>Session Tokens</h2>';
     my ($dev,$ino,$mode,$nlink,      $cleaned=0;
  $uid,$gid,$rdev,$size,      $active=0;
  $atime,$mtime,$ctime,      while ($fname=<$perlvar{'lonIDsDir'}/*>) {
  $blksize,$blocks)=stat($fname);   my ($dev,$ino,$mode,$nlink,
       $uid,$gid,$rdev,$size,
       $atime,$mtime,$ctime,
       $blksize,$blocks)=stat($fname);
    $now=time;
    $since=$now-$mtime;
    if ($since>$perlvar{'lonExpire'}) {
       $cleaned++;
       print $fh "Unlinking $fname<br>";
       unlink("$fname");
    } else {
       $active++;
    }
   
     if ($size>200000) {  
  print $fh "Rotating logs ...<p>";  
  rename("$fname.2","$fname.3");  
  rename("$fname.1","$fname.2");  
  rename("$fname","$fname.1");  
     }      }
       print $fh "<p>Cleaned up ".$cleaned." stale session token(s).";
       print $fh "<h3>$active open session(s)</h3>";
   
     &errout($fh);  # ----------------------------------------------------------------------- httpd
 # ------------------------------------------------------------------------ lond  
   
 print $fh '<hr><a name="lond"><h2>lond</h2><h3>Log</h3><pre>';  
 print "lond\n";  
   
 if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){      print $fh '<hr><a name="httpd"><h2>httpd</h2><h3>Access Log</h3><pre>';
 open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/lond.log|");      
 while ($line=<DFH>) {       open (DFH,"tail -n25 /etc/httpd/logs/access_log|");
    print $fh "$line";      while ($line=<DFH>) { print $fh "$line" };
    if ($line=~/INFO/) { $notices++; }      close (DFH);
    if ($line=~/WARNING/) { $notices++; }  
    if ($line=~/CRITICAL/) { $warnings++; }  
 };  
 close (DFH);  
 }  
 print $fh "</pre>";  
   
 my $londfile="$perlvar{'lonDaemons'}/logs/lond.pid";      print $fh "</pre><h3>Error Log</h3><pre>";
   
 $restartflag=1;      open (DFH,"tail -n25 /etc/httpd/logs/error_log|");
 if (-e $londfile) {      
    my $lfh=IO::File->new("$londfile");  
    my $londpid=<$lfh>;  
    chomp($londpid);  
    if (kill 0 => $londpid) {  
       print $fh "<h3>lond at pid $londpid responding, sending USR1</h3>";  
       kill USR1 => $londpid;  
       $restartflag=0;  
    } else {  
       $errors++;  
       print $fh "<h3>lond at pid $londpid not responding</h3>";  
       $restartflag=1;  
       print $fh   
   "<h3>Decided to clean up stale .pid file and restart lond</h3>";  
    }  
 }   
 if ($restartflag==1) {  
    $errors++;  
   print $fh '<br><font color="red">Killall lond: '.  
                     system('killall lond').' - ';  
           sleep 2;  
           print $fh unlink($londfile).' - '.system('killall -9 lond').  
                     '</font><br>';  
    print $fh "<h3>lond not running, trying to start</h3>";  
    system(  
      "$perlvar{'lonDaemons'}/lond 2>>$perlvar{'lonDaemons'}/logs/lond_errors");  
    sleep 2;  
    if (-e $londfile) {  
        print $fh "Seems like it started ...<p>";  
        my $lfh=IO::File->new("$londfile");  
        my $londpid=<$lfh>;  
        chomp($londpid);  
        sleep 2;  
        if (kill 0 => $londpid) {  
           print $fh "<h3>lond at pid $londpid responding</h3>";  
        } else {  
           $errors++; $errors++;  
           print $fh "<h3>lond at pid $londpid not responding</h3>";  
           print $fh "Give it one more try ...<p>";  
   system(  
  "$perlvar{'lonDaemons'}/lond 2>>$perlvar{'lonDaemons'}/logs/lond_errors");  
           sleep 2;  
        }  
    } else {  
        print $fh "Seems like that did not work!<p>";  
        $errors++;  
    }  
    if (-e "$perlvar{'lonDaemons'}/logs/lond.log"){  
     print $fh "<p><pre>";  
     open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lond.log|");  
     while ($line=<DFH>) {       while ($line=<DFH>) { 
       print $fh "$line";   print $fh "$line";
       if ($line=~/WARNING/) { $notices++; }   if ($line=~/\[error\]/) { $notices++; } 
       if ($line=~/CRITICAL/) { $notices++; }  
     };      };
     close (DFH);      close (DFH);
     print $fh "</pre>";      print $fh "</pre>";
    }      &errout($fh);
 }  
   
 $fname="$perlvar{'lonDaemons'}/logs/lond.log";  
   
                           my ($dev,$ino,$mode,$nlink,  
                               $uid,$gid,$rdev,$size,  
                               $atime,$mtime,$ctime,  
                               $blksize,$blocks)=stat($fname);  
   
 if ($size>40000) {  
     print $fh "Rotating logs ...<p>";  
     rename("$fname.2","$fname.3");  
     rename("$fname.1","$fname.2");  
     rename("$fname","$fname.1");  
 }  
   
 &errout($fh);  
 # ------------------------------------------------------------------------ lonc  
   
 print $fh '<hr><a name="lonc"><h2>lonc</h2><h3>Log</h3><pre>';  # ---------------------------------------------------------------------- lonsql
 print "lonc\n";  
   
 if (-e "$perlvar{'lonDaemons'}/logs/lonc.log"){      &checkon_daemon($fh,'lonsql',200000);
 open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/lonc.log|");  
 while ($line=<DFH>) {   
    print $fh "$line";  
    if ($line=~/INFO/) { $notices++; }  
    if ($line=~/WARNING/) { $notices++; }  
    if ($line=~/CRITICAL/) { $warnings++; }  
 };  
 close (DFH);  
 }  
 print $fh "</pre>";  
   
 my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid";  # ------------------------------------------------------------------------ lond
   
 $restartflag=1;      &checkon_daemon($fh,'lond',40000,1);
 if (-e $loncfile) {  
    my $lfh=IO::File->new("$loncfile");  
    my $loncpid=<$lfh>;  
    chomp($loncpid);  
    if (kill 0 => $loncpid) {  
       print $fh "<h3>lonc at pid $loncpid responding, sending USR1</h3>";  
       kill USR1 => $loncpid;  
       $restartflag=0;  
    } else {  
       $errors++;  
       print $fh "<h3>lonc at pid $loncpid not responding</h3>";  
       # Solution: kill parent and children processes, remove .pid and restart  
   $restartflag=1;  
       print $fh   
   "<h3>Decided to clean up stale .pid file and restart lonc</h3>";  
    }  
 }   
 if ($restartflag==1) {  
    $errors++;  
   print $fh '<br><font color="red">Killall lonc: '.  
             system('killall lonc').' - ';  
           sleep 2;  
           print $fh unlink($loncfile).' - '.system('killall -9 lonc').  
                     '</font><br>';  
    print $fh "<h3>lonc not running, trying to start</h3>";  
  system(  
  "$perlvar{'lonDaemons'}/lonc 2>>$perlvar{'lonDaemons'}/logs/lonc_errors");  
    sleep 2;  
    if (-e $loncfile) {  
        print $fh "Seems like it started ...<p>";  
        my $lfh=IO::File->new("$loncfile");  
        my $loncpid=<$lfh>;  
        chomp($loncpid);  
        sleep 2;  
        if (kill 0 => $loncpid) {  
           print $fh "<h3>lonc at pid $loncpid responding</h3>";  
        } else {  
           $errors++; $errors++;  
           print $fh "<h3>lonc at pid $loncpid not responding</h3>";  
           print $fh "Give it one more try ...<p>";  
    system(  
  "$perlvar{'lonDaemons'}/lonc 2>>$perlvar{'lonDaemons'}/logs/lonc_errors");  
           sleep 2;  
        }  
    } else {  
        print $fh "Seems like that did not work!<p>";  
        $errors++;  
    }  
    if (-e "$perlvar{'lonDaemons'}/logs/lonc.log") {  
     print $fh "<p><pre>";  
     open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonc.log|");  
     while ($line=<DFH>) {   
       print $fh "$line";  
       if ($line=~/WARNING/) { $notices++; }  
       if ($line=~/CRITICAL/) { $notices++; }  
     };  
     close (DFH);  
     print $fh "</pre>";  
    }  
 }  
   
 $fname="$perlvar{'lonDaemons'}/logs/lonc.log";  # ------------------------------------------------------------------------ lonc
   
                           my ($dev,$ino,$mode,$nlink,      &checkon_daemon($fh,'lonc',40000,1);
                               $uid,$gid,$rdev,$size,  
                               $atime,$mtime,$ctime,  
                               $blksize,$blocks)=stat($fname);  
   
 if ($size>40000) {  
     print $fh "Rotating logs ...<p>";  
     rename("$fname.2","$fname.3");  
     rename("$fname.1","$fname.2");  
     rename("$fname","$fname.1");  
 }  
   
      
 &errout($fh);  
 # -------------------------------------------------------------------- lonhttpd  # -------------------------------------------------------------------- lonhttpd
   
 print $fh '<hr><a name="lonhttpd"><h2>lonhttpd</h2><h3>Log</h3><pre>';      &checkon_daemon($fh,'lonhttpd',40000);
 print "lonhttpd\n";  
   
 if (-e "$perlvar{'lonDaemons'}/logs/lonhttpd.log"){  
 open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/lonhttpd.log|");  
 while ($line=<DFH>) {   
    print $fh "$line";  
    if ($line=~/INFO/) { $notices++; }  
    if ($line=~/WARNING/) { $notices++; }  
    if ($line=~/CRITICAL/) { $warnings++; }  
 };  
 close (DFH);  
 }  
 print $fh "</pre>";  
   
 my $lonhttpdfile="$perlvar{'lonDaemons'}/logs/lonhttpd.pid";  
   
 $restartflag=1;  # ---------------------------------------------------------------------- lonnet
 if (-e $lonhttpdfile) {  
    my $lfh=IO::File->new("$lonhttpdfile");  
    my $lonhttpdpid=<$lfh>;  
    chomp($lonhttpdpid);  
    if (kill 0 => $lonhttpdpid) {  
       print $fh "<h3>lonhttpd at pid $lonhttpdpid responding</h3>";  
       $restartflag=0;  
    } else {  
       $errors++;  
       print $fh "<h3>lonhttpd at pid $lonhttpdpid not responding</h3>";  
       # Solution: kill parent and children processes, remove .pid and restart  
   $restartflag=1;  
       print $fh   
   "<h3>Decided to clean up stale .pid file and restart lonhttpd</h3>";  
    }  
 }   
 if ($restartflag==1) {  
    $errors++;  
   print $fh '<br><font color="red">Killall lonhttpd: '.  
             system('killall lonhttpd').' - ';  
           sleep 2;  
           print $fh unlink($lonhttpdfile).' - '.system('killall -9 lonhttpd').  
                     '</font><br>';  
    print $fh "<h3>lonhttpd not running, trying to start</h3>";  
  system(  
  "$perlvar{'lonDaemons'}/lonhttpd 2>>$perlvar{'lonDaemons'}/logs/lonhttpd_errors");  
    sleep 2;  
    if (-e $lonhttpdfile) {  
        print $fh "Seems like it started ...<p>";  
        my $lfh=IO::File->new("$lonhttpdfile");  
        my $lonhttpdpid=<$lfh>;  
        chomp($lonhttpdpid);  
        sleep 2;  
        if (kill 0 => $lonhttpdpid) {  
           print $fh "<h3>lonhttpd at pid $lonhttpdpid responding</h3>";  
        } else {  
           $errors++; $errors++;  
           print $fh "<h3>lonhttpd at pid $lonhttpdpid not responding</h3>";  
           print $fh "Give it one more try ...<p>";  
    system(  
  "$perlvar{'lonDaemons'}/lonhttpd 2>>$perlvar{'lonDaemons'}/logs/lonhttpd_errors");  
           sleep 2;  
        }  
    } else {  
        print $fh "Seems like that did not work!<p>";  
        $errors++;  
    }  
    if (-e "$perlvar{'lonDaemons'}/logs/lonhttpd.log") {  
     print $fh "<p><pre>";  
     open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/lonhttpd.log|");  
     while ($line=<DFH>) {   
       print $fh "$line";  
       if ($line=~/WARNING/) { $notices++; }  
       if ($line=~/CRITICAL/) { $notices++; }  
     };  
     close (DFH);  
     print $fh "</pre>";  
    }  
 }  
   
 $fname="$perlvar{'lonDaemons'}/logs/lonhttpd.log";  
   
                           my ($dev,$ino,$mode,$nlink,      print $fh '<hr><a name="lonnet"><h2>lonnet</h2><h3>Temp Log</h3><pre>';
                               $uid,$gid,$rdev,$size,      print "checking logs\n";
                               $atime,$mtime,$ctime,      if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
                               $blksize,$blocks)=stat($fname);   open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
    while ($line=<DFH>) { 
 if ($size>40000) {      print $fh "$line";
     print $fh "Rotating logs ...<p>";   };
     rename("$fname.2","$fname.3");   close (DFH);
     rename("$fname.1","$fname.2");      }
     rename("$fname","$fname.1");      print $fh "</pre><h3>Perm Log</h3><pre>";
 }      
       if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") {
    open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|");
    while ($line=<DFH>) { 
       print $fh "$line";
    };
    close (DFH);
       } else { print $fh "No perm log\n" }
   
          $fname="$perlvar{'lonDaemons'}/logs/lonnet.log";
 &errout($fh);  
 # ---------------------------------------------------------------------- lonnet  
   
 print $fh '<hr><a name="lonnet"><h2>lonnet</h2><h3>Temp Log</h3><pre>';      my ($dev,$ino,$mode,$nlink,
 print "lonnet\n";   $uid,$gid,$rdev,$size,
 if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){   $atime,$mtime,$ctime,
 open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");   $blksize,$blocks)=stat($fname);
 while ($line=<DFH>) {   
     print $fh "$line";  
 };  
 close (DFH);  
 }  
 print $fh "</pre><h3>Perm Log</h3><pre>";  
   
 if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") {      if ($size>40000) {
     open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|");   print $fh "Rotating logs ...<p>";
 while ($line=<DFH>) {    rename("$fname.2","$fname.3");
    print $fh "$line";   rename("$fname.1","$fname.2");
 };   rename("$fname","$fname.1");
 close (DFH);      }
 } else { print $fh "No perm log\n" }  
   
 $fname="$perlvar{'lonDaemons'}/logs/lonnet.log";  
   
                           my ($dev,$ino,$mode,$nlink,  
                               $uid,$gid,$rdev,$size,  
                               $atime,$mtime,$ctime,  
                               $blksize,$blocks)=stat($fname);  
   
 if ($size>40000) {  
     print $fh "Rotating logs ...<p>";  
     rename("$fname.2","$fname.3");  
     rename("$fname.1","$fname.2");  
     rename("$fname","$fname.1");  
 }  
   
 print $fh "</pre>";      print $fh "</pre>";
 &errout($fh);      &errout($fh);
 # ----------------------------------------------------------------- Connections  # ----------------------------------------------------------------- Connections
   
 print $fh '<hr><a name="connections"><h2>Connections</h2>';      print $fh '<hr><a name="connections"><h2>Connections</h2>';
       print "testing connections\n";
 print $fh "<table border=2>";      print $fh "<table border=2>";
 foreach $tryserver (sort(keys(%hostname))) {      foreach $tryserver (sort(keys(%hostname))) {
    print(".");
    $answer=reply("pong",$tryserver);
    if ($answer eq "$tryserver:$perlvar{'lonHostID'}") {
       $result="<b>ok</b>";
    } else {
       $result=$answer;
       $warnings++;
       if ($answer eq 'con_lost') { $warnings++; }
    }
    if ($answer =~ /con_lost/) { print(" $tryserver down\n"); }
    print $fh "<tr><td>$tryserver</td><td>$result</td></tr>\n";
   
     $answer=reply("pong",$tryserver);  
     if ($answer eq "$tryserver:$perlvar{'lonHostID'}") {  
  $result="<b>ok</b>";  
     } else {  
         $result=$answer;  
         $warnings++;  
         if ($answer eq 'con_lost') { $warnings++; }  
     }      }
     print $fh "<tr><td>$tryserver</td><td>$result</td></tr>\n";      print $fh "</table>";
   
 }  
 print $fh "</table>";  
   
 &errout($fh);      &errout($fh);
 # ------------------------------------------------------------ Delayed messages  # ------------------------------------------------------------ Delayed messages
   
 print $fh '<hr><a name="delayed"><h2>Delayed Messages</h2>';      print $fh '<hr><a name="delayed"><h2>Delayed Messages</h2>';
 print "buffers\n";      print "checking buffers\n";
   
 print $fh '<h3>Scanning Permanent Log</h3>';      print $fh '<h3>Scanning Permanent Log</h3>';
   
 $unsend=0;      $unsend=0;
 {      {
     my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log");   my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log");
     while ($line=<$dfh>) {   while ($line=<$dfh>) {
  ($time,$sdf,$dserv,$dcmd)=split(/:/,$line);      ($time,$sdf,$dserv,$dcmd)=split(/:/,$line);
         if ($sdf eq 'F') {       if ($sdf eq 'F') { 
     $local=localtime($time);   $local=localtime($time);
             print "<b>Failed: $time, $dserv, $dcmd</b><br>";   print $fh "<b>Failed: $time, $dserv, $dcmd</b><br>";
             $warnings++;   $warnings++;
         }      }
         if ($sdf eq 'S') { $unsend--; }      if ($sdf eq 'S') { $unsend--; }
         if ($sdf eq 'D') { $unsend++; }      if ($sdf eq 'D') { $unsend++; }
    }
     }      }
 }      print $fh "Total unsend messages: <b>$unsend</b><p>\n";
 print $fh "Total unsend messages: <b>$unsend</b><p>\n";      $warnings=$warnings+5*$unsend;
 $warnings=$warnings+5*$unsend;  
   
 print $fh "<h3>Outgoing Buffer</h3>";      if ($unsend) { $simplestatus{'unsend'}=$unsend; }
       print $fh "<h3>Outgoing Buffer</h3>";
   
 open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|");      open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|");
 while ($line=<DFH>) {       while ($line=<DFH>) { 
     print $fh "$line<br>";   print $fh "$line<br>";
 };      };
 close (DFH);      close (DFH);
   
 # ------------------------------------------------------------------------- End  # ------------------------------------------------------------------------- End
 print $fh "<a name=errcount>\n";      print $fh "<a name=errcount>\n";
 $totalcount=$notices+4*$warnings+100*$errors;      $totalcount=$notices+4*$warnings+100*$errors;
 &errout($fh);      &errout($fh);
 print $fh "<h1>Total Error Count: $totalcount</h1>";      print $fh "<h1>Total Error Count: $totalcount</h1>";
 $now=time;      $now=time;
 $date=localtime($now);      $date=localtime($now);
 print $fh "<hr>$date ($now)</body></html>\n";      print $fh "<hr>$date ($now)</body></html>\n";
 print "writing done\n";      print "lon-status webpage updated\n";
 }      $fh->close();
   }
   if ($errors) { $simplestatus{'errors'}=$errors; }
   if ($warnings) { $simplestatus{'warnings'}=$warnings; }
   if ($notices) { $simplestatus{'notices'}=$notices; }
   $simplestatus{'time'}=time;
   
 rename ("$statusdir/newstatus.html","$statusdir/index.html");  rename ("$statusdir/newstatus.html","$statusdir/index.html");
   {
       my $sfh=IO::File->new(">$statusdir/loncron_simple.txt");
       foreach (keys %simplestatus) {
    print $sfh $_.'='.$simplestatus{$_}.'&';
       }
       print $sfh "\n";
       $sfh->close();
   }
 if ($totalcount>200) {  if ($totalcount>200) {
    print "mailing\n";      print "sending mail\n";
    $emailto="$perlvar{'lonAdmEMail'}";      $emailto="$perlvar{'lonAdmEMail'}";
    if ($totalcount>1000) {      if ($totalcount>1000) {
       $emailto.=",$perlvar{'lonSysEMail'}";   $emailto.=",$perlvar{'lonSysEMail'}";
    }      }
    $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices";       $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; 
    system(      system("metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html");
  "metasend -b -t $emailto -s '$subj' -f $statusdir/index.html -m text/html");  
 }  }
 1;  1;
   
Removed from v.1.40  
changed lines
  Added in v.1.43

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
500 Internal Server Error

Internal Server Error

The server encountered an internal error or misconfiguration and was unable to complete your request.

Please contact the server administrator at root@localhost to inform them of the time this error occurred, and the actions you performed just before this error.

More information about this error may be available in the server error log.