File:  [LON-CAPA] / loncom / loncron
Revision 1.75: download - view: text, annotated - select for diffs
Tue Jun 12 23:50:39 2007 UTC (17 years ago) by albertel
Branches: MAIN
CVS tags: version_2_4_X, version_2_4_2, version_2_4_1, HEAD
- some maxima's want to figure out what dir we are in, and blowup on
  intialization if it's not in a directory in can read, so chdir to the
  daemons directory before we start doing stuff

    1: #!/usr/bin/perl
    2: 
    3: # Housekeeping program, started by cron, loncontrol and loncron.pl
    4: #
    5: # $Id: loncron,v 1.75 2007/06/12 23:50:39 albertel Exp $
    6: #
    7: # Copyright Michigan State University Board of Trustees
    8: #
    9: # This file is part of the LearningOnline Network with CAPA (LON-CAPA).
   10: #
   11: # LON-CAPA is free software; you can redistribute it and/or modify
   12: # it under the terms of the GNU General Public License as published by
   13: # the Free Software Foundation; either version 2 of the License, or
   14: # (at your option) any later version.
   15: #
   16: # LON-CAPA is distributed in the hope that it will be useful,
   17: # but WITHOUT ANY WARRANTY; without even the implied warranty of
   18: # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   19: # GNU General Public License for more details.
   20: #
   21: # You should have received a copy of the GNU General Public License
   22: # along with LON-CAPA; if not, write to the Free Software
   23: # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
   24: #
   25: # /home/httpd/html/adm/gpl.txt
   26: #
   27: # http://www.lon-capa.org/
   28: #
   29: 
   30: $|=1;
   31: use strict;
   32: 
   33: use lib '/home/httpd/lib/perl/';
   34: use LONCAPA::Configuration;
   35: use Apache::lonnet;
   36: 
   37: use IO::File;
   38: use IO::Socket;
   39: use HTML::Entities;
   40: use Getopt::Long;
   41: #globals
   42: use vars qw (%perlvar %simplestatus $errors $warnings $notices $totalcount);
   43: 
   44: my $statusdir="/home/httpd/html/lon-status";
   45: 
   46: 
   47: # --------------------------------------------------------- Output error status
   48: 
   49: sub log {
   50:     my $fh=shift;
   51:     if ($fh) {	print $fh @_  }
   52: }
   53: 
   54: sub errout {
   55:    my $fh=shift;
   56:    &log($fh,(<<ENDERROUT));
   57:      <table border="2" bgcolor="#CCCCCC">
   58:      <tr><td>Notices</td><td>$notices</td></tr>
   59:      <tr><td>Warnings</td><td>$warnings</td></tr>
   60:      <tr><td>Errors</td><td>$errors</td></tr>
   61:      </table><p><a href="#top">Top</a></p>
   62: ENDERROUT
   63: }
   64: 
   65: sub rotate_logfile {
   66:     my ($file,$fh,$description) = @_;
   67:     my $size=(stat($file))[7];
   68:     if ($size>40000) {
   69: 	&log($fh,"<p>Rotating $description ...</p>");
   70: 	rename("$file.2","$file.3");
   71: 	rename("$file.1","$file.2");
   72: 	rename("$file","$file.1");
   73:     } 
   74: }
   75: 
   76: sub start_daemon {
   77:     my ($fh,$daemon,$pidfile,$args) = @_;
   78:     my $progname=$daemon;
   79:     if ($daemon eq 'lonc') {
   80: 	$progname='loncnew'; 
   81:     }
   82:     my $error_fname="$perlvar{'lonDaemons'}/logs/${daemon}_errors";
   83:     &rotate_logfile($error_fname,$fh,'error logs');
   84:     if ($daemon eq 'lonc') {
   85: 	&clean_sockets($fh);
   86:     }
   87:     system("$perlvar{'lonDaemons'}/$progname 2>$perlvar{'lonDaemons'}/logs/${daemon}_errors");
   88:     sleep 1;
   89:     if (-e $pidfile) {
   90: 	&log($fh,"<p>Seems like it started ...</p>");
   91: 	my $lfh=IO::File->new("$pidfile");
   92: 	my $daemonpid=<$lfh>;
   93: 	chomp($daemonpid);
   94: 	if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) {
   95: 	    return 1;
   96: 	} else {
   97: 	    return 0;
   98: 	}
   99:     }
  100:     &log($fh,"<p>Seems like that did not work!</p>");
  101:     $errors++;
  102:     return 0;
  103: }
  104: 
  105: sub checkon_daemon {
  106:     my ($fh,$daemon,$maxsize,$send,$args)=@_;
  107: 
  108:     my $result;
  109:     &log($fh,'<hr /><a name="'.$daemon.'" /><h2>'.$daemon.'</h2><h3>Log</h3><p style="white-space: pre;"><tt>');
  110:     printf("%-15s ",$daemon);
  111:     if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){
  112: 	open (DFH,"tail -n25 $perlvar{'lonDaemons'}/logs/$daemon.log|");
  113: 	while (my $line=<DFH>) { 
  114: 	    &log($fh,"$line");
  115: 	    if ($line=~/INFO/) { $notices++; }
  116: 	    if ($line=~/WARNING/) { $notices++; }
  117: 	    if ($line=~/CRITICAL/) { $warnings++; }
  118: 	};
  119: 	close (DFH);
  120:     }
  121:     &log($fh,"</tt></p>");
  122:     
  123:     my $pidfile="$perlvar{'lonDaemons'}/logs/$daemon.pid";
  124:     
  125:     my $restartflag=1;
  126:     my $daemonpid;
  127:     if (-e $pidfile) {
  128: 	my $lfh=IO::File->new("$pidfile");
  129: 	$daemonpid=<$lfh>;
  130: 	chomp($daemonpid);
  131: 	if ($daemonpid =~ /^\d+$/ && kill 0 => $daemonpid) {
  132: 	    &log($fh,"<h3>$daemon at pid $daemonpid responding");
  133: 	    if ($send) { &log($fh,", sending $send"); }
  134: 	    &log($fh,"</h3>");
  135: 	    if ($send eq 'USR1') { kill USR1 => $daemonpid; }
  136: 	    if ($send eq 'USR2') { kill USR2 => $daemonpid; }
  137: 	    $restartflag=0;
  138: 	    if ($send eq 'USR2') {
  139: 		$result = 'reloaded';
  140: 		print "reloaded\n";
  141: 	    } else {
  142: 		$result = 'running';
  143: 		print "running\n";
  144: 	    }
  145: 	} else {
  146: 	    $errors++;
  147: 	    &log($fh,"<h3>$daemon at pid $daemonpid not responding</h3>");
  148: 	    $restartflag=1;
  149: 	    &log($fh,"<h3>Decided to clean up stale .pid file and restart $daemon</h3>");
  150: 	}
  151:     }
  152:     if ($restartflag==1) {
  153: 	$simplestatus{$daemon}='off';
  154: 	$errors++;
  155: 	my $kadaemon=$daemon;
  156: 	if ($kadaemon eq 'lonmemcached') { $kadaemon='memcached'; }
  157: 	&log($fh,'<br><font color="red">Killall '.$daemon.': '.
  158: 	    `killall $kadaemon 2>&1`.' - ');
  159: 	sleep 1;
  160: 	&log($fh,unlink($pidfile).' - '.
  161: 	    `killall -9 $kadaemon 2>&1`.
  162: 	    '</font><br>');
  163: 	&log($fh,"<h3>$daemon not running, trying to start</h3>");
  164: 	
  165: 	if (&start_daemon($fh,$daemon,$pidfile,$args)) {
  166: 	    &log($fh,"<h3>$daemon at pid $daemonpid responding</h3>");
  167: 	    $simplestatus{$daemon}='restarted';
  168: 	    $result = 'started';
  169: 	    print "started\n";
  170: 	} else {
  171: 	    $errors++;
  172: 	    &log($fh,"<h3>$daemon at pid $daemonpid not responding</h3>");
  173: 	    &log($fh,"<p>Give it one more try ...</p>");
  174: 	    print " ";
  175: 	    if (&start_daemon($fh,$daemon,$pidfile,$args)) {
  176: 		&log($fh,"<h3>$daemon at pid $daemonpid responding</h3>");
  177: 		$simplestatus{$daemon}='restarted';
  178: 		$result = 'started';
  179: 		print "started\n";
  180: 	    } else {
  181: 		$result = 'failed';
  182: 		print " failed\n";
  183: 		$simplestatus{$daemon}='failed';
  184: 		$errors++; $errors++;
  185: 		&log($fh,"<h3>$daemon at pid $daemonpid not responding</h3>");
  186: 		&log($fh,"<p>Unable to start $daemon</p>");
  187: 	    }
  188: 	}
  189: 
  190: 	if (-e "$perlvar{'lonDaemons'}/logs/$daemon.log"){
  191: 	    &log($fh,"<p><pre>");
  192: 	    open (DFH,"tail -n100 $perlvar{'lonDaemons'}/logs/$daemon.log|");
  193: 	    while (my $line=<DFH>) { 
  194: 		&log($fh,"$line");
  195: 		if ($line=~/WARNING/) { $notices++; }
  196: 		if ($line=~/CRITICAL/) { $notices++; }
  197: 	    };
  198: 	    close (DFH);
  199: 	    &log($fh,"</pre></p>");
  200: 	}
  201:     }
  202:     
  203:     my $fname="$perlvar{'lonDaemons'}/logs/$daemon.log";
  204:     &rotate_logfile($fname,$fh,'logs');
  205: 
  206:     &errout($fh);
  207:     return $result;
  208: }
  209: 
  210: # --------------------------------------------------------------------- Machine
  211: sub log_machine_info {
  212:     my ($fh)=@_;
  213:     &log($fh,'<hr /><a name="machine" /><h2>Machine Information</h2>');
  214:     &log($fh,"<h3>loadavg</h3>");
  215: 	
  216:     open (LOADAVGH,"/proc/loadavg");
  217:     my $loadavg=<LOADAVGH>;
  218:     close (LOADAVGH);
  219:     
  220:     &log($fh,"<tt>$loadavg</tt>");
  221:     
  222:     my @parts=split(/\s+/,$loadavg);
  223:     if ($parts[1]>4.0) {
  224: 	$errors++;
  225:     } elsif ($parts[1]>2.0) {
  226: 	$warnings++;
  227:     } elsif ($parts[1]>1.0) {
  228: 	$notices++;
  229:     }
  230: 
  231:     &log($fh,"<h3>df</h3>");
  232:     &log($fh,"<pre>");
  233: 
  234:     open (DFH,"df|");
  235:     while (my $line=<DFH>) { 
  236: 	&log($fh,&encode_entities($line,'<>&"')); 
  237: 	@parts=split(/\s+/,$line);
  238: 	my $usage=$parts[4];
  239: 	$usage=~s/\W//g;
  240: 	if ($usage>90) { 
  241: 	    $warnings++;
  242: 	    $notices++; 
  243: 	} elsif ($usage>80) {
  244: 	    $warnings++;
  245: 	} elsif ($usage>60) {
  246: 	    $notices++;
  247: 	}
  248: 	if ($usage>95) { $warnings++; $warnings++; $simplestatus{'diskfull'}++; }
  249:     }
  250:     close (DFH);
  251:     &log($fh,"</pre>");
  252: 
  253: 
  254:     &log($fh,"<h3>ps</h3>");
  255:     &log($fh,"<pre>");
  256:     my $psproc=0;
  257: 
  258:     open (PSH,"ps aux --cols 140 |");
  259:     while (my $line=<PSH>) { 
  260: 	&log($fh,&encode_entities($line,'<>&"')); 
  261: 	$psproc++;
  262:     }
  263:     close (PSH);
  264:     &log($fh,"</pre>");
  265: 
  266:     if ($psproc>200) { $notices++; }
  267:     if ($psproc>250) { $notices++; }
  268: 
  269:     &log($fh,"<h3>distprobe</h3>");
  270:     &log($fh,"<pre>");
  271:     open(DSH,"$perlvar{'lonDaemons'}/distprobe |");
  272:     while (my $line=<DSH>) { 
  273: 	&log($fh,&encode_entities($line,'<>&"')); 
  274: 	$psproc++;
  275:     }
  276:     close(DSH);
  277:     &log($fh,"</pre>");
  278: 
  279:     &errout($fh);
  280: }
  281: 
  282: sub start_logging {
  283:     my $fh=IO::File->new(">$statusdir/newstatus.html");
  284:     my %simplestatus=();
  285:     my $now=time;
  286:     my $date=localtime($now);
  287:     
  288: 
  289:     &log($fh,(<<ENDHEADERS));
  290: <html>
  291: <head>
  292: <title>LON Status Report $perlvar{'lonHostID'}</title>
  293: </head>
  294: <body bgcolor="#AAAAAA">
  295: <a name="top" />
  296: <h1>LON Status Report $perlvar{'lonHostID'}</h1>
  297: <h2>$date ($now)</h2>
  298: <ol>
  299: <li><a href="#configuration">Configuration</a></li>
  300: <li><a href="#machine">Machine Information</a></li>
  301: <li><a href="#tmp">Temporary Files</a></li>
  302: <li><a href="#tokens">Session Tokens</a></li>
  303: <li><a href="#httpd">httpd</a></li>
  304: <li><a href="#lonsql">lonsql</a></li>
  305: <li><a href="#lond">lond</a></li>
  306: <li><a href="#lonc">lonc</a></li>
  307: <li><a href="#lonhttpd">lonhttpd</a></li>
  308: <li><a href="#lonnet">lonnet</a></li>
  309: <li><a href="#connections">Connections</a></li>
  310: <li><a href="#delayed">Delayed Messages</a></li>
  311: <li><a href="#errcount">Error Count</a></li>
  312: </ol>
  313: <hr />
  314: <a name="configuration" />
  315: <h2>Configuration</h2>
  316: <h3>PerlVars</h3>
  317: <table border="2">
  318: ENDHEADERS
  319: 
  320:     foreach my $varname (sort(keys(%perlvar))) {
  321: 	&log($fh,"<tr><td>$varname</td><td>".
  322: 	     &encode_entities($perlvar{$varname},'<>&"')."</td></tr>\n");
  323:     }
  324:     &log($fh,"</table><h3>Hosts</h3><table border='2'>");
  325:     my %hostname = &Apache::lonnet::all_hostnames();
  326:     foreach my $id (sort(keys(%hostname))) {
  327: 	my $role = (&Apache::lonnet::is_library($id) ? 'library'
  328: 		                                     : 'access');
  329: 	&log($fh,
  330: 	    "<tr><td>$id</td><td>".&Apache::lonnet::host_domain($id).
  331: 	    "</td><td>".$role.
  332: 	    "</td><td>".&Apache::lonnet::hostname($id)."</td></tr>\n");
  333:     }
  334:     &log($fh,"</table><h3>Spare Hosts</h3><ul>");
  335:     foreach my $type (sort(keys(%Apache::lonnet::spareid))) {
  336: 	&log($fh,"<li>$type\n<ol>");
  337: 	foreach my $id (@{ $Apache::lonnet::spareid{$type} }) {
  338: 	    &log($fh,"<li>$id</li>\n");
  339: 	}
  340: 	&log($fh,"</ol>\n</li>\n");
  341:     }
  342:     &log($fh,"</ul>\n");
  343:     return $fh;
  344: }
  345: 
  346: # --------------------------------------------------------------- clean out tmp
  347: sub clean_tmp {
  348:     my ($fh)=@_;
  349:     &log($fh,'<hr /><a name="tmp" /><h2>Temporary Files</h2>');
  350:     my $cleaned=0;
  351:     my $old=0;
  352:     while (my $fname=<$perlvar{'lonDaemons'}/tmp/*>) {
  353: 	my ($dev,$ino,$mode,$nlink,
  354: 	    $uid,$gid,$rdev,$size,
  355: 	    $atime,$mtime,$ctime,
  356: 	    $blksize,$blocks)=stat($fname);
  357: 	my $now=time;
  358: 	my $since=$now-$mtime;
  359: 	if ($since>$perlvar{'lonExpire'}) {
  360: 	    my $line='';
  361: 	    if (open(PROBE,$fname)) {
  362: 		$line=<PROBE>;
  363: 		close(PROBE);
  364: 	    }
  365: 	    unless ($line=~/^CHECKOUTTOKEN\&/) {
  366: 		$cleaned++;
  367: 		unlink("$fname");
  368: 	    } else {
  369: 		if ($since>365*$perlvar{'lonExpire'}) {
  370: 		    $cleaned++;
  371: 		    unlink("$fname");
  372: 		} else { $old++; }
  373: 	    }
  374: 	}
  375:     }
  376:     &log($fh,"Cleaned up ".$cleaned." files (".$old." old checkout tokens).");
  377: }
  378: 
  379: # ------------------------------------------------------------ clean out lonIDs
  380: sub clean_lonIDs {
  381:     my ($fh)=@_;
  382:     &log($fh,'<hr /><a name="tokens" /><h2>Session Tokens</h2>');
  383:     my $cleaned=0;
  384:     my $active=0;
  385:     while (my $fname=<$perlvar{'lonIDsDir'}/*>) {
  386: 	my ($dev,$ino,$mode,$nlink,
  387: 	    $uid,$gid,$rdev,$size,
  388: 	    $atime,$mtime,$ctime,
  389: 	    $blksize,$blocks)=stat($fname);
  390: 	my $now=time;
  391: 	my $since=$now-$mtime;
  392: 	if ($since>$perlvar{'lonExpire'}) {
  393: 	    $cleaned++;
  394: 	    &log($fh,"Unlinking $fname<br>");
  395: 	    unlink("$fname");
  396: 	} else {
  397: 	    $active++;
  398: 	}
  399:     }
  400:     &log($fh,"<p>Cleaned up ".$cleaned." stale session token(s).</p>");
  401:     &log($fh,"<h3>$active open session(s)</h3>");
  402: }
  403: 
  404: # ----------------------------------------------------------- clean out sockets
  405: sub clean_sockets {
  406:     my ($fh)=@_;
  407:     my $cleaned=0;
  408:     opendir(SOCKETS,$perlvar{'lonSockDir'});
  409:     while (my $fname=readdir(SOCKETS)) {
  410: 	next if (-d $fname 
  411: 		 || $fname=~/(mysqlsock|maximasock|\Q$perlvar{'lonSockDir'}\E)/);
  412: 	$cleaned++;
  413: 	&log($fh,"Unlinking $fname<br />");
  414: 	unlink("/home/httpd/sockets/$fname");
  415:     }
  416:     &log($fh,"<p>Cleaned up ".$cleaned." stale sockets.</p>");
  417: }
  418: 
  419: 
  420: # ----------------------------------------------------------------------- httpd
  421: sub check_httpd_logs {
  422:     my ($fh)=@_;
  423:     &log($fh,'<hr /><a name="httpd" /><h2>httpd</h2><h3>Access Log</h3><pre>');
  424:     
  425:     open (DFH,"tail -n25 /etc/httpd/logs/access_log|");
  426:     while (my $line=<DFH>) { &log($fh,&encode_entities($line,'<>&"')) };
  427:     close (DFH);
  428: 	
  429:     &log($fh,"</pre><h3>Error Log</h3><pre>");
  430: 	
  431:     open (DFH,"tail -n25 /etc/httpd/logs/error_log|");
  432:     while (my $line=<DFH>) { 
  433: 	&log($fh,"$line");
  434: 	if ($line=~/\[error\]/) { $notices++; } 
  435:     }
  436:     close (DFH);
  437:     &log($fh,"</pre>");
  438:     &errout($fh);
  439: }
  440: 
  441: # ---------------------------------------------------------------------- lonnet
  442: 
  443: sub rotate_lonnet_logs {
  444:     my ($fh)=@_;
  445:     &log($fh,'<hr /><a name="lonnet" /><h2>lonnet</h2><h3>Temp Log</h3><pre>');
  446:     print "checking logs\n";
  447:     if (-e "$perlvar{'lonDaemons'}/logs/lonnet.log"){
  448: 	open (DFH,"tail -n50 $perlvar{'lonDaemons'}/logs/lonnet.log|");
  449: 	while (my $line=<DFH>) { 
  450: 	    &log($fh,&encode_entities($line,'<>&"'));
  451: 	}
  452: 	close (DFH);
  453:     }
  454:     &log($fh,"</pre><h3>Perm Log</h3><pre>");
  455:     
  456:     if (-e "$perlvar{'lonDaemons'}/logs/lonnet.perm.log") {
  457: 	open(DFH,"tail -n10 $perlvar{'lonDaemons'}/logs/lonnet.perm.log|");
  458: 	while (my $line=<DFH>) { 
  459: 	    &log($fh,&encode_entities($line,'<>&"'));
  460: 	}
  461: 	close (DFH);
  462:     } else { &log($fh,"No perm log\n") }
  463: 
  464:     my $fname="$perlvar{'lonDaemons'}/logs/lonnet.log";
  465:     &rotate_logfile($fname,$fh,'lonnet log');
  466: 
  467:     &log($fh,"</pre>");
  468:     &errout($fh);
  469: }
  470: 
  471: sub rotate_other_logs {
  472:     my ($fh) = @_;
  473:     my $fname="$perlvar{'lonDaemons'}/logs/autoenroll.log";
  474:     &rotate_logfile($fname,$fh,'Auto Enroll log');
  475:     $fname="$perlvar{'lonDaemons'}/logs/autocreate.log";
  476:     &rotate_logfile($fname,$fh,'Create Course log');
  477:     $fname="$perlvar{'lonDaemons'}/logs/searchcat.log";
  478:     &rotate_logfile($fname,$fh,'Search Cataloguing log');
  479: }
  480: 
  481: # ----------------------------------------------------------------- Connections
  482: sub test_connections {
  483:     my ($fh)=@_;
  484:     &log($fh,'<hr /><a name="connections" /><h2>Connections</h2>');
  485:     print "testing connections\n";
  486:     &log($fh,"<table border='2'>");
  487:     my ($good,$bad)=(0,0);
  488:     my %hostname = &Apache::lonnet::all_hostnames();
  489:     foreach my $tryserver (sort(keys(%hostname))) {
  490: 	print(".");
  491: 	my $result;
  492: 	my $answer=&Apache::lonnet::reply("ping",$tryserver);
  493: 	if ($answer eq "$tryserver:$perlvar{'lonHostID'}") {
  494: 	    $result="<b>ok</b>";
  495: 	    $good++;
  496: 	} else {
  497: 	    $result=$answer;
  498: 	    $warnings++;
  499: 	    if ($answer eq 'con_lost') {
  500: 		$bad++;
  501: 		$warnings++;
  502: 	    } else {
  503: 		$good++; #self connection
  504: 	    }
  505: 	}
  506: 	if ($answer =~ /con_lost/) { print(" $tryserver down\n"); }
  507: 	&log($fh,"<tr><td>$tryserver</td><td>$result</td></tr>\n");
  508:     }
  509:     &log($fh,"</table>");
  510:     print "\n$good good, $bad bad connections\n";
  511:     &errout($fh);
  512: }
  513: 
  514: 
  515: # ------------------------------------------------------------ Delayed messages
  516: sub check_delayed_msg {
  517:     my ($fh)=@_;
  518:     &log($fh,'<hr /><a name="delayed" /><h2>Delayed Messages</h2>');
  519:     print "checking buffers\n";
  520:     
  521:     &log($fh,'<h3>Scanning Permanent Log</h3>');
  522: 
  523:     my $unsend=0;
  524: 
  525:     my $dfh=IO::File->new("$perlvar{'lonDaemons'}/logs/lonnet.perm.log");
  526:     while (my $line=<$dfh>) {
  527: 	my ($time,$sdf,$dserv,$dcmd)=split(/:/,$line);
  528: 	if ($sdf eq 'F') { 
  529: 	    my $local=localtime($time);
  530: 	    &log($fh,"<b>Failed: $time, $dserv, $dcmd</b><br>");
  531: 	    $warnings++;
  532: 	}
  533: 	if ($sdf eq 'S') { $unsend--; }
  534: 	if ($sdf eq 'D') { $unsend++; }
  535:     }
  536: 
  537:     &log($fh,"<p>Total unsend messages: <b>$unsend</b></p>\n");
  538:     $warnings=$warnings+5*$unsend;
  539: 
  540:     if ($unsend) { $simplestatus{'unsend'}=$unsend; }
  541:     &log($fh,"<h3>Outgoing Buffer</h3>\n<pre>");
  542: # list directory with delayed messages and remember offline servers
  543:     my %servers=();
  544:     open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|");
  545:     while (my $line=<DFH>) {
  546:         my ($server)=($line=~/\.(\w+)$/);
  547:         if ($server) { $servers{$server}=1; }
  548: 	&log($fh,&encode_entities($line,'<>&"'));
  549:     }
  550:     &log($fh,"</pre>\n");
  551:     close (DFH);
  552: # pong to all servers that have delayed messages
  553: # this will trigger a reverse connection, which should flush the buffers
  554:     foreach my $tryserver (keys %servers) {
  555: 	my $answer=&Apache::lonnet::reply("pong",$tryserver);
  556: 	&log($fh,"Pong to $tryserver: $answer<br />");
  557:     }
  558: }
  559: 
  560: sub finish_logging {
  561:     my ($fh)=@_;
  562:     &log($fh,"<a name='errcount' />\n");
  563:     $totalcount=$notices+4*$warnings+100*$errors;
  564:     &errout($fh);
  565:     &log($fh,"<h1>Total Error Count: $totalcount</h1>");
  566:     my $now=time;
  567:     my $date=localtime($now);
  568:     &log($fh,"<hr />$date ($now)</body></html>\n");
  569:     print "lon-status webpage updated\n";
  570:     $fh->close();
  571: 
  572:     if ($errors) { $simplestatus{'errors'}=$errors; }
  573:     if ($warnings) { $simplestatus{'warnings'}=$warnings; }
  574:     if ($notices) { $simplestatus{'notices'}=$notices; }
  575:     $simplestatus{'time'}=time;
  576: }
  577: 
  578: sub log_simplestatus {
  579:     rename("$statusdir/newstatus.html","$statusdir/index.html");
  580:     
  581:     my $sfh=IO::File->new(">$statusdir/loncron_simple.txt");
  582:     foreach (keys %simplestatus) {
  583: 	print $sfh $_.'='.$simplestatus{$_}.'&';
  584:     }
  585:     print $sfh "\n";
  586:     $sfh->close();
  587: }
  588: 
  589: sub send_mail {
  590:     print "sending mail\n";
  591:     my $emailto="$perlvar{'lonAdmEMail'}";
  592:     if ($totalcount>2500) {
  593: 	$emailto.=",$perlvar{'lonSysEMail'}";
  594:     }
  595:     my $subj="LON: $perlvar{'lonHostID'} E:$errors W:$warnings N:$notices"; 
  596: 
  597:     my $result=system("metasend -b -S 4000000 -t $emailto -s '$subj' -f $statusdir/index.html -m text/html >& /dev/null");
  598:     if ($result != 0) {
  599: 	$result=system("mail -s '$subj' $emailto < $statusdir/index.html");
  600:     }
  601: }
  602: 
  603: sub usage {
  604:     print(<<USAGE);
  605: loncron - housekeeping program that checks up on various parts of Lon-CAPA
  606: 
  607: Options:
  608:    --help     Display 
  609:    --noemail  Do not send the status email
  610:    --justcheckconnections  Only check the current status of the lonc/d
  611:                                 connections, do not send emails do not
  612:                                 check if the daemons are running, do not
  613:                                 generate lon-status
  614:    --justcheckdaemons      Only check that all of the Lon-CAPA daemons are
  615:                                 running, do not send emails do not
  616:                                 check the lonc/d connections, do not
  617:                                 generate lon-status
  618:    --justreload            Only tell the daemons to reload the config files,
  619: 				do not send emails do not
  620:                                 check if the daemons are running, do not
  621:                                 generate lon-status
  622:                            
  623: USAGE
  624: }
  625: 
  626: # ================================================================ Main Program
  627: sub main () {
  628:     my ($help,$justcheckdaemons,$noemail,$justcheckconnections,
  629: 	$justreload);
  630:     &GetOptions("help"                 => \$help,
  631: 		"justcheckdaemons"     => \$justcheckdaemons,
  632: 		"noemail"              => \$noemail,
  633: 		"justcheckconnections" => \$justcheckconnections,
  634: 		"justreload"           => \$justreload
  635: 		);
  636:     if ($help) { &usage(); return; }
  637: # --------------------------------- Read loncapa_apache.conf and loncapa.conf
  638:     my $perlvarref=LONCAPA::Configuration::read_conf('loncapa.conf');
  639:     %perlvar=%{$perlvarref};
  640:     undef $perlvarref;
  641:     delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed
  642:     delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed
  643:     chdir($perlvar{'lonDaemons'});
  644: # --------------------------------------- Make sure that LON-CAPA is configured
  645: # I only test for one thing here (lonHostID).  This is just a safeguard.
  646:     if ('{[[[[lonHostID]]]]}' eq $perlvar{'lonHostID'}) {
  647: 	print("Unconfigured machine.\n");
  648: 	my $emailto=$perlvar{'lonSysEMail'};
  649: 	my $hostname=`/bin/hostname`;
  650: 	chop $hostname;
  651: 	$hostname=~s/[^\w\.]//g; # make sure is safe to pass through shell
  652: 	my $subj="LON: Unconfigured machine $hostname";
  653: 	system("echo 'Unconfigured machine $hostname.' |\
  654:  mailto $emailto -s '$subj' > /dev/null");
  655: 	exit 1;
  656:     }
  657: 
  658: # ----------------------------- Make sure this process is running from user=www
  659:     my $wwwid=getpwnam('www');
  660:     if ($wwwid!=$<) {
  661: 	print("User ID mismatch.  This program must be run as user 'www'\n");
  662: 	my $emailto="$perlvar{'lonAdmEMail'},$perlvar{'lonSysEMail'}";
  663: 	my $subj="LON: $perlvar{'lonHostID'} User ID mismatch";
  664: 	system("echo 'User ID mismatch.  loncron must be run as user www.' |\
  665:  mailto $emailto -s '$subj' > /dev/null");
  666: 	exit 1;
  667:     }
  668: 
  669: # -------------------------------------------- Force reload of host information
  670:     &Apache::lonnet::load_hosts_tab(1);
  671:     &Apache::lonnet::load_domain_tab(1);
  672:     &Apache::lonnet::get_iphost(1);
  673: 
  674: # ---------------------------------------------------------------- Start report
  675: 
  676:     $errors=0;
  677:     $warnings=0;
  678:     $notices=0;
  679: 
  680: 	
  681:     my $fh;
  682:     if (!$justcheckdaemons && !$justcheckconnections && !$justreload) {
  683: 	$fh=&start_logging();
  684: 
  685: 	&log_machine_info($fh);
  686: 	&clean_tmp($fh);
  687: 	&clean_lonIDs($fh);
  688: 	&check_httpd_logs($fh);
  689: 	&rotate_lonnet_logs($fh);
  690: 	&rotate_other_logs($fh);
  691:     }
  692:     if (!$justcheckconnections && !$justreload) {
  693: 	&checkon_daemon($fh,'lonsql',200000);
  694: 	if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') {
  695: 	    &checkon_daemon($fh,'lond',40000,'USR2');
  696: 	}
  697: 	&checkon_daemon($fh,'lonc',40000,'USR1');
  698: 	&checkon_daemon($fh,'lonhttpd',40000);
  699: 	&checkon_daemon($fh,'lonmemcached',40000);
  700:         &checkon_daemon($fh,'lonmaxima',40000);
  701:     }
  702:     if ($justreload) {
  703: 	&checkon_daemon($fh,'lond',40000,'USR2');
  704: 	&checkon_daemon($fh,'lonc',40000,'USR2');
  705:     }
  706:     if ($justcheckconnections) {
  707: 	&test_connections($fh);
  708:     }
  709:     if (!$justcheckdaemons && !$justcheckconnections && !$justreload) {
  710: 	&check_delayed_msg($fh);
  711: 	&finish_logging($fh);
  712: 	&log_simplestatus();
  713: 	
  714: 	if ($totalcount>200 && !$noemail) { &send_mail(); }
  715:     }
  716: }
  717: 
  718: &main();
  719: 1;
  720: 
  721: 
  722: 
  723: 
  724: 
  725: 
  726: 
  727: 

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
500 Internal Server Error

Internal Server Error

The server encountered an internal error or misconfiguration and was unable to complete your request.

Please contact the server administrator at root@localhost to inform them of the time this error occurred, and the actions you performed just before this error.

More information about this error may be available in the server error log.