--- loncom/loncron 2006/01/27 21:27:15 1.63 +++ loncom/loncron 2009/06/11 00:15:27 1.81 @@ -2,7 +2,7 @@ # Housekeeping program, started by cron, loncontrol and loncron.pl # -# $Id: loncron,v 1.63 2006/01/27 21:27:15 albertel Exp $ +# $Id: loncron,v 1.81 2009/06/11 00:15:27 raeburn Exp $ # # Copyright Michigan State University Board of Trustees # @@ -32,6 +32,8 @@ use strict; use lib '/home/httpd/lib/perl/'; use LONCAPA::Configuration; +use Apache::lonnet; +use Apache::loncommon; use IO::File; use IO::Socket; @@ -43,21 +45,6 @@ use vars qw (%perlvar %simplestatus $err my $statusdir="/home/httpd/html/lon-status"; -# -------------------------------------------------- Non-critical communication -sub reply { - my ($cmd,$server)=@_; - my $peerfile="$perlvar{'lonSockDir'}/$server"; - my $client=IO::Socket::UNIX->new(Peer =>"$peerfile", - Type => SOCK_STREAM, - Timeout => 10) - or return "con_lost"; - print $client "$cmd\n"; - my $answer=<$client>; - chomp($answer); - if (!$answer) { $answer="con_lost"; } - return $answer; -} - # --------------------------------------------------------- Output error status sub log { @@ -76,20 +63,27 @@ sub errout { ENDERROUT } +sub rotate_logfile { + my ($file,$fh,$description) = @_; + my $size=(stat($file))[7]; + if ($size>40000) { + &log($fh,"

Rotating $description ...

"); + rename("$file.2","$file.3"); + rename("$file.1","$file.2"); + rename("$file","$file.1"); + } +} + sub start_daemon { my ($fh,$daemon,$pidfile,$args) = @_; my $progname=$daemon; - if ($daemon eq 'lonc' && $args eq 'new') { + if ($daemon eq 'lonc') { $progname='loncnew'; - print "new "; } my $error_fname="$perlvar{'lonDaemons'}/logs/${daemon}_errors"; - my $size=(stat($error_fname))[7]; - if ($size>40000) { - &log($fh,"

Rotating error logs ...

"); - rename("$error_fname.2","$error_fname.3"); - rename("$error_fname.1","$error_fname.2"); - rename("$error_fname","$error_fname.1"); + &rotate_logfile($error_fname,$fh,'error logs'); + if ($daemon eq 'lonc') { + &clean_sockets($fh); } system("$perlvar{'lonDaemons'}/$progname 2>$perlvar{'lonDaemons'}/logs/${daemon}_errors"); sleep 1; @@ -208,18 +202,7 @@ sub checkon_daemon { } my $fname="$perlvar{'lonDaemons'}/logs/$daemon.log"; - - my ($dev,$ino,$mode,$nlink, - $uid,$gid,$rdev,$size, - $atime,$mtime,$ctime, - $blksize,$blocks)=stat($fname); - - if ($size>$maxsize) { - &log($fh,"

Rotating logs ...

"); - rename("$fname.2","$fname.3"); - rename("$fname.1","$fname.2"); - rename("$fname","$fname.1"); - } + &rotate_logfile($fname,$fh,'logs'); &errout($fh); return $result; @@ -298,7 +281,6 @@ sub log_machine_info { } sub start_logging { - my ($hostdom,$hostrole,$hostname,$spareid)=@_; my $fh=IO::File->new(">$statusdir/newstatus.html"); my %simplestatus=(); my $now=time; @@ -323,7 +305,6 @@ sub start_logging {
  • lonsql
  • lond
  • lonc
  • -
  • lonhttpd
  • lonnet
  • Connections
  • Delayed Messages
  • @@ -341,17 +322,24 @@ ENDHEADERS &encode_entities($perlvar{$varname},'<>&"')."\n"); } &log($fh,"

    Hosts

    "); - foreach my $id (sort(keys(%{$hostname}))) { + my %hostname = &Apache::lonnet::all_hostnames(); + foreach my $id (sort(keys(%hostname))) { + my $role = (&Apache::lonnet::is_library($id) ? 'library' + : 'access'); &log($fh, - "\n"); - } - &log($fh,"
    $id".$hostdom->{$id}. - "".$hostrole->{$id}. - "".$hostname->{$id}."

    Spare Hosts

      "); - foreach my $id (sort(keys(%{$spareid}))) { - &log($fh,"
    1. $id\n
    2. "); + "$id".&Apache::lonnet::host_domain($id). + "".$role. + "".&Apache::lonnet::hostname($id)."\n"); + } + &log($fh,"

      Spare Hosts

    \n"); + &log($fh,"\n"); return $fh; } @@ -413,6 +401,21 @@ sub clean_lonIDs { &log($fh,"

    $active open session(s)

    "); } +# ----------------------------------------------------------- clean out sockets +sub clean_sockets { + my ($fh)=@_; + my $cleaned=0; + opendir(SOCKETS,$perlvar{'lonSockDir'}); + while (my $fname=readdir(SOCKETS)) { + next if (-d $fname + || $fname=~/(mysqlsock|maximasock|rsock|\Q$perlvar{'lonSockDir'}\E)/); + $cleaned++; + &log($fh,"Unlinking $fname
    "); + unlink("/home/httpd/sockets/$fname"); + } + &log($fh,"

    Cleaned up ".$cleaned." stale sockets.

    "); +} + # ----------------------------------------------------------------------- httpd sub check_httpd_logs { @@ -459,34 +462,34 @@ sub rotate_lonnet_logs { } else { &log($fh,"No perm log\n") } my $fname="$perlvar{'lonDaemons'}/logs/lonnet.log"; - - my ($dev,$ino,$mode,$nlink, - $uid,$gid,$rdev,$size, - $atime,$mtime,$ctime, - $blksize,$blocks)=stat($fname); - - if ($size>40000) { - &log($fh,"

    Rotating logs ...

    "); - rename("$fname.2","$fname.3"); - rename("$fname.1","$fname.2"); - rename("$fname","$fname.1"); - } + &rotate_logfile($fname,$fh,'lonnet log'); &log($fh,""); &errout($fh); } +sub rotate_other_logs { + my ($fh) = @_; + my $fname="$perlvar{'lonDaemons'}/logs/autoenroll.log"; + &rotate_logfile($fname,$fh,'Auto Enroll log'); + $fname="$perlvar{'lonDaemons'}/logs/autocreate.log"; + &rotate_logfile($fname,$fh,'Create Course log'); + $fname="$perlvar{'lonDaemons'}/logs/searchcat.log"; + &rotate_logfile($fname,$fh,'Search Cataloguing log'); +} + # ----------------------------------------------------------------- Connections sub test_connections { - my ($fh,$hostname)=@_; + my ($fh)=@_; &log($fh,'

    Connections

    '); print "testing connections\n"; &log($fh,""); my ($good,$bad)=(0,0); - foreach my $tryserver (sort(keys(%{$hostname}))) { + my %hostname = &Apache::lonnet::all_hostnames(); + foreach my $tryserver (sort(keys(%hostname))) { print("."); my $result; - my $answer=reply("ping",$tryserver); + my $answer=&Apache::lonnet::reply("ping",$tryserver); if ($answer eq "$tryserver:$perlvar{'lonHostID'}") { $result="ok"; $good++; @@ -536,13 +539,22 @@ sub check_delayed_msg { if ($unsend) { $simplestatus{'unsend'}=$unsend; } &log($fh,"

    Outgoing Buffer

    \n
    ");
    -
    +# list directory with delayed messages and remember offline servers
    +    my %servers=();
         open (DFH,"ls -lF $perlvar{'lonSockDir'}/delayed|");
    -    while (my $line=) { 
    +    while (my $line=) {
    +        my ($server)=($line=~/\.(\w+)$/);
    +        if ($server) { $servers{$server}=1; }
     	&log($fh,&encode_entities($line,'<>&"'));
         }
         &log($fh,"
    \n"); close (DFH); +# pong to all servers that have delayed messages +# this will trigger a reverse connection, which should flush the buffers + foreach my $tryserver (keys %servers) { + my $answer=&Apache::lonnet::reply("pong",$tryserver); + &log($fh,"Pong to $tryserver: $answer
    "); + } } sub finish_logging { @@ -564,7 +576,7 @@ sub finish_logging { } sub log_simplestatus { - rename ("$statusdir/newstatus.html","$statusdir/index.html"); + rename("$statusdir/newstatus.html","$statusdir/index.html"); my $sfh=IO::File->new(">$statusdir/loncron_simple.txt"); foreach (keys %simplestatus) { @@ -576,7 +588,10 @@ sub log_simplestatus { sub send_mail { print "sending mail\n"; - my $emailto="$perlvar{'lonAdmEMail'}"; + my $defdom = $perlvar{'lonDefDomain'}; + my $origmail = $perlvar{'lonAdmEMail'}; + my $emailto = &Apache::loncommon::build_recipient_list(undef, + 'lonstatusmail',$defdom,$origmail); if ($totalcount>2500) { $emailto.=",$perlvar{'lonSysEMail'}"; } @@ -593,8 +608,7 @@ sub usage { loncron - housekeeping program that checks up on various parts of Lon-CAPA Options: - --help Display help - --oldlonc When starting the lonc daemon use 'lonc' not 'loncnew' + --help Display --noemail Do not send the status email --justcheckconnections Only check the current status of the lonc/d connections, do not send emails do not @@ -614,10 +628,9 @@ USAGE # ================================================================ Main Program sub main () { - my ($oldlonc,$help,$justcheckdaemons,$noemail,$justcheckconnections, + my ($help,$justcheckdaemons,$noemail,$justcheckconnections, $justreload); &GetOptions("help" => \$help, - "oldlonc" => \$oldlonc, "justcheckdaemons" => \$justcheckdaemons, "noemail" => \$noemail, "justcheckconnections" => \$justcheckconnections, @@ -630,7 +643,7 @@ sub main () { undef $perlvarref; delete $perlvar{'lonReceipt'}; # remove since sensitive and not needed delete $perlvar{'lonSqlAccess'}; # remove since sensitive and not needed - + chdir($perlvar{'lonDaemons'}); # --------------------------------------- Make sure that LON-CAPA is configured # I only test for one thing here (lonHostID). This is just a safeguard. if ('{[[[[lonHostID]]]]}' eq $perlvar{'lonHostID'}) { @@ -656,32 +669,28 @@ sub main () { exit 1; } -# ------------------------------------------------------------- Read hosts file - my $config=IO::File->new("$perlvar{'lonTabDir'}/hosts.tab"); - - my (%hostname,%hostdom,%hostrole,%spareid); - while (my $configline=<$config>) { - next if ($configline =~ /^(\#|\s*\$)/); - my ($id,$domain,$role,$name)=split(/:/,$configline); - if ($id && $domain && $role && $name) { - $name=~s/\s//g; - $hostname{$id}=$name; - $hostdom{$id}=$domain; - $hostrole{$id}=$role; - } - } - undef $config; - -# ------------------------------------------------------ Read spare server file - $config=IO::File->new("$perlvar{'lonTabDir'}/spare.tab"); - - while (my $configline=<$config>) { - chomp($configline); - if (($configline) && ($configline ne $perlvar{'lonHostID'})) { - $spareid{$configline}=1; - } +# -------------------------------------------- Force reload of host information + &Apache::lonnet::load_hosts_tab(1); + &Apache::lonnet::load_domain_tab(1); + &Apache::lonnet::get_iphost(1); + +# ----------------------------------------- Force firewall update for lond port + + if ((!$justcheckdaemons) && (!$justreload)) { + my $now = time; + my $tmpfile = $perlvar{'lonDaemons'}.'/tmp/lciptables_iphost_'. + $now.$$.int(rand(10000)); + if (open(my $fh,">$tmpfile")) { + my %iphosts = &Apache::lonnet::get_iphost(); + foreach my $key (keys(%iphosts)) { + print $fh "$key\n"; + } + close($fh); + my $execpath = $perlvar{'lonDaemons'}.'/lciptables'; + system("$execpath $tmpfile"); + unlink($fh); + } } - undef $config; # ---------------------------------------------------------------- Start report @@ -692,35 +701,31 @@ sub main () { my $fh; if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { - $fh=&start_logging(\%hostdom,\%hostrole,\%hostname,\%spareid); + $fh=&start_logging(); &log_machine_info($fh); &clean_tmp($fh); &clean_lonIDs($fh); &check_httpd_logs($fh); &rotate_lonnet_logs($fh); + &rotate_other_logs($fh); } if (!$justcheckconnections && !$justreload) { + &checkon_daemon($fh,'lonmemcached',40000); &checkon_daemon($fh,'lonsql',200000); if ( &checkon_daemon($fh,'lond',40000,'USR1') eq 'running') { &checkon_daemon($fh,'lond',40000,'USR2'); } - my $args='new'; - if ($oldlonc) { $args = ''; } - if ( &checkon_daemon($fh,'lonc',40000,'USR1',$args) eq 'running') { - &checkon_daemon($fh,'lond',40000,'USR2',$args); - } - &checkon_daemon($fh,'lonhttpd',40000); - &checkon_daemon($fh,'lonmemcached',40000); + &checkon_daemon($fh,'lonc',40000,'USR1'); + &checkon_daemon($fh,'lonmaxima',40000); + &checkon_daemon($fh,'lonr',40000); } if ($justreload) { &checkon_daemon($fh,'lond',40000,'USR2'); - my $args='new'; - if ($oldlonc) { $args = ''; } - &checkon_daemon($fh,'lonc',40000,'USR2',$args); + &checkon_daemon($fh,'lonc',40000,'USR2'); } if ($justcheckconnections) { - &test_connections($fh,\%hostname); + &test_connections($fh); } if (!$justcheckdaemons && !$justcheckconnections && !$justreload) { &check_delayed_msg($fh); 500 Internal Server Error

    Internal Server Error

    The server encountered an internal error or misconfiguration and was unable to complete your request.

    Please contact the server administrator at root@localhost to inform them of the time this error occurred, and the actions you performed just before this error.

    More information about this error may be available in the server error log.