File:  [LON-CAPA] / loncom / interface / loncoursedata.pm
Revision 1.43: download - view: text, annotated - select for diffs
Fri Jan 31 22:34:38 2003 UTC (21 years, 5 months ago) by matthew
Branches: MAIN
CVS tags: HEAD
Caching enabled version.

    1: # The LearningOnline Network with CAPA
    2: # (Publication Handler
    3: #
    4: # $Id: loncoursedata.pm,v 1.43 2003/01/31 22:34:38 matthew Exp $
    5: #
    6: # Copyright Michigan State University Board of Trustees
    7: #
    8: # This file is part of the LearningOnline Network with CAPA (LON-CAPA).
    9: #
   10: # LON-CAPA is free software; you can redistribute it and/or modify
   11: # it under the terms of the GNU General Public License as published by
   12: # the Free Software Foundation; either version 2 of the License, or
   13: # (at your option) any later version.
   14: #
   15: # LON-CAPA is distributed in the hope that it will be useful,
   16: # but WITHOUT ANY WARRANTY; without even the implied warranty of
   17: # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   18: # GNU General Public License for more details.
   19: #
   20: # You should have received a copy of the GNU General Public License
   21: # along with LON-CAPA; if not, write to the Free Software
   22: # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
   23: #
   24: # /home/httpd/html/adm/gpl.txt
   25: #
   26: # http://www.lon-capa.org/
   27: #
   28: ###
   29: 
   30: =pod
   31: 
   32: =head1 NAME
   33: 
   34: loncoursedata
   35: 
   36: =head1 SYNOPSIS
   37: 
   38: Set of functions that download and process student and course information.
   39: 
   40: =head1 PACKAGES USED
   41: 
   42:  Apache::Constants qw(:common :http)
   43:  Apache::lonnet()
   44:  Apache::lonhtmlcommon
   45:  HTML::TokeParser
   46:  GDBM_File
   47: 
   48: =cut
   49: 
   50: package Apache::loncoursedata;
   51: 
   52: use strict;
   53: use Apache::Constants qw(:common :http);
   54: use Apache::lonnet();
   55: use Apache::lonhtmlcommon;
   56: use HTML::TokeParser;
   57: use GDBM_File;
   58: 
   59: =pod
   60: 
   61: =head1 DOWNLOAD INFORMATION
   62: 
   63: This section contains all the functions that get data from other servers 
   64: and/or itself.
   65: 
   66: =cut
   67: 
   68: # ----- DOWNLOAD INFORMATION -------------------------------------------
   69: 
   70: =pod
   71: 
   72: =item &DownloadClasslist()
   73: 
   74: Collects lastname, generation, middlename, firstname, PID, and section for each
   75: student from their environment database.  The section data is also download, though
   76: it is in a rough format, and is processed later.  The list of students is built from
   77: collecting a classlist for the course that is to be displayed.  Once the classlist
   78: has been downloaded, its date stamp is recorded.  Unless the datestamp for the
   79: class database is reset or is modified, this data will not be downloaded again.  
   80: Also, there was talk about putting the fullname and section
   81: and perhaps other pieces of data into the classlist file.  This would
   82: reduce the number of different file accesses and reduce the amount of 
   83: processing on this side.
   84: 
   85: =over 4
   86: 
   87: Input: $courseID, $lastDownloadTime, $c
   88: 
   89: $courseID:  The id of the course
   90: 
   91: $lastDownloadTime:  This is the date stamp for when this information was
   92: last gathered.  If it is set to Not downloaded, it will gather the data
   93: again, though it currently does not remove the old data.
   94: 
   95: $c: The connection class that can determine if the browser has aborted.  It
   96: is used to short circuit this function so that it does not continue to 
   97: get information when there is no need.
   98: 
   99: Output: \%classlist
  100: 
  101: \%classlist: A pointer to a hash containing the following data:
  102: 
  103: -A list of student name:domain (as keys) (known below as $name)
  104: 
  105: -A hash pointer for each student containing lastname, generation, firstname,
  106: middlename, and PID : Key is $name.studentInformation
  107: 
  108: -A hash pointer to each students section data : Key is $name.section
  109: 
  110: -If there was an error in dump, it will be returned in the hash.  See
  111: the error codes for dump in lonnet.  Also, an error key will be 
  112: generated if an abort occurs.
  113: 
  114: =back
  115: 
  116: =cut
  117: 
  118: sub DownloadClasslist {
  119:     my ($courseID, $lastDownloadTime, $c)=@_;
  120:     my ($courseDomain,$courseNumber)=split(/\_/,$courseID);
  121:     my %classlist;
  122: 
  123:     my $modifiedTime = &Apache::lonnet::GetFileTimestamp($courseDomain, $courseNumber,
  124:                                                          'classlist.db', 
  125:                                                          $Apache::lonnet::perlvar{'lonUsersDir'});
  126: 
  127:     # Always download the information if lastDownloadTime is set to
  128:     # Not downloaded, otherwise it is only downloaded if the file
  129:     # has been updated and has a more recent date stamp
  130:     if($lastDownloadTime ne 'Not downloaded' &&
  131:        $lastDownloadTime >= $modifiedTime && $modifiedTime >= 0) {
  132:         # Data is not gathered so return UpToDate as true.  This
  133:         # will be interpreted in ProcessClasslist
  134:         $classlist{'lastDownloadTime'}=time;
  135:         $classlist{'UpToDate'} = 'true';
  136:         return \%classlist;
  137:     }
  138: 
  139:     %classlist=&Apache::lonnet::dump('classlist',$courseDomain, $courseNumber);
  140:     foreach(keys (%classlist)) {
  141:         if(/^(con_lost|error|no_such_host)/i) {
  142: 	    return;
  143:         }
  144:     }
  145: 
  146:     foreach my $name (keys(%classlist)) {
  147:         if(defined($c) && ($c->aborted())) {
  148:             $classlist{'error'}='aborted';
  149:             return \%classlist;
  150:         }
  151: 
  152:         my ($studentName,$studentDomain) = split(/\:/,$name);
  153:         # Download student environment data, specifically the full name and id.
  154:         my %studentInformation=&Apache::lonnet::get('environment',
  155:                                                     ['lastname','generation',
  156:                                                      'firstname','middlename',
  157:                                                      'id'],
  158:                                                     $studentDomain,
  159:                                                     $studentName);
  160:         $classlist{$name.':studentInformation'}=\%studentInformation;
  161: 
  162:         if($c->aborted()) {
  163:             $classlist{'error'}='aborted';
  164:             return \%classlist;
  165:         }
  166: 
  167:         #Section
  168:         my %section=&Apache::lonnet::dump('roles',$studentDomain,$studentName);
  169:         $classlist{$name.':sections'}=\%section;
  170:     }
  171: 
  172:     $classlist{'UpToDate'} = 'false';
  173:     $classlist{'lastDownloadTime'}=time;
  174: 
  175:     return \%classlist;
  176: }
  177: 
  178: =pod
  179: 
  180: =item &DownloadCourseInformation()
  181: 
  182: Dump of all the course information for a single student.  The data can be
  183: pruned by making use of dumps regular expression arguement.  This function
  184: also takes a regular expression which it passes straight through to dump.  
  185: The data is no escaped, because it is done elsewhere.  It also
  186: checks the timestamp of the students course database file and only downloads
  187: if it has been modified since the last download.
  188: 
  189: =over 4
  190: 
  191: Input: $namedata, $courseID, $lastDownloadTime, $WhatIWant
  192: 
  193: $namedata: student name:domain
  194: 
  195: $courseID:  The id of the course
  196: 
  197: $lastDownloadTime:  This is the date stamp for when this information was
  198: last gathered.  If it is set to Not downloaded, it will gather the data
  199: again, though it currently does not remove the old data.
  200: 
  201: $WhatIWant:  Regular expression used to get selected data with dump
  202: 
  203: Output: \%courseData
  204: 
  205: \%courseData:  A hash pointer to the raw data from the students course
  206: database.
  207: 
  208: =back
  209: 
  210: =cut
  211: 
  212: sub DownloadCourseInformation {
  213:     my ($namedata,$courseID,$lastDownloadTime,$WhatIWant)=@_;
  214:     my %courseData;
  215:     my ($name,$domain) = split(/\:/,$namedata);
  216: 
  217:     my $modifiedTime = &Apache::lonnet::GetFileTimestamp($domain, $name,
  218:                                       $courseID.'.db', 
  219:                                       $Apache::lonnet::perlvar{'lonUsersDir'});
  220: 
  221:     if($lastDownloadTime ne 'Not downloaded' && 
  222:        $lastDownloadTime >= $modifiedTime && $modifiedTime >= 0) {
  223:         # Data is not gathered so return UpToDate as true.  This
  224:         # will be interpreted in ProcessClasslist
  225:         $courseData{$namedata.':lastDownloadTime'}=time;
  226:         $courseData{$namedata.':UpToDate'} = 'true';
  227:         return \%courseData;
  228:     }
  229: 
  230:     # Download course data
  231:     if(!defined($WhatIWant)) {
  232:         # set the regular expression to everything by setting it to period
  233:         $WhatIWant = '.';
  234:     }
  235:     %courseData=&Apache::lonnet::dump($courseID, $domain, $name, $WhatIWant);
  236:     $courseData{'UpToDate'} = 'false';
  237:     $courseData{'lastDownloadTime'}=time;
  238: 
  239:     my %newData;
  240:     foreach (keys(%courseData)) {
  241:         # need to have the keys to be prepended with the name:domain of the
  242:         # student to reduce data collision later.
  243:         $newData{$namedata.':'.$_} = $courseData{$_};
  244:     }
  245: 
  246:     return \%newData;
  247: }
  248: 
  249: # ----- END DOWNLOAD INFORMATION ---------------------------------------
  250: 
  251: =pod
  252: 
  253: =head1 PROCESSING FUNCTIONS
  254: 
  255: These functions process all the data for all the students.  Also, they
  256: are the functions that access the cache database for writing the majority of
  257: the time.  The downloading and caching were separated to reduce problems 
  258: with stopping downloading then can not tie hash to database later.
  259: 
  260: =cut
  261: 
  262: # ----- PROCESSING FUNCTIONS ---------------------------------------
  263: 
  264: =pod
  265: 
  266: =item &ProcessTopResourceMap()
  267: 
  268: Trace through the "big hash" created in rat/lonuserstate.pm::loadmap.  
  269: Basically, this function organizes a subset of the data and stores it in
  270: cached data.  The data stored is the problems, sequences, sequence titles,
  271: parts of problems, and their ordering.  Column width information is also 
  272: partially handled here on a per sequence basis.
  273: 
  274: =over 4
  275: 
  276: Input: $cache, $c
  277: 
  278: $cache:  A pointer to a hash to store the information
  279: 
  280: $c:  The connection class used to determine if an abort has been sent to the 
  281: browser
  282: 
  283: Output: A string that contains an error message or "OK" if everything went 
  284: smoothly.
  285: 
  286: =back
  287: 
  288: =cut
  289: 
  290: sub ProcessTopResourceMap {
  291:     my ($cache,$c)=@_;
  292:     my %hash;
  293:     my $fn=$ENV{'request.course.fn'};
  294:     if(-e "$fn.db") {
  295: 	my $tieTries=0;
  296: 	while($tieTries < 3) {
  297:             if($c->aborted()) {
  298:                 return;
  299:             }
  300: 	    if(tie(%hash,'GDBM_File',"$fn.db",&GDBM_READER(),0640)) {
  301: 		last;
  302: 	    }
  303: 	    $tieTries++;
  304: 	    sleep 1;
  305: 	}
  306: 	if($tieTries >= 3) {
  307:             return 'Coursemap undefined.';
  308:         }
  309:     } else {
  310:         return 'Can not open Coursemap.';
  311:     }
  312: 
  313:     my $oldkeys;
  314:     delete $cache->{'OptionResponses'};
  315:     if(defined($cache->{'ResourceKeys'})) {
  316:         $oldkeys = $cache->{'ResourceKeys'};
  317:         foreach (split(':::', $cache->{'ResourceKeys'})) {
  318:             delete $cache->{$_};
  319:         }
  320:         delete $cache->{'ResourceKeys'};
  321:     }
  322: 
  323:     # Initialize state machine.  Set information pointing to top level map.
  324:     my (@sequences, @currentResource, @finishResource);
  325:     my ($currentSequence, $currentResourceID, $lastResourceID);
  326: 
  327:     $currentResourceID=$hash{'ids_'.
  328:       &Apache::lonnet::clutter($ENV{'request.course.uri'})};
  329:     push(@currentResource, $currentResourceID);
  330:     $lastResourceID=-1;
  331:     $currentSequence=-1;
  332:     my $topLevelSequenceNumber = $currentSequence;
  333: 
  334:     my %sequenceRecord;
  335:     my %allkeys;
  336:     while(1) {
  337:         if($c->aborted()) {
  338:             last;
  339:         }
  340: 	# HANDLE NEW SEQUENCE!
  341: 	#if page || sequence
  342: 	if(defined($hash{'map_pc_'.$hash{'src_'.$currentResourceID}}) &&
  343:            !defined($sequenceRecord{$currentResourceID})) {
  344:             $sequenceRecord{$currentResourceID}++;
  345: 	    push(@sequences, $currentSequence);
  346: 	    push(@currentResource, $currentResourceID);
  347: 	    push(@finishResource, $lastResourceID);
  348: 
  349: 	    $currentSequence=$hash{'map_pc_'.$hash{'src_'.$currentResourceID}};
  350: 
  351:             # Mark sequence as containing problems.  If it doesn't, then
  352:             # it will be removed when processing for this sequence is
  353:             # complete.  This allows the problems in a sequence
  354:             # to be outputed before problems in the subsequences
  355:             if(!defined($cache->{'orderedSequences'})) {
  356:                 $cache->{'orderedSequences'}=$currentSequence;
  357:             } else {
  358:                 $cache->{'orderedSequences'}.=':'.$currentSequence;
  359:             }
  360:             $allkeys{'orderedSequences'}++;
  361: 
  362: 	    $lastResourceID=$hash{'map_finish_'.
  363: 				  $hash{'src_'.$currentResourceID}};
  364: 	    $currentResourceID=$hash{'map_start_'.
  365: 				     $hash{'src_'.$currentResourceID}};
  366: 
  367: 	    if(!($currentResourceID) || !($lastResourceID)) {
  368: 		$currentSequence=pop(@sequences);
  369: 		$currentResourceID=pop(@currentResource);
  370: 		$lastResourceID=pop(@finishResource);
  371: 		if($currentSequence eq $topLevelSequenceNumber) {
  372: 		    last;
  373: 		}
  374: 	    }
  375:             next;
  376: 	}
  377: 
  378: 	# Handle gradable resources: exams, problems, etc
  379: 	$currentResourceID=~/(\d+)\.(\d+)/;
  380:         my $partA=$1;
  381:         my $partB=$2;
  382: 	if($hash{'src_'.$currentResourceID}=~
  383: 	   /\.(problem|exam|quiz|assess|survey|form)$/ &&
  384: 	   $partA eq $currentSequence && 
  385:            !defined($sequenceRecord{$currentSequence.':'.
  386:                                     $currentResourceID})) {
  387:             $sequenceRecord{$currentSequence.':'.$currentResourceID}++;
  388: 	    my $Problem = &Apache::lonnet::symbclean(
  389: 			  &Apache::lonnet::declutter($hash{'map_id_'.$partA}).
  390: 			  '___'.$partB.'___'.
  391: 			  &Apache::lonnet::declutter($hash{'src_'.
  392: 							 $currentResourceID}));
  393: 
  394: 	    $cache->{$currentResourceID.':problem'}=$Problem;
  395:             $allkeys{$currentResourceID.':problem'}++;
  396: 	    if(!defined($cache->{$currentSequence.':problems'})) {
  397: 		$cache->{$currentSequence.':problems'}=$currentResourceID;
  398: 	    } else {
  399: 		$cache->{$currentSequence.':problems'}.=
  400: 		    ':'.$currentResourceID;
  401: 	    }
  402:             $allkeys{$currentSequence.':problems'}++;
  403: 
  404: 	    my $meta=$hash{'src_'.$currentResourceID};
  405: #            $cache->{$currentResourceID.':title'}=
  406: #                &Apache::lonnet::metdata($meta,'title');
  407:             $cache->{$currentResourceID.':title'}=
  408:                 $hash{'title_'.$currentResourceID};
  409:             $allkeys{$currentResourceID.':title'}++;
  410:             $cache->{$currentResourceID.':source'}=
  411:                 $hash{'src_'.$currentResourceID};
  412:             $allkeys{$currentResourceID.':source'}++;
  413: 
  414:             # Get Parts for problem
  415:             my %beenHere;
  416:             foreach (split(/\,/,&Apache::lonnet::metadata($meta,'packages'))) {
  417:                 if(/^\w+response_\d+.*/) {
  418:                     my (undef, $partId, $responseId) = split(/_/,$_);
  419:                     if($beenHere{'p:'.$partId} ==  0) {
  420:                         $beenHere{'p:'.$partId}++;
  421:                         if(!defined($cache->{$currentSequence.':'.
  422:                                             $currentResourceID.':parts'})) {
  423:                             $cache->{$currentSequence.':'.$currentResourceID.
  424:                                      ':parts'}=$partId;
  425:                         } else {
  426:                             $cache->{$currentSequence.':'.$currentResourceID.
  427:                                      ':parts'}.=':'.$partId;
  428:                         }
  429:                         $allkeys{$currentSequence.':'.$currentResourceID.
  430:                                   ':parts'}++;
  431:                     }
  432:                     if($beenHere{'r:'.$partId.':'.$responseId} == 0) {
  433:                         $beenHere{'r:'.$partId.':'.$responseId}++;
  434:                         if(!defined($cache->{$currentSequence.':'.
  435:                                              $currentResourceID.':'.$partId.
  436:                                              ':responseIDs'})) {
  437:                             $cache->{$currentSequence.':'.$currentResourceID.
  438:                                      ':'.$partId.':responseIDs'}=$responseId;
  439:                         } else {
  440:                             $cache->{$currentSequence.':'.$currentResourceID.
  441:                                      ':'.$partId.':responseIDs'}.=':'.
  442:                                                                   $responseId;
  443:                         }
  444:                         $allkeys{$currentSequence.':'.$currentResourceID.':'.
  445:                                      $partId.':responseIDs'}++;
  446:                     }
  447:                     if(/^optionresponse/ && 
  448:                        $beenHere{'o:'.$partId.':'.$currentResourceID} == 0) {
  449:                         $beenHere{'o:'.$partId.$currentResourceID}++;
  450:                         if(defined($cache->{'OptionResponses'})) {
  451:                             $cache->{'OptionResponses'}.= ':::'.
  452:                                 $currentSequence.':'.$currentResourceID.':'.
  453:                                 $partId.':'.$responseId;
  454:                         } else {
  455:                             $cache->{'OptionResponses'}= $currentSequence.':'.
  456:                                 $currentResourceID.':'.
  457:                                 $partId.':'.$responseId;
  458:                         }
  459:                         $allkeys{'OptionResponses'}++;
  460:                     }
  461:                 }
  462:             }
  463:         }
  464: 
  465: 	# if resource == finish resource, then it is the end of a sequence/page
  466: 	if($currentResourceID eq $lastResourceID) {
  467: 	    # pop off last resource of sequence
  468: 	    $currentResourceID=pop(@currentResource);
  469: 	    $lastResourceID=pop(@finishResource);
  470: 
  471: 	    if(defined($cache->{$currentSequence.':problems'})) {
  472: 		# Capture sequence information here
  473: 		$cache->{$currentSequence.':title'}=
  474: 		    $hash{'title_'.$currentResourceID};
  475:                 $allkeys{$currentSequence.':title'}++;
  476:                 $cache->{$currentSequence.':source'}=
  477:                     $hash{'src_'.$currentResourceID};
  478:                 $allkeys{$currentSequence.':source'}++;
  479: 
  480:                 my $totalProblems=0;
  481:                 foreach my $currentProblem (split(/\:/,
  482:                                                $cache->{$currentSequence.
  483:                                                ':problems'})) {
  484:                     foreach (split(/\:/,$cache->{$currentSequence.':'.
  485:                                                    $currentProblem.
  486:                                                    ':parts'})) {
  487:                         $totalProblems++;
  488:                     }
  489:                 }
  490: 		my @titleLength=split(//,$cache->{$currentSequence.
  491:                                                     ':title'});
  492:                 # $extra is 5 for problems correct and 3 for space
  493:                 # between problems correct and problem output
  494:                 my $extra = 8;
  495: 		if(($totalProblems + $extra) > (scalar @titleLength)) {
  496: 		    $cache->{$currentSequence.':columnWidth'}=
  497:                         $totalProblems + $extra;
  498: 		} else {
  499: 		    $cache->{$currentSequence.':columnWidth'}=
  500:                         (scalar @titleLength);
  501: 		}
  502:                 $allkeys{$currentSequence.':columnWidth'}++;
  503: 	    } else {
  504:                 # Remove sequence from list, if it contains no problems to
  505:                 # display.
  506:                 $cache->{'orderedSequences'}=~s/$currentSequence//;
  507:                 $cache->{'orderedSequences'}=~s/::/:/g;
  508:                 $cache->{'orderedSequences'}=~s/^:|:$//g;
  509:             }
  510: 
  511: 	    $currentSequence=pop(@sequences);
  512: 	    if($currentSequence eq $topLevelSequenceNumber) {
  513: 		last;
  514: 	    }
  515:         }
  516: 
  517: 	# MOVE!!!
  518: 	# move to next resource
  519: 	unless(defined($hash{'to_'.$currentResourceID})) {
  520: 	    # big problem, need to handle.  Next is probably wrong
  521:             my $errorMessage = 'Big problem in ';
  522:             $errorMessage .= 'loncoursedata::ProcessTopLevelMap.';
  523:             $errorMessage .= "  bighash to_$currentResourceID not defined!";
  524:             &Apache::lonnet::logthis($errorMessage);
  525: 	    last;
  526: 	}
  527: 	my @nextResources=();
  528: 	foreach (split(/\,/,$hash{'to_'.$currentResourceID})) {
  529:             if(!defined($sequenceRecord{$currentSequence.':'.
  530:                                         $hash{'goesto_'.$_}})) {
  531:                 push(@nextResources, $hash{'goesto_'.$_});
  532:             }
  533: 	}
  534: 	push(@currentResource, @nextResources);
  535: 	# Set the next resource to be processed
  536: 	$currentResourceID=pop(@currentResource);
  537:     }
  538: 
  539:     my @theKeys = keys(%allkeys);
  540:     my $newkeys = join(':::', @theKeys);
  541:     $cache->{'ResourceKeys'} = join(':::', $newkeys);
  542:     if($newkeys ne $oldkeys) {
  543:         $cache->{'ResourceUpdated'} = 'true';
  544:     } else {
  545:         $cache->{'ResourceUpdated'} = 'false';
  546:     }
  547: 
  548:     unless (untie(%hash)) {
  549:         &Apache::lonnet::logthis("<font color=blue>WARNING: ".
  550:                                  "Could not untie coursemap $fn (browse)".
  551:                                  ".</font>"); 
  552:     }
  553: 
  554:     return 'OK';
  555: }
  556: 
  557: =pod
  558: 
  559: =item &ProcessClasslist()
  560: 
  561: Taking the class list dumped from &DownloadClasslist(), all the 
  562: students and their non-class information is processed using the 
  563: &ProcessStudentInformation() function.  A date stamp is also recorded for
  564: when the data was processed.
  565: 
  566: Takes data downloaded for a student and breaks it up into managable pieces and 
  567: stored in cache data.  The username, domain, class related date, PID, 
  568: full name, and section are all processed here.
  569: 
  570: =over 4
  571: 
  572: Input: $cache, $classlist, $courseID, $ChartDB, $c
  573: 
  574: $cache: A hash pointer to store the data
  575: 
  576: $classlist:  The hash of data collected about a student from 
  577: &DownloadClasslist().  The hash contains a list of students, a pointer 
  578: to a hash of student information for each student, and each students section 
  579: number.
  580: 
  581: $courseID:  The course ID
  582: 
  583: $ChartDB:  The name of the cache database file.
  584: 
  585: $c:  The connection class used to determine if an abort has been sent to the 
  586: browser
  587: 
  588: Output: @names
  589: 
  590: @names:  An array of students whose information has been processed, and are to 
  591: be considered in an arbitrary order.  The entries in @names are of the form
  592: username:domain.
  593: 
  594: The values in $cache are as follows:
  595: 
  596:  *NOTE: for the following $name implies username:domain
  597:  $name.':error'                  only defined if an error occured.  Value
  598:                                  contains the error message
  599:  $name.':lastDownloadTime'       unconverted time of the last update of a
  600:                                  student\'s course data
  601:  $name.'updateTime'              coverted time of the last update of a 
  602:                                  student\'s course data
  603:  $name.':username'               username of a student
  604:  $name.':domain'                 domain of a student
  605:  $name.':fullname'               full name of a student
  606:  $name.':id'                     PID of a student
  607:  $name.':Status'                 active/expired status of a student
  608:  $name.':section'                section of a student
  609: 
  610: =back
  611: 
  612: =cut
  613: 
  614: sub ProcessClasslist {
  615:     my ($cache,$classlist,$courseID,$c)=@_;
  616:     my @names=();
  617: 
  618:     $cache->{'ClasslistTimeStamp'}=$classlist->{'lastDownloadTime'};
  619:     if($classlist->{'UpToDate'} eq 'true') {
  620:         return split(/:::/,$cache->{'NamesOfStudents'});;
  621:     }
  622: 
  623:     foreach my $name (keys(%$classlist)) {
  624:         if($name =~ /\:section/ || $name =~ /\:studentInformation/ ||
  625:            $name eq '' || $name eq 'UpToDate' || $name eq 'lastDownloadTime') {
  626:             next;
  627:         }
  628:         if($c->aborted()) {
  629:             return ();
  630:         }
  631:         my $studentInformation = $classlist->{$name.':studentInformation'};
  632:         my $date = $classlist->{$name};
  633:         my ($studentName,$studentDomain) = split(/\:/,$name);
  634: 
  635:         $cache->{$name.':username'}=$studentName;
  636:         $cache->{$name.':domain'}=$studentDomain;
  637:         # Initialize timestamp for student
  638:         if(!defined($cache->{$name.':lastDownloadTime'})) {
  639:             $cache->{$name.':lastDownloadTime'}='Not downloaded';
  640:             $cache->{$name.':updateTime'}=' Not updated';
  641:         }
  642: 
  643:         my $error = 0;
  644:         foreach(keys(%$studentInformation)) {
  645:             if(/^(con_lost|error|no_such_host)/i) {
  646:                 $cache->{$name.':error'}=
  647:                     'Could not download student environment data.';
  648:                 $cache->{$name.':fullname'}='';
  649:                 $cache->{$name.':id'}='';
  650:                 $error = 1;
  651:             }
  652:         }
  653:         next if($error);
  654:         push(@names,$name);
  655:         $cache->{$name.':fullname'}=&ProcessFullName(
  656:                                           $studentInformation->{'lastname'},
  657:                                           $studentInformation->{'generation'},
  658:                                           $studentInformation->{'firstname'},
  659:                                           $studentInformation->{'middlename'});
  660:         $cache->{$name.':id'}=$studentInformation->{'id'};
  661: 
  662:         my ($end, $start)=split(':',$date);
  663:         $courseID=~s/\_/\//g;
  664:         $courseID=~s/^(\w)/\/$1/;
  665: 
  666:         my $sec='';
  667:         my $sectionData = $classlist->{$name.':sections'};
  668:         foreach my $key (keys (%$sectionData)) {
  669:             my $value = $sectionData->{$key};
  670:             if ($key=~/^$courseID(?:\/)*(\w+)*\_st$/) {
  671:                 my $tempsection=$1;
  672:                 if($key eq $courseID.'_st') {
  673:                     $tempsection='';
  674:                 }
  675:                 my (undef,$roleend,$rolestart)=split(/\_/,$value);
  676:                 if($roleend eq $end && $rolestart eq $start) {
  677:                     $sec = $tempsection;
  678:                     last;
  679:                 }
  680:             }
  681:         }
  682: 
  683:         my $status='Expired';
  684:         if(((!$end) || time < $end) && ((!$start) || (time > $start))) {
  685:             $status='Active';
  686:         }
  687:         $cache->{$name.':Status'}=$status;
  688:         $cache->{$name.':section'}=$sec;
  689: 
  690:         if($sec eq '' || !defined($sec) || $sec eq ' ') {
  691:             $sec = 'none';
  692:         }
  693:         if(defined($cache->{'sectionList'})) {
  694:             if($cache->{'sectionList'} !~ /(^$sec:|^$sec$|:$sec$|:$sec:)/) {
  695:                 $cache->{'sectionList'} .= ':'.$sec;
  696:             }
  697:         } else {
  698:             $cache->{'sectionList'} = $sec;
  699:         }
  700:     }
  701: 
  702:     $cache->{'ClasslistTimestamp'}=time;
  703:     $cache->{'NamesOfStudents'}=join(':::',@names);
  704: 
  705:     return @names;
  706: }
  707: 
  708: =pod
  709: 
  710: =item &ProcessStudentData()
  711: 
  712: Takes the course data downloaded for a student in 
  713: &DownloadCourseInformation() and breaks it up into key value pairs
  714: to be stored in the cached data.  The keys are comprised of the 
  715: $username:$domain:$keyFromCourseDatabase.  The student username:domain is
  716: stored away signifying that the students information has been downloaded and 
  717: can be reused from cached data.
  718: 
  719: =over 4
  720: 
  721: Input: $cache, $courseData, $name
  722: 
  723: $cache: A hash pointer to store data
  724: 
  725: $courseData:  A hash pointer that points to the course data downloaded for a 
  726: student.
  727: 
  728: $name:  username:domain
  729: 
  730: Output: None
  731: 
  732: *NOTE:  There is no output, but an error message is stored away in the cache 
  733: data.  This is checked in &FormatStudentData().  The key username:domain:error 
  734: will only exist if an error occured.  The error is an error from 
  735: &DownloadCourseInformation().
  736: 
  737: =back
  738: 
  739: =cut
  740: 
  741: sub ProcessStudentData {
  742:     my ($cache,$courseData,$name)=@_;
  743: 
  744:     if(!&CheckDateStampError($courseData, $cache, $name)) {
  745:         return;
  746:     }
  747: 
  748:     # This little delete thing, should not be here.  Move some other
  749:     # time though.
  750:     if(defined($cache->{$name.':keys'})) {
  751: 	foreach (split(':::', $cache->{$name.':keys'})) {
  752: 	    delete $cache->{$name.':'.$_};
  753: 	}
  754:         delete $cache->{$name.':keys'};
  755:     }
  756: 
  757:     my %courseKeys;
  758:     # user name:domain was prepended earlier in DownloadCourseInformation
  759:     foreach (keys %$courseData) {
  760: 	my $currentKey = $_;
  761: 	$currentKey =~ s/^$name//;
  762: 	$courseKeys{$currentKey}++;
  763:         $cache->{$_}=$courseData->{$_};
  764:     }
  765: 
  766:     $cache->{$name.':keys'} = join(':::', keys(%courseKeys));
  767: 
  768:     return;
  769: }
  770: 
  771: =pod
  772: 
  773: =item &ExtractStudentData()
  774: 
  775: HISTORY: This function originally existed in every statistics module,
  776: and performed different tasks, the had some overlap.  Due to the need
  777: for the data from the different modules, they were combined into
  778: a single function.
  779: 
  780: This function now extracts all the necessary course data for a student
  781: from what was downloaded from their homeserver.  There is some extra
  782: time overhead compared to the ProcessStudentInformation function, but
  783: it would have had to occurred at some point anyways.  This is now
  784: typically called while downloading the data it will process.  It is
  785: the brother function to ProcessStudentInformation.
  786: 
  787: =over 4
  788: 
  789: Input: $input, $output, $data, $name
  790: 
  791: $input: A hash that contains the input data to be processed
  792: 
  793: $output: A hash to contain the processed data
  794: 
  795: $data: A hash containing the information on what is to be
  796: processed and how (basically).
  797: 
  798: $name:  username:domain
  799: 
  800: The input is slightly different here, but is quite simple.
  801: It is currently used where the $input, $output, and $data
  802: can and are often the same hashes, but they do not need
  803: to be.
  804: 
  805: Output: None
  806: 
  807: *NOTE:  There is no output, but an error message is stored away in the cache 
  808: data.  This is checked in &FormatStudentData().  The key username:domain:error 
  809: will only exist if an error occured.  The error is an error from 
  810: &DownloadCourseInformation().
  811: 
  812: =back
  813: 
  814: =cut
  815: 
  816: sub ExtractStudentData {
  817:     my ($input, $output, $data, $name)=@_;
  818: 
  819:     if(!&CheckDateStampError($input, $data, $name)) {
  820:         return;
  821:     }
  822: 
  823:     # This little delete thing, should not be here.  Move some other
  824:     # time though.
  825:     my %allkeys;
  826:     if(defined($output->{$name.':keys'})) {
  827: 	foreach (split(':::', $output->{$name.':keys'})) {
  828: 	    delete $output->{$name.':'.$_};
  829: 	}
  830:         delete $output->{$name.':keys'};
  831:     }
  832: 
  833:     my ($username,$domain)=split(':',$name);
  834: 
  835:     my $Version;
  836:     my $problemsCorrect = 0;
  837:     my $totalProblems   = 0;
  838:     my $problemsSolved  = 0;
  839:     my $numberOfParts   = 0;
  840:     my $totalAwarded    = 0;
  841:     foreach my $sequence (split(':', $data->{'orderedSequences'})) {
  842:         foreach my $problemID (split(':', $data->{$sequence.':problems'})) {
  843:             my $problem = $data->{$problemID.':problem'};
  844:             my $LatestVersion = $input->{$name.':version:'.$problem};
  845: 
  846:             # Output dashes for all the parts of this problem if there
  847:             # is no version information about the current problem.
  848:             $output->{$name.':'.$problemID.':NoVersion'} = 'false';
  849:             $allkeys{$name.':'.$problemID.':NoVersion'}++;
  850:             if(!$LatestVersion) {
  851:                 foreach my $part (split(/\:/,$data->{$sequence.':'.
  852:                                                       $problemID.
  853:                                                       ':parts'})) {
  854:                     $output->{$name.':'.$problemID.':'.$part.':tries'} = 0;
  855:                     $output->{$name.':'.$problemID.':'.$part.':awarded'} = 0;
  856:                     $output->{$name.':'.$problemID.':'.$part.':code'} = ' ';
  857: 		    $allkeys{$name.':'.$problemID.':'.$part.':tries'}++;
  858: 		    $allkeys{$name.':'.$problemID.':'.$part.':awarded'}++;
  859: 		    $allkeys{$name.':'.$problemID.':'.$part.':code'}++;
  860:                     $totalProblems++;
  861:                 }
  862:                 $output->{$name.':'.$problemID.':NoVersion'} = 'true';
  863:                 next;
  864:             }
  865: 
  866:             my %partData=undef;
  867:             # Initialize part data, display skips correctly
  868:             # Skip refers to when a student made no submissions on that
  869:             # part/problem.
  870:             foreach my $part (split(/\:/,$data->{$sequence.':'.
  871:                                                  $problemID.
  872:                                                  ':parts'})) {
  873:                 $partData{$part.':tries'}=0;
  874:                 $partData{$part.':code'}=' ';
  875:                 $partData{$part.':awarded'}=0;
  876:                 $partData{$part.':timestamp'}=0;
  877:                 foreach my $response (split(':', $data->{$sequence.':'.
  878:                                                          $problemID.':'.
  879:                                                          $part.':responseIDs'})) {
  880:                     $partData{$part.':'.$response.':submission'}='';
  881:                 }
  882:             }
  883: 
  884:             # Looping through all the versions of each part, starting with the
  885:             # oldest version.  Basically, it gets the most recent 
  886:             # set of grade data for each part.
  887:             my @submissions = ();
  888: 	    for(my $Version=1; $Version<=$LatestVersion; $Version++) {
  889:                 foreach my $part (split(/\:/,$data->{$sequence.':'.
  890:                                                      $problemID.
  891:                                                      ':parts'})) {
  892: 
  893:                     if(!defined($input->{"$name:$Version:$problem".
  894:                                          ":resource.$part.solved"})) {
  895:                         # No grade for this submission, so skip
  896:                         next;
  897:                     }
  898: 
  899:                     my $tries=0;
  900:                     my $code=' ';
  901:                     my $awarded=0;
  902: 
  903:                     $tries = $input->{$name.':'.$Version.':'.$problem.
  904:                                       ':resource.'.$part.'.tries'};
  905:                     $awarded = $input->{$name.':'.$Version.':'.$problem.
  906:                                         ':resource.'.$part.'.awarded'};
  907: 
  908:                     $partData{$part.':awarded'}=($awarded) ? $awarded : 0;
  909:                     $partData{$part.':tries'}=($tries) ? $tries : 0;
  910: 
  911:                     $partData{$part.':timestamp'}=$input->{$name.':'.$Version.':'.
  912:                                                            $problem.
  913:                                                            ':timestamp'};
  914:                     if(!$input->{$name.':'.$Version.':'.$problem.':resource.'.$part.
  915:                                  '.previous'}) {
  916:                         foreach my $response (split(':',
  917:                                                    $data->{$sequence.':'.
  918:                                                            $problemID.':'.
  919:                                                            $part.':responseIDs'})) {
  920:                             @submissions=($input->{$name.':'.$Version.':'.
  921:                                                    $problem.
  922:                                                    ':resource.'.$part.'.'.
  923:                                                    $response.'.submission'},
  924:                                           @submissions);
  925:                         }
  926:                     }
  927: 
  928:                     my $val = $input->{$name.':'.$Version.':'.$problem.
  929:                                        ':resource.'.$part.'.solved'};
  930:                     if    ($val eq 'correct_by_student')   {$code = '*';} 
  931:                     elsif ($val eq 'correct_by_override')  {$code = '+';}
  932:                     elsif ($val eq 'incorrect_attempted')  {$code = '.';} 
  933:                     elsif ($val eq 'incorrect_by_override'){$code = '-';}
  934:                     elsif ($val eq 'excused')              {$code = 'x';}
  935:                     elsif ($val eq 'ungraded_attempted')   {$code = '#';}
  936:                     else                                   {$code = ' ';}
  937:                     $partData{$part.':code'}=$code;
  938:                 }
  939:             }
  940: 
  941:             foreach my $part (split(/\:/,$data->{$sequence.':'.$problemID.
  942:                                                  ':parts'})) {
  943:                 $output->{$name.':'.$problemID.':'.$part.':wrong'} = 
  944:                     $partData{$part.':tries'};
  945: 		$allkeys{$name.':'.$problemID.':'.$part.':wrong'}++;
  946: 
  947:                 if($partData{$part.':code'} eq '*') {
  948:                     $output->{$name.':'.$problemID.':'.$part.':wrong'}--;
  949:                     $problemsCorrect++;
  950:                 } elsif($partData{$part.':code'} eq '+') {
  951:                     $output->{$name.':'.$problemID.':'.$part.':wrong'}--;
  952:                     $problemsCorrect++;
  953:                 }
  954: 
  955:                 $output->{$name.':'.$problemID.':'.$part.':tries'} = 
  956:                     $partData{$part.':tries'};
  957:                 $output->{$name.':'.$problemID.':'.$part.':code'} =
  958:                     $partData{$part.':code'};
  959:                 $output->{$name.':'.$problemID.':'.$part.':awarded'} =
  960:                     $partData{$part.':awarded'};
  961: 		$allkeys{$name.':'.$problemID.':'.$part.':tries'}++;
  962: 		$allkeys{$name.':'.$problemID.':'.$part.':code'}++;
  963: 		$allkeys{$name.':'.$problemID.':'.$part.':awarded'}++;
  964: 
  965:                 $totalAwarded += $partData{$part.':awarded'};
  966:                 $output->{$name.':'.$problemID.':'.$part.':timestamp'} =
  967:                     $partData{$part.':timestamp'};
  968: 		$allkeys{$name.':'.$problemID.':'.$part.':timestamp'}++;
  969: 
  970:                 foreach my $response (split(':', $data->{$sequence.':'.
  971:                                                          $problemID.':'.
  972:                                                          $part.':responseIDs'})) {
  973:                     $output->{$name.':'.$problemID.':'.$part.':'.$response.
  974:                               ':submission'}=join(':::',@submissions);
  975: 		    $allkeys{$name.':'.$problemID.':'.$part.':'.$response.
  976: 			     ':submission'}++;
  977:                 }
  978: 
  979:                 if($partData{$part.':code'} ne 'x') {
  980:                     $totalProblems++;
  981:                 }
  982:             }
  983:         }
  984: 
  985:         $output->{$name.':'.$sequence.':problemsCorrect'} = $problemsCorrect;
  986: 	$allkeys{$name.':'.$sequence.':problemsCorrect'}++;
  987:         $problemsSolved += $problemsCorrect;
  988: 	$problemsCorrect=0;
  989:     }
  990: 
  991:     $output->{$name.':problemsSolved'} = $problemsSolved;
  992:     $output->{$name.':totalProblems'} = $totalProblems;
  993:     $output->{$name.':totalAwarded'} = $totalAwarded;
  994:     $allkeys{$name.':problemsSolved'}++;
  995:     $allkeys{$name.':totalProblems'}++;
  996:     $allkeys{$name.':totalAwarded'}++;
  997: 
  998:     $output->{$name.':keys'} = join(':::', keys(%allkeys));
  999: 
 1000:     return;
 1001: }
 1002: 
 1003: sub LoadDiscussion {
 1004:     my ($courseID)=@_;
 1005:     my %Discuss=();
 1006:     my %contrib=&Apache::lonnet::dump(
 1007:                 $courseID,
 1008:                 $ENV{'course.'.$courseID.'.domain'},
 1009:                 $ENV{'course.'.$courseID.'.num'});
 1010: 				 
 1011:     #my %contrib=&DownloadCourseInformation($name, $courseID, 0);
 1012: 
 1013:     foreach my $temp(keys %contrib) {
 1014: 	if ($temp=~/^version/) {
 1015: 	    my $ver=$contrib{$temp};
 1016: 	    my ($dummy,$prb)=split(':',$temp);
 1017: 	    for (my $idx=1; $idx<=$ver; $idx++ ) {
 1018: 		my $name=$contrib{"$idx:$prb:sendername"};
 1019: 		$Discuss{"$name:$prb"}=$idx;	
 1020: 	    }
 1021: 	}
 1022:     }       
 1023: 
 1024:     return \%Discuss;
 1025: }
 1026: 
 1027: # ----- END PROCESSING FUNCTIONS ---------------------------------------
 1028: 
 1029: =pod
 1030: 
 1031: =head1 HELPER FUNCTIONS
 1032: 
 1033: These are just a couple of functions do various odd and end 
 1034: jobs.  There was also a couple of bulk functions added.  These are
 1035: &DownloadStudentCourseData(), &DownloadStudentCourseDataSeparate(), and
 1036: &CheckForResidualDownload().  These functions now act as the interface
 1037: for downloading student course data.  The statistical modules should
 1038: no longer make the calls to dump and download and process etc.  They
 1039: make calls to these bulk functions to get their data.
 1040: 
 1041: =cut
 1042: 
 1043: # ----- HELPER FUNCTIONS -----------------------------------------------
 1044: 
 1045: sub CheckDateStampError {
 1046:     my ($courseData, $cache, $name)=@_;
 1047:     if($courseData->{$name.':UpToDate'} eq 'true') {
 1048:         $cache->{$name.':lastDownloadTime'} = 
 1049:             $courseData->{$name.':lastDownloadTime'};
 1050:         if($courseData->{$name.':lastDownloadTime'} eq 'Not downloaded') {
 1051:             $cache->{$name.':updateTime'} = ' Not updated';
 1052:         } else {
 1053:             $cache->{$name.':updateTime'}=
 1054:                 localtime($courseData->{$name.':lastDownloadTime'});
 1055:         }
 1056:         return 0;
 1057:     }
 1058: 
 1059:     $cache->{$name.':lastDownloadTime'}=$courseData->{$name.':lastDownloadTime'};
 1060:     if($courseData->{$name.':lastDownloadTime'} eq 'Not downloaded') {
 1061:         $cache->{$name.':updateTime'} = ' Not updated';
 1062:     } else {
 1063:         $cache->{$name.':updateTime'}=
 1064:             localtime($courseData->{$name.':lastDownloadTime'});
 1065:     }
 1066: 
 1067:     if(defined($courseData->{$name.':error'})) {
 1068:         $cache->{$name.':error'}=$courseData->{$name.':error'};
 1069:         return 0;
 1070:     }
 1071: 
 1072:     return 1;
 1073: }
 1074: 
 1075: =pod
 1076: 
 1077: =item &ProcessFullName()
 1078: 
 1079: Takes lastname, generation, firstname, and middlename (or some partial
 1080: set of this data) and returns the full name version as a string.  Format
 1081: is Lastname generation, firstname middlename or a subset of this.
 1082: 
 1083: =cut
 1084: 
 1085: sub ProcessFullName {
 1086:     my ($lastname, $generation, $firstname, $middlename)=@_;
 1087:     my $Str = '';
 1088: 
 1089:     # Strip whitespace preceeding & following name components.
 1090:     $lastname   =~ s/(\s+$|^\s+)//g;
 1091:     $generation =~ s/(\s+$|^\s+)//g;
 1092:     $firstname  =~ s/(\s+$|^\s+)//g;
 1093:     $middlename =~ s/(\s+$|^\s+)//g;
 1094: 
 1095:     if($lastname ne '') {
 1096: 	$Str .= $lastname;
 1097: 	$Str .= ' '.$generation if ($generation ne '');
 1098: 	$Str .= ',';
 1099:         $Str .= ' '.$firstname  if ($firstname ne '');
 1100:         $Str .= ' '.$middlename if ($middlename ne '');
 1101:     } else {
 1102:         $Str .= $firstname      if ($firstname ne '');
 1103:         $Str .= ' '.$middlename if ($middlename ne '');
 1104:         $Str .= ' '.$generation if ($generation ne '');
 1105:     }
 1106: 
 1107:     return $Str;
 1108: }
 1109: 
 1110: =pod
 1111: 
 1112: =item &TestCacheData()
 1113: 
 1114: Determine if the cache database can be accessed with a tie.  It waits up to
 1115: ten seconds before returning failure.  This function exists to help with
 1116: the problems with stopping the data download.  When an abort occurs and the
 1117: user quickly presses a form button and httpd child is created.  This
 1118: child needs to wait for the other to finish (hopefully within ten seconds).
 1119: 
 1120: =over 4
 1121: 
 1122: Input: $ChartDB
 1123: 
 1124: $ChartDB: The name of the cache database to be opened
 1125: 
 1126: Output: -1, 0, 1
 1127: 
 1128: -1: Could not tie database
 1129:  0: Use cached data
 1130:  1: New cache database created, use that.
 1131: 
 1132: =back
 1133: 
 1134: =cut
 1135: 
 1136: sub TestCacheData {
 1137:     my ($ChartDB,$isRecalculate,$totalDelay)=@_;
 1138:     my $isCached=-1;
 1139:     my %testData;
 1140:     my $tieTries=0;
 1141: 
 1142:     if(!defined($totalDelay)) {
 1143:         $totalDelay = 10;
 1144:     }
 1145: 
 1146:     if ((-e "$ChartDB") && (!$isRecalculate)) {
 1147: 	$isCached = 1;
 1148:     } else {
 1149: 	$isCached = 0;
 1150:     }
 1151: 
 1152:     while($tieTries < $totalDelay) {
 1153:         my $result=0;
 1154:         if($isCached) {
 1155:             $result=tie(%testData,'GDBM_File',$ChartDB,&GDBM_READER(),0640);
 1156:         } else {
 1157:             $result=tie(%testData,'GDBM_File',$ChartDB,&GDBM_NEWDB(),0640);
 1158:         }
 1159:         if($result) {
 1160:             last;
 1161:         }
 1162:         $tieTries++;
 1163:         sleep 1;
 1164:     }
 1165:     if($tieTries >= $totalDelay) {
 1166:         return -1;
 1167:     }
 1168: 
 1169:     untie(%testData);
 1170: 
 1171:     return $isCached;
 1172: }
 1173: 
 1174: sub DownloadStudentCourseData {
 1175:     my ($students,$checkDate,$cacheDB,$extract,$status,$courseID,$r,$c)=@_;
 1176: 
 1177:     my $title = 'LON-CAPA Statistics';
 1178:     my $heading = 'Download and Process Course Data';
 1179:     my $studentCount = scalar(@$students);
 1180: 
 1181:     my $WhatIWant;
 1182:     $WhatIWant = '(^version:|';
 1183:     $WhatIWant .= '^\d+:.+?:(resource\.\d+\.';
 1184:     $WhatIWant .= '(solved|tries|previous|awarded|(\d+\.submission))\s*$';#'
 1185:     $WhatIWant .= '|timestamp)';
 1186:     $WhatIWant .= ')';
 1187: #    $WhatIWant = '.';
 1188: 
 1189:     if($status eq 'true') {
 1190:         &Apache::lonhtmlcommon::Create_PrgWin($r, $title, $heading);
 1191:     }
 1192: 
 1193:     my $displayString;
 1194:     my $count=0;
 1195:     foreach (@$students) {
 1196:         my %cache;
 1197: 
 1198:         if($c->aborted()) { return 'Aborted'; }
 1199: 
 1200:         if($status eq 'true') {
 1201:             $count++;
 1202:             my $displayString = $count.'/'.$studentCount.': '.$_;
 1203:             &Apache::lonhtmlcommon::Update_PrgWin($displayString, $r);
 1204:         }
 1205: 
 1206:         my $downloadTime='Not downloaded';
 1207:         my $needUpdate = 'false';
 1208:         if($checkDate eq 'true'  && 
 1209:            tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) {
 1210:             $downloadTime = $cache{$_.':lastDownloadTime'};
 1211:             $needUpdate = $cache{'ResourceUpdated'};
 1212:             untie(%cache);
 1213:         }
 1214: 
 1215:         if($c->aborted()) { return 'Aborted'; }
 1216: 
 1217:         if($needUpdate eq 'true') {
 1218:             $downloadTime = 'Not downloaded';
 1219: 	}
 1220: 	my $courseData = 
 1221: 	    &DownloadCourseInformation($_, $courseID, $downloadTime, 
 1222: 				       $WhatIWant);
 1223: 	if(tie(%cache,'GDBM_File',$cacheDB,&GDBM_WRCREAT(),0640)) {
 1224: 	    foreach my $key (keys(%$courseData)) {
 1225: 		if($key =~ /^(con_lost|error|no_such_host)/i) {
 1226: 		    $courseData->{$_.':error'} = 'No course data for '.$_;
 1227: 		    last;
 1228: 		}
 1229: 	    }
 1230: 	    if($extract eq 'true') {
 1231: 		&ExtractStudentData($courseData, \%cache, \%cache, $_);
 1232: 	    } else {
 1233: 		&ProcessStudentData(\%cache, $courseData, $_);
 1234: 	    }
 1235: 	    untie(%cache);
 1236: 	} else {
 1237: 	    next;
 1238: 	}
 1239:     }
 1240:     if($status eq 'true') { &Apache::lonhtmlcommon::Close_PrgWin($r); }
 1241: 
 1242:     return 'OK';
 1243: }
 1244: 
 1245: sub DownloadStudentCourseDataSeparate {
 1246:     my ($students,$checkDate,$cacheDB,$extract,$status,$courseID,$r,$c)=@_;
 1247:     my $residualFile = '/home/httpd/perl/tmp/'.$courseID.'DownloadFile.db';
 1248:     my $title = 'LON-CAPA Statistics';
 1249:     my $heading = 'Download Course Data';
 1250: 
 1251:     my $WhatIWant;
 1252:     $WhatIWant = '(^version:|';
 1253:     $WhatIWant .= '^\d+:.+?:(resource\.\d+\.';
 1254:     $WhatIWant .= '(solved|tries|previous|awarded|(\d+\.submission))\s*$';
 1255:     $WhatIWant .= '|timestamp)';
 1256:     $WhatIWant .= ')';
 1257: 
 1258:     &CheckForResidualDownload($cacheDB, 'true', 'true', $courseID, $r, $c);
 1259: 
 1260:     my $studentCount = scalar(@$students);
 1261:     if($status eq 'true') {
 1262:         &Apache::lonhtmlcommon::Create_PrgWin($r, $title, $heading);
 1263:     }
 1264:     my $count=0;
 1265:     my $displayString='';
 1266:     foreach (@$students) {
 1267:         if($c->aborted()) {
 1268:             return 'Aborted';
 1269:         }
 1270: 
 1271:         if($status eq 'true') {
 1272:             $count++;
 1273:             $displayString = $count.'/'.$studentCount.': '.$_;
 1274:             &Apache::lonhtmlcommon::Update_PrgWin($displayString, $r);
 1275:         }
 1276: 
 1277:         my %cache;
 1278:         my $downloadTime='Not downloaded';
 1279:         my $needUpdate = 'false';
 1280:         if($checkDate eq 'true'  && 
 1281:            tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) {
 1282:             $downloadTime = $cache{$_.':lastDownloadTime'};
 1283:             $needUpdate = $cache{'ResourceUpdated'};
 1284:             untie(%cache);
 1285:         }
 1286: 
 1287:         if($c->aborted()) {
 1288:             return 'Aborted';
 1289:         }
 1290: 
 1291:         if($needUpdate eq 'true') {
 1292:             $downloadTime = 'Not downloaded';
 1293: 	}
 1294: 
 1295:         my $error = 0;
 1296:         my $courseData = 
 1297:             &DownloadCourseInformation($_, $courseID, $downloadTime,
 1298:                                        $WhatIWant);
 1299:         my %downloadData;
 1300:         unless(tie(%downloadData,'GDBM_File',$residualFile,
 1301:                    &GDBM_WRCREAT(),0640)) {
 1302:             return 'Failed to tie temporary download hash.';
 1303:         }
 1304:         foreach my $key (keys(%$courseData)) {
 1305:             $downloadData{$key} = $courseData->{$key};
 1306:             if($key =~ /^(con_lost|error|no_such_host)/i) {
 1307:                 $error = 1;
 1308:                 last;
 1309:             }
 1310:         }
 1311:         if($error) {
 1312:             foreach my $deleteKey (keys(%$courseData)) {
 1313:                 delete $downloadData{$deleteKey};
 1314:             }
 1315:             $downloadData{$_.':error'} = 'No course data for '.$_;
 1316:         }
 1317:         untie(%downloadData);
 1318:     }
 1319:     if($status eq 'true') { &Apache::lonhtmlcommon::Close_PrgWin($r); }
 1320: 
 1321:     return &CheckForResidualDownload($cacheDB, 'true', 'true', 
 1322:                                      $courseID, $r, $c);
 1323: }
 1324: 
 1325: sub CheckForResidualDownload {
 1326:     my ($cacheDB,$extract,$status,$courseID,$r,$c)=@_;
 1327: 
 1328:     my $residualFile = '/home/httpd/perl/tmp/'.$courseID.'DownloadFile.db';
 1329:     if(!-e $residualFile) {
 1330:         return 'OK';
 1331:     }
 1332: 
 1333:     my %downloadData;
 1334:     my %cache;
 1335:     unless(tie(%downloadData,'GDBM_File',$residualFile,&GDBM_READER(),0640)) {
 1336:         return 'Can not tie database for check for residual download: tempDB';
 1337:     }
 1338:     unless(tie(%cache,'GDBM_File',$cacheDB,&GDBM_WRCREAT(),0640)) {
 1339:         untie(%downloadData);
 1340:         return 'Can not tie database for check for residual download: cacheDB';
 1341:     }
 1342: 
 1343:     my @students=();
 1344:     my %checkStudent;
 1345:     my $key;
 1346:     while(($key, undef) = each %downloadData) {
 1347:         my @temp = split(':', $key);
 1348:         my $student = $temp[0].':'.$temp[1];
 1349:         if(!defined($checkStudent{$student})) {
 1350:             $checkStudent{$student}++;
 1351:             push(@students, $student);
 1352:         }
 1353:     }
 1354: 
 1355:     my $heading = 'Process Course Data';
 1356:     my $title = 'LON-CAPA Statistics';
 1357:     my $studentCount = scalar(@students);
 1358:     if($status eq 'true') {
 1359:         &Apache::lonhtmlcommon::Create_PrgWin($r, $title, $heading);
 1360:     }
 1361: 
 1362:     my $count=1;
 1363:     foreach my $name (@students) {
 1364:         last if($c->aborted());
 1365: 
 1366:         if($status eq 'true') {
 1367:             my $displayString = $count.'/'.$studentCount.': '.$name;
 1368:             &Apache::lonhtmlcommon::Update_PrgWin($displayString, $r);
 1369:         }
 1370: 
 1371:         if($extract eq 'true') {
 1372:             &ExtractStudentData(\%downloadData, \%cache, \%cache, $name);
 1373:         } else {
 1374:             &ProcessStudentData(\%cache, \%downloadData, $name);
 1375:         }
 1376:         $count++;
 1377:     }
 1378: 
 1379:     if($status eq 'true') { &Apache::lonhtmlcommon::Close_PrgWin($r); }
 1380: 
 1381:     untie(%cache);
 1382:     untie(%downloadData);
 1383: 
 1384:     if(!$c->aborted()) {
 1385:         my @files = ($residualFile);
 1386:         unlink(@files);
 1387:     }
 1388: 
 1389:     return 'OK';
 1390: }
 1391: 
 1392: ################################################
 1393: ################################################
 1394: 
 1395: =pod
 1396: 
 1397: =item &get_classlist();
 1398: 
 1399: Retrieve the classist of a given class or of the current class.  Student
 1400: information is returned from the classlist.db file and, if needed,
 1401: from the students environment.
 1402: 
 1403: Optional arguments are $cid, $cdom, and $cnum (course id, course domain,
 1404: and course number, respectively).  Any omitted arguments will be taken 
 1405: from the current environment ($ENV{'request.course.id'},
 1406: $ENV{'course.'.$cid.'.domain'}, and $ENV{'course.'.$cid.'.num'}).
 1407: 
 1408: Returns a reference to a hash which contains:
 1409:  keys    '$sname:$sdom'
 1410:  values  [$end,$start,$id,$section,$fullname]
 1411: 
 1412: =cut
 1413: 
 1414: ################################################
 1415: ################################################
 1416: 
 1417: sub get_classlist {
 1418:     my ($cid,$cdom,$cnum) = @_;
 1419:     $cid = $cid || $ENV{'request.course.id'};
 1420:     $cdom = $cdom || $ENV{'course.'.$cid.'.domain'};
 1421:     $cnum = $cnum || $ENV{'course.'.$cid.'.num'};
 1422:    my $now = time;
 1423:     #
 1424:     my %classlist=&Apache::lonnet::dump('classlist',$cdom,$cnum);
 1425:     while (my ($student,$info) = each(%classlist)) {
 1426:         return undef if ($student =~ /^(con_lost|error|no_such_host)/i);
 1427:         my ($sname,$sdom) = split(/:/,$student);
 1428:         my @Values = split(/:/,$info);
 1429:         my ($end,$start,$id,$section,$fullname);
 1430:         if (@Values > 2) {
 1431:             ($end,$start,$id,$section,$fullname) = @Values;
 1432:         } else { # We have to get the data ourselves
 1433:             ($end,$start) = @Values;
 1434:             $section = &Apache::lonnet::getsection($sdom,$sname,$cid);
 1435:             my %info=&Apache::lonnet::get('environment',
 1436:                                           ['firstname','middlename',
 1437:                                            'lastname','generation','id'],
 1438:                                           $sdom, $sname);
 1439:             my ($tmp) = keys(%info);
 1440:             if ($tmp =~/^(con_lost|error|no_such_host)/i) {
 1441:                 $fullname = 'not available';
 1442:                 $id = 'not available';
 1443:                 &Apache::lonnet::logthis('unable to retrieve environment '.
 1444:                                          'for '.$sname.':'.$sdom);
 1445:             } else {
 1446:                 $fullname = &ProcessFullName(@info{qw/lastname generation 
 1447:                                                        firstname middlename/});
 1448:                 $id = $info{'id'};
 1449:             }
 1450:             # Update the classlist with this students information
 1451:             if ($fullname ne 'not available') {
 1452:                 my $enrolldata = join(':',$end,$start,$id,$section,$fullname);
 1453:                 my $reply=&Apache::lonnet::cput('classlist',
 1454:                                                 {$student => $enrolldata},
 1455:                                                 $cdom,$cnum);
 1456:                 if ($reply !~ /^(ok|delayed)/) {
 1457:                     &Apache::lonnet::logthis('Unable to update classlist for '.
 1458:                                              'student '.$sname.':'.$sdom.
 1459:                                              ' error:'.$reply);
 1460:                 }
 1461:             }
 1462:         }
 1463:         my $status='Expired';
 1464:         if(((!$end) || $now < $end) && ((!$start) || ($now > $start))) {
 1465:             $status='Active';
 1466:         }
 1467:         $classlist{$student} = 
 1468:             [$sdom,$sname,$end,$start,$id,$section,$fullname,$status];
 1469:     }
 1470:     if (wantarray()) {
 1471:         return (\%classlist,['domain','username','end','start','id',
 1472:                              'section','fullname','status']);
 1473:     } else {
 1474:         return \%classlist;
 1475:     }
 1476: }
 1477: 
 1478: # ----- END HELPER FUNCTIONS --------------------------------------------
 1479: 
 1480: 1;
 1481: __END__
 1482: 
 1483: 

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>