loncom/interface/loncoursedata.pm - diff

Return to loncoursedata.pm CVS log

Up to [LON-CAPA] / loncom / interface

Diff for /loncom/interface/loncoursedata.pm between versions 1.120 and 1.207

-version 1.120, 2004/02/18 20:48:21
+version 1.207, 2021/03/05 17:17:57
  Line 25
  # http://www.lon-capa.org/
  #
  ###
  =pod
  =head1 NAME
- loncoursedata
+ Apache::loncoursedata
  =head1 SYNOPSIS
- Line 38  Set of functions that download and proce
+ Line 37  Set of functions that download and proce
  =head1 PACKAGES USED
-  Apache::Constants qw(:common :http)
+   Apache::lonnet
-  Apache::lonnet()
+   Apache::longroup
-  Apache::lonhtmlcommon
+   Time::HiRes
-  HTML::TokeParser
+   Apache::lonmysql
-  GDBM_File
+   LONCAPA
+   Digest::MD5
  =cut
  package Apache::loncoursedata;
  use strict;
- use Apache::Constants qw(:common :http);
+ use Apache::lonnet;
- use Apache::lonnet();
+ use Apache::longroup();
- use Apache::lonhtmlcommon;
+ use Time::HiRes();
- use Time::HiRes;
+ use Apache::lonmysql();
- use Apache::lonmysql;
+ use LONCAPA;
- use HTML::TokeParser;
+ use Digest::MD5();
- use GDBM_File;
+ =pod
- =pod
+ =head2 make_into_hash
- =head1 DOWNLOAD INFORMATION
+ Turn a colon separated string into a hash and return a reference
- This section contains all the functions that get data from other servers
+ to it.  Numbering from 0 even elements are keys and odd elements
- and/or itself.
+ are values e.g. a:b:c:d creates a hash like
+   a => b, c =>d
- =cut
- ####################################################
- ####################################################
- =pod
- =item &get_sequence_assessment_data()
- Use lonnavmaps to build a data structure describing the order and
- assessment contents of each sequence in the current course.
- The returned structure is a hash reference.
- { title => 'title',
-   symb  => 'symb',
-   src   => '/s/o/u/r/c/e',
-   type  => (container|assessment),
-   num_assess   => 2,               # only for container
-   parts        => [11,13,15],      # only for assessment
-   response_ids => [12,14,16],      # only for assessment
-   contents     => [........]       # only for container
- }
- $hash->{'contents'} is a reference to an array of hashes of the same structure.
- Also returned are array references to the sequences and assessments contained
- in the course.
  =cut
- ####################################################
- ####################################################
- sub get_sequence_assessment_data {
-     my $fn=$ENV{'request.course.fn'};
-     ##
-     ## use navmaps
-     my $navmap = Apache::lonnavmaps::navmap->new();
-     if (!defined($navmap)) {
-         return 'Can not open Coursemap';
-     }
-     # We explicity grab the top level map because I am not sure we
-     # are pulling it from the iterator.
-     my $top_level_map = $navmap->getById('0.0');
-     #
-     my $iterator = $navmap->getIterator(undef, undef, undef, 1);
-     my $curRes = $iterator->next(); # Top level sequence
-     ##
-     ## Prime the pump
-     ##
-     ## We are going to loop until we run out of sequences/pages to explore for
-     ## resources.  This means we have to start out with something to look
-     ## at.
-     my $title = $ENV{'course.'.$ENV{'request.course.id'}.'.description'};
-     my $symb  = $top_level_map->symb();
-     my $src   = $top_level_map->src();
-     my $randompick = $top_level_map->randompick();
-     #
-     my @Sequences;
-     my @Assessments;
-     my @Nested_Sequences = ();   # Stack of sequences, keeps track of depth
-     my $top = { title    => $title,
-                 src      => $src,
-                 symb     => $symb,
-                 type     => 'container',
-                 num_assess => 0,
-                 num_assess_parts => 0,
-                 contents   => [],
-                 randompick => $randompick,
-             };
-     push (@Sequences,$top);
-     push (@Nested_Sequences, $top);
-     #
-     # We need to keep track of which sequences contain homework problems
-     #
-     my $previous_too;
-     my $previous;
-     while (scalar(@Nested_Sequences)) {
-         $previous_too = $previous;
-         $previous = $curRes;
-         $curRes = $iterator->next();
-         my $currentmap = $Nested_Sequences[-1]; # Last one on the stack
-         if ($curRes == $iterator->BEGIN_MAP()) {
-             if (! ref($previous)) {
-                 $previous = $previous_too;
-             }
-             if (! ref($previous)) {
-                 next;
-             }
-             # get the map itself, instead of BEGIN_MAP
-             $title = $previous->title();
-             $title =~ s/\&colon;/\&\#058;/g;
-             $symb  = $previous->symb();
-             $src   = $previous->src();
-             # pick up the filename if there is no title available
-             if (! defined($title) || $title eq '') {
-                 ($title) = ($src=~/\/([^\/]*)$/);
-             }
-             $randompick = $previous->randompick();
-             my $newmap = { title    => $title,
-                            src      => $src,
-                            symb     => $symb,
-                            type     => 'container',
-                            num_assess => 0,
-                            randompick => $randompick,
-                            contents   => [],
-                        };
-             push (@{$currentmap->{'contents'}},$newmap); # this is permanent
-             push (@Sequences,$newmap);
-             push (@Nested_Sequences, $newmap); # this is a stack
-             next;
-         }
-         if ($curRes == $iterator->END_MAP()) {
-             pop(@Nested_Sequences);
-             next;
-         }
-         next if (! ref($curRes));
-         next if (! $curRes->is_problem());# && !$curRes->randomout);
-         # Okay, from here on out we only deal with assessments
-         $title = $curRes->title();
-         $title =~ s/\&colon;/\&\#058;/g;
-         $symb  = $curRes->symb();
-         $src   = $curRes->src();
-         # Grab the filename if there is not title available
-         if (! defined($title) || $title eq '') {
-             ($title) = ($src=~ m:/([^/]*)$:);
-         }
-         my $parts = $curRes->parts();
-         my %partdata;
-         foreach my $part (@$parts) {
-             my @Responses = $curRes->responseType($part);
-             my @Ids       = $curRes->responseIds($part);
-             $partdata{$part}->{'ResponseTypes'}= \@Responses;
-             $partdata{$part}->{'ResponseIds'}  = \@Ids;
-             # Count how many responses of each type there are in this part
-             foreach (@Responses) {
-                 $partdata{$part}->{$_}++;
-             }
-         }
-         my $assessment = { title => $title,
-                            src   => $src,
-                            symb  => $symb,
-                            type  => 'assessment',
-                            parts => $parts,
-                            num_parts => scalar(@$parts),
-                            partdata => \%partdata,
-                        };
-         push(@Assessments,$assessment);
-         push(@{$currentmap->{'contents'}},$assessment);
-         $currentmap->{'num_assess'}++;
-         $currentmap->{'num_assess_parts'}+= scalar(@$parts);
-     }
-     $navmap->untieHashes();
-     return ($top,\@Sequences,\@Assessments);
- }
- sub LoadDiscussion {
-     my ($courseID)=@_;
-     my %Discuss=();
-     my %contrib=&Apache::lonnet::dump(
-                 $courseID,
-                 $ENV{'course.'.$courseID.'.domain'},
-                 $ENV{'course.'.$courseID.'.num'});
-     #my %contrib=&DownloadCourseInformation($name, $courseID, 0);
-     foreach my $temp(keys %contrib) {
- 	if ($temp=~/^version/) {
- 	    my $ver=$contrib{$temp};
- 	    my ($dummy,$prb)=split(':',$temp);
- 	    for (my $idx=1; $idx<=$ver; $idx++ ) {
- 		my $name=$contrib{"$idx:$prb:sendername"};
- 		$Discuss{"$name:$prb"}=$idx;
- 	    }
- 	}
-     }
-     return \%Discuss;
- }
- ################################################
- ################################################
- =pod
- =item &GetUserName(username,userdomain)
- Returns a hash with the following entries:
-    'firstname', 'middlename', 'lastname', 'generation', and 'fullname'
-    'fullname' is the result of &Apache::loncoursedata::ProcessFullName.
- =cut
- ################################################
- ################################################
- sub GetUserName {
-     my ($username,$userdomain) = @_;
-     $username = $ENV{'user.name'} if (! defined($username));
-     $userdomain = $ENV{'user.domain'} if (! defined($username));
-     my %userenv = &Apache::lonnet::get('environment',
-                            ['firstname','middlename','lastname','generation'],
-                                        $userdomain,$username);
-     $userenv{'fullname'} = &ProcessFullName($userenv{'lastname'},
-                                             $userenv{'generation'},
-                                             $userenv{'firstname'},
-                                             $userenv{'middlename'});
-     return %userenv;
- }
- ################################################
- ################################################
- =pod
- =item &ProcessFullName()
- Takes lastname, generation, firstname, and middlename (or some partial
- set of this data) and returns the full name version as a string.  Format
- is Lastname generation, firstname middlename or a subset of this.
- =cut
- ################################################
- ################################################
- sub ProcessFullName {
-     my ($lastname, $generation, $firstname, $middlename)=@_;
-     my $Str = '';
-     # Strip whitespace preceeding & following name components.
-     $lastname   =~ s/(\s+$|^\s+)//g;
-     $generation =~ s/(\s+$|^\s+)//g;
-     $firstname  =~ s/(\s+$|^\s+)//g;
-     $middlename =~ s/(\s+$|^\s+)//g;
-     if($lastname ne '') {
- 	$Str .= $lastname;
- 	$Str .= ' '.$generation if ($generation ne '');
- 	$Str .= ',';
-         $Str .= ' '.$firstname  if ($firstname ne '');
-         $Str .= ' '.$middlename if ($middlename ne '');
-     } else {
-         $Str .= $firstname      if ($firstname ne '');
-         $Str .= ' '.$middlename if ($middlename ne '');
-         $Str .= ' '.$generation if ($generation ne '');
-     }
-     return $Str;
- }
- ################################################
- ################################################
- =pod
- =item &make_into_hash($values);
- Returns a reference to a hash as described by $values.  $values is
- assumed to be the result of
-     join(':',map {&Apache::lonnet::escape($_)} %orighash);
- This is a helper function for get_current_state.
- =cut
- ################################################
- ################################################
  sub make_into_hash {
      my $values = shift;
-     my %tmp = map { &Apache::lonnet::unescape($_); }
+     my %tmp = map { &unescape($_); } split(':',$values);
-                                            split(':',$values);
      return \%tmp;
  }
- ################################################
- ################################################
- =pod
- =head1 LOCAL DATA CACHING SUBROUTINES
- The local caching is done using MySQL.  There is no fall-back implementation
- if MySQL is not running.
- The programmers interface is to call &get_current_state() or some other
- primary interface subroutine (described below).  The internals of this
- storage system are documented here.
- There are six tables used to store student performance data (the results of
- a dumpcurrent).  Each of these tables is created in MySQL with a name of
- $courseid_*****, where ***** is 'symb', 'part', or whatever is appropriate
- for the table.  The tables and their purposes are described below.
- Some notes before we get started.
- Each table must have a PRIMARY KEY, which is a column or set of columns which
- will serve to uniquely identify a row of data.  NULL is not allowed!
- INDEXes work best on integer data.
- JOIN is used to combine data from many tables into one output.
- lonmysql.pm is used for some of the interface, specifically the table creation
- calls.  The inserts are done in bulk by directly calling the database handler.
- The SELECT ... JOIN statement used to retrieve the data does not have an
- interface in lonmysql.pm and I shudder at the thought of writing one.
- =head3 Table Descriptions
- =over 4
- =item Tables used to store meta information
- The following tables hold data required to keep track of the current status
- of a students data in the tables or to look up the students data in the tables.
- =over 4
- =item $symb_table
- The symb_table has two columns.  The first is a 'symb_id' and the second
- is the text name for the 'symb' (limited to 64k).  The 'symb_id' is generated
- automatically by MySQL so inserts should be done on this table with an
- empty first element.  This table has its PRIMARY KEY on the 'symb_id'.
- =item $part_table
- The part_table has two columns.  The first is a 'part_id' and the second
- is the text name for the 'part' (limited to 100 characters).  The 'part_id' is
- generated automatically by MySQL so inserts should be done on this table with
- an empty first element.  This table has its PRIMARY KEY on the 'part' (100
- characters) and a KEY on 'part_id'.
- =item $student_table
- The student_table has 7 columns.  The first is a 'student_id' assigned by
- MySQL.  The second is 'student' which is username:domain.  The third through
- fifth are 'section', 'status' (enrollment status), and 'classification'
- (to be used in the future).  The sixth and seventh ('updatetime' and
- 'fullupdatetime') contain the time of last update and full update of student
- data.  This table has its PRIMARY KEY on the 'student_id' column and is indexed
- on 'student', 'section', and 'status'.
- =back
- =item Tables used to store current status data
- The following tables store data only about the students current status on
- a problem, meaning only the data related to the last attempt on a problem.
- =over 4
- =item $performance_table
- The performance_table has 9 columns.  The first three are 'symb_id',
- 'student_id', and 'part_id'.  These comprise the PRIMARY KEY for this table
- and are directly related to the $symb_table, $student_table, and $part_table
- described above.  MySQL does better indexing on numeric items than text,
- so we use these three "index tables".  The remaining columns are
- 'solved', 'tries', 'awarded', 'award', 'awarddetail', and 'timestamp'.
- These are either the MySQL type TINYTEXT or various integers ('tries' and
- 'timestamp').  This table has KEYs of 'student_id' and 'symb_id'.
- For use of this table, see the functions described below.
- =item $parameters_table
- The parameters_table holds the data that does not fit neatly into the
- performance_table.  The parameters table has four columns: 'symb_id',
- 'student_id', 'parameter', and 'value'.  'symb_id', 'student_id', and
- 'parameter' comprise the PRIMARY KEY for this table.  'parameter' is
- limited to 255 characters.  'value' is limited to 64k characters.
- =back
- =item Tables used for storing historic data
- The following tables are used to store almost all of the transactions a student
- has made on a homework problem.  See loncapa/docs/homework/datastorage for
- specific information about each of the parameters stored.
- =over 4
- =item $fulldump_response_table
- The response table holds data (documented in loncapa/docs/homework/datastorage)
- associated with a particular response id which is stored when a student
- attempts a problem.  The following are the columns of the table, in order:
- 'symb_id','part_id','response_id','student_id','transaction','tries',
- 'awarddetail', 'response_specific' (data particular to the response
- type), 'response_specific_value', and 'submission (the text of the students
- submission).  The primary key is based on the first five columns listed above.
- =item $fulldump_part_table
- The part table holds data (documented in loncapa/docs/homework/datastorage)
- associated with a particular part id which is stored when a student attempts
- a problem.  The following are the columns of the table, in order:
- 'symb_id','part_id','student_id','transaction','tries','award','awarded',
- and 'previous'.  The primary key is based on the first five columns listed
- above.
- =item $fulldump_timestamp_table
- The timestamp table holds the timestamps of the transactions which are
- stored in $fulldump_response_table and $fulldump_part_table.  This data is
- about both the response and part data.  Columns: 'symb_id','student_id',
- 'transaction', and 'timestamp'.
- The primary key is based on the first 3 columns.
- =back
- =back
- =head3 Important Subroutines
- Here is a brief overview of the subroutines which are likely to be of
- interest:
- =over 4
- =item &get_current_state(): programmers interface.
- =item &init_dbs(): table creation
- =item &update_student_data(): data storage calls
- =item &get_student_data_from_performance_cache(): data retrieval
- =back
- =head3 Main Documentation
- =over 4
- =cut
- ################################################
- ################################################
- ################################################
- ################################################
  { # Begin scope of table identifiers
  my $current_course ='';
  my $symb_table;
  my $part_table;
  my $student_table;
+ my $groupnames_table;
+ my $students_groups_table;
  my $performance_table;
  my $parameters_table;
  my $fulldump_response_table;
  my $fulldump_part_table;
  my $fulldump_timestamp_table;
+ my $weight_table;
  my @Tables;
- ################################################
- ################################################
- =pod
- =item &init_dbs()
- Input: course id
- Output: 0 on success, positive integer on error
- This routine issues the calls to lonmysql to create the tables used to
- store student data.
- =cut
- ################################################
- ################################################
  sub init_dbs {
-     my $courseid = shift;
+     my ($courseid,$drop) = @_;
      &setup_table_names($courseid);
      #
      # Drop any of the existing tables
-     foreach my $table (@Tables) {
+     if ($drop) {
-         &Apache::lonmysql::drop_table($table);
+         foreach my $table (@Tables) {
+             &Apache::lonmysql::drop_table($table);
+         }
      }
      #
      # Note - changes to this table must be reflected in the code that
- Line 572  sub init_dbs {
+ Line 129  sub init_dbs {
                        restrictions => 'NOT NULL',
                        auto_inc     => 'yes', },
                      { name => 'part',
-                       type => 'VARCHAR(100)',
+                       type => 'VARCHAR(100) BINARY',
                        restrictions => 'NOT NULL'},
                      ],
          'PRIMARY KEY' => ['part (100)'],
- Line 587  sub init_dbs {
+ Line 144  sub init_dbs {
                        restrictions => 'NOT NULL',
                        auto_inc     => 'yes', },
                      { name => 'student',
-                       type => 'VARCHAR(100)',
+                       type => 'VARCHAR(100) BINARY',
                        restrictions => 'NOT NULL UNIQUE'},
                      { name => 'section',
-                       type => 'VARCHAR(100)',
+                       type => 'VARCHAR(100) BINARY',
                        restrictions => 'NOT NULL'},
-                     { name => 'status',
+                     { name => 'start',
-                       type => 'VARCHAR(15)',
+                       type => 'INT',
+                       restrictions => 'NOT NULL'},
+                     { name => 'end',
+                       type => 'INT',
                        restrictions => 'NOT NULL'},
                      { name => 'classification',
-                       type => 'varchar(100)', },
+                       type => 'VARCHAR(100) BINARY', },
                      { name => 'updatetime',
                        type => 'INT UNSIGNED'},
                      { name => 'fullupdatetime',
- Line 605  sub init_dbs {
+ Line 165  sub init_dbs {
          'PRIMARY KEY' => ['student_id'],
          'KEY' => [{ columns => ['student (100)',
                                  'section (100)',
-                                 'status (15)',]},],
+                                 'start',
+ 				'end']},],
+     };
+     #
+     my $groupnames_table_def = {
+         id => $groupnames_table,
+         permanent => 'no',
+         columns => [{ name => 'group_id',
+                       type => 'MEDIUMINT UNSIGNED',
+                       restrictions => 'NOT NULL',
+                       auto_inc => 'yes', },
+                     { name => 'groupname',
+                       type => 'VARCHAR(100) BINARY',
+                       restrictions => 'NOT NULL UNIQUE'},
+                    ],
+         'PRIMARY KEY' => ['group_id'],
+         'KEY' => [{ columns => ['groupname (100)',]},],
+     };
+     #
+     my $students_groups_table_def = {
+         id => $students_groups_table,
+         permanent => 'no',
+         columns => [{ name => 'student_id',
+                       type => 'MEDIUMINT UNSIGNED',
+                       restrictions => 'NOT NULL', },
+                     { name => 'group_id',
+                       type => 'MEDIUMINT UNSIGNED',
+                       restrictions => 'NOT NULL', },
+                    ],
+         'PRIMARY KEY' => ['student_id','group_id'],
+         'KEY' => [{ columns => ['student_id'] },
+                   { columns => ['group_id'] },],
      };
      #
      my $performance_table_def = {
- Line 621  sub init_dbs {
+ Line 212  sub init_dbs {
                        type => 'MEDIUMINT UNSIGNED',
                        restrictions => 'NOT NULL' },
                      { name => 'part',
-                       type => 'VARCHAR(100)',
+                       type => 'VARCHAR(100) BINARY',
                        restrictions => 'NOT NULL'},
                      { name => 'solved',
                        type => 'TINYTEXT' },
                      { name => 'tries',
                        type => 'SMALLINT UNSIGNED' },
                      { name => 'awarded',
-                       type => 'TINYTEXT' },
+                       type => 'REAL' },
                      { name => 'award',
                        type => 'TINYTEXT' },
                      { name => 'awarddetail',
- Line 663  sub init_dbs {
+ Line 254  sub init_dbs {
                      { name => 'award',
                        type => 'TINYTEXT' },
                      { name => 'awarded',
-                       type => 'TINYTEXT' },
+                       type => 'REAL' },
                      { name => 'previous',
                        type => 'SMALLINT UNSIGNED' },
  #                    { name => 'regrader',
- Line 701  sub init_dbs {
+ Line 292  sub init_dbs {
                      { name => 'awarddetail',
                        type => 'TINYTEXT' },
  #                    { name => 'message',
- #                      type => 'CHAR' },
+ #                      type => 'CHAR BINARY'},
                      { name => 'response_specific',
                        type => 'TINYTEXT' },
                      { name => 'response_specific_value',
                        type => 'TINYTEXT' },
+                     { name => 'response_specific_2',
+                       type => 'TINYTEXT' },
+                     { name => 'response_specific_value_2',
+                       type => 'TINYTEXT' },
                      { name => 'submission',
                        type => 'TEXT'},
                      ],
- Line 740  sub init_dbs {
+ Line 335  sub init_dbs {
                    { columns=>['transaction'] },
                    ],
      };
      #
      my $parameters_table_def = {
          id => $parameters_table,
- Line 760  sub init_dbs {
+ Line 354  sub init_dbs {
          'PRIMARY KEY' => ['symb_id','student_id','parameter (255)'],
      };
      #
+     my $weight_table_def = {
+         id => $weight_table,
+         permanent => 'no',
+         columns => [{ name => 'symb_id',
+                       type => 'MEDIUMINT UNSIGNED',
+                       restrictions => 'NOT NULL'  },
+                     { name => 'part_id',
+                       type => 'MEDIUMINT UNSIGNED',
+                       restrictions => 'NOT NULL'  },
+                     { name => 'weight',
+                       type => 'REAL',
+                       restrictions => 'NOT NULL'  },
+                     ],
+         'PRIMARY KEY' => ['symb_id','part_id'],
+     };
+     #
      # Create the tables
      my $tableid;
      $tableid = &Apache::lonmysql::create_table($symb_table_def);
- Line 816  sub init_dbs {
+ Line 426  sub init_dbs {
                                   &Apache::lonmysql::get_error());
          return 9;
      }
+     $tableid = &Apache::lonmysql::create_table($weight_table_def);
+     if (! defined($tableid)) {
+         &Apache::lonnet::logthis("error creating weight_table: ".
+                                  &Apache::lonmysql::get_error());
+         return 10;
+     }
+     $tableid = &Apache::lonmysql::create_table($groupnames_table_def);
+     if (! defined($tableid)) {
+         &Apache::lonnet::logthis("error creating groupnames_table: ".
+                                  &Apache::lonmysql::get_error());
+         return 11;
+     }
+     $tableid = &Apache::lonmysql::create_table($students_groups_table_def);
+     if (! defined($tableid)) {
+         &Apache::lonnet::logthis("error creating student_groups_table: ".
+                                  &Apache::lonmysql::get_error());
+         return 12;
+     }
      return 0;
  }
- ################################################
- ################################################
  =pod
- =item &delete_caches()
+ =head2 delete_caches
- =cut
+ Drops all of the tables in the local mysql cache associated with the
+ specified course id.
- ################################################
+ TODO:  The drops should be pushed into lonmysql to further isolate
- ################################################
+ mysql code from other modules.
+ =cut
  sub delete_caches {
      my $courseid = shift;
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      #
      &setup_table_names($courseid);
      #
- Line 847  sub delete_caches {
+ Line 475  sub delete_caches {
      return;
  }
- ################################################
- ################################################
- =pod
- =item &get_part_id()
- Get the MySQL id of a problem part string.
- Input: $part
- Output: undef on error, integer $part_id on success.
- =item &get_part()
- Get the string describing a part from the MySQL id of the problem part.
- Input: $part_id
- Output: undef on error, $part string on success.
- =cut
- ################################################
- ################################################
  my $have_read_part_table = 0;
  my %ids_by_part;
- Line 913  sub get_part {
+ Line 516  sub get_part {
      return undef; # error
  }
- ################################################
- ################################################
- =pod
- =item &get_symb_id()
- Get the MySQL id of a symb.
- Input: $symb
- Output: undef on error, integer $symb_id on success.
- =item &get_symb()
- Get the symb associated with a MySQL symb_id.
- Input: $symb_id
- Output: undef on error, $symb on success.
- =cut
- ################################################
- ################################################
  my $have_read_symb_table = 0;
  my %ids_by_symb;
- Line 978  sub get_symb {
+ Line 556  sub get_symb {
      return undef; # error
  }
- ################################################
- ################################################
- =pod
- =item &get_student_id()
- Get the MySQL id of a student.
- Input: $sname, $dom
- Output: undef on error, integer $student_id on success.
- =item &get_student()
- Get student username:domain associated with the MySQL student_id.
- Input: $student_id
- Output: undef on error, string $student (username:domain) on success.
- =cut
- ################################################
- ################################################
  my $have_read_student_table = 0;
  my %ids_by_student;
  my %students_by_id;
- Line 1048  sub get_student {
+ Line 600  sub get_student {
  sub populate_student_table {
      my ($courseid) = @_;
      if (! defined($courseid)) {
-         $courseid = $ENV{'request.course.id'};
+         $courseid = $env{'request.course.id'};
      }
      #
      &setup_table_names($courseid);
+     &init_dbs($courseid,0);
      my $dbh = &Apache::lonmysql::get_dbh();
      my $request = 'INSERT IGNORE INTO '.$student_table.
-         "(student,section,status) VALUES ";
+         "(student,section,start,end) VALUES ";
-     my $classlist = &get_classlist($courseid);
+     my $cdom = $env{'course.'.$courseid.'.domain'};
+     my $cnum = $env{'course.'.$courseid.'.num'};
+     my $classlist = &get_classlist($cdom,$cnum);
      my $student_count=0;
      while (my ($student,$data) = each %$classlist) {
-         my ($section,$status) = ($data->[&CL_SECTION()],
+         my ($section,$start,$end) = ($data->[&CL_SECTION()],
-                                  $data->[&CL_STATUS()]);
+ 				     $data->[&CL_START()],
+ 				     $data->[&CL_END()]);
          if ($section eq '' || $section =~ /^\s*$/) {
              $section = 'none';
          }
-         $request .= "('".$student."','".$section."','".$status."'),";
+ 	if (!defined($start)) { $start = 0; }
+ 	if (!defined($end))   { $end   = 0; }
+         $request .= "('".$student."','".$section."','".$start."','".$end."'),";
          $student_count++;
      }
      return if ($student_count == 0);
- Line 1071  sub populate_student_table {
+ Line 629  sub populate_student_table {
      $dbh->do($request);
      if ($dbh->err()) {
          &Apache::lonnet::logthis("error ".$dbh->errstr().
-                                  " occured executing \n".
+                                  " occurred executing \n".
                                   $request);
      }
      return;
  }
+ my $have_read_groupnames_table = 0;
+ my %ids_by_groupname;
- ################################################
+ sub get_group_id {
- ################################################
+     my ($groupname) = @_;
+     if (! $have_read_groupnames_table) {
+         my @Result = &Apache::lonmysql::get_rows($groupnames_table);
+         foreach (@Result) {
+             $ids_by_groupname{$_->[1]}=$_->[0];
+         }
+         $have_read_groupnames_table = 1;
+     }
+     if (! exists($ids_by_groupname{$groupname})) {
+         &populate_groupnames_table();
+         undef(%ids_by_groupname);
+         my @Result = &Apache::lonmysql::get_rows($groupnames_table);
+         foreach (@Result) {
+             $ids_by_groupname{$_->[1]}=$_->[0];
+         }
+     }
+     if (exists($ids_by_groupname{$groupname})) {
+         return $ids_by_groupname{$groupname};
+     }
+     return undef; # error
+ }
- =pod
+ sub populate_groupnames_table {
+     my ($courseid) = @_;
+     if (! defined($courseid)) {
+         $courseid = $env{'request.course.id'};
+     }
+     &setup_table_names($courseid);
+     &init_dbs($courseid,0);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     my $cdom = $env{'course.'.$courseid.'.domain'};
+     my $cnum = $env{'course.'.$courseid.'.num'};
+     my %curr_groups = &Apache::longroup::coursegroups($cdom,$cnum);
+     return if (!%curr_groups);
+     my $request = 'INSERT IGNORE INTO '.$groupnames_table.
+                   '(groupname) VALUES ';
+     foreach my $groupname (sort(keys(%curr_groups)),'none') {
+         $request .= "('".$groupname."'),";
+     }
+     chop($request);
+     $dbh->do($request);
+     if ($dbh->err()) {
+         &Apache::lonnet::logthis("error ".$dbh->errstr().
+                                  " occurred executing \n".
+                                  $request);
+     }
+     return;
+ }
- =item &clear_internal_caches()
+ my $have_read_studentsgroups_table = 0;
+ my %groupids_by_studentid;
- Causes the internal caches used in get_student_id, get_student,
+ sub get_students_groupids {
- get_symb_id, get_symb, get_part_id, and get_part to be undef'd.
+     my ($student_id) = @_;
+     if (! $have_read_studentsgroups_table) {
+         my @Result = &Apache::lonmysql::get_rows($students_groups_table);
+         foreach (@Result) {
+             push(@{$groupids_by_studentid{$_->[0]}},$_->[1]);
+         }
+         $have_read_studentsgroups_table = 1;
+     }
+     if (! exists($groupids_by_studentid{$student_id})) {
+         &populate_students_groups_table();
+         undef(%groupids_by_studentid);
+         my @Result = &Apache::lonmysql::get_rows($students_groups_table);
+         foreach (@Result) {
+             push(@{$groupids_by_studentid{$_->[0]}},$_->[1]);
+         }
+     }
+     if (exists($groupids_by_studentid{$student_id})) {
+         if (ref($groupids_by_studentid{$student_id}) eq 'ARRAY') {
+             return @{$groupids_by_studentid{$student_id}};
+         }
+     }
+     return undef; # error
+ }
- Needs to be called before the first operation with the MySQL database
- for a given Apache request.
- =cut
+ sub populate_students_groups_table {
+     my ($courseid) = @_;
+     if (! defined($courseid)) {
+         $courseid = $env{'request.course.id'};
+     }
+     #
+     &setup_table_names($courseid);
+     &init_dbs($courseid,0);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     my $request = 'INSERT IGNORE INTO '.$students_groups_table.
+         "(student_id,group_id) VALUES ";
+     my $cdom = $env{'course.'.$courseid.'.domain'};
+     my $cnum = $env{'course.'.$courseid.'.num'};
+     my ($classlist,$keylist) = &get_classlist($cdom,$cnum);
+     my ($classgroups,$studentgroups) = &get_group_memberships($classlist,
+                                                               $keylist,
+                                                               $cdom,$cnum);
+     my $record_count = 0;
+     foreach my $student (sort(keys(%{$classgroups}))) {
+         my $student_id = &get_student_id(split(':',$student));
+         my @studentsgroups = &get_students_groups($student,'Active',$classgroups);
+         if (@studentsgroups < 1) {
+             @studentsgroups = ('none');
+         }
+         foreach my $groupname (@studentsgroups) {
+             my $group_id = &get_group_id($groupname);
+             $request .= "('".$student_id."','".$group_id."'),";
+             $record_count++;
+         }
+     }
+     return if ($record_count == 0);
+     chop($request);
+     $dbh->do($request);
+     if ($dbh->err()) {
+         &Apache::lonnet::logthis("error ".$dbh->errstr().
+                                  " occurred executing \n".
+                                  $request);
+     }
+     return;
+ }
- ################################################
- ################################################
  sub clear_internal_caches {
      $have_read_part_table = 0;
      undef(%ids_by_part);
- Line 1105  sub clear_internal_caches {
+ Line 768  sub clear_internal_caches {
      $have_read_student_table = 0;
      undef(%ids_by_student);
      undef(%students_by_id);
+     $have_read_groupnames_table = 0;
+     undef(%ids_by_groupname);
  }
+ sub symb_is_for_task {
+     my ($symb) = @_;
+     return ($symb =~ /\.task$/);
+ }
- ################################################
+ my $requested_max_packet = 0;
- ################################################
+ my $max_allowed_packet;
- =pod
- =item &update_full_student_data($sname,$sdom,$courseid)
- Does a lonnet::dump on a student to populate the courses tables.
- Input: $sname, $sdom, $courseid
- Output: $returnstatus
- $returnstatus is a string describing any errors that occured.  'okay' is the
- default.
- This subroutine loads a students data using lonnet::dump and inserts
- it into the MySQL database.  The inserts are done on three tables,
- $fulldump_response_table, $fulldump_part_table, and $fulldump_timestamp_table.
- The INSERT calls are made directly by this subroutine, not through lonmysql
- because we do a 'bulk'insert which takes advantage of MySQLs non-SQL
- compliant INSERT command to insert multiple rows at a time.
- If anything has gone wrong during this process, $returnstatus is updated with
- a description of the error.
- Once the "fulldump" tables are updated, the tables used for chart and
- spreadsheet (which hold only the current state of the student on their
- homework, not historical data) are updated.  If all updates have occured
- successfully, $student_table is updated to reflect the time of the update.
- Notice we do not insert the data and immediately query it.  This means it
- is possible for there to be data returned this first time that is not
- available the second time.  CYA.
- =cut
- ################################################
- ################################################
  sub update_full_student_data {
      my ($sname,$sdom,$courseid) = @_;
      #
- Line 1159  sub update_full_student_data {
+ Line 793  sub update_full_student_data {
      #
      # Download students data
      my $time_of_retrieval = time;
-     my @tmp = &Apache::lonnet::dump($courseid,$sdom,$sname);
+     my @tmp = &Apache::lonnet::dumpstore($courseid,$sdom,$sname);
      if (@tmp && $tmp[0] =~ /^error/) {
          $returnstatus = 'error retrieving full student data';
          return $returnstatus;
- Line 1184  sub update_full_student_data {
+ Line 818  sub update_full_student_data {
      while (my ($key,$value) = each(%studentdata)) {
          next if ($key =~ /^(\d+):(resource$|subnum$|keys:)/);
          my ($transaction,$symb,$parameter) = split(':',$key);
+ 	$symb = &unescape($symb);
+ 	$parameter = &unescape($parameter);
          my $symb_id = &get_symb_id($symb);
          if ($parameter eq 'timestamp') {
              # We can deal with 'timestamp' right away
- Line 1199  sub update_full_student_data {
+ Line 835  sub update_full_student_data {
              next;
          } elsif ($parameter eq 'version') {
              next;
-         } elsif ($parameter =~ /^resource\.(.*)\.(tries|
+ 	} elsif (&symb_is_for_task($symb)) {
+ 	    next if ($parameter !~ /^resource\.(.*)\.(award|
+ 						      awarded|
+ 						      solved|
+ 						      submission|
+ 						      portfiles|
+ 						      status|
+ 						      version|
+ 						      regrader)\s*$/x);
+ 	    my ($version_and_part_id, $field) = ($1,$2);
+ 	    next if ($version_and_part_id !~ /\./
+ 		     && $field ne 'regrader' && $field ne 'version');
+ 	    my ($version, $part, $instance) =
+ 		split(/\./,$version_and_part_id);
+ 	    #skip and instance dimension or criteria specific data
+ 	    next if (defined($instance)
+ 		     && $instance ne $field
+ 		     && $instance ne 'bridgetask');
+ 	    if (!defined($part)) {
+ 		$part = $version;
+ 	    }
+ 	    my $resp_id = &get_part_id('0');
+ 	    my $part_id = &get_part_id($part);
+ 	    if ($field eq 'version') {
+ 		# for tasks each version is an attempt at it thus
+ 		#     version -> tries
+ 		$partdata->{$symb_id}{$part_id}{$transaction}{'tries'}=
+ 		    $value;
+ 		# at new version time the record gets reset thus adding a
+ 		# virtual response awarddetail of 'new_version'
+ 		$respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific'}='status';
+ 		$respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific_value'}='new_version';
+ 	    } elsif ($field eq 'award' || $field eq 'awarded'
+ 		     || $field eq 'solved') {
+ 		$partdata->{$symb_id}{$part_id}{$transaction}{$field}=
+ 		    $value;
+ 	    } elsif ($field eq 'portfiles') {
+ 		# tasks only accepts portfolio submissions
+ 		$value = $dbh->quote($value);
+ 		$respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'submission'}=$value;
+ 	    } elsif ($field eq 'status') {
+ 		$respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific'}=$field;
+ 		$respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific_value'}=$value;
+ 	    } elsif ($field eq 'regrader') {
+ 		$respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific_2'}=$field;
+ 		$respdata->{$symb_id}{$part_id}{$resp_id}{$transaction}{'response_specific_value_2'}=$value;
+ 	    }
+ 	} elsif ($parameter =~ /^resource\.(.*)\.(tries|
                                                    award|
                                                    awarded|
                                                    previous|
- Line 1270  sub update_full_student_data {
+ Line 959  sub update_full_student_data {
          chop($store_command);
          $dbh->do($store_command);
          if ($dbh->err) {
-             $returnstatus = 'error storing part data';
+             $returnstatus = 'error saving part data';
              &Apache::lonnet::logthis('insert error '.$dbh->errstr());
              &Apache::lonnet::logthis("While attempting\n".$store_command);
          }
      }
      ##
      ## Store the response data
-     $store_command = 'INSERT IGNORE INTO '.$fulldump_response_table.
+     my $store_prefix = 'INSERT IGNORE INTO '.$fulldump_response_table.
          ' VALUES '."\n";
      $store_rows = 0;
+     unless ($requested_max_packet) {
+         (undef,$max_allowed_packet) = $dbh->selectrow_array(
+                                              qq{show variables LIKE ? },
+                                              undef,
+                                              "max_allowed_packet");
+         if ($max_allowed_packet !~ /^\d+$/) {
+             $max_allowed_packet = '';
+         }
+         $requested_max_packet = 1;
+     }
+     my @store_values = ();
+     my $curr_values = '';
+     my $curr_length = 0;
+     my ($max_values);
+     if ($max_allowed_packet) {
+         $max_values = $max_allowed_packet - length($store_prefix);
+     }
      while (my ($symb_id,$hash1) = each (%$respdata)) {
          while (my ($part_id,$hash2) = each (%$hash1)) {
              while (my ($resp_id,$hash3) = each (%$hash2)) {
- Line 1291  sub update_full_student_data {
+ Line 997  sub update_full_student_data {
                      # and odd number of '\' cause insert errors to occur.
                      # Best trap this somehow...
                      $submission = $dbh->quote($submission);
-                     $store_command .= "('".
+                     my $sql_values = "('".
                          join("','",$symb_id,$part_id,
                               $resp_id,$student_id,
                               $transaction,
                               $data->{'awarddetail'},
                               $data->{'response_specific'},
-                              $data->{'response_specific_value'}).
+                              $data->{'response_specific_value'},
-                              "',".$submission."),";
+                              $data->{'response_specific_2'},
+                              $data->{'response_specific_value_2'})."',";
+                     if ($max_values) {
+                         my $length = length($sql_values) + length($submission."),");
+                         if ($length > $max_values) {
+                             &Apache::lonnet::logthis("SQL responsedata insert for student: $sname would exceed max_allowed_packet size");
+                             &Apache::lonnet::logthis("symb_id: $symb_id, part_id: $part_id, resp_id: $resp_id");
+                             &Apache::lonnet::logthis("You may want to increase the max_allowed_packet size from the current: $max_allowed_packet");
+                             $sql_values .= $dbh->quote('WARNING: Submission too large -- see grading interface for actual submission')."),";
+                             $length = length($sql_values);
+                             &Apache::lonnet::logthis("Placeholder inserted instead of value of actual submission");
+                             &Apache::lonnet::logthis("See grading interface for the actual submission");
+                         } else {
+                             $sql_values .= $submission."),";
+                         }
+                         if ($length + $curr_length > $max_values) {
+                             push(@store_values,$curr_values);
+                             $curr_values = $sql_values;
+                             $curr_length = $length;
+                         } else {
+                             $curr_values .= $sql_values;
+                             $curr_length += $length;
+                         }
+                     } else {
+                         $curr_values .= $sql_values.$submission."),";
+                     }
                      $store_rows++;
                  }
              }
          }
      }
      if ($store_rows) {
-         chop($store_command);
+         if ($curr_values ne '') {
-         $dbh->do($store_command);
+             push(@store_values,$curr_values);
-         if ($dbh->err) {
+         }
-             $returnstatus = 'error storing response data';
+         foreach my $item (@store_values) {
-             &Apache::lonnet::logthis('insert error '.$dbh->errstr());
+             chop($item);
-             &Apache::lonnet::logthis("While attempting\n".$store_command);
+             if ($item ne '') {
+                 $dbh->do($store_prefix.$item);
+                 if ($dbh->err) {
+                     $returnstatus = 'error saving response data';
+                     &Apache::lonnet::logthis('insert error '.$dbh->errstr());
+                     &Apache::lonnet::logthis("While attempting\n".$store_prefix.$item);
+                     last;
+                 }
+             }
          }
      }
      ##
- Line 1320  sub update_full_student_data {
+ Line 1059  sub update_full_student_data {
          ($sname,$sdom,$courseid,
           &Apache::lonnet::convert_dump_to_currentdump(\%studentdata));
      if ($returnstatus eq 'okay' && $status ne 'okay') {
-         $returnstatus = 'error storing current data:'.$status;
+         $returnstatus = 'error saving current data:'.$status;
      } elsif ($status ne 'okay') {
-         $returnstatus .= ' error storing current data:'.$status;
+         $returnstatus .= ' error saving current data:'.$status;
      }
      ##
      ## Update the students time......
      if ($returnstatus eq 'okay') {
- Line 1339  sub update_full_student_data {
+ Line 1078  sub update_full_student_data {
      return $returnstatus;
  }
- ################################################
- ################################################
- =pod
- =item &update_student_data()
- Input: $sname, $sdom, $courseid
- Output: $returnstatus, \%student_data
- $returnstatus is a string describing any errors that occured.  'okay' is the
- default.
- \%student_data is the data returned by a call to lonnet::currentdump.
- This subroutine loads a students data using lonnet::currentdump and inserts
- it into the MySQL database.  The inserts are done on two tables,
- $performance_table and $parameters_table.  $parameters_table holds the data
- that is not included in $performance_table.  See the description of
- $performance_table elsewhere in this file.  The INSERT calls are made
- directly by this subroutine, not through lonmysql because we do a 'bulk'
- insert which takes advantage of MySQLs non-SQL compliant INSERT command to
- insert multiple rows at a time.  If anything has gone wrong during this
- process, $returnstatus is updated with a description of the error and
- \%student_data is returned.
- Notice we do not insert the data and immediately query it.  This means it
- is possible for there to be data returned this first time that is not
- available the second time.  CYA.
- =cut
- ################################################
- ################################################
  sub update_student_data {
      my ($sname,$sdom,$courseid) = @_;
      #
- Line 1386  sub update_student_data {
+ Line 1092  sub update_student_data {
      #
      # Download students data
      my $time_of_retrieval = time;
-     my @tmp = &Apache::lonnet::currentdump($courseid,$sdom,$sname);
+     my %student_data = &Apache::lonnet::currentdump($courseid,$sdom,$sname);
-     if ((scalar(@tmp) > 0) && ($tmp[0] =~ /^error:/)) {
+     if (&Apache::lonnet::error(%student_data)) {
          &Apache::lonnet::logthis('error getting data for '.
                                   $sname.':'.$sdom.' in course '.$courseid.
-                                  ':'.$tmp[0]);
+                                  ':'.(%student_data)[0]);
-         $returnstatus = 'error getting data';
+         $returnstatus =(%student_data)[0] ;
          return ($returnstatus,undef);
      }
-     if (scalar(@tmp) < 1) {
+     if (scalar(keys(%student_data)) < 1) {
          return ('no data',undef);
      }
-     my %student_data = @tmp;
      my @Results = &store_student_data($sname,$sdom,$courseid,\%student_data);
      #
      # Set the students update time
      if ($Results[0] eq 'okay') {
-         &store_updatetime($student_id,$time_of_retrieval,$time_of_retrieval);
+         &store_updatetime($student_id,$time_of_retrieval);
      }
      #
      return @Results;
- Line 1447  sub store_student_data {
+ Line 1152  sub store_student_data {
      my $starttime = Time::HiRes::time;
      my $elapsed = 0;
      my $rows_stored;
-     my $store_parameters_command  = 'INSERT IGNORE INTO '.$parameters_table.
+     my $store_parameters_prefix  = 'INSERT IGNORE INTO '.$parameters_table.
          ' VALUES '."\n";
      my $num_parameters = 0;
-     my $store_performance_command = 'INSERT IGNORE INTO '.$performance_table.
+     my $store_performance_prefix = 'INSERT IGNORE INTO '.$performance_table.
          ' VALUES '."\n";
      return ('error',undef) if (! defined($dbh));
+     unless ($requested_max_packet) {
+         (undef,$max_allowed_packet) = $dbh->selectrow_array(
+                                              qq{show variables LIKE ? },
+                                              undef,
+                                              "max_allowed_packet");
+         if ($max_allowed_packet !~ /^\d+$/) {
+             $max_allowed_packet = '';
+         }
+         $requested_max_packet = 1;
+     }
+     my @store_parameters_values = ();
+     my $curr_params_values = '';
+     my $curr_params_length = 0;
+     my @store_performance_values = ();
+     my $curr_perf_values = '';
+     my $curr_perf_length = 0;
+     my ($max_param,$max_perf);
+     if ($max_allowed_packet) {
+         $max_param = $max_allowed_packet - length($store_parameters_prefix);
+         $max_perf = $max_allowed_packet - length($store_performance_prefix);
+     }
      while (my ($current_symb,$param_hash) = each(%{$student_data})) {
          #
          # make sure the symb is set up properly
          my $symb_id = &get_symb_id($current_symb);
          #
-         # Load data into the tables
+         # Parameters
          while (my ($parameter,$value) = each(%$param_hash)) {
-             my $newstring;
              if ($parameter !~ /(timestamp|resource\.(.*)\.(solved|tries|awarded|award|awarddetail|previous))/) {
-                 $newstring = "('".join("','",
+                 my $sql_parameter = "('".join("','",
-                                        $symb_id,$student_id,
+                                               $symb_id,$student_id,
-                                        $parameter)."',".
+                                               $parameter)."',".
-                                            $dbh->quote($value)."),\n";
+                                                   $dbh->quote($value)."),\n";
-                 $num_parameters ++;
+                 if ($sql_parameter !~ /''/) {
-                 if ($newstring !~ /''/) {
+                     if ($max_param) {
-                     $store_parameters_command .= $newstring;
+                         my $length = length($sql_parameter);
-                     $rows_stored++;
+                         if ($length > $max_param) {
+                             &Apache::lonnet::logthis("SQL parameter insert for student: $sname for parameter: $parameter would exceed max_allowed_packet size");
+                             &Apache::lonnet::logthis("symb_id: $symb_id");
+                             &Apache::lonnet::logthis("You may want to increase the max_allowed_packet size from the current: $max_allowed_packet");
+                             if ($parameter =~ /\.submission$/) {
+                                 $sql_parameter = "('".join("','",
+                                               $symb_id,$student_id,
+                                               $parameter)."',".
+                                                   $dbh->quote('WARNING: Submission too large -- see grading interface for actual submission')."),\n";
+                                 $length = length($sql_parameter);
+                                 &Apache::lonnet::logthis("Placeholder inserted instead of value of actual submission");
+                                 &Apache::lonnet::logthis("See grading interface for the actual submission");
+                             } else {
+                                 &Apache::lonnet::logthis("Skipping this item");
+                                 next;
+                             }
+                         }
+                         if ($length + $curr_params_length > $max_param) {
+                             push(@store_parameters_values,$curr_params_values);
+                             $curr_params_values = $sql_parameter;
+                             $curr_params_length = $length;
+                         } else {
+                             $curr_params_values .= $sql_parameter;
+                             $curr_params_length += $length;
+                         }
+                     } else {
+                         $curr_params_values .= $sql_parameter;
+                     }
+                     #$rows_stored++;
+                     $num_parameters ++;
                  }
              }
-             next if ($parameter !~ /^resource\.(.*)\.solved$/);
+         }
+         # Performance
+         my %stored;
+         while (my ($parameter,$value) = each(%$param_hash)) {
+             next if ($parameter !~ /^resource\.(.*)\.(solved|awarded)$/);
+             my $part  = $1;
+ 	    my $which = $2;
+ 	    next if ($part =~ /\./);
+             next if (exists($stored{$part}));
+             $stored{$part}++;
              #
-             my $part = $1;
              my $part_id = &get_part_id($part);
              next if (!defined($part_id));
-             my $solved  = $value;
-             my $tries   = $param_hash->{'resource.'.$part.'.tries'};
+             my ($solved,$awarded);
-             my $awarded = $param_hash->{'resource.'.$part.'.awarded'};
+ 	    if ($which eq 'solved') {
+ 		$solved  = $value;
+ 		$awarded = $param_hash->{'resource.'.$part.'.awarded'};
+ 	    } else {
+ 		$solved  = $param_hash->{'resource.'.$part.'.solved'};
+ 		$awarded = $value;
+ 	    }
              my $award   = $param_hash->{'resource.'.$part.'.award'};
              my $awarddetail = $param_hash->{'resource.'.$part.'.awarddetail'};
              my $timestamp = $param_hash->{'timestamp'};
+ 	    my $tries   = $param_hash->{'resource.'.$part.'.tries'};
+ 	    if (&symb_is_for_task($current_symb)) {
+ 		$tries   = $param_hash->{'resource.'.$part.'.version'};
+ 	    }
              #
              $solved      = '' if (! defined($solved));
              $tries       = '' if (! defined($tries));
              $awarded     = '' if (! defined($awarded));
              $award       = '' if (! defined($award));
              $awarddetail = '' if (! defined($awarddetail));
-             $newstring = "('".join("','",$symb_id,$student_id,$part_id,$part,
+             my $sql_performance =
-                                    $solved,$tries,$awarded,$award,
+                 "('".join("','",$symb_id,$student_id,$part_id,$part,
-                                    $awarddetail,$timestamp)."'),\n";
+                                 $solved,$tries,$awarded,$award,
-             $store_performance_command .= $newstring;
+                                 $awarddetail,$timestamp)."'),\n";
+             if ($max_perf) {
+                 my $length = length($sql_performance);
+                 if ($length > $max_perf) {
+                             &Apache::lonnet::logthis("SQL performance insert for student: $sname would exceed max_allowed_packet size");
+                             &Apache::lonnet::logthis("symb_id: $symb_id");
+                             &Apache::lonnet::logthis("Skipping this item.  You may want to increase the max_allowed_packet size from the current: $max_allowed_packet");
+                             next;
+                 } else {
+                     if ($length + $curr_perf_length > $max_perf) {
+                         push(@store_performance_values,$curr_perf_values);
+                         $curr_perf_values = $sql_performance;
+                         $curr_perf_length = $length;
+                     } else {
+                         $curr_perf_values .= $sql_performance;
+                         $curr_perf_length += $length;
+                     }
+                 }
+             } else {
+                 $curr_perf_values .= $sql_performance;
+             }
              $rows_stored++;
          }
      }
-     chop $store_parameters_command;
+     if ($curr_params_values ne '') {
-     chop $store_parameters_command;
+         push(@store_parameters_values,$curr_params_values);
-     chop $store_performance_command;
+     }
-     chop $store_performance_command;
+     if ($curr_perf_values ne '') {
+         push(@store_performance_values,$curr_perf_values);
+     }
+     if (! $rows_stored) { return ($returnstatus, undef); }
      my $start = Time::HiRes::time;
-     $dbh->do($store_performance_command);
+     foreach my $item (@store_performance_values) {
-     if ($dbh->err()) {
+         $item =~ s|,\n$||;
-         &Apache::lonnet::logthis(' bigass insert error:'.$dbh->errstr());
+         if ($item ne '') {
-         &Apache::lonnet::logthis('command = '.$store_performance_command);
+             $dbh->do($store_performance_prefix.$item);
-         $returnstatus = 'error: unable to insert performance into database';
+             if ($dbh->err()) {
-         return ($returnstatus,$student_data);
+                 &Apache::lonnet::logthis('performance insert error:'.
+                                          $dbh->errstr());
+                 &Apache::lonnet::logthis('command = '.$/.$store_performance_prefix.$item);
+                 $returnstatus = 'error: unable to insert performance into database';
+                 return ($returnstatus,$student_data);
+             }
+         }
      }
-     $dbh->do($store_parameters_command) if ($num_parameters>0);
+     if ($num_parameters > 0) {
-     if ($dbh->err()) {
+         foreach my $item (@store_parameters_values) {
-         &Apache::lonnet::logthis(' bigass insert error:'.$dbh->errstr());
+             $item =~ s|,\n$||;
-         &Apache::lonnet::logthis('command = '.$store_parameters_command);
+             if ($item ne '') {
-         &Apache::lonnet::logthis('rows_stored = '.$rows_stored);
+                 $dbh->do($store_parameters_prefix.$item);
-         &Apache::lonnet::logthis('student_id = '.$student_id);
+                 if ($dbh->err()) {
-         $returnstatus = 'error: unable to insert parameters into database';
+                      &Apache::lonnet::logthis('parameters insert error:'.
-         return ($returnstatus,$student_data);
+                                               $dbh->errstr());
+                      &Apache::lonnet::logthis('command = '.$/.$store_parameters_prefix.$item);
+                      &Apache::lonnet::logthis('rows_stored = '.$rows_stored);
+                      &Apache::lonnet::logthis('student_id = '.$student_id);
+                      $returnstatus = 'error: unable to insert parameters into database';
+                      return ($returnstatus,$student_data);
+                 }
+             }
+         }
      }
      $elapsed += Time::HiRes::time - $start;
      return ($returnstatus,$student_data);
  }
- ######################################
- ######################################
- =pod
- =item &ensure_tables_are_set_up($courseid)
- Checks to be sure the MySQL tables for the given class are set up.
- If $courseid is omitted it will be obtained from the environment.
- Returns nothing on success and 'error' on failure
- =cut
- ######################################
- ######################################
  sub ensure_tables_are_set_up {
      my ($courseid) = @_;
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      #
      # Clean out package variables
      &setup_table_names($courseid);
      #
      # if the tables do not exist, make them
      my @CurrentTable = &Apache::lonmysql::tables_in_db();
-     my ($found_symb,$found_student,$found_part,
+     my ($found_symb,$found_student,$found_groups,$found_groupnames,$found_part,
          $found_performance,$found_parameters,$found_fulldump_part,
-         $found_fulldump_response,$found_fulldump_timestamp);
+         $found_fulldump_response,$found_fulldump_timestamp,
+         $found_weight);
      foreach (@CurrentTable) {
          $found_symb        = 1 if ($_ eq $symb_table);
          $found_student     = 1 if ($_ eq $student_table);
+         $found_groups      = 1 if ($_ eq $students_groups_table);
+         $found_groupnames  = 1 if ($_ eq $groupnames_table);
          $found_part        = 1 if ($_ eq $part_table);
          $found_performance = 1 if ($_ eq $performance_table);
          $found_parameters  = 1 if ($_ eq $parameters_table);
          $found_fulldump_part      = 1 if ($_ eq $fulldump_part_table);
          $found_fulldump_response  = 1 if ($_ eq $fulldump_response_table);
          $found_fulldump_timestamp = 1 if ($_ eq $fulldump_timestamp_table);
+         $found_weight      = 1 if ($_ eq $weight_table);
      }
-     if (!$found_symb        ||
+     if (!$found_symb          ||
-         !$found_student     || !$found_part   ||
+         !$found_student       || !$found_part              ||
-         !$found_performance || !$found_parameters ||
+         !$found_performance   || !$found_parameters        ||
          !$found_fulldump_part || !$found_fulldump_response ||
-         !$found_fulldump_timestamp ) {
+         !$found_fulldump_timestamp || !$found_weight ) {
-         if (&init_dbs($courseid)) {
+         if (&init_dbs($courseid,1)) {
              return 'error';
          }
      }
  }
- ################################################
- ################################################
- =pod
- =item &ensure_current_data()
- Input: $sname, $sdom, $courseid
- Output: $status, $data
- This routine ensures the data for a given student is up to date.
- The $student_table is queried to determine the time of the last update.
- If the students data is out of date, &update_student_data() is called.
- The return values from the call to &update_student_data() are returned.
- =cut
- ################################################
- ################################################
  sub ensure_current_data {
      my ($sname,$sdom,$courseid) = @_;
      my $status = 'okay';   # return value
      #
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      &ensure_tables_are_set_up($courseid);
      #
      # Get the update time for the user
      my $updatetime = 0;
+     my $getuserdir = 1;
      my $modifiedtime = &Apache::lonnet::GetFileTimestamp
-         ($sdom,$sname,$courseid.'.db',
+         ($sdom,$sname,$courseid.'.db',$getuserdir);
-          $Apache::lonnet::perlvar{'lonUsersDir'});
      #
+     if ($modifiedtime == -1) {
+ 	return ('no data',undef);
+     }
      my $student_id = &get_student_id($sname,$sdom);
+     &get_students_groupids($student_id);
      my @Result = &Apache::lonmysql::get_rows($student_table,
                                               "student_id ='$student_id'");
      my $data = undef;
      if (@Result) {
-         $updatetime = $Result[0]->[5];  # Ack!  This is dumb!
+         $updatetime = $Result[0]->[6];  # Ack!  This is dumb!
      }
      if ($modifiedtime > $updatetime) {
          ($status,$data) = &update_student_data($sname,$sdom,$courseid);
- Line 1616  sub ensure_current_data {
+ Line 1399  sub ensure_current_data {
      return ($status,$data);
  }
- ################################################
- ################################################
- =pod
- =item &ensure_current_full_data($sname,$sdom,$courseid)
- Input: $sname, $sdom, $courseid
- Output: $status
- This routine ensures the fulldata (the data from a lonnet::dump, not a
- lonnet::currentdump) for a given student is up to date.
- The $student_table is queried to determine the time of the last update.
- If the students fulldata is out of date, &update_full_student_data() is
- called.
- The return value from the call to &update_full_student_data() is returned.
- =cut
- ################################################
- ################################################
  sub ensure_current_full_data {
      my ($sname,$sdom,$courseid) = @_;
      my $status = 'okay';   # return value
      #
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      &ensure_tables_are_set_up($courseid);
      #
      # Get the update time for the user
+     my $getuserdir = 1;
      my $modifiedtime = &Apache::lonnet::GetFileTimestamp
-         ($sdom,$sname,$courseid.'.db',
+         ($sdom,$sname,$courseid.'.db',$getuserdir);
-          $Apache::lonnet::perlvar{'lonUsersDir'});
      #
      my $student_id = &get_student_id($sname,$sdom);
+     &get_students_groupids($student_id);
      my @Result = &Apache::lonmysql::get_rows($student_table,
                                               "student_id ='$student_id'");
      my $updatetime;
      if (@Result && ref($Result[0]) eq 'ARRAY') {
-         $updatetime = $Result[0]->[6];
+         $updatetime = $Result[0]->[7];
      }
      if (! defined($updatetime) || $modifiedtime > $updatetime) {
          $status = &update_full_student_data($sname,$sdom,$courseid);
- Line 1664  sub ensure_current_full_data {
+ Line 1426  sub ensure_current_full_data {
      return $status;
  }
- ################################################
+ sub ensure_current_groups {
- ################################################
+     my ($courseid) = @_;
+     my ($cdom,$cnum);
- =pod
+     if (defined($courseid)) {
+         my %coursehash = &Apache::lonnet::coursedescription($courseid);
- =item &get_student_data_from_performance_cache()
+         $cdom = $coursehash{'domain'};
+         $cnum = $coursehash{'num'};
- Input: $sname, $sdom, $symb, $courseid
+     } elsif ($env{'request.course.id'}) {
+         $courseid = $env{'request.course.id'};
- Output: hash reference containing the data for the given student.
+         $cdom = $env{'course.'.$courseid.'.domain'};
- If $symb is undef, all the students data is returned.
+         $cnum = $env{'course.'.$courseid.'.num'};
+     }
+     if ($cdom eq '' || $cnum eq '') {
+         return 'error: invalid course';
+     }
+     &setup_table_names($courseid);
+     my @CurrentTables = &Apache::lonmysql::tables_in_db();
+     unless (grep(/^\Q$groupnames_table\E$/,@CurrentTables)) {
+         return;
+     }
+     # Get the update time for the groupnames table
+     my $getuserdir = 1;
+     my $modifiedtime = &Apache::lonnet::GetFileTimestamp
+         ($cdom,$cnum,'coursegroups.db',$getuserdir);
+     my %tableinfo = &Apache::lonmysql::table_information($groupnames_table);
+     my $updatetime;
+     if ($tableinfo{'Update_time'}) {
+         $updatetime = $tableinfo{'Update_time'};
+     }
+     if (! defined($updatetime) || $modifiedtime > $updatetime) {
+         my (%groups_in_sql,%removegroups,$addgroup);
+         my %curr_groups = &Apache::longroup::coursegroups($cdom,$cnum);
+         my @Result = &Apache::lonmysql::get_rows($groupnames_table);
+         foreach my $row (@Result) {
+             my ($id,$name) = @{$row};
+             unless (exists($curr_groups{$name})) {
+                 $groups_in_sql{$name}=$id;
+             } elsif ($id) {
+                 $removegroups{$id} = $name;
+             }
+         }
+         foreach my $group (keys(%curr_groups)) {
+             unless (exists($groups_in_sql{$group})) {
+                 $addgroup = 1;
+                 last;
+             }
+         }
+         if (keys(%removegroups)) {
+             my $dbh = &Apache::lonmysql::get_dbh();
+             foreach my $group_id (keys(%removegroups)) {
+                 my $command = 'DELETE FROM '.$groupnames_table.' WHERE group_id='.
+                               $group_id;
+                 $dbh->do($command);
+                 if ($dbh->err()) {
+                     &Apache::lonnet::logthis("error ".$dbh->errstr().
+                                              " occurred executing \n".
+                                              "SQL command: $command");
+                 }
+             }
+         }
+         if ($addgroup) {
+             &populate_groupnames_table($courseid);
+         }
+     }
+     return;
+ }
- This routine is the heart of the local caching system.  See the description
+ sub ensure_current_students_groups {
- of $performance_table, $symb_table, $student_table, and $part_table.  The
+     my ($courseid) = @_;
- main task is building the MySQL request.  The tables appear in the request
+     my ($cdom,$cnum);
- in the order in which they should be parsed by MySQL.  When searching
+     if (defined($courseid)) {
- on a student the $student_table is used to locate the 'student_id'.  All
+         my %coursehash = &Apache::lonnet::coursedescription($courseid);
- rows in $performance_table which have a matching 'student_id' are returned,
+         $cdom = $coursehash{'domain'};
- with data from $part_table and $symb_table which match the entries in
+         $cnum = $coursehash{'num'};
- $performance_table, 'part_id' and 'symb_id'.  When searching on a symb,
+     } elsif ($env{'request.course.id'}) {
- the $symb_table is processed first, with matching rows grabbed from
+         $courseid = $env{'request.course.id'};
- $performance_table and filled in from $part_table and $student_table in
+         $cdom = $env{'course.'.$courseid.'.domain'};
- that order.
+         $cnum = $env{'course.'.$courseid.'.num'};
+     }
+     &setup_table_names($courseid);
+     my @CurrentTables = &Apache::lonmysql::tables_in_db();
+     unless (grep(/^\Q$students_groups_table\E$/,@CurrentTables)) {
+         return;
+     }
+     # Get the update time for the groupnames table
+     my $getuserdir = 1;
+     my $modifiedtime = &Apache::lonnet::GetFileTimestamp
+         ($cdom,$cnum,'groupmembership.db',$getuserdir);
+     my %tableinfo = &Apache::lonmysql::table_information($students_groups_table);
+     my $updatetime;
+     if ($tableinfo{'Update_time'}) {
+         $updatetime = $tableinfo{'Update_time'};
+     }
+     if ((!defined($updatetime)) || ($modifiedtime > $updatetime)) {
+         if (&Apache::lonmysql::drop_table($students_groups_table)) {
+             if (&init_dbs($courseid)) {
+                 return "error creating $students_groups_table\n";
+             } else {
+                 &populate_students_groups_table($courseid);
+             }
+         }
+     }
+     return;
+ }
- Running 'EXPLAIN ' on the 'SELECT' statements generated can be quite
+ sub ensure_current_sections {
- interesting, especially if you play with the order the tables are listed.
+     my ($courseid) = @_;
+     my ($cdom,$cnum);
+     if (defined($courseid)) {
+         my %coursehash = &Apache::lonnet::coursedescription($courseid);
+         $cdom = $coursehash{'domain'};
+         $cnum = $coursehash{'num'};
+     } elsif ($env{'request.course.id'}) {
+         $courseid = $env{'request.course.id'};
+         $cdom = $env{'course.'.$courseid.'.domain'};
+         $cnum = $env{'course.'.$courseid.'.num'};
+     }
+     &setup_table_names($courseid);
+     my @CurrentTables = &Apache::lonmysql::tables_in_db();
+     unless (grep(/^\Q$student_table\E$/,@CurrentTables)) {
+         return;
+     }
+     # Get the update time for the student table
+     my $getuserdir = 1;
+     my $modifiedtime = &Apache::lonnet::GetFileTimestamp
+         ($cdom,$cnum,'classlist.db',$getuserdir);
+     my %tableinfo = &Apache::lonmysql::table_information($student_table);
+     my $updatetime;
+     if ($tableinfo{'Update_time'}) {
+         $updatetime = $tableinfo{'Update_time'};
+     }
+     if ((!defined($updatetime)) || ($modifiedtime > $updatetime)) {
+         &update_student_table($cdom,$cnum);
+     }
+     return;
+ }
- =cut
+ sub update_student_table {
+     my ($cdom,$cnum) = @_;
+     return unless (($cdom ne '') && ($cnum ne ''));
+     my (%roster,%sqldata);
+     my $classlist = &get_classlist($cdom,$cnum);
+     while (my ($student,$data) = each (%$classlist)) {
+         my ($section,$start,$end) = ($data->[&CL_SECTION()],
+                                      $data->[&CL_START()],
+                                      $data->[&CL_END()]);
+         if ($section eq '' || $section =~ /^\s*$/) {
+             $section = 'none';
+         }
+         if ($start eq '') { $start = 0; }
+         if ($end eq '')   { $end   = 0; }
+         $roster{$student}{'section'} = $section;
+         $roster{$student}{'start'} = $start;
+         $roster{$student}{'end'} = $end;
+     }
+     my $dbh = &Apache::lonmysql::get_dbh();
+     my $statement = "SELECT student_id,student,section,start,end FROM $student_table";
+     my $sth = $dbh->prepare($statement);
+     $sth->execute();
+     if ($sth->err()) {
+         &Apache::lonnet::logthis("Unable to execute MySQL request:");
+         &Apache::lonnet::logthis("\n".$statement."\n");
+         &Apache::lonnet::logthis("error is:".$sth->errstr());
+         return undef;
+     }
+     foreach my $row (@{$sth->fetchall_arrayref}) {
+         my ($id,$student,$section,$start,$end) = (@$row);
+         if (ref($roster{$student}) eq 'HASH') {
+             if (($roster{$student}{'section'} ne $section) ||
+                 ($roster{$student}{'start'} ne $start) ||
+                 ($roster{$student}{'end'} ne $end)) {
+                 $sqldata{$id} = {
+                                   section => $roster{$student}{'section'},
+                                   start   => $roster{$student}{'start'},
+                                   end     => $roster{$student}{'end'},
+                                 };
+             }
+         }
+     }
+     $sth->finish();
+     if (keys(%sqldata)) {
+         foreach my $id (sort { $a <=> $b } keys(%sqldata)) {
+             my $request = "UPDATE $student_table SET section='$sqldata{$id}{section}'".
+                           ", start='$sqldata{$id}{start}'".
+                           ", end='$sqldata{$id}{end}' WHERE student_id='$id'";
+             $dbh->do($request);
+         }
+     }
+     return;
+ }
- ################################################
- ################################################
  sub get_student_data_from_performance_cache {
      my ($sname,$sdom,$symb,$courseid)=@_;
      my $student = $sname.':'.$sdom if (defined($sname) && defined($sdom));
- Line 1778  sub get_student_data_from_performance_ca
+ Line 1700  sub get_student_data_from_performance_ca
      return $studentdata;
  }
- ################################################
- ################################################
- =pod
- =item &get_current_state()
- Input: $sname,$sdom,$symb,$courseid
- Output: Described below
- Retrieve the current status of a students performance.  $sname and
- $sdom are the only required parameters.  If $symb is undef the results
- of an &Apache::lonnet::currentdump() will be returned.
- If $courseid is undef it will be retrieved from the environment.
- The return structure is based on &Apache::lonnet::currentdump.  If
- $symb is unspecified, all the students data is returned in a hash of
- the form:
- (
-   symb1 => { param1 => value1, param2 => value2 ... },
-   symb2 => { param1 => value1, param2 => value2 ... },
- )
- If $symb is specified, a hash of
- (
-   param1 => value1,
-   param2 => value2,
- )
- is returned.
- If no data is found for $symb, or if the student has no performance data,
- an empty list is returned.
- =cut
- ################################################
- ################################################
  sub get_current_state {
      my ($sname,$sdom,$symb,$courseid,$forcedownload)=@_;
      #
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      #
      return () if (! defined($sname) || ! defined($sdom));
      #
- Line 1846  sub get_current_state {
+ Line 1731  sub get_current_state {
      } else {
          if ($status ne 'okay' && $status ne '') {
              &Apache::lonnet::logthis('status = '.$status);
-             return ();
+             return ('error: '.$status,undef);
          }
          my $returnhash = &get_student_data_from_performance_cache($sname,$sdom,
                                                        $symb,$courseid);
- Line 1855  sub get_current_state {
+ Line 1740  sub get_current_state {
      return ();
  }
- ################################################
- ################################################
- =pod
- =item &get_problem_statistics()
- Gather data on a given problem.  The database is assumed to be
- populated and all local caching variables are assumed to be set
- properly.  This means you need to call &ensure_current_data for
- the students you are concerned with prior to calling this routine.
- Inputs: $students, $symb, $part, $courseid
- =over 4
- =item $students is an array of hash references.
- Each hash must contain at least the 'username' and 'domain' of a student.
- =item $symb is the symb for the problem.
- =item $part is the part id you need statistics for
- =item $courseid is the course id, of course!
- =back
- Outputs: See the code for up to date information.  A hash reference is
- returned.  The hash has the following keys defined:
- =over 4
- =item num_students The number of students attempting the problem
- =item tries The total number of tries for the students
- =item max_tries The maximum number of tries taken
- =item mean_tries The average number of tries
- =item num_solved The number of students able to solve the problem
- =item num_override The number of students whose answer is 'correct_by_override'
- =item deg_of_diff The degree of difficulty of the problem
- =item std_tries The standard deviation of the number of tries
- =item skew_tries The skew of the number of tries
- =item per_wrong The number of students attempting the problem who were not
- able to answer it correctly.
- =back
- =cut
- ################################################
- ################################################
  sub get_problem_statistics {
-     my ($Sections,$status,$symb,$part,$courseid) = @_;
+     my ($Sections,$Groups,$status,$symb,$part,$courseid,$starttime,$endtime) = @_;
      return if (! defined($symb) || ! defined($part));
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      #
      &setup_table_names($courseid);
      my $symb_id = &get_symb_id($symb);
      my $part_id = &get_part_id($part);
-     my $stats_table = $courseid.'_problem_stats';
+     my $stats_table = &temp_table_name($courseid,'problem_stats');
      #
      my $dbh = &Apache::lonmysql::get_dbh();
      return undef if (! defined($dbh));
      #
+     # Clean out the table
      $dbh->do('DROP TABLE '.$stats_table);  # May return an error
      my $request =
          'CREATE TEMPORARY TABLE '.$stats_table.' '.
          'SELECT a.student_id,a.solved,a.award,a.awarded,a.tries '.
          'FROM '.$performance_table.' AS a ';
+     #
+     # See if we need to include some requirements on the students
      if ((defined($Sections) && lc($Sections->[0]) ne 'all') ||
          (defined($status)   && lc($status)        ne 'any')) {
          $request .= 'NATURAL LEFT JOIN '.$student_table.' AS b ';
      }
+     my ($groups_join,$group_limits) = &limit_by_group($Groups,'b','c','d');
+     if (defined($groups_join)) {
+         $request .= $groups_join;
+     }
      $request .= ' WHERE a.symb_id='.$symb_id.' AND a.part_id='.$part_id;
-     if (defined($Sections) && lc($Sections->[0]) ne 'all') {
+     #
-         $request .= ' AND ('.
+     # Limit the students included to those specified
-             join(' OR ', map { "b.section='".$_."'" } @$Sections
+     my ($section_limits,$enrollment_limits)=
-                  ).')';
+         &limit_by_section_and_status($Sections,$status,'b');
+     #
+     # Limit by starttime and endtime
+     my $time_requirements = undef;
+     if (defined($starttime)) {
+         $time_requirements .= 'a.timestamp>='.$starttime;
+         if (defined($endtime)) {
+             $time_requirements .= ' AND a.timestamp<='.$endtime;
+         }
+     } elsif (defined($endtime)) {
+         $time_requirements .= 'a.timestamp<='.$endtime;
+     }
+     if (defined($time_requirements)) {
+         $request .= ' AND '.$time_requirements;
+     }
+     if (defined($section_limits)) {
+         $request .= ' AND '.$section_limits;
      }
-     if (defined($status) && lc($status) ne 'any') {
+     if (defined($enrollment_limits)) {
-         $request .= " AND b.status='".$status."'";
+         $request .= ' AND '.$enrollment_limits;
      }
+     # Limit by group, as required
+     if (defined($group_limits)) {
+         $request .= ' AND '.$group_limits;
+     }
+     #
+     # Finally, execute the request to create the temporary table
      $dbh->do($request);
- #    &Apache::lonnet::logthis('request = '.$/.$request);
+     #
-     $request = 'SELECT COUNT(*),SUM(tries),MAX(tries),AVG(tries),STD(tries) '.
+     # Collect the first suite of statistics
+     $request = 'SELECT COUNT(*),SUM(tries),'.
+         'AVG(tries),STD(tries) '.
          'FROM '.$stats_table;
-     my ($num,$tries,$mod,$mean,$STD) = &execute_SQL_request
+     my ($num,$tries,$mean,$STD) = &execute_SQL_request
          ($dbh,$request);
- #    &Apache::lonnet::logthis('request = '.$/.$request);
+     #
+     $request = 'SELECT MAX(tries),MIN(tries) FROM '.$stats_table.
+         ' WHERE awarded>0';
+     my ($max,$min) = &execute_SQL_request($dbh,$request);
+     #
      $request = 'SELECT SUM(awarded) FROM '.$stats_table;
      my ($Solved) = &execute_SQL_request($dbh,$request);
- #    &Apache::lonnet::logthis('request = '.$/.$request);
+     #
      $request = 'SELECT SUM(awarded) FROM '.$stats_table.
          " WHERE solved='correct_by_override'";
- #    &Apache::lonnet::logthis('request = '.$/.$request);
      my ($solved) = &execute_SQL_request($dbh,$request);
- #    $Solved = int($Solved);
+     #
- #    $solved = int($solved);
+     $Solved -= $solved;
      #
      $num    = 0 if (! defined($num));
      $tries  = 0 if (! defined($tries));
-     $mod    = 0 if (! defined($mod));
+     $max    = 0 if (! defined($max));
+     $min    = 0 if (! defined($min));
      $STD    = 0 if (! defined($STD));
-     $Solved = 0 if (! defined($Solved));
+     $Solved = 0 if (! defined($Solved) || $Solved < 0);
      $solved = 0 if (! defined($solved));
      #
+     # Compute the more complicated statistics
      my $DegOfDiff = 'nan';
      $DegOfDiff = 1-($Solved)/$tries if ($tries>0);
+     #
      my $SKEW = 'nan';
      my $wrongpercent = 0;
+     my $numwrong = 'nan';
      if ($num > 0) {
          ($SKEW) = &execute_SQL_request($dbh,'SELECT SQRT(SUM('.
                                       'POWER(tries - '.$STD.',3)'.
                                       '))/'.$num.' FROM '.$stats_table);
-         $wrongpercent=int(10*100*($num-$Solved+$solved)/$num)/10;
+         $numwrong = $num-$Solved;
+         $wrongpercent=int(10*100*$numwrong/$num)/10;
      }
      #
- #    $dbh->do('DROP TABLE '.$stats_table);  # May return an error
+     # Drop the temporary table
-     #
+     $dbh->do('DROP TABLE '.$stats_table);  # May return an error
-     # Store in metadata
-     #
-     if ($num) {
- 	my %storestats=();
-         my $urlres=(&Apache::lonnet::decode_symb($symb))[2];
- 	$storestats{$courseid.'___'.$urlres.'___timestamp'}=time;
- 	$storestats{$courseid.'___'.$urlres.'___stdno'}=$num;
- 	$storestats{$courseid.'___'.$urlres.'___avetries'}=$mean;
- 	$storestats{$courseid.'___'.$urlres.'___difficulty'}=$DegOfDiff;
- 	$urlres=~/^(\w+)\/(\w+)/;
- 	&Apache::lonnet::put('nohist_resevaldata',\%storestats,$1,$2);
-     }
      #
      # Return result
-     #
      return { num_students => $num,
               tries        => $tries,
-              max_tries    => $mod,
+              max_tries    => $max,
+              min_tries    => $min,
               mean_tries   => $mean,
               std_tries    => $STD,
               skew_tries   => $SKEW,
               num_solved   => $Solved,
               num_override => $solved,
+              num_wrong    => $numwrong,
               per_wrong    => $wrongpercent,
               deg_of_diff  => $DegOfDiff };
  }
+ ##
+ ## This is a helper for get_statistics
  sub execute_SQL_request {
      my ($dbh,$request)=@_;
  #    &Apache::lonnet::logthis($request);
      my $sth = $dbh->prepare($request);
+     if (!$sth) {
+ 	die($dbh->errstr . " SQL: $request");
+     }
      $sth->execute();
      my $row = $sth->fetchrow_arrayref();
      if (ref($row) eq 'ARRAY' && scalar(@$row)>0) {
- Line 2025  sub execute_SQL_request {
+ Line 1881  sub execute_SQL_request {
      return ();
  }
+ sub populate_weight_table {
+     my ($courseid) = @_;
+     if (! defined($courseid)) {
+         $courseid = $env{'request.course.id'};
+     }
+     #
+     &setup_table_names($courseid);
+     my $navmap = Apache::lonnavmaps::navmap->new();
+     if (!defined($navmap)) {
+         &Apache::lonnet::logthis('loncoursedata::populate_weight_table:'.$/.
+                                  '  unable to get navmaps resource'.$/.
+                                  '  '.join(' ',(caller)));
+         return;
+     }
+     my @sequences = $navmap->retrieveResources(undef,
+                                                sub { shift->is_map(); },1,0,1);
+     my @resources;
+     foreach my $seq (@sequences) {
+         push(@resources,$navmap->retrieveResources($seq,
+                                                    sub {shift->is_gradable();},
+,0,0));
+     }
+     if (! scalar(@resources)) {
+         &Apache::lonnet::logthis('loncoursedata::populate_weight_table:'.$/.
+                                  ' no resources returned for '.$courseid);
+         return;
+     }
+     #       Since we use lonnet::EXT to retrieve problem weights,
+     #       to ensure current data we must clear the caches out.
+     &Apache::lonnet::clear_EXT_cache_status();
+     my $dbh = &Apache::lonmysql::get_dbh();
+     my $request = 'INSERT IGNORE INTO '.$weight_table.
+         "(symb_id,part_id,weight) VALUES ";
+     my $weight;
+     foreach my $res (@resources) {
+         my $symb_id = &get_symb_id($res->symb);
+         foreach my $part (@{$res->parts}) {
+             my $part_id = &get_part_id($part);
+             $weight = &Apache::lonnet::EXT('resource.'.$part.'.weight',
+                                            $res->symb,
+                                            undef,undef,undef);
+             if (!defined($weight) || ($weight eq '')) {
+                 $weight=1;
+             }
+             $request .= "('".$symb_id."','".$part_id."','".$weight."'),";
+         }
+     }
+     $request =~ s/(,)$//;
+ #    &Apache::lonnet::logthis('request = '.$/.$request);
+     $dbh->do($request);
+     if ($dbh->err()) {
+         &Apache::lonnet::logthis("error ".$dbh->errstr().
+                                  " occurred executing \n".
+                                  $request);
+     }
+     return;
+ }
+ sub limit_by_start_end_time {
+     my ($starttime,$endtime,$table) = @_;
+     my $time_requirements = undef;
+     if (defined($starttime)) {
+         $time_requirements .= $table.".timestamp>='".$starttime."'";
+         if (defined($endtime)) {
+             $time_requirements .= " AND ".$table.".timestamp<='".$endtime."'";
+         }
+     } elsif (defined($endtime)) {
+         $time_requirements .= $table.".timestamp<='".$endtime."'";
+     }
+     return $time_requirements;
+ }
+ sub limit_by_section_and_status {
+     my ($Sections,$enrollment,$tablename) = @_;
+     my $student_requirements = undef;
+     if ( (defined($Sections) && $Sections->[0] ne 'all')) {
+         $student_requirements = '('.
+             join(' OR ', map { $tablename.".section='".$_."'" } @$Sections
+                  ).')';
+     }
+     my $enrollment_requirements=undef;
+     if (defined($enrollment) && $enrollment ne 'Any') {
+ 	my $now = time();
+ 	if ( $enrollment eq 'Future' ) {
+ 	    $enrollment_requirements =
+ 		"( $tablename.start > $now AND ".
+ 		"( $tablename.end = 0 OR $tablename.end > $now))";
+ 	} elsif ( $enrollment eq 'Active' ) {
+ 	    $enrollment_requirements =
+ 		"(( $tablename.start = 0 OR $tablename.start < $now )  AND ".
+ 		" ( $tablename.end   = 0 OR $tablename.end   > $now ))";
+ 	} elsif ( $enrollment eq 'Expired' ) {
+ 	    $enrollment_requirements =
+ 		"(( $tablename.start < $now )  AND ".
+ 		" ( $tablename.end   < $now ))";
+ 	}
+     }
+     return ($student_requirements,$enrollment_requirements);
+ }
+ sub limit_by_group {
+     my ($Groups,$stutable,$grptable,$stugrptab) = @_;
+     my $groups_join = undef;
+     my $group_limits = undef;
+     if ( (defined($Groups) && $Groups->[0] ne 'all')) {
+         $groups_join =
+           ' LEFT JOIN '.$students_groups_table.
+                      ' AS '.$stugrptab.' ON '.
+                      $stugrptab.'.student_id = '.$stutable.'.student_id'.
+           ' LEFT JOIN '.$groupnames_table.
+                      ' AS '.$grptable.' ON '.
+                      $stugrptab.'.group_id = '.$grptable.'.group_id ';
+         $group_limits =
+           ' ('.
+              join(' OR ', map {  "$grptable.groupname='".$_."'" } @$Groups
+            ).')';
+     }
+     return ($groups_join,$group_limits);
+ }
+ sub RNK_student { return 0; };
+ sub RNK_score   { return 1; };
+ sub rank_students_by_scores_on_resources {
+     my ($resources,$Sections,$Groups,$enrollment,$courseid,$starttime,$endtime,
+         $has_award_for) = @_;
+     return if (! defined($resources) || ! ref($resources) eq 'ARRAY');
+     if (! defined($courseid)) {
+         $courseid = $env{'request.course.id'};
+     }
+     #
+     &setup_table_names($courseid);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     my ($section_limits,$enrollment_limits)=
+         &limit_by_section_and_status($Sections,$enrollment,'b');
+     my ($groups_join,$group_limits) = &limit_by_group($Groups,'b','c','d');
+     my $symb_limits = '('.join(' OR ',map {'a.symb_id='.&get_symb_id($_);
+                                        } @$resources
+                                ).')';
+     my ($award_col, $award_join, $award_clause) = ('', '', '');
+     if ($has_award_for) {
+         my $resource_id = &get_symb_id($has_award_for);
+         $award_col = ", perf.awarded";
+         $award_join = "LEFT JOIN $performance_table AS perf ON perf.symb_id"
+             ." = $resource_id AND perf.student_id = b.student_id ";
+         $award_clause = "AND perf.awarded IS NOT NULL";
+     }
+     my $time_limits = &limit_by_start_end_time($starttime,$endtime,'a');
+     my $request = "SELECT b.student,SUM(a.awarded*w.weight) AS score "
+         ."$award_col FROM $performance_table AS a ".
+         "NATURAL LEFT JOIN $weight_table AS w ".
+         "LEFT JOIN $student_table AS b ON a.student_id=b.student_id ".
+         "$award_join $groups_join ";
+     my $limits;
+     if (defined($section_limits)) {
+         $limits .= $section_limits.' AND ';
+     }
+     if (defined($enrollment_limits)) {
+         $limits .= $enrollment_limits.' AND ';
+     }
+     if (defined($time_limits)) {
+         $limits .= $time_limits.' AND ';
+     }
+     if ($symb_limits ne '()') {
+         $limits .= $symb_limits.' AND ';
+     }
+     if (defined($group_limits)) {
+         $limits .= $group_limits.' AND ';
+     }
+     if ($limits) {
+         $limits =~ s/( AND )$//;   # Remove extra conjunction
+         $request .= "WHERE $limits";
+     }
+     $request .= " $award_clause GROUP BY a.student_id ORDER BY score, b.student";
+     #&Apache::lonnet::logthis('request = '.$/.$request);
+     my $sth = $dbh->prepare($request) or die "Can't prepare $request";
+     $sth->execute();
+     my $rows = $sth->fetchall_arrayref();
+     return ($rows);
+ }
+ sub get_sum_of_scores {
+     my ($symb,$part,$students,$courseid,$starttime,$endtime) = @_;
+     if (! defined($courseid)) {
+         $courseid = $env{'request.course.id'};
+     }
+     if (defined($students) &&
+         ((@$students == 0) ||
+          (@$students == 1 && (! defined($students->[0]) ||
+                               $students->[0] eq ''))
+          )
+         ){
+         undef($students);
+     }
+     #
+     &setup_table_names($courseid);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     my $time_limits = &limit_by_start_end_time($starttime,$endtime,'a');
+     my $request = 'SELECT SUM(a.awarded*w.weight),SUM(w.weight) FROM '.
+         $performance_table.' AS a '.
+         'NATURAL LEFT JOIN '.$weight_table.' AS w ';
+     $request .= 'WHERE a.symb_id='.&get_symb_id($symb).
+         ' AND a.part_id='.&get_part_id($part);
+     if (defined($time_limits)) {
+         $request .= ' AND '.$time_limits;
+     }
+     if (defined($students)) {
+         $request .= ' AND ('.
+             join(' OR ',map {'a.student_id='.&get_student_id(split(':',$_));
+                          } @$students).
+                              ')';
+     }
+     my $sth = $dbh->prepare($request);
+     $sth->execute();
+     my $rows = $sth->fetchrow_arrayref();
+     if ($dbh->err) {
+         &Apache::lonnet::logthis('error 1 = '.$dbh->errstr());
+         &Apache::lonnet::logthis('prepared then executed, fetchrow_arrayrefed'.
+                                  $/.$request);
+         return (undef,undef);
+     }
+     return ($rows->[0],$rows->[1]);
+ }
+ sub score_stats {
+     my ($Sections,$Groups,$enrollment,$symbs,$starttime,$endtime,$courseid)=@_;
+     if (! defined($courseid)) {
+         $courseid = $env{'request.course.id'};
+     }
+     #
+     &setup_table_names($courseid);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     #
+     my ($section_limits,$enrollment_limits)=
+         &limit_by_section_and_status($Sections,$enrollment,'b');
+     my ($groups_join,$group_limits) = &limit_by_group($Groups,'b','c','d');
+     my $time_limits = &limit_by_start_end_time($starttime,$endtime,'a');
+     my @Symbids = map { &get_symb_id($_); } @{$symbs};
+     #
+     my $stats_table = &temp_table_name($courseid,'problem_stats');
+     my $symb_restriction = join(' OR ',map {'a.symb_id='.$_;} @Symbids);
+     my $request = 'DROP TABLE '.$stats_table;
+     $dbh->do($request);
+     $request =
+         'CREATE TEMPORARY TABLE '.$stats_table.' '.
+         'SELECT a.student_id,'.
+         'SUM(a.awarded*w.weight) AS score FROM '.
+         $performance_table.' AS a '.
+         'NATURAL LEFT JOIN '.$weight_table.' AS w '.
+         'LEFT JOIN '.$student_table.' AS b ON a.student_id=b.student_id '.
+         $groups_join;
+     my $limit = ' WHERE ('.$symb_restriction.')';
+     if ($time_limits) {
+         $limit .= ' AND '.$time_limits;
+     }
+     if ($section_limits) {
+         $limit .= ' AND '.$section_limits;
+     }
+     if ($enrollment_limits) {
+         $limit .= ' AND '.$enrollment_limits;
+     }
+     if ($group_limits) {
+         $limit .= ' AND '.$group_limits;
+     }
+     $request .= $limit.' GROUP BY a.student_id';
+ #    &Apache::lonnet::logthis('request = '.$/.$request);
+     my $sth = $dbh->prepare($request);
+     $sth->execute();
+     $request =
+         'SELECT AVG(score),STD(score),MAX(score),MIN(score),COUNT(score) '.
+         'FROM '.$stats_table;
+     my ($ave,$std,$max,$min,$count) = &execute_SQL_request($dbh,$request);
+ #    &Apache::lonnet::logthis('request = '.$/.$request);
+     $request = 'SELECT SUM(weight) FROM '.$weight_table.
+         ' AS a WHERE ('.$symb_restriction.')';
+     my ($max_possible) = &execute_SQL_request($dbh,$request);
+     # &Apache::lonnet::logthis('request = '.$/.$request);
+     return($min,$max,$ave,$std,$count,$max_possible);
+ }
+ sub count_stats {
+     my ($Sections,$Groups,$enrollment,$symbs,$starttime,$endtime,$courseid)=@_;
+     if (! defined($courseid)) {
+         $courseid = $env{'request.course.id'};
+     }
+     #
+     &setup_table_names($courseid);
+     my $dbh = &Apache::lonmysql::get_dbh();
+     #
+     my ($section_limits,$enrollment_limits)=
+         &limit_by_section_and_status($Sections,$enrollment,'b');
+     my ($groups_join,$group_limits) = &limit_by_group($Groups,'b','c','d');
+     my $time_limits = &limit_by_start_end_time($starttime,$endtime,'a');
+     my @Symbids = map { &get_symb_id($_); } @{$symbs};
+     #
+     my $stats_table = &temp_table_name($courseid,'problem_stats');
+     my $symb_restriction = join(' OR ',map {'a.symb_id='.$_;} @Symbids);
+     my $request = 'DROP TABLE '.$stats_table;
+     $dbh->do($request);
+     $request =
+         'CREATE TEMPORARY TABLE '.$stats_table.' '.
+         'SELECT a.student_id,'.
+         'SUM(a.awarded) AS count FROM '.
+         $performance_table.' AS a '.
+         'LEFT JOIN '.$student_table.' AS b ON a.student_id=b.student_id '.
+         $groups_join;
+     my $limit =  ' WHERE ('.$symb_restriction.')';
+     if ($time_limits) {
+         $limit .= ' AND '.$time_limits;
+     }
+     if ($section_limits) {
+         $limit .= ' AND '.$section_limits;
+     }
+     if ($enrollment_limits) {
+         $limit .= ' AND '.$enrollment_limits;
+     }
+     if ($group_limits) {
+         $limit .= ' AND '.$group_limits;
+     }
+     $request .= $limit.' GROUP BY a.student_id';
+ #    &Apache::lonnet::logthis('request = '.$/.$request);
+     my $sth = $dbh->prepare($request);
+     $sth->execute();
+     $request =
+         'SELECT AVG(count),STD(count),MAX(count),MIN(count),COUNT(count) '.
+         'FROM '.$stats_table;
+     my ($ave,$std,$max,$min,$count) = &execute_SQL_request($dbh,$request);
+ #    &Apache::lonnet::logthis('request = '.$/.$request);
+     return($min,$max,$ave,$std,$count);
+ }
  sub get_student_data {
      my ($students,$courseid) = @_;
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      &setup_table_names($courseid);
      my $dbh = &Apache::lonmysql::get_dbh();
      return undef if (! defined($dbh));
- Line 2046  sub get_student_data {
+ Line 2243  sub get_student_data {
      my $sth = $dbh->prepare($request);
      $sth->execute();
      if ($dbh->err) {
-         &Apache::lonnet::logthis('error = '.$dbh->errstr());
+         &Apache::lonnet::logthis('error 2 = '.$dbh->errstr());
+         &Apache::lonnet::logthis('prepared then executed '.$/.$request);
          return undef;
      }
      my $dataset = $sth->fetchall_arrayref();
- Line 2055  sub get_student_data {
+ Line 2253  sub get_student_data {
      }
  }
- sub RD_student_id    { return 0; }
+ sub RD_student_id      { return 0; }
- sub RD_awarddetail   { return 1; }
+ sub RD_awarddetail     { return 1; }
- sub RD_response_eval { return 2; }
+ sub RD_response_eval   { return 2; }
- sub RD_submission    { return 3; }
+ sub RD_response_eval_2 { return 3; }
- sub RD_timestamp     { return 4; }
+ sub RD_submission      { return 4; }
- sub RD_tries         { return 5; }
+ sub RD_timestamp       { return 5; }
- sub RD_sname         { return 6; }
+ sub RD_tries           { return 6; }
+ sub RD_sname           { return 7; }
  sub get_response_data {
-     my ($students,$symb,$response,$courseid) = @_;
+     my ($Sections,$Groups,$enrollment,$symb,$response,$courseid) = @_;
      return undef if (! defined($symb) ||
                 ! defined($response));
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      #
      &setup_table_names($courseid);
      my $symb_id = &get_symb_id($symb);
+     if (! defined($symb_id)) {
+         &Apache::lonnet::logthis('Unable to find symb for '.$symb.' in '.$courseid);
+         return undef;
+     }
      my $response_id = &get_part_id($response);
+     if (! defined($response_id)) {
+         &Apache::lonnet::logthis('Unable to find id for '.$response.' in '.$courseid);
+         return undef;
+     }
      #
      my $dbh = &Apache::lonmysql::get_dbh();
      return undef if (! defined($dbh));
+     #
+     my ($student_requirements,$enrollment_requirements) =
+         &limit_by_section_and_status($Sections,$enrollment,'d');
+     my ($groups_join,$group_limits) = &limit_by_group($Groups,'d','e','f');
      my $request = 'SELECT '.
          'a.student_id, a.awarddetail, a.response_specific_value, '.
-         'a.submission, b.timestamp, c.tries, d.student '.
+         'a.response_specific_value_2, a.submission, b.timestamp, '.
+ 	'c.tries, d.student '.
          'FROM '.$fulldump_response_table.' AS a '.
          'LEFT JOIN '.$fulldump_timestamp_table.' AS b '.
          'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '.
- Line 2087  sub get_response_data {
+ Line 2299  sub get_response_data {
          'a.part_id=c.part_id AND a.transaction = c.transaction '.
          'LEFT JOIN '.$student_table.' AS d '.
          'ON a.student_id=d.student_id '.
-         'WHERE '.
+         $groups_join;
+     my $limit = ' WHERE '.
          'a.symb_id='.$symb_id.' AND a.response_id='.$response_id;
-     if (defined($students)) {
+     if (defined($student_requirements)) {
-         $request .= ' AND ('.
+         $limit .= ' AND '.$student_requirements;
-             join(' OR ', map {'a.student_id='.
+     }
-                                   &get_student_id($_->{'username'},
+     if (defined($enrollment_requirements)) {
-                                                   $_->{'domain'})
+         $limit .= ' AND '.$enrollment_requirements;
-                               } @$students
-                  ).')';
      }
-     $request .= ' ORDER BY b.timestamp';
+     if (defined($group_limits)) {
+         $limit .= ' AND '.$group_limits;
+     }
+     $request .= $limit.' ORDER BY b.timestamp';
  #    &Apache::lonnet::logthis("request =\n".$request);
      my $sth = $dbh->prepare($request);
      $sth->execute();
      if ($dbh->err) {
-         &Apache::lonnet::logthis('error = '.$dbh->errstr());
+         &Apache::lonnet::logthis('error 3 = '.$dbh->errstr());
+         &Apache::lonnet::logthis('prepared then executed '.$/.$request);
          return undef;
      }
      my $dataset = $sth->fetchall_arrayref();
      if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
          # Clear the \'s from around the submission
          for (my $i =0;$i<scalar(@$dataset);$i++) {
-             $dataset->[$i]->[3] =~ s/(\'$|^\')//g;
+             $dataset->[$i]->[&RD_submission()] =~ s/(\'$|^\')//g;
          }
          return $dataset;
      }
  }
- sub RDs_awarddetail   { return 3; }
+ sub RDs_awarddetail     { return 3; }
- sub RDs_submission    { return 2; }
+ sub RDs_submission      { return 2; }
- sub RDs_timestamp     { return 1; }
+ sub RDs_timestamp       { return 1; }
- sub RDs_tries         { return 0; }
+ sub RDs_tries           { return 0; }
- sub RDs_awarded       { return 4; }
+ sub RDs_awarded         { return 4; }
+ sub RDs_response_eval   { return 5; }
+ sub RDs_response_eval_2 { return 6; }
+ sub RDs_part_award      { return 7; }
  sub get_response_data_by_student {
      my ($student,$symb,$response,$courseid) = @_;
      return undef if (! defined($symb) ||
                       ! defined($response));
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      #
      &setup_table_names($courseid);
      my $symb_id = &get_symb_id($symb);
- Line 2138  sub get_response_data_by_student {
+ Line 2356  sub get_response_data_by_student {
      my $dbh = &Apache::lonmysql::get_dbh();
      return undef if (! defined($dbh));
      my $request = 'SELECT '.
-         'c.tries, b.timestamp, a.submission, a.awarddetail, e.awarded '.
+         'c.tries, b.timestamp, a.submission, a.awarddetail, c.awarded, '.
+ 	'a.response_specific_value, a.response_specific_value_2, c.award '.
          'FROM '.$fulldump_response_table.' AS a '.
          'LEFT JOIN '.$fulldump_timestamp_table.' AS b '.
          'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '.
- Line 2154  sub get_response_data_by_student {
+ Line 2373  sub get_response_data_by_student {
          'WHERE '.
          'a.symb_id='.$symb_id.' AND a.response_id='.$response_id.
          ' AND a.student_id='.$student_id.' ORDER BY b.timestamp';
-     # &Apache::lonnet::logthis("request =\n".$request);
+ #    &Apache::lonnet::logthis("request =\n".$request);
      my $sth = $dbh->prepare($request);
      $sth->execute();
      if ($dbh->err) {
-         &Apache::lonnet::logthis('error = '.$dbh->errstr());
+         &Apache::lonnet::logthis('error 4 = '.$dbh->errstr());
+         &Apache::lonnet::logthis('prepared then executed '.$/.$request);
          return undef;
      }
      my $dataset = $sth->fetchall_arrayref();
      if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) {
          # Clear the \'s from around the submission
          for (my $i =0;$i<scalar(@$dataset);$i++) {
-             $dataset->[$i]->[2] =~ s/(\'$|^\')//g;
+             $dataset->[$i]->[&RDs_submission] =~ s/(\'$|^\')//g;
          }
          return $dataset;
      }
- Line 2178  sub RT_tries      { return 2; }
+ Line 2398  sub RT_tries      { return 2; }
  sub RT_timestamp  { return 3; }
  sub get_response_time_data {
-     my ($students,$symb,$part,$courseid) = @_;
+     my ($sections,$groups,$enrollment,$symb,$part,$courseid) = @_;
      return undef if (! defined($symb) ||
                       ! defined($part));
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      #
      &setup_table_names($courseid);
      my $symb_id = &get_symb_id($symb);
+     if (! defined($symb_id)) {
+         &Apache::lonnet::logthis('Unable to find symb for '.$symb.' in '.$courseid);
+         return undef;
+     }
      my $part_id = &get_part_id($part);
+     if (! defined($part_id)) {
+         &Apache::lonnet::logthis('Unable to find id for '.$part.' in '.$courseid);
+         return undef;
+     }
      #
      my $dbh = &Apache::lonmysql::get_dbh();
      return undef if (! defined($dbh));
+     my ($student_requirements,$enrollment_requirements) =
+         &limit_by_section_and_status($sections,$enrollment,'d');
+     my ($groups_join,$group_limits) = &limit_by_group($groups,'d','e','f');
      my $request = 'SELECT '.
          'a.student_id, a.awarded, a.tries, b.timestamp '.
          'FROM '.$fulldump_part_table.' AS a '.
-         'NATURAL LEFT JOIN '.$fulldump_timestamp_table.' AS b '.
+         'LEFT JOIN '.$fulldump_timestamp_table.' AS b '.
- #        'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '.
+         'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '.
- #        'a.transaction = b.transaction '.
+         'a.transaction = b.transaction '.
-         'WHERE '.
+         'LEFT JOIN '.$student_table.' as d '.
+         'ON a.student_id=d.student_id '.
+         $groups_join;
+     my $limit = ' WHERE '.
          'a.symb_id='.$symb_id.' AND a.part_id='.$part_id;
-     if (defined($students)) {
+     if (defined($student_requirements)) {
-         $request .= ' AND ('.
+         $limit .= ' AND '.$student_requirements;
-             join(' OR ', map {'a.student_id='.
+     }
-                                   &get_student_id($_->{'username'},
+     if (defined($enrollment_requirements)) {
-                                                   $_->{'domain'})
+         $limit .= ' AND '.$enrollment_requirements;
-                               } @$students
-                  ).')';
      }
-     $request .= ' ORDER BY b.timestamp';
+     if (defined($group_limits)) {
+         $limit .= ' AND '.$group_limits;
+     }
+     $request .= $limit.' ORDER BY b.timestamp';
  #    &Apache::lonnet::logthis("request =\n".$request);
      my $sth = $dbh->prepare($request);
      $sth->execute();
      if ($dbh->err) {
-         &Apache::lonnet::logthis('error = '.$dbh->errstr());
+         &Apache::lonnet::logthis('error 5 = '.$dbh->errstr());
+         &Apache::lonnet::logthis('prepared then executed '.$/.$request);
          return undef;
      }
      my $dataset = $sth->fetchall_arrayref();
- Line 2220  sub get_response_time_data {
+ Line 2456  sub get_response_time_data {
  }
- ################################################
- ################################################
- =pod
- =item &get_student_scores($Sections,$Symbs,$enrollment,$courseid)
- =cut
- ################################################
- ################################################
  sub get_student_scores {
-     my ($Sections,$Symbs,$enrollment,$courseid) = @_;
+     my ($sections,$groups,$Symbs,$enrollment,$courseid,$starttime,$endtime) = @_;
-     $courseid = $ENV{'request.course.id'} if (! defined($courseid));
+     $courseid = $env{'request.course.id'} if (! defined($courseid));
      &setup_table_names($courseid);
      my $dbh = &Apache::lonmysql::get_dbh();
      return (undef) if (! defined($dbh));
-     my $tmptable = $courseid.'_temp_'.time;
+     my $tmptable = &temp_table_name($courseid,'temp_'.time);
+     my $request = 'DROP TABLE IF EXISTS '.$tmptable;
+ #    &Apache::lonnet::logthis('request = '.$/.$request);
+     $dbh->do($request);
      #
      my $symb_requirements;
      if (defined($Symbs)  && @$Symbs) {
          $symb_requirements = '('.
              join(' OR ', map{ "(a.symb_id='".&get_symb_id($_->{'symb'}).
-                                   "' AND a.part_id='".&get_part_id($_->{'part'}).
+                               "' AND a.part_id='".&get_part_id($_->{'part'}).
-                                   "')"
+                               "')"
                                } @$Symbs).')';
      }
      #
-     my $student_requirements;
+     my ($student_requirements,$enrollment_requirements) =
-     if ( (defined($Sections) && $Sections->[0] ne 'all')) {
+         &limit_by_section_and_status($sections,$enrollment,'b');
-         $student_requirements = '('.
-             join(' OR ', map { "b.section='".$_."'" } @$Sections
-                  ).')';
-     }
      #
-     my $enrollment_requirements=undef;
+     my ($groups_join,$group_limits) = &limit_by_group($groups,'b','d','e');
-     if (defined($enrollment) && $enrollment ne 'Any') {
+     my $time_requirements = &limit_by_start_end_time($starttime,$endtime,'a');
-         $enrollment_requirements = "b.status='".$enrollment."'";
-     }
      ##
-     ##
+     $request = 'CREATE TEMPORARY TABLE IF NOT EXISTS '.$tmptable.
-     my $request = 'CREATE TEMPORARY TABLE IF NOT EXISTS '.$tmptable.
+         ' SELECT a.student_id,SUM(a.awarded*c.weight) AS score FROM '.
-         ' SELECT a.student_id,SUM(a.awarded) AS score FROM '.
          $performance_table.' AS a ';
+     $request .= "LEFT JOIN ".$weight_table.' AS c ON a.symb_id=c.symb_id AND a.part_id=c.part_id ';
      if (defined($student_requirements) || defined($enrollment_requirements)) {
-         $request .= ' NATURAL LEFT JOIN '.$student_table.' AS b ';
+         $request .= ' LEFT JOIN '.$student_table.' AS b ON a.student_id=b.student_id ';
+     }
+     if (defined($groups_join)) {
+         $request .= $groups_join;
      }
-     if (defined($symb_requirements)      ||
+     if (defined($symb_requirements)       ||
-         defined($student_requirements)   ||
+         defined($student_requirements)    ||
-         defined($enrollment_requirements) ) {
+         defined($enrollment_requirements) ||
+         defined($group_limits) ) {
          $request .= ' WHERE ';
      }
      if (defined($symb_requirements)) {
- Line 2281  sub get_student_scores {
+ Line 2507  sub get_student_scores {
      if (defined($enrollment_requirements)) {
          $request .= $enrollment_requirements.' AND ';
      }
-     $request =~ s/ AND $//;
+     if (defined($time_requirements)) {
+         $request .= $time_requirements.' AND ';
+     }
+     $request =~ s/ AND $//; # Strip of the trailing ' AND '.
      $request .= ' GROUP BY a.student_id';
  #    &Apache::lonnet::logthis("request = \n".$request);
      my $sth = $dbh->prepare($request);
      $sth->execute();
      if ($dbh->err) {
-         &Apache::lonnet::logthis('error = '.$dbh->errstr());
+         &Apache::lonnet::logthis('error 6 = '.$dbh->errstr());
+         &Apache::lonnet::logthis('prepared then executed '.$/.$request);
          return undef;
      }
      $request = 'SELECT score,COUNT(*) FROM '.$tmptable.' GROUP BY score';
- Line 2295  sub get_student_scores {
+ Line 2525  sub get_student_scores {
      $sth = $dbh->prepare($request);
      $sth->execute();
      if ($dbh->err) {
-         &Apache::lonnet::logthis('error = '.$dbh->errstr());
+         &Apache::lonnet::logthis('error 7 = '.$dbh->errstr());
+         &Apache::lonnet::logthis('prepared then executed '.$/.$request);
          return undef;
      }
      my $dataset = $sth->fetchall_arrayref();
      return $dataset;
  }
- ################################################
- ################################################
- =pod
- =item &setup_table_names()
- input: course id
- output: none
- Cleans up the package variables for local caching.
- =cut
- ################################################
- ################################################
  sub setup_table_names {
      my ($courseid) = @_;
      if (! defined($courseid)) {
-         $courseid = $ENV{'request.course.id'};
+         $courseid = $env{'request.course.id'};
      }
      #
      if (! defined($current_course) || $current_course ne $courseid) {
- Line 2336  sub setup_table_names {
+ Line 2552  sub setup_table_names {
          $have_read_student_table = 0;
          undef(%ids_by_student);
          undef(%students_by_id);
+         $have_read_groupnames_table = 0;
+         undef(%ids_by_groupname);
          #
          $current_course = $courseid;
      }
      #
      # Set up database names
-     my $base_id = $courseid;
+     my $base_id = 'md5_'.&Digest::MD5::md5_hex($courseid);
-     $symb_table        = $base_id.'_'.'symb';
+     $symb_table               = $base_id.'_'.'symb';
-     $part_table        = $base_id.'_'.'part';
+     $part_table               = $base_id.'_'.'part';
-     $student_table     = $base_id.'_'.'student';
+     $student_table            = $base_id.'_'.'student';
-     $performance_table = $base_id.'_'.'performance';
+     $groupnames_table         = $base_id.'_'.'groupnames';
-     $parameters_table  = $base_id.'_'.'parameters';
+     $students_groups_table    = $base_id.'_'.'studentgroups';
+     $performance_table        = $base_id.'_'.'performance';
+     $parameters_table         = $base_id.'_'.'parameters';
      $fulldump_part_table      = $base_id.'_'.'partdata';
      $fulldump_response_table  = $base_id.'_'.'responsedata';
      $fulldump_timestamp_table = $base_id.'_'.'timestampdata';
+     $weight_table             = $base_id.'_'.'weight';
      #
      @Tables = (
                 $symb_table,
                 $part_table,
                 $student_table,
+                $groupnames_table,
+                $students_groups_table,
                 $performance_table,
                 $parameters_table,
                 $fulldump_part_table,
                 $fulldump_response_table,
                 $fulldump_timestamp_table,
+                $weight_table,
                 );
      return;
  }
- ################################################
+ sub temp_table_name {
- ################################################
+     my ($courseid,$affix) = @_;
+     my $base_id = 'md5_'.&Digest::MD5::md5_hex($courseid);
- =pod
+     return $base_id.'_'.$affix;
+ }
- =back
- =item End of Local Data Caching Subroutines
- =cut
- ################################################
- ################################################
  } # End scope of table identifiers
- ################################################
- ################################################
- =pod
- =head3 Classlist Subroutines
- =item &get_classlist();
- Retrieve the classist of a given class or of the current class.  Student
- information is returned from the classlist.db file and, if needed,
- from the students environment.
- Optional arguments are $cid, $cdom, and $cnum (course id, course domain,
- and course number, respectively).  Any omitted arguments will be taken
- from the current environment ($ENV{'request.course.id'},
- $ENV{'course.'.$cid.'.domain'}, and $ENV{'course.'.$cid.'.num'}).
- Returns a reference to a hash which contains:
-  keys    '$sname:$sdom'
-  values  [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type]
- The constant values CL_SDOM, CL_SNAME, CL_END, etc. can be used
- as indices into the returned list to future-proof clients against
- changes in the list order.
- =cut
- ################################################
- ################################################
  sub CL_SDOM     { return 0; }
  sub CL_SNAME    { return 1; }
- Line 2420  sub CL_SECTION  { return 5; }
+ Line 2608  sub CL_SECTION  { return 5; }
  sub CL_FULLNAME { return 6; }
  sub CL_STATUS   { return 7; }
  sub CL_TYPE     { return 8; }
+ sub CL_LOCKEDTYPE   { return 9; }
+ sub CL_CREDITS  { return 10; }
+ sub CL_INSTSEC { return 11; }
+ sub CL_GROUP    { return 12; }
+ sub CL_PERMANENTEMAIL { return 13; }
+ sub CL_ROLE     { return 14; }
+ sub CL_EXTENT   { return 15; }
+ sub CL_PHOTO   { return 16; }
+ sub CL_THUMBNAIL { return 17; }
+ sub CL_AUTHORQUOTA { return 18; }
+ sub CL_AUTHORUSAGE { return 19; }
  sub get_classlist {
-     my ($cid,$cdom,$cnum) = @_;
+     my ($cdom,$cnum) = @_;
-     $cid = $cid || $ENV{'request.course.id'};
+     my $cid = $cdom.'_'.$cnum;
-     $cdom = $cdom || $ENV{'course.'.$cid.'.domain'};
+     if (!defined($cdom) || !defined($cnum)) {
-     $cnum = $cnum || $ENV{'course.'.$cid.'.num'};
+ 	$cid =  $env{'request.course.id'};
+ 	$cdom = $env{'course.'.$cid.'.domain'};
+ 	$cnum = $env{'course.'.$cid.'.num'};
+     }
      my $now = time;
      #
      my %classlist=&Apache::lonnet::dump('classlist',$cdom,$cnum);
- Line 2436  sub get_classlist {
+ Line 2638  sub get_classlist {
          }
          my ($sname,$sdom) = split(/:/,$student);
          my @Values = split(/:/,$info);
-         my ($end,$start,$id,$section,$fullname,$type);
+         my ($end,$start,$id,$section,$fullname,$type,$lockedtype,$credits,$instsec);
          if (@Values > 2) {
-             ($end,$start,$id,$section,$fullname,$type) = @Values;
+             ($end,$start,$id,$section,$fullname,$type,$lockedtype,$credits,$instsec) = @Values;
          } else { # We have to get the data ourselves
              ($end,$start) = @Values;
              $section = &Apache::lonnet::getsection($sdom,$sname,$cid);
- Line 2453  sub get_classlist {
+ Line 2655  sub get_classlist {
                  &Apache::lonnet::logthis('unable to retrieve environment '.
                                           'for '.$sname.':'.$sdom);
              } else {
-                 $fullname = &ProcessFullName(@info{qw/lastname generation
+                 $fullname = &Apache::lonnet::format_name(@info{qw/firstname middlename lastname generation/},'lastname');
-                                                        firstname middlename/});
                  $id = $info{'id'};
              }
              # Update the classlist with this students information
              if ($fullname ne 'not available') {
-                 my $enrolldata = join(':',$end,$start,$id,$section,$fullname);
+ 		my $enrolldata = join(':',$end,$start,$id,$section,$fullname);
-                 my $reply=&Apache::lonnet::cput('classlist',
+ 		my $reply=&Apache::lonnet::cput('classlist',
                                                  {$student => $enrolldata},
                                                  $cdom,$cnum);
                  if ($reply !~ /^(ok|delayed)/) {
- Line 2474  sub get_classlist {
+ Line 2675  sub get_classlist {
          if(((!$end) || $now < $end) && ((!$start) || ($now > $start))) {
              $status='Active';
          }
+         if(($now < $start) && ((!$end) || $now < $end )) {
+             $status='Future';
+         }
          $classlist{$student} =
-             [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type];
+             [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type,
+              $lockedtype,$credits,$instsec];
      }
      if (wantarray()) {
          return (\%classlist,['domain','username','end','start','id',
-                              'section','fullname','status','type']);
+                              'section','fullname','status','type',
+                              'lockedtype','credits','instsec']);
      } else {
          return \%classlist;
      }
  }
+ sub get_group_memberships {
+     my ($classlist,$keylist,$cdom,$cnum) = @_;
+     return ({},{}) if (!ref($classlist) || !ref($keylist));
+     my $cid = $cdom.'_'.$cnum;
+     if (!defined($cdom) || !defined($cnum)) {
+         $cid =  $env{'request.course.id'};
+         $cdom = $env{'course.'.$cid.'.domain'};
+         $cnum = $env{'course.'.$cid.'.num'};
+     }
+     my (%classgroups,%studentgroups);
+     my $now = time;
+     my $access_end = $env{'course.'.$cid.'.default_enrollment_end_date'};
+     my %curr_groups =&Apache::longroup::coursegroups($cdom,$cnum);
+     if (%curr_groups) {
+         my $grpindex = &CL_GROUP();
+         my %groupmemberhash =
+ 	    &Apache::lonnet::get_group_membership($cdom,$cnum);
+         foreach my $student (keys(%{$classlist})) {
+             %{$classgroups{$student}} = ();
+             my $hasgroup = 0;
+             foreach my $status ('previous','future','active','aftercourse') {
+                 %{$classgroups{$student}{$status}} = ();
+             }
+             foreach my $group (keys(%curr_groups)) {
+                 if (defined($groupmemberhash{$group.':'.$student})) {
+                     my ($end,$start) = split(/:/,$groupmemberhash{$group.':'.
+                                                                     $student});
+                     if ($start == -1) {
+                         next;
+                     } else {
+                         $studentgroups{$group} ++;
+                         $hasgroup = 1;
+                         if ($end && $end < $now) {
+                             $classgroups{$student}{'previous'}{$group} =
+                                          $groupmemberhash{$group.':'.$student};
+                             if ($classlist->{$student}[&CL_STATUS()] eq 'Expired') {
+                                 if ($access_end && $access_end < $now) {
+                                     if ($access_end - $end < 86400) {
+                                         $classgroups{$student}{'aftercourse'}{$group} = $groupmemberhash{$group.':'.$student};
+                                     }
+                                 }
+                             }
+                         } elsif ($now > $start) {
+                             if (!$end || $end > $now) {
+                                 $classgroups{$student}{'active'}{$group} =
+                                          $groupmemberhash{$group.':'.$student};
+                             }
+                         } else {
+                             $classgroups{$student}{'future'}{$group} =
+                                          $groupmemberhash{$group.':'.$student};
+                         }
+                     }
+                 }
+             }
+             if (!$hasgroup) {
+                 $studentgroups{'none'} ++;
+             } else {
+                 $classlist->{$student}->[$grpindex] = join(',',
+                               sort(keys(%{$classgroups{$student}{'active'}})));
+             }
+         }
+     }
+     return (\%classgroups,\%studentgroups);
+ }
+ sub get_students_groups {
+     my ($student,$enrollment_status,$classgroups) = @_;
+     my @studentsgroups = ();
+     if (ref($$classgroups{$student}{'active'}) eq 'HASH') {
+         push(@studentsgroups,keys(%{$$classgroups{$student}{'active'}}));
+     }
+     if ($enrollment_status eq 'Any') {
+         foreach my $status ('previous','future') {
+             if (ref($$classgroups{$student}{$status}) eq 'HASH') {
+                 push(@studentsgroups,keys(%{$$classgroups{$student}{$status}}));
+             }
+         }
+     } else {
+         if (ref($$classgroups{$student}{'aftercourse'}) eq 'HASH') {
+             push(@studentsgroups,keys(%{$$classgroups{$student}{'aftercourse'}}));
+         }
+     }
+     return @studentsgroups;
+ }
  # ----- END HELPER FUNCTIONS --------------------------------------------
 ;
  __END__
+ =pod
+ =head1 NAME
+ Apache::loncoursedata
+ =head1 SYNOPSIS
+ Set of functions that download and process student and course information.
+ =head1 PACKAGES USED
+   Apache::lonnet
+   Apache::longroup
+   Time::HiRes
+   Apache::lonmysql
+   LONCAPA
+   Digest::MD5
+ =head1 DOWNLOAD INFORMATION
+ This section contains all the functions that get data from other servers
+ and/or itself.
+ =head1 LOCAL DATA CACHING SUBROUTINES
+ The local caching is done using MySQL.  There is no fall-back implementation
+ if MySQL is not running.
+ The programmers interface is to call &get_current_state() or some other
+ primary interface subroutine (described below).  The internals of this
+ storage system are documented here.
+ There are six tables used to store student performance data (the results of
+ a dumpcurrent).  Each of these tables is created in MySQL with a name of
+ $courseid_*****, where ***** is 'symb', 'part', or whatever is appropriate
+ for the table.  The tables and their purposes are described below.
+ Some notes before we get started.
+ Each table must have a PRIMARY KEY, which is a column or set of columns which
+ will serve to uniquely identify a row of data.  NULL is not allowed!
+ INDEXes work best on integer data.
+ JOIN is used to combine data from many tables into one output.
+ lonmysql.pm is used for some of the interface, specifically the table creation
+ calls.  The inserts are done in bulk by directly calling the database handler.
+ The SELECT ... JOIN statement used to retrieve the data does not have an
+ interface in lonmysql.pm and I shudder at the thought of writing one.
+ =head2 Table Descriptions
+ =over 4
+ =head2 Tables used to store meta information
+ The following tables hold data required to keep track of the current status
+ of a students data in the tables or to look up the students data in the tables.
+ =over 4
+ =item C<$symb_table>
+ The symb_table has two columns.  The first is a 'symb_id' and the second
+ is the text name for the 'symb' (limited to 64k).  The 'symb_id' is generated
+ automatically by MySQL so inserts should be done on this table with an
+ empty first element.  This table has its PRIMARY KEY on the 'symb_id'.
+ =item C<$part_table>
+ The part_table has two columns.  The first is a 'part_id' and the second
+ is the text name for the 'part' (limited to 100 characters).  The 'part_id' is
+ generated automatically by MySQL so inserts should be done on this table with
+ an empty first element.  This table has its PRIMARY KEY on the 'part' (100
+ characters) and a KEY on 'part_id'.
+ =item C<$student_table>
+ The student_table has 7 columns.  The first is a 'student_id' assigned by
+ MySQL.  The second is 'student' which is username:domain.  The third through
+ fifth are 'section', 'status' (enrollment status), and 'classification'
+ (to be used in the future).  The sixth and seventh ('updatetime' and
+ 'fullupdatetime') contain the time of last update and full update of student
+ data.  This table has its PRIMARY KEY on the 'student_id' column and is indexed
+ on 'student', 'section', and 'status'.
+ =item C<$groupnames_table>
+ The groupnames_table has 2 columns.  The first is a 'group_id' assigned by
+ MySQL.  The second is 'groupname' which is the name of the group in the course.
+ =item C<$students_groups_table>
+ The students_groups_table has 2 columns.  The first is the 'student_id', and the
+ second is the 'group_id'. These two columns comprise the PRIMARY KEY for this
+ table, as an individual student may be affiliated with more than one group at
+ any time. This table is indexed on both student_id and group_id.
+ =back
+ =head2 Tables used to store current status data
+ The following tables store data only about the students current status on
+ a problem, meaning only the data related to the last attempt on a problem.
+ =over 4
+ =item C<$performance_table>
+ The performance_table has 9 columns.  The first three are 'symb_id',
+ 'student_id', and 'part_id'.  These comprise the PRIMARY KEY for this table
+ and are directly related to the $symb_table, $student_table, and $part_table
+ described above.  MySQL does better indexing on numeric items than text,
+ so we use these three "index tables".  The remaining columns are
+ 'solved', 'tries', 'awarded', 'award', 'awarddetail', and 'timestamp'.
+ These are either the MySQL type TINYTEXT or various integers ('tries' and
+ 'timestamp').  This table has KEYs of 'student_id' and 'symb_id'.
+ For use of this table, see the functions described below.
+ =item C<$parameters_table>
+ The parameters_table holds the data that does not fit neatly into the
+ performance_table.  The parameters table has four columns: 'symb_id',
+ 'student_id', 'parameter', and 'value'.  'symb_id', 'student_id', and
+ 'parameter' comprise the PRIMARY KEY for this table.  'parameter' is
+ limited to 255 characters.  'value' is limited to 64k characters.
+ =back
+ =head2 Tables used for storing historic data
+ The following tables are used to store almost all of the transactions a student
+ has made on a homework problem.  See loncapa/docs/homework/datastorage for
+ specific information about each of the parameters stored.
+ =over 4
+ =item C<$fulldump_response_table>
+ The response table holds data (documented in loncapa/docs/homework/datastorage)
+ associated with a particular response id which is stored when a student
+ attempts a problem.  The following are the columns of the table, in order:
+ 'symb_id','part_id','response_id','student_id','transaction','tries',
+ 'awarddetail', 'response_specific', 'response_specific_value',
+ 'response_specific_2', 'response_specific_value_2', and 'submission
+ (the text of the students submission).  The primary key is based on the
+ first five columns listed above.
+ =item C<$fulldump_part_table()>
+ The part table holds data (documented in loncapa/docs/homework/datastorage)
+ associated with a particular part id which is stored when a student attempts
+ a problem.  The following are the columns of the table, in order:
+ 'symb_id','part_id','student_id','transaction','tries','award','awarded',
+ and 'previous'.  The primary key is based on the first five columns listed
+ above.
+ =item C<$fulldump_timestamp_table()>
+ The timestamp table holds the timestamps of the transactions which are
+ stored in $fulldump_response_table and $fulldump_part_table.  This data is
+ about both the response and part data.  Columns: 'symb_id','student_id',
+ 'transaction', and 'timestamp'.
+ The primary key is based on the first 3 columns.
+ =item C<$weight_table()>
+ The weight table holds the weight for the problems used in the class.
+ Whereas the weight of a problem can vary by section and student the data
+ here is applied to the class as a whole.
+ Columns: 'symb_id','part_id','response_id','weight'.
+ =back
+ =head1 IMPORTANT SUBROUTINES
+ Here is a brief overview of the subroutines which are likely to be of
+ interest:
+ =over 4
+ =item C<&get_current_state()>
+ programmers interface.
+ =item C<&init_dbs()>
+ table creation
+ =item C<&update_student_data()>
+ data storage calls
+ =item C<&get_student_data_from_performance_cache()>
+ data retrieval
+ =back
+ =head1 OTHER SUBROUTINES
+ =over 4
+ =item C<&make_into_hash($values)>
+ Returns a reference to a hash as described by $values.  $values is
+ assumed to be the result of
+     join(':',map {&escape($_)} %orighash);
+ This is a helper function for get_current_state.
+ =item C<&init_dbs()>
+ Input: course id
+ Output: 0 on success, positive integer on error
+ This routine issues the calls to lonmysql to create the tables used to
+ store student data.
+ item C<&delete_caches()>
+ This routine drops all the tables associated with a course from the
+ MySQL database.
+ Input: course id (optional, determined by environment if omitted)
+ Returns: nothing
+ =item C<&get_part_id()>
+ Get the MySQL id of a problem part string.
+ Input: $part
+ Output: undef on error, integer $part_id on success.
+ =item C<&get_part()>
+ Get the string describing a part from the MySQL id of the problem part.
+ Input: $part_id
+ Output: undef on error, $part string on success.
+ =item C<&get_symb_id()>
+ Get the MySQL id of a symb.
+ Input: $symb
+ Output: undef on error, integer $symb_id on success.
+ =item C<&get_symb()>
+ Get the symb associated with a MySQL symb_id.
+ Input: $symb_id
+ Output: undef on error, $symb on success.
+ =item C<&get_student_id()>
+ Get the MySQL id of a student.
+ Input: $sname, $dom
+ Output: undef on error, integer $student_id on success.
+ =item C<&get_student()>
+ Get student username:domain associated with the MySQL student_id.
+ Input: $student_id
+ Output: undef on error, string $student (username:domain) on success.
+ =item C<&clear_internal_caches()>
+ Causes the internal caches used in get_student_id, get_student,
+ get_symb_id, get_symb, get_part_id, and get_part to be undef'd.
+ Needs to be called before the first operation with the MySQL database
+ for a given Apache request.
+ =item C<&update_full_student_data($sname,$sdom,$courseid)>
+ Does a lonnet::dump on a student to populate the courses tables.
+ Input: $sname, $sdom, $courseid
+ Output: $returnstatus
+ $returnstatus is a string describing any errors that occurred.  'okay' is the
+ default.
+ This subroutine loads a students data using lonnet::dump and inserts
+ it into the MySQL database.  The inserts are done on three tables,
+ $fulldump_response_table, $fulldump_part_table, and $fulldump_timestamp_table.
+ The INSERT calls are made directly by this subroutine, not through lonmysql
+ because we do a 'bulk'insert which takes advantage of MySQLs non-SQL
+ compliant INSERT command to insert multiple rows at a time.
+ If anything has gone wrong during this process, $returnstatus is updated with
+ a description of the error.
+ Once the "fulldump" tables are updated, the tables used for chart and
+ spreadsheet (which hold only the current state of the student on their
+ homework, not historical data) are updated.  If all updates have occurred
+ successfully, $student_table is updated to reflect the time of the update.
+ Notice we do not insert the data and immediately query it.  This means it
+ is possible for there to be data returned this first time that is not
+ available the second time.  CYA.
+ =item C<&update_student_data()>
+ Input: $sname, $sdom, $courseid
+ Output: $returnstatus, \%student_data
+ $returnstatus is a string describing any errors that occurred.  'okay' is the
+ default.
+ \%student_data is the data returned by a call to lonnet::currentdump.
+ This subroutine loads a students data using lonnet::currentdump and inserts
+ it into the MySQL database.  The inserts are done on two tables,
+ $performance_table and $parameters_table.  $parameters_table holds the data
+ that is not included in $performance_table.  See the description of
+ $performance_table elsewhere in this file.  The INSERT calls are made
+ directly by this subroutine, not through lonmysql because we do a 'bulk'
+ insert which takes advantage of MySQLs non-SQL compliant INSERT command to
+ insert multiple rows at a time.  If anything has gone wrong during this
+ process, $returnstatus is updated with a description of the error and
+ \%student_data is returned.
+ Notice we do not insert the data and immediately query it.  This means it
+ is possible for there to be data returned this first time that is not
+ available the second time.  CYA.
+ =item &ensure_tables_are_set_up($courseid)
+ Checks to be sure the MySQL tables for the given class are set up.
+ If $courseid is omitted it will be obtained from the environment.
+ Returns nothing on success and 'error' on failure
+ =item C<&ensure_current_data()>
+ Input: $sname, $sdom, $courseid
+ Output: $status, $data
+ This routine ensures the data for a given student is up to date.
+ The $student_table is queried to determine the time of the last update.
+ If the students data is out of date, &update_student_data() is called.
+ The return values from the call to &update_student_data() are returned.
+ =item C<&ensure_current_full_data($sname,$sdom,$courseid)>
+ Input: $sname, $sdom, $courseid
+ Output: $status
+ This routine ensures the fulldata (the data from a lonnet::dump, not a
+ lonnet::currentdump) for a given student is up to date.
+ The $student_table is queried to determine the time of the last update.
+ If the students fulldata is out of date, &update_full_student_data() is
+ called.
+ The return value from the call to &update_full_student_data() is returned.
+ =item C<&get_student_data_from_performance_cache()>
+ Input: $sname, $sdom, $symb, $courseid
+ Output: hash reference containing the data for the given student.
+ If $symb is undef, all the students data is returned.
+ This routine is the heart of the local caching system.  See the description
+ of $performance_table, $symb_table, $student_table, and $part_table.  The
+ main task is building the MySQL request.  The tables appear in the request
+ in the order in which they should be parsed by MySQL.  When searching
+ on a student the $student_table is used to locate the 'student_id'.  All
+ rows in $performance_table which have a matching 'student_id' are returned,
+ with data from $part_table and $symb_table which match the entries in
+ $performance_table, 'part_id' and 'symb_id'.  When searching on a symb,
+ the $symb_table is processed first, with matching rows grabbed from
+ $performance_table and filled in from $part_table and $student_table in
+ that order.
+ Running 'EXPLAIN ' on the 'SELECT' statements generated can be quite
+ interesting, especially if you play with the order the tables are listed.
+ =item C<&get_current_state()>
+ Input: $sname,$sdom,$symb,$courseid
+ Output: Described below
+ Retrieve the current status of a students performance.  $sname and
+ $sdom are the only required parameters.  If $symb is undef the results
+ of an &Apache::lonnet::currentdump() will be returned.
+ If $courseid is undef it will be retrieved from the environment.
+ The return structure is based on &Apache::lonnet::currentdump.  If
+ $symb is unspecified, all the students data is returned in a hash of
+ the form:
+ (
+   symb1 => { param1 => value1, param2 => value2 ... },
+   symb2 => { param1 => value1, param2 => value2 ... },
+ )
+ If $symb is specified, a hash of
+ (
+   param1 => value1,
+   param2 => value2,
+ )
+ is returned.
+ If no data is found for $symb, or if the student has no performance data,
+ an empty list is returned.
+ =item C<&get_problem_statistics()>
+ Gather data on a given problem.  The database is assumed to be
+ populated and all local caching variables are assumed to be set
+ properly.  This means you need to call &ensure_current_data for
+ the students you are concerned with prior to calling this routine.
+ Inputs: $Sections, Groups, $status, $symb, $part, $courseid, $starttime,
+         $endtime
+ =over 4
+ =item $Sections Array ref containing section names for students.
+ 'all' is allowed to be the first (and only) item in the array.
+ =item $Groups Array ref containing group names for students.
+ 'all' is allowed to be the first (and only) item in the array.
+ =item $status String describing the status of students
+ =item $symb is the symb for the problem.
+ =item $part is the part id you need statistics for
+ =item $courseid is the course id, of course!
+ =item $starttime and $endtime are unix times which to use to limit
+ the statistical data.
+ =back
+ Outputs: See the code for up to date information.  A hash reference is
+ returned.  The hash has the following keys defined:
+ =over 4
+ =item * num_students
+ The number of students attempting the problem
+ =item tries
+ The total number of tries for the students
+ =item max_tries
+ The maximum number of tries taken
+ =item mean_tries
+ The average number of tries
+ =item num_solved T
+ he number of students able to solve the problem
+ =item num_override
+ The number of students whose answer is 'correct_by_override'
+ =item deg_of_diff
+ The degree of difficulty of the problem
+ =item std_tries
+ The standard deviation of the number of tries
+ =item skew_tries
+ The skew of the number of tries
+ =item per_wrong
+ The number of students attempting the problem who were not
+ able to answer it correctly.
+ =back
+ =item C<&populate_weight_table()>
+ =item C<&limit_by_start_end_times()>
+ Build SQL WHERE condition which limits the data collected by the start
+ and end times provided
+ Inputs: $starttime, $endtime, $table
+ Returns: $time_limits
+ =item C<&limit_by_section_and_status()C>
+ Build SQL WHERE condition which limits the data collected by section and
+ student status.
+ Inputs: $Sections (array ref)
+     $enrollment (string: 'any', 'expired', 'active')
+     $tablename The name of the table that holds the student data
+ Returns: $student_requirements,$enrollment_requirements
+ =item C<&limit_by_group()>
+ Build SQL LEFT JOIN statement to include students_groups and groupnames tables and SQL WHERE condition which limits the data collected by group.
+ Inputs: $Groups (array ref)
+     $stutable   The name of the table which holds the student data.
+     $grptable   The name of the table which maps group_id to groupname.
+     $stugrptab  The name of the table which holds student group affiliations.
+ Returns: $groups_join,$group_limits
+    $groups_join  JOIN part of SQL statement (to include group related tables)
+    $group_limits SQL WHERE condition limiting to requested groups
+ =item C<rank_students_by_scores_on_resources()>
+ Inputs:
+     $resources: array ref of hash ref.  Each hash ref needs key 'symb'.
+     $Sections: array ref of sections to include,
+     $Groups: array ref of groups to include.
+     $enrollment: string,
+     $courseid (may be omitted)
+     $starttime (may be omitted)
+     $endtime (may be omitted)
+     $has_award_for (may be omitted)
+ Returns; An array of arrays.  The sub arrays contain a student name and
+ their score on the resources. $starttime and $endtime constrain the
+ list to awards obtained during the given time limits. $has_score_on
+ constrains the list to those students who at least attempted the
+ resource identified by the given symb, which is used to filter out
+ such students for statistics that would be adversely affected by such
+ students.
+ =item C<&get_sum_of_scores>
+ Inputs: $resource (hash ref, needs {'symb'} key),
+ $part, (the part id),
+ $students (array ref, contents of array are scalars holding 'sname:sdom'),
+ $courseid
+ Returns: the sum of the score on the problem part over the students and the
+    maximum possible value for the sum (taken from the weight table).
+ =item C<&score_stats()>
+ Inputs: $Sections, $enrollment, $symbs, $starttime,
+         $endtime, $courseid
+ $Sections, $enrollment, $starttime, $endtime, and $courseid are the same as
+ elsewhere in this module.
+ $symbs is an array ref of symbs
+ Returns: minimum, maximum, mean, s.d., number of students, and maximum
+   possible of student scores on the given resources
+ =item C<&count_stats()>
+ Inputs: $Sections, $Groups, $enrollment, $symbs, $starttime,
+         $endtime, $courseid
+ $Sections, $Groups $enrollment, $starttime, $endtime, and $courseid are the
+ same as elsewhere in this module.
+ $symbs is an array ref of symbs
+ Returns: minimum, maximum, mean, s.d., and number of students
+   of the number of items correct on the given resources
+ =item C<get_student_data()>
+ =item C<&get_student_scores($Sections,$Groups,$Symbs,$enrollment,$courseid)>
+ =item C<&setup_table_names()>
+ input: course id
+ output: none
+ =back
+ =head3 End of Local Data Caching Subroutines
+ =head3 Classlist Subroutines
+ =over
+ =item &get_classlist();
+ Retrieve the classist of a given class or of the current class.  Student
+ information is returned from the classlist.db file and, if needed,
+ from the students environment.
+ Optional arguments are $cdom, and $cnum (course domain,
+ and course number, respectively).  If either is ommitted the course
+ will be taken from the current environment ($env{'request.course.id'},
+ $env{'course.'.$cid.'.domain'}, and $env{'course.'.$cid.'.num'}).
+ Returns a reference to a hash which contains:
+  keys    '$sname:$sdom'
+  values  [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type,
+           $lockedtype,$credits,$instsec]
+ The constant values CL_SDOM, CL_SNAME, CL_END, etc. can be used
+ as indices into the returned list to future-proof clients against
+ changes in the list order.
+ =back
+ =cut

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>

Removed from v.1.120
changed lines
	Added in v.1.207