version 1.56, 2004/04/09 22:04:53
|
version 1.62, 2005/03/11 03:25:18
|
Line 77 use HTML::TokeParser;
|
Line 77 use HTML::TokeParser;
|
use GDBM_File; |
use GDBM_File; |
use POSIX qw(strftime mktime); |
use POSIX qw(strftime mktime); |
|
|
|
use Sys::Hostname; |
|
|
use File::Find; |
use File::Find; |
|
|
# |
# |
# Set up configuration options |
# Set up configuration options |
my ($simulate,$oneuser,$help,$verbose,$logfile,$debug); |
my ($simulate,$oneuser,$help,$verbose,$logfile,$debug,$multidom); |
GetOptions ( |
GetOptions ( |
'help' => \$help, |
'help' => \$help, |
'simulate' => \$simulate, |
'simulate' => \$simulate, |
'only=s' => \$oneuser, |
'only=s' => \$oneuser, |
'verbose=s' => \$verbose, |
'verbose=s' => \$verbose, |
'debug' => \$debug, |
'debug' => \$debug, |
|
'multi_domain' => \$multidom, |
); |
); |
|
|
if ($help) { |
if ($help) { |
Line 100 Options:
|
Line 103 Options:
|
-only=user Only compute for the given user. Implies -simulate |
-only=user Only compute for the given user. Implies -simulate |
-verbose=val Sets logging level, val must be a number |
-verbose=val Sets logging level, val must be a number |
-debug Turns on debugging output |
-debug Turns on debugging output |
|
-multi_domain Parse the hosts.tab file domain(s) to use. |
ENDHELP |
ENDHELP |
exit 0; |
exit 0; |
} |
} |
Line 119 if (defined($oneuser)) {
|
Line 123 if (defined($oneuser)) {
|
## |
## |
## Use variables for table names so we can test this routine a little easier |
## Use variables for table names so we can test this routine a little easier |
my $oldname = 'metadata'; |
my $oldname = 'metadata'; |
my $newname = 'newmetadata'; |
my $newname = 'newmetadata'.$$; # append pid to have unique temporary table |
|
|
# |
# |
# Read loncapa_apache.conf and loncapa.conf |
# Read loncapa_apache.conf and loncapa.conf |
Line 142 if ($wwwid!=$<) {
|
Line 146 if ($wwwid!=$<) {
|
} |
} |
# |
# |
# Let people know we are running |
# Let people know we are running |
open(LOG,'>'.$perlvar{'lonDaemons'}.'/logs/searchcat.log'); |
open(LOG,'>>'.$perlvar{'lonDaemons'}.'/logs/searchcat.log'); |
&log(0,'==== Searchcat Run '.localtime()."===="); |
&log(0,'==== Searchcat Run '.localtime()."===="); |
|
|
|
|
if ($debug) { |
if ($debug) { |
&log(0,'simulating') if ($simulate); |
&log(0,'simulating') if ($simulate); |
&log(0,'only processing user '.$oneuser) if ($oneuser); |
&log(0,'only processing user '.$oneuser) if ($oneuser); |
Line 171 if ($dbh->err) {
|
Line 177 if ($dbh->err) {
|
} |
} |
# |
# |
# find out which users we need to examine |
# find out which users we need to examine |
my $dom = $perlvar{'lonDefDomain'}; |
my @domains; |
opendir(RESOURCES,"$perlvar{'lonDocRoot'}/res/$dom"); |
if (defined($multidom)) { |
my @homeusers = |
&log(1,'====multi domain setup===='); |
grep { |
# Peek into the hosts.tab and look for matches of our hostname |
&ishome("$perlvar{'lonDocRoot'}/res/$dom/$_"); |
my $host = hostname(); |
} grep { |
&log(9,'hostname = "'.$host.'"'); |
!/^\.\.?$/; |
open(HOSTFILE,$perlvar{'lonTabDir'}.'/hosts.tab') || |
} readdir(RESOURCES); |
die ("Unable to determine domain(s) of multi-domain server"); |
closedir RESOURCES; |
my %domains; |
# |
while (<HOSTFILE>) { |
if ($oneuser) { |
next if (/^\#/); |
@homeusers=($oneuser); |
next if (!/:\Q$host\E/); |
} |
&log(9,$_); |
# |
$domains{(split(':',$_))[1]}++; |
# Loop through the users |
} |
foreach my $user (@homeusers) { |
close HOSTFILE; |
&log(0,"=== User: ".$user); |
@domains = sort(keys(%domains)); |
&process_dynamic_metadata($user,$dom); |
&log(9,join(',',@domains)); |
# |
if (! scalar(@domains)) { |
# Use File::Find to get the files we need to read/modify |
die ("Unable to find any domains in the hosts.tab that match ".$host); |
find( |
} |
{preprocess => \&only_meta_files, |
} else { |
# wanted => \&print_filename, |
push(@domains,$perlvar{'lonDefDomain'}); |
# wanted => \&log_metadata, |
} |
wanted => \&process_meta_file, |
|
}, |
foreach my $dom (@domains) { |
"$perlvar{'lonDocRoot'}/res/$perlvar{'lonDefDomain'}/$user"); |
&log(9,'domain = '.$dom); |
|
opendir(RESOURCES,"$perlvar{'lonDocRoot'}/res/$dom"); |
|
my @homeusers = |
|
grep { |
|
&ishome("$perlvar{'lonDocRoot'}/res/$dom/$_"); |
|
} grep { |
|
!/^\.\.?$/; |
|
} readdir(RESOURCES); |
|
closedir RESOURCES; |
|
&log(5,'users = '.$dom.':'.join(',',@homeusers)); |
|
# |
|
if ($oneuser) { |
|
@homeusers=($oneuser); |
|
} |
|
# |
|
# Loop through the users |
|
foreach my $user (@homeusers) { |
|
&log(0,"=== User: ".$user); |
|
&process_dynamic_metadata($user,$dom); |
|
# |
|
# Use File::Find to get the files we need to read/modify |
|
find( |
|
{preprocess => \&only_meta_files, |
|
#wanted => \&print_filename, |
|
#wanted => \&log_metadata, |
|
wanted => \&process_meta_file, |
|
}, join('/',($perlvar{'lonDocRoot'},'res',$dom,$user)) ); |
|
} |
} |
} |
# |
# |
# Rename the table |
# Rename the table |
Line 210 if (! $simulate) {
|
Line 243 if (! $simulate) {
|
&log(1,"MySQL table rename successful."); |
&log(1,"MySQL table rename successful."); |
} |
} |
} |
} |
|
|
if (! $dbh->disconnect) { |
if (! $dbh->disconnect) { |
&log(0,"MySQL Error Disconnect: ".$dbh->errstr); |
&log(0,"MySQL Error Disconnect: ".$dbh->errstr); |
die $dbh->errstr; |
die $dbh->errstr; |
Line 305 sub log_metadata {
|
Line 337 sub log_metadata {
|
$_=$file; |
$_=$file; |
} |
} |
|
|
|
|
## |
## |
## process_meta_file |
## process_meta_file |
## Called by File::Find. |
## Called by File::Find. |
Line 320 sub process_meta_file {
|
Line 351 sub process_meta_file {
|
my $ref=&metadata($filename); |
my $ref=&metadata($filename); |
# |
# |
# $url is the original file url, not the metadata file |
# $url is the original file url, not the metadata file |
my $url='/res/'.&declutter($filename); |
my $target = $filename; |
$url=~s/\.meta$//; |
$target =~ s/\.meta$//; |
|
my $url='/res/'.&declutter($target); |
&log(3," ".$url) if ($debug); |
&log(3," ".$url) if ($debug); |
# |
# |
# Ignore some files based on their metadata |
# Ignore some files based on their metadata |
Line 346 sub process_meta_file {
|
Line 378 sub process_meta_file {
|
&count_type($url); |
&count_type($url); |
} |
} |
# |
# |
|
if (! defined($ref->{'creationdate'}) || |
|
$ref->{'creationdate'} =~ /^\s*$/) { |
|
$ref->{'creationdate'} = (stat($target))[9]; |
|
} |
|
if (! defined($ref->{'lastrevisiondate'}) || |
|
$ref->{'lastrevisiondate'} =~ /^\s*$/) { |
|
$ref->{'lastrevisiondate'} = (stat($target))[9]; |
|
} |
$ref->{'creationdate'} = &sqltime($ref->{'creationdate'}); |
$ref->{'creationdate'} = &sqltime($ref->{'creationdate'}); |
$ref->{'lastrevisiondate'} = &sqltime($ref->{'lastrevisiondate'}); |
$ref->{'lastrevisiondate'} = &sqltime($ref->{'lastrevisiondate'}); |
my %Data = ( |
my %Data = ( |
Line 358 sub process_meta_file {
|
Line 398 sub process_meta_file {
|
\%Data); |
\%Data); |
if ($err) { |
if ($err) { |
&log(0,"MySQL Error Insert: ".$err); |
&log(0,"MySQL Error Insert: ".$err); |
die $err; |
|
} |
} |
if ($count < 1) { |
if ($count < 1) { |
&log(0,"Unable to insert record into MySQL database for $url"); |
&log(0,"Unable to insert record into MySQL database for $url"); |
die "Unable to insert record into MySQl database for $url"; |
|
} |
} |
} |
} |
# |
# |
Line 444 sub getfile {
|
Line 482 sub getfile {
|
######################################################## |
######################################################## |
######################################################## |
######################################################## |
## |
## |
## Dynamic metadata description |
## Dynamic metadata description (incomplete) |
|
## |
|
## For a full description of all fields, |
|
## see LONCAPA::lonmetadata |
## |
## |
## Field Type |
## Field Type |
##----------------------------------------------------------- |
##----------------------------------------------------------- |
## count integer |
## count integer |
## course integer |
## course integer |
## course_list comma seperated list of course ids |
## course_list comma separated list of course ids |
## avetries real |
## avetries real |
## avetries_list comma seperated list of real numbers |
## avetries_list comma separated list of real numbers |
## stdno real |
## stdno real |
## stdno_list comma seperated list of real numbers |
## stdno_list comma separated list of real numbers |
## usage integer |
## usage integer |
## usage_list comma seperated list of resources |
## usage_list comma separated list of resources |
## goto scalar |
## goto scalar |
## goto_list comma seperated list of resources |
## goto_list comma separated list of resources |
## comefrom scalar |
## comefrom scalar |
## comefrom_list comma seperated list of resources |
## comefrom_list comma separated list of resources |
## difficulty real |
## difficulty real |
## difficulty_list comma seperated list of real numbers |
## difficulty_list comma separated list of real numbers |
## sequsage scalar |
## sequsage scalar |
## sequsage_list comma seperated list of resources |
## sequsage_list comma separated list of resources |
## clear real |
## clear real |
## technical real |
## technical real |
## correct real |
## correct real |
Line 491 sub process_dynamic_metadata {
|
Line 532 sub process_dynamic_metadata {
|
return 0; |
return 0; |
} |
} |
# |
# |
# Process every stored element |
%DynamicData = &LONCAPA::lonmetadata::process_reseval_data(\%evaldata); |
while (my ($storedkey,$value) = each(%evaldata)) { |
|
my ($source,$file,$type) = split('___',$storedkey); |
|
$source = &unescape($source); |
|
$file = &unescape($file); |
|
$value = &unescape($value); |
|
" got ".$file."\n ".$type." ".$source."\n"; |
|
if ($type =~ /^(avetries|count|difficulty|stdno|timestamp)$/) { |
|
# |
|
# Statistics: $source is course id |
|
$DynamicData{$file}->{'statistics'}->{$source}->{$type}=$value; |
|
} elsif ($type =~ /^(clear|comments|depth|technical|helpful)$/){ |
|
# |
|
# Evaluation $source is username, check if they evaluated it |
|
# more than once. If so, pad the entry with a space. |
|
while(exists($DynamicData{$file}->{'evaluation'}->{$type}->{$source})) { |
|
$source .= ' '; |
|
} |
|
$DynamicData{$file}->{'evaluation'}->{$type}->{$source}=$value; |
|
} elsif ($type =~ /^(course|comefrom|goto|usage)$/) { |
|
# |
|
# Context $source is course id or resource |
|
push(@{$DynamicData{$file}->{$type}},&unescape($source)); |
|
} else { |
|
&log(0," ".$user."@".$dom.":Process metadata: Unable to decode ".$type); |
|
} |
|
} |
|
untie(%evaldata); |
untie(%evaldata); |
|
$DynamicData{'domain'} = $dom; |
|
print('user = '.$user.' domain = '.$dom.$/); |
# |
# |
# Read in the access count data |
# Read in the access count data |
&log(7,'Reading access count data') if ($debug); |
&log(7,'Reading access count data') if ($debug); |
Line 547 sub process_dynamic_metadata {
|
Line 564 sub process_dynamic_metadata {
|
sub get_dynamic_metadata { |
sub get_dynamic_metadata { |
my ($url) = @_; |
my ($url) = @_; |
$url =~ s:^/res/::; |
$url =~ s:^/res/::; |
if (! exists($DynamicData{$url})) { |
my %data = &LONCAPA::lonmetadata::process_dynamic_metadata($url, |
&log(7,' No dynamic data for '.$url) if ($debug); |
\%DynamicData); |
return (); |
|
} |
|
my %data; |
|
my $resdata = $DynamicData{$url}; |
|
# |
|
# Get the statistical data |
|
foreach my $type (qw/avetries difficulty stdno/) { |
|
my $count; |
|
my $sum; |
|
my @Values; |
|
foreach my $coursedata (values(%{$resdata->{'statistics'}})) { |
|
if (ref($coursedata) eq 'HASH' && exists($coursedata->{$type})) { |
|
$count++; |
|
$sum += $coursedata->{$type}; |
|
push(@Values,$coursedata->{$type}); |
|
} |
|
} |
|
if ($count) { |
|
$data{$type} = $sum/$count; |
|
$data{$type.'_list'} = join(',',@Values); |
|
} |
|
} |
|
# find the count |
# find the count |
$data{'count'} = $Counts{$url}; |
$data{'count'} = $Counts{$url}; |
# |
# |
# Get the context data |
|
foreach my $type (qw/course goto comefrom/) { |
|
if (defined($resdata->{$type}) && |
|
ref($resdata->{$type}) eq 'ARRAY') { |
|
$data{$type} = scalar(@{$resdata->{$type}}); |
|
$data{$type.'_list'} = join(',',@{$resdata->{$type}}); |
|
} |
|
} |
|
if (defined($resdata->{'usage'}) && |
|
ref($resdata->{'usage'}) eq 'ARRAY') { |
|
$data{'sequsage'} = scalar(@{$resdata->{'usage'}}); |
|
$data{'sequsage_list'} = join(',',@{$resdata->{'usage'}}); |
|
} |
|
# |
|
# Get the evaluation data |
|
foreach my $type (qw/clear technical correct helpful depth/) { |
|
my $count; |
|
my $sum; |
|
foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{$type}})){ |
|
$sum += $resdata->{'evaluation'}->{$type}->{$evaluator}; |
|
$count++; |
|
} |
|
if ($count > 0) { |
|
$data{$type}=$sum/$count; |
|
} |
|
} |
|
# |
|
# put together comments |
|
my $comments = '<div class="LCevalcomments">'; |
|
foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{'comments'}})){ |
|
$comments .= $evaluator.':'. |
|
$resdata->{'evaluation'}->{'comments'}->{$evaluator}.'<hr />'; |
|
} |
|
$comments .= '</div>'; |
|
# |
|
# Log the dynamic metadata |
# Log the dynamic metadata |
if ($debug) { |
if ($debug) { |
while (my($k,$v)=each(%data)) { |
while (my($k,$v)=each(%data)) { |
&log(8," ".$k." => ".$v); |
&log(8," ".$k." => ".$v); |
} |
} |
} |
} |
# |
|
return %data; |
return %data; |
} |
} |
|
|