--- loncom/publisher/lonpublisher.pm 2001/08/13 12:53:06 1.37 +++ loncom/publisher/lonpublisher.pm 2014/08/03 13:52:59 1.292 @@ -1,148 +1,452 @@ # The LearningOnline Network with CAPA # Publication Handler -# -# (TeX Content Handler # -# 05/29/00,05/30,10/11 Gerd Kortemeyer) +# $Id: lonpublisher.pm,v 1.292 2014/08/03 13:52:59 raeburn Exp $ # -# 11/28,11/29,11/30,12/01,12/02,12/04,12/23 Gerd Kortemeyer -# 03/23 Guy Albertelli -# 03/24,03/29,04/03 Gerd Kortemeyer -# 04/16/2001 Scott Harrison -# 05/03,05/05,05/07 Gerd Kortemeyer -# 05/28/2001 Scott Harrison -# 06/23,08/07,08/11,8/13 Gerd Kortemeyer +# Copyright Michigan State University Board of Trustees +# +# This file is part of the LearningOnline Network with CAPA (LON-CAPA). +# +# LON-CAPA is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# LON-CAPA is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with LON-CAPA; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# /home/httpd/html/adm/gpl.txt +# +# http://www.lon-capa.org/ +# +### + +############################################################################### +## ## +## ORGANIZATION OF THIS PERL MODULE ## +## ## +## 1. Modules used by this module ## +## 2. Various subroutines ## +## 3. Publication Step One ## +## 4. Phase Two ## +## 5. Main Handler ## +## ## +############################################################################### + + +###################################################################### +###################################################################### + +=pod + +=head1 NAME + +lonpublisher - LON-CAPA publishing handler + +=head1 SYNOPSIS + +B is used by B inside B. This is the +invocation by F: + + + PerlAccessHandler Apache::lonacc + SetHandler perl-script + PerlHandler Apache::lonpublisher + ErrorDocument 403 /adm/login + ErrorDocument 404 /adm/notfound.html + ErrorDocument 406 /adm/unauthorized.html + ErrorDocument 500 /adm/errorhandler + + +=head1 OVERVIEW + +Authors can only write-access the C space. +They can copy resources into the resource area through the +publication step, and move them back through a recover step. +Authors do not have direct write-access to their resource space. + +During the publication step, several events will be +triggered. Metadata is gathered, where a wizard manages default +entries on a hierarchical per-directory base: The wizard imports the +metadata (including access privileges and royalty information) from +the most recent published resource in the current directory, and if +that is not available, from the next directory above, etc. The Network +keeps all previous versions of a resource and makes them available by +an explicit version number, which is inserted between the file name +and extension, for example C, while the most recent +version does not carry a version number (C). Servers +subscribing to a changed resource are notified that a new version is +available. + +=head1 DESCRIPTION + +B takes the proper steps to add resources to the LON-CAPA +digital library. This includes updating the metadata table in the +LON-CAPA database. + +B is many things to many people. + +This module publishes a file. This involves gathering metadata, +versioning the file, copying file from construction space to +publication space, and copying metadata from construction space +to publication space. + +=head2 SUBROUTINES + +Many of the undocumented subroutines implement various magical +parsing shortcuts. + +=cut + +###################################################################### +###################################################################### + package Apache::lonpublisher; +# ------------------------------------------------- modules used by this module use strict; use Apache::File; use File::Copy; use Apache::Constants qw(:common :http :methods); -use HTML::TokeParser; +use HTML::LCParser; +use HTML::Entities; +use Encode::Encoder; use Apache::lonxml; -use Apache::lonhomework; -use Apache::loncacc; use DBI; +use Apache::lonnet; +use Apache::loncommon(); +use Apache::lonhtmlcommon; +use Apache::lonmysql; +use Apache::lonlocal; +use Apache::loncfile; +use LONCAPA::lonmetadata; +use Apache::lonmsg; +use vars qw(%metadatafields %metadatakeys); +use LONCAPA qw(:DEFAULT :match); + my %addid; my %nokey; -my %language; -my %cprtag; - -my %metadatafields; -my %metadatakeys; my $docroot; my $cuname; my $cudom; -# ----------------------------------------------- Evaluate string with metadata +my $registered_cleanup; +my $modified_urls; + +my $lock; + +=pod + +=over 4 + +=item B + +Evaluates a string that contains metadata. This subroutine +stores values inside I<%metadatafields> and I<%metadatakeys>. +The hash key is a I<$unikey> corresponding to a unique id +that is descriptive of the parser location inside the XML tree. + +Parameters: + +=over 4 + +=item I<$metastring> + +A string that contains metadata. + +=back + +Returns: + +nothing + +=cut + +######################################### +######################################### +# +# Modifies global %metadatafields %metadatakeys +# sub metaeval { - my $metastring=shift; + my ($metastring,$prefix)=@_; - my $parser=HTML::TokeParser->new(\$metastring); - my $token; - while ($token=$parser->get_token) { - if ($token->[0] eq 'S') { - my $entry=$token->[1]; - my $unikey=$entry; - if (defined($token->[2]->{'package'})) { - $unikey.='_package_'.$token->[2]->{'package'}; - } - if (defined($token->[2]->{'part'})) { - $unikey.='_'.$token->[2]->{'part'}; - } - if (defined($token->[2]->{'id'})) { - $unikey.='_'.$token->[2]->{'id'}; - } - if (defined($token->[2]->{'name'})) { - $unikey.='_'.$token->[2]->{'name'}; - } - map { - $metadatafields{$unikey.'.'.$_}=$token->[2]->{$_}; - if ($metadatakeys{$unikey}) { - $metadatakeys{$unikey}.=','.$_; - } else { - $metadatakeys{$unikey}=$_; - } - } @{$token->[3]}; - if ($metadatafields{$unikey}) { - my $newentry=$parser->get_text('/'.$entry); - unless ($metadatafields{$unikey}=~/$newentry/) { - $metadatafields{$unikey}.=', '.$newentry; - } - } else { - $metadatafields{$unikey}=$parser->get_text('/'.$entry); - } - } - } + my $parser=HTML::LCParser->new(\$metastring); + my $token; + while ($token=$parser->get_token) { + if ($token->[0] eq 'S') { + my $entry=$token->[1]; + my $unikey=$entry; + next if ($entry =~ m/^(?:parameter|stores)_/); + if (defined($token->[2]->{'package'})) { + $unikey.="\0package\0".$token->[2]->{'package'}; + } + if (defined($token->[2]->{'part'})) { + $unikey.="\0".$token->[2]->{'part'}; + } + if (defined($token->[2]->{'id'})) { + $unikey.="\0".$token->[2]->{'id'}; + } + if (defined($token->[2]->{'name'})) { + $unikey.="\0".$token->[2]->{'name'}; + } + foreach (@{$token->[3]}) { + $metadatafields{$unikey.'.'.$_}=$token->[2]->{$_}; + if ($metadatakeys{$unikey}) { + $metadatakeys{$unikey}.=','.$_; + } else { + $metadatakeys{$unikey}=$_; + } + } + my $newentry=$parser->get_text('/'.$entry); + if (($entry eq 'customdistributionfile') || + ($entry eq 'sourcerights')) { + $newentry=~s/^\s*//; + if ($newentry !~m|^/res|) { $newentry=$prefix.$newentry; } + } +# actually store + if ( $entry eq 'rule' && exists($metadatafields{$unikey})) { + $metadatafields{$unikey}.=','.$newentry; + } else { + $metadatafields{$unikey}=$newentry; + } + } + } } -# -------------------------------------------------------- Read a metadata file +######################################### +######################################### + +=pod + +=item B + +Read a metadata file + +Parameters: + +=over + +=item I<$logfile> + +File output stream to output errors and warnings to. + +=item I<$fn> + +File name (including path). +=back + +Returns: + +=over 4 + +=item Scalar string (if successful) + +XHTML text that indicates successful reading of the metadata. + +=back + +=cut + +######################################### +######################################### sub metaread { - my ($logfile,$fn)=@_; + my ($logfile,$fn,$prefix)=@_; unless (-e $fn) { - print $logfile 'No file '.$fn."\n"; - return '
No file: '.$fn.''; + print($logfile 'No file '.$fn."\n"); + return '

' + .&mt('No file: [_1]',&Apache::loncfile::display($fn)) + .'

'; } - print $logfile 'Processing '.$fn."\n"; + print($logfile 'Processing '.$fn."\n"); my $metastring; { - my $metafh=Apache::File->new($fn); - $metastring=join('',<$metafh>); + my $metafh=Apache::File->new($fn); + $metastring=join('',<$metafh>); } - &metaeval($metastring); - return '
Processed file: '.$fn.''; + &metaeval($metastring,$prefix); + return '

' + .&mt('Processed file: [_1]',&Apache::loncfile::display($fn)) + .'

'; } -# ---------------------------- convert 'time' format into a datetime sql format -sub sqltime { - my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = - localtime(@_[0]); - $mon++; $year+=1900; - return "$year-$mon-$mday $hour:$min:$sec"; +######################################### +######################################### + +sub coursedependencies { + my $url=&Apache::lonnet::declutter(shift); + $url=~s/\.meta$//; + my ($adomain,$aauthor)=($url=~ m{^($match_domain)/($match_username)/}); + my $regexp=quotemeta($url); + $regexp='___'.$regexp.'___course'; + my %evaldata=&Apache::lonnet::dump('nohist_resevaldata',$adomain, + $aauthor,$regexp); + my %courses=(); + foreach (keys %evaldata) { + if ($_=~/^([a-zA-Z0-9]+_[a-zA-Z0-9]+)___.+___course$/) { + $courses{$1}=1; + } + } + return %courses; } +######################################### +######################################### + + +=pod + +=item Form-field-generating subroutines. + +For input parameters, these subroutines take in values +such as I<$name>, I<$value> and other form field metadata. +The output (scalar string that is returned) is an XHTML +string which presents the form field (foreseeably inside +
tags). + +=over 4 -# --------------------------------------------------------- Various form fields +=item B +=item B + +=item B + +=back + +=cut + +######################################### +######################################### sub textfield { - my ($title,$name,$value)=@_; - return "\n

$title:
". - ''; + my ($title,$name,$value,$noline)=@_; + $value=~s/^\s+//gs; + $value=~s/\s+$//gs; + $value=~s/\s+/ /gs; + $title=&mt($title); + $env{'form.'.$name}=$value; + return "\n".&Apache::lonhtmlcommon::row_title($title) + .'' + .&Apache::lonhtmlcommon::row_closure($noline); +} + +sub text_with_browse_field { + my ($title,$name,$value,$restriction,$noline)=@_; + $value=~s/^\s+//gs; + $value=~s/\s+$//gs; + $value=~s/\s+/ /gs; + $title=&mt($title); + $env{'form.'.$name}=$value; + return "\n".&Apache::lonhtmlcommon::row_title($title) + .'' + .'
' + .'' + .&mt('Select') + .' ' + .'' + .&mt('Search') + .'' + .&Apache::lonhtmlcommon::row_closure($noline); } sub hiddenfield { my ($name,$value)=@_; - return "\n".''; + $env{'form.'.$name}=$value; + return "\n".''; +} + +sub checkbox { + my ($name,$text)=@_; + return "\n
"; } sub selectbox { - my ($title,$name,$value,%options)=@_; - my $selout="\n

$title:
".''; + foreach (@idlist) { $selout.=''; + } + $selout.=''.&Apache::lonhtmlcommon::row_closure(); + return $selout; } -# -------------------------------------------------------- Publication Step One +sub select_level_form { + my ($value,$name)=@_; + $env{'form.'.$name}=$value; + if (!defined($value)) { $env{'form.'.$name}=0; } + return &Apache::loncommon::select_level_form($value,$name); +} +######################################### +######################################### + +=pod + +=item B +Fix up a url? First step of publication + +=cut + +######################################### +######################################### sub urlfixup { my ($url,$target)=@_; - my ($host)=($url=~/(?:http\:\/\/)*([^\/]+)/); - map { - if ($_ eq $host) { - $url=~s/^http\:\/\///; - $url=~s/^$host//; - } - } values %Apache::lonnet::hostname; - $url=~s/\~$cuname/res\/$cudom\/$cuname/; + unless ($url) { return ''; } + #javascript code needs no fixing + if ($url =~ /^javascript:/i) { return $url; } + if ($url =~ /^mailto:/i) { return $url; } + #internal document links need no fixing + if ($url =~ /^\#/) { return $url; } + my ($host)=($url=~m{(?:(?:http|https|ftp)://)*([^/]+)}); + my @lonids = &Apache::lonnet::machine_ids($host); + if (@lonids) { + $url=~s{^(?:http|https|ftp)://}{}; + $url=~s/^\Q$host\E//; + } + if ($url=~m{^(?:http|https|ftp)://}) { return $url; } + $url=~s{\Q~$cuname\E}{res/$cudom/$cuname}; + return $url; +} + +######################################### +######################################### + +=pod + +=item B + +Currently undocumented. + +=cut + +######################################### +######################################### +sub absoluteurl { + my ($url,$target)=@_; + unless ($url) { return ''; } if ($target) { $target=~s/\/[^\/]+$//; $url=&Apache::lonnet::hreflocation($target,$url); @@ -150,24 +454,590 @@ sub urlfixup { return $url; } +######################################### +######################################### + +=pod + +=item B + +Currently undocumented + +=cut + +######################################### +######################################### +sub set_allow { + my ($allow,$logfile,$target,$tag,$oldurl,$type)=@_; + my $newurl=&urlfixup($oldurl,$target); + my $return_url=$oldurl; + print $logfile 'GUYURL: '.$tag.':'.$oldurl.' - '.$newurl."\n"; + if ($newurl ne $oldurl) { + $return_url=$newurl; + print $logfile 'URL: '.$tag.':'.$oldurl.' - '.$newurl."\n"; + } + if (($newurl !~ /^javascript:/i) && + ($newurl !~ /^mailto:/i) && + ($newurl !~ /^(?:http|https|ftp):/i) && + ($newurl !~ /^\#/)) { + if (($type eq 'src') || ($type eq 'href')) { + if ($newurl =~ /^([^?]+)\?[^?]*$/) { + $newurl = $1; + } + } + $$allow{&absoluteurl($newurl,$target)}=1; + } + return $return_url; +} + +######################################### +######################################### + +=pod + +=item B + +Currently undocumented + +=cut + +######################################### +######################################### +sub get_subscribed_hosts { + my ($target)=@_; + my @subscribed; + my $filename; + $target=~/(.*)\/([^\/]+)$/; + my $srcf=$2; + opendir(DIR,$1); + # cycle through listed files, subscriptions used to exist + # as "filename.lonid" + while ($filename=readdir(DIR)) { + if ($filename=~/\Q$srcf\E\.($match_lonid)$/) { + my $subhost=$1; + if (($subhost ne 'meta' + && $subhost ne 'subscription' + && $subhost ne 'meta.subscription' + && $subhost ne 'tmp') && + ($subhost ne $Apache::lonnet::perlvar{'lonHostID'})) { + push(@subscribed,$subhost); + } + } + } + closedir(DIR); + my $sh; + if ( $sh=Apache::File->new("$target.subscription") ) { + while (my $subline=<$sh>) { + if ($subline =~ /^($match_lonid):/) { + if ($1 ne $Apache::lonnet::perlvar{'lonHostID'}) { + push(@subscribed,$1); + } + } + } + } + return @subscribed; +} + + +######################################### +######################################### + +=pod + +=item B + +Currently undocumented + +=cut + +######################################### +######################################### +sub get_max_ids_indices { + my ($content)=@_; + my $maxindex=10; + my $maxid=10; + my $needsfixup=0; + my $duplicateids=0; + + my %allids; + my %duplicatedids; + + my $parser=HTML::LCParser->new($content); + $parser->xml_mode(1); + my $token; + while ($token=$parser->get_token) { + if ($token->[0] eq 'S') { + my $counter; + if ($counter=$addid{$token->[1]}) { + if ($counter eq 'id') { + if (defined($token->[2]->{'id'}) && + $token->[2]->{'id'} !~ /^\s*$/) { + $maxid=($token->[2]->{'id'}>$maxid)?$token->[2]->{'id'}:$maxid; + if (exists($allids{$token->[2]->{'id'}})) { + $duplicateids=1; + $duplicatedids{$token->[2]->{'id'}}=1; + } else { + $allids{$token->[2]->{'id'}}=1; + } + } else { + $needsfixup=1; + } + } else { + if (defined($token->[2]->{'index'}) && + $token->[2]->{'index'} !~ /^\s*$/) { + $maxindex=($token->[2]->{'index'}>$maxindex)?$token->[2]->{'index'}:$maxindex; + } else { + $needsfixup=1; + } + } + } + } + } + return ($needsfixup,$maxid,$maxindex,$duplicateids, + (keys(%duplicatedids))); +} + +######################################### +######################################### + +=pod + +=item B + +Currently undocumented + +=cut + +######################################### +######################################### +sub get_all_text_unbalanced { + #there is a copy of this in lonxml.pm + my($tag,$pars)= @_; + my $token; + my $result=''; + $tag='<'.$tag.'>'; + while ($token = $$pars[-1]->get_token) { + if (($token->[0] eq 'T')||($token->[0] eq 'C')||($token->[0] eq 'D')) { + $result.=$token->[1]; + } elsif ($token->[0] eq 'PI') { + $result.=$token->[2]; + } elsif ($token->[0] eq 'S') { + $result.=$token->[4]; + } elsif ($token->[0] eq 'E') { + $result.=$token->[2]; + } + if ($result =~ /\Q$tag\E/s) { + ($result,my $redo)=$result =~ /(.*)\Q$tag\E(.*)/is; + #&Apache::lonnet::logthis('Got a winner with leftovers ::'.$2); + #&Apache::lonnet::logthis('Result is :'.$1); + $redo=$tag.$redo; + push (@$pars,HTML::LCParser->new(\$redo)); + $$pars[-1]->xml_mode('1'); + last; + } + } + return $result +} + +######################################### +######################################### + +=pod + +=item B + +Currently undocumented + +=cut + +######################################### +######################################### +#Arguably this should all be done as a lonnet::ssi instead +sub fix_ids_and_indices { + my ($logfile,$source,$target)=@_; + + my %allow; + my $content; + { + my $org=Apache::File->new($source); + $content=join('',<$org>); + } + + my ($needsfixup,$maxid,$maxindex,$duplicateids,@duplicatedids)= + &get_max_ids_indices(\$content); + + print $logfile ("Got $needsfixup,$maxid,$maxindex,$duplicateids--". + join(', ',@duplicatedids)); + if ($duplicateids) { + print $logfile "Duplicate ID(s) exist, ".join(', ',@duplicatedids)."\n"; + my $outstring=''.&mt('Unable to publish file, it contains duplicated ID(s), ID(s) need to be unique. The duplicated ID(s) are').': '.join(', ',@duplicatedids).''; + return ($outstring,1); + } + if ($needsfixup) { + print $logfile "Needs ID and/or index fixup\n". + "Max ID : $maxid (min 10)\n". + "Max Index: $maxindex (min 10)\n"; + } + my $outstring=''; + my $responsecounter=1; + my @parser; + $parser[0]=HTML::LCParser->new(\$content); + $parser[-1]->xml_mode(1); + my $token; + while (@parser) { + while ($token=$parser[-1]->get_token) { + if ($token->[0] eq 'S') { + my $counter; + my $tag=$token->[1]; + my $lctag=lc($tag); + if ($lctag eq 'allow') { + $allow{$token->[2]->{'src'}}=1; + next; + } + if ($lctag eq 'base') { next; } + if (($lctag eq 'part') || ($lctag eq 'problem')) { + $responsecounter=0; + } + if ($lctag=~/response$/) { $responsecounter++; } + if ($lctag eq 'import') { $responsecounter++; } + my %parms=%{$token->[2]}; + $counter=$addid{$tag}; + if (!$counter) { $counter=$addid{$lctag}; } + if ($counter) { + if ($counter eq 'id') { + unless (defined($parms{'id'}) && + $parms{'id'}!~/^\s*$/) { + $maxid++; + $parms{'id'}=$maxid; + print $logfile 'ID(new) : '.$tag.':'.$maxid."\n"; + } else { + print $logfile 'ID(kept): '.$tag.':'.$parms{'id'}."\n"; + } + } elsif ($counter eq 'index') { + unless (defined($parms{'index'}) && + $parms{'index'}!~/^\s*$/) { + $maxindex++; + $parms{'index'}=$maxindex; + print $logfile 'Index: '.$tag.':'.$maxindex."\n"; + } + } + } + unless ($parms{'type'} eq 'zombie') { + foreach my $type ('src','href','background','bgimg') { + foreach my $key (keys(%parms)) { + if ($key =~ /^$type$/i) { + next if (($lctag eq 'img') && ($type eq 'src') && + ($parms{$key} =~ m{^data\:image/gif;base64,})); + $parms{$key}=&set_allow(\%allow,$logfile, + $target,$tag, + $parms{$key},$type); + } + } + } + } + # probably a image type

.html or
_player.html files, +# so add this file to %allow (where
is name user gave to file/archive). +# + my $spritesheet = $1.'express_show/spritesheet.png'; + $allow{&absoluteurl($spritesheet,$target)}=1; + } + } elsif ($srctype eq 'PosterImageSrc') { + if ($url =~ m{^(.+)_First_Frame\.png$}) { + my $prefix = $1; +# +# Camtasia 8.1:
_Thumbnails.png needed, and included in zip archive. +# Not referenced directly in
.html or
_player.html files, +# so add this file to %allow (where
is name user gave to file/archive). +# + my $thumbnail = $prefix.'_Thumbnails.png'; + $allow{&absoluteurl($thumbnail,$target)}=1; + } + } + } + } + } + } + } + if ($script =~ /\(document,\s*(['"])script\1,\s*\[([^\]]+)\]\);/s) { + my $scriptslist = $2; + my @srcs = split(/\s*,\s*/,$scriptslist); + foreach my $src (@srcs) { + if ($src =~ /(["'])(?:(?!\1).)+\.js\1/) { + my $quote = $1; + my ($url) = ($src =~ m/\Q$quote\E([^$quote]+)\Q$quote\E/); + $url = &urlfixup($url); + unless ($url=~m{^(?:http|https|ftp)://}) { + $allow{&absoluteurl($url,$target)}=1; + } + } + } + } + $outstring .= $script; + } + } + } elsif ($token->[0] eq 'E') { + if ($token->[2]) { + unless ($token->[1] eq 'allow') { + $outstring.='[1].'>'; + } + } + if ((($token->[1] eq 'part') || ($token->[1] eq 'problem')) + && (!$responsecounter)) { + my $outstring=''.&mt('Found [_1] without responses. This resource cannot be published.',$token->[1]).''; + return ($outstring,1); + } + } else { + $outstring.=$token->[1]; + } + } + pop(@parser); + } + + if ($needsfixup) { + print $logfile "End of ID and/or index fixup\n". + "Max ID : $maxid (min 10)\n". + "Max Index: $maxindex (min 10)\n"; + } else { + print $logfile "Does not need ID and/or index fixup\n"; + } + + return ($outstring,0,%allow); +} + +######################################### +######################################### + +=pod + +=item B + +Store the metadata in the metadata table in the loncapa database. +Uses lonmysql to access the database. + +Inputs: \%metadata + +Returns: (error,status). error is undef on success, status is undef on error. + +=cut + +######################################### +######################################### +sub store_metadata { + my %metadata = @_; + my $error; + # Determine if the table exists + my $status = &Apache::lonmysql::check_table('metadata'); + if (! defined($status)) { + $error='' + .&mt('WARNING: Cannot connect to database!') + .''; + &Apache::lonnet::logthis($error); + return ($error,undef); + } + if ($status == 0) { + # It would be nice to actually create the table.... + $error ='' + .&mt('WARNING: The metadata table does not exist in the LON-CAPA database!') + .''; + &Apache::lonnet::logthis($error); + return ($error,undef); + } + my $dbh = &Apache::lonmysql::get_dbh(); + if (($metadata{'obsolete'}) || ($metadata{'copyright'} eq 'priv')) { + # remove this entry + my $delitem = 'url = '.$dbh->quote($metadata{'url'}); + $status = &LONCAPA::lonmetadata::delete_metadata($dbh,undef,$delitem); + + } else { + $status = &LONCAPA::lonmetadata::update_metadata($dbh,undef,undef, + \%metadata); + } + if (defined($status) && $status ne '') { + $error='' + .&mt('Error occurred saving new values in metadata table in LON-CAPA database!') + .''; + &Apache::lonnet::logthis($error); + &Apache::lonnet::logthis($status); + return ($error,undef); + } + return (undef,'success'); +} + + +# ========================================== Parse file for errors and warnings + +sub checkonthis { + my ($r,$source)=@_; + my $uri=&Apache::lonnet::hreflocation($source); + $uri=~s/\/$//; + my $result=&Apache::lonnet::ssi_body($uri, + ('grade_target'=>'web', + 'return_only_error_and_warning_counts' => 1)); + my ($errorcount,$warningcount)=split(':',$result); + if (($errorcount) || ($warningcount)) { + $r->print('

'.&mt('Warnings and Errors').'

'); + $r->print(''.$uri.':'); + $r->print('
    '); + if ($warningcount) { + $r->print('
  • ' + .&mt('[quant,_1,warning]',$warningcount) + .'
    • '); + } + if ($errorcount) { + $r->print('
  • ' + .&mt('[quant,_1,error]',$errorcount) + .' ' + .'
    • '); + } + $r->print('
'); + } else { + #$r->print(''.&mt('ok').''); + } + $r->rflush(); + return ($warningcount,$errorcount); +} + +# ============================================== Parse file itself for metadata +# +# parses a file with target meta, sets global %metadatafields %metadatakeys + +sub parseformeta { + my ($source,$style)=@_; + my $allmeta=''; + if (($style eq 'ssi') || ($style eq 'prv')) { + my $dir=$source; + $dir=~s-/[^/]*$--; + my $file=$source; + $file=(split('/',$file))[-1]; + $source=&Apache::lonnet::hreflocation($dir,$file); + $allmeta=&Apache::lonnet::ssi_body($source,('grade_target' => 'meta')); + &metaeval($allmeta); + } + return $allmeta; +} + +######################################### +######################################### + +=pod + +=item B + +This is the workhorse function of this module. This subroutine generates +backup copies, performs any automatic processing (prior to publication, +especially for rat and ssi files), + +Returns a 2 element array, the first is the string to be shown to the +user, the second is an error code, either 1 (an error occurred) or 0 +(no error occurred) + +I + +=cut + +######################################### +######################################### sub publish { - my ($source,$target,$style)=@_; + my ($source,$target,$style,$batch)=@_; my $logfile; my $scrout=''; my $allmeta=''; my $content=''; my %allow=(); - undef %allow; unless ($logfile=Apache::File->new('>>'.$source.'.log')) { - return - 'No write permission to user directory, FAIL'; + return (''.&mt('No write permission to user directory, FAIL').'',1); } print $logfile -"\n\n================= Publish ".localtime()." Phase One ================\n"; +"\n\n================= Publish ".localtime()." Phase One ================\n".$env{'user.name'}.':'.$env{'user.domain'}."\n"; - if (($style eq 'ssi') || ($style eq 'rat')) { + if (($style eq 'ssi') || ($style eq 'rat') || ($style eq 'prv')) { # ------------------------------------------------------- This needs processing # ----------------------------------------------------------------- Backup Copy @@ -176,793 +1046,1291 @@ sub publish { print $logfile "Copied original file to ".$copyfile."\n"; } else { print $logfile "Unable to write backup ".$copyfile.':'.$!."\n"; - return "Failed to write backup copy, $!,FAIL"; + return ("".&mt("Failed to write backup copy, [_1], FAIL",$1)."",1); } # ------------------------------------------------------------- IDs and indices + + my ($outstring,$error); + ($outstring,$error,%allow)=&fix_ids_and_indices($logfile,$source, + $target); + if ($error) { return ($outstring,$error); } +# ------------------------------------------------------------ Construct Allows + + my $outdep=''; # Collect dependencies output data + my $allowstr=''; + foreach my $thisdep (sort(keys(%allow))) { + if ($thisdep !~ /[^\s]/) { next; } + if ($thisdep =~/\$/) { + $outdep.='
' + .&mt('The resource depends on another resource with variable filename, i.e., [_1].',''.$thisdep.'').'
' + .&mt('You likely need to explicitly allow access to all possible dependencies using the [_1]-tag.','<allow>') + ."
\n"; + } + unless ($style eq 'rat') { + $allowstr.="\n".''; + } + $outdep.='
'; + if ($thisdep!~/[\*\$]/ && $thisdep!~m|^/adm/|) { + $outdep.=''; + } + $outdep.=''.$thisdep.''; + if ($thisdep!~/[\*\$]/ && $thisdep!~m|^/adm/|) { + $outdep.=''; + if ( + &Apache::lonnet::getfile($Apache::lonnet::perlvar{'lonDocRoot'}.'/'. + $thisdep.'.meta') eq '-1') { + $outdep.= ' - '.&mt('Currently not available'). + ''; + } else { +# +# Store the fact that the dependency has been used by the target file +# Unfortunately, usage is erroneously named sequsage in lonmeta.pm +# The translation happens in lonmetadata.pm +# + my %temphash=(&Apache::lonnet::declutter($target).'___'. + &Apache::lonnet::declutter($thisdep).'___usage' + => time); + $thisdep=~m{^/res/($match_domain)/($match_username)/}; + if ((defined($1)) && (defined($2))) { + &Apache::lonnet::put('nohist_resevaldata',\%temphash, + $1,$2); + } + } + } + $outdep.='

'; + } - my $maxindex=10; - my $maxid=10; - - my $needsfixup=0; - - { - my $org=Apache::File->new($source); - $content=join('',<$org>); + if ($outdep) { + $scrout.='

'.&mt('Dependencies').'

' + .$outdep } - { - my $parser=HTML::TokeParser->new(\$content); - my $token; - while ($token=$parser->get_token) { - if ($token->[0] eq 'S') { - my $counter; - if ($counter=$addid{$token->[1]}) { - if ($counter eq 'id') { - if (defined($token->[2]->{'id'})) { - $maxid= - ($token->[2]->{'id'}>$maxid)?$token->[2]->{'id'}:$maxid; - } else { - $needsfixup=1; - } - } else { - if (defined($token->[2]->{'index'})) { - $maxindex= - ($token->[2]->{'index'}>$maxindex)?$token->[2]->{'index'}:$maxindex; - } else { - $needsfixup=1; - } - } - } - } - } - } - if ($needsfixup) { - print $logfile "Needs ID and/or index fixup\n". - "Max ID : $maxid (min 10)\n". - "Max Index: $maxindex (min 10)\n"; - } - my $outstring=''; - my $parser=HTML::TokeParser->new(\$content); - my $token; - while ($token=$parser->get_token) { - if ($token->[0] eq 'S') { - my $counter; - my $tag=$token->[1]; - unless ($tag eq 'allow') { - my %parms=%{$token->[2]}; - if ($counter=$addid{$tag}) { - if ($counter eq 'id') { - unless (defined($parms{'id'})) { - $maxid++; - $parms{'id'}=$maxid; - print $logfile 'ID: '.$tag.':'.$maxid."\n"; - } - } elsif ($counter eq 'index') { - unless (defined($parms{'index'})) { - $maxindex++; - $parms{'index'}=$maxindex; - print $logfile 'Index: '.$tag.':'.$maxindex."\n"; - } - } - } - - map { - if (defined($parms{$_})) { - my $oldurl=$parms{$_}; - my $newurl=&urlfixup($oldurl,$target); - if ($newurl ne $oldurl) { - $parms{$_}=$newurl; - print $logfile 'URL: '.$tag.':'.$oldurl.' - '. - $newurl."\n"; - } - $allow{$newurl}=1; - } - } ('src','href','codebase'); - - my $newparmstring=''; - my $endtag=''; - map { - if ($_ eq '/') { - $endtag=' /'; - } else { - my $quote=($parms{$_}=~/\"/?"'":'"'); - $newparmstring.=' '.$_.'='.$quote.$parms{$_}.$quote; - } - } keys %parms; - - $outstring.='<'.$tag.$newparmstring.$endtag.'>'; - } else { - $allow{$token->[2]->{'src'}}=1; - } - } elsif ($token->[0] eq 'E') { - unless ($token->[1] eq 'allow') { - $outstring.=$token->[2]; - } - } else { - $outstring.=$token->[1]; - } - } -# ------------------------------------------------------------ Construct Allows - unless ($style eq 'rat') { - my $allowstr="\n"; - map { - $allowstr.=''."\n"; - } keys %allow; - $outstring=~s/(\<\/[^\>]+\>\s*)$/$allowstr$1/s; - } -# ------------------------------------------------------------- Write modified + $outstring=~s/\n*(\<\/[^\>]+\>[^<]*)$/$allowstr\n$1\n/s; + +# ------------------------------------------------------------- Write modified. { my $org; unless ($org=Apache::File->new('>'.$source)) { print $logfile "No write permit to $source\n"; - return - "No write permission to $source, FAIL"; + return (''.&mt('No write permission to'). + ' '.$source. + ', '.&mt('FAIL').'',1); } - print $org $outstring; + print($org $outstring); } $content=$outstring; - if ($needsfixup) { - print $logfile "End of ID and/or index fixup\n". - "Max ID : $maxid (min 10)\n". - "Max Index: $maxindex (min 10)\n"; - } else { - print $logfile "Does not need ID and/or index fixup\n"; - } } -# --------------------------------------------- Initial step done, now metadata - -# ---------------------------------------- Storage for metadata keys and fields +# -------------------------------------------- Initial step done, now metadata. +# --------------------------------------- Storage for metadata keys and fields. +# these are globals +# %metadatafields=(); %metadatakeys=(); my %oldparmstores=(); + + unless ($batch) { + $scrout.='

'.&mt('Metadata').' ' . + &Apache::loncommon::help_open_topic("Metadata_Description") + . '

'; + } # ------------------------------------------------ First, check out environment - unless (-e $source.'.meta') { - $metadatafields{'author'}=$ENV{'environment.firstname'}.' '. - $ENV{'environment.middlename'}.' '. - $ENV{'environment.lastname'}.' '. - $ENV{'environment.generation'}; + if ((!(-e $source.'.meta')) || ($env{'form.forceoverride'})) { + $metadatafields{'author'}=$env{'environment.firstname'}.' '. + $env{'environment.middlename'}.' '. + $env{'environment.lastname'}.' '. + $env{'environment.generation'}; $metadatafields{'author'}=~s/\s+/ /g; $metadatafields{'author'}=~s/\s+$//; - $metadatafields{'owner'}=$cuname.'@'.$cudom; + $metadatafields{'owner'}=$cuname.':'.$cudom; # ------------------------------------------------ Check out directory hierachy my $thisdisfn=$source; - $thisdisfn=~s/^\/home\/$cuname\///; - my @urlparts=split(/\//,$thisdisfn); + $thisdisfn=~s/^\Q$docroot\E\/priv\/\Q$cudom\E\/\Q$cuname\E\///; + my @urlparts=('.',split(/\//,$thisdisfn)); $#urlparts--; - my $currentpath='/home/'.$cuname.'/'; + my $currentpath=$docroot.'/priv/'.$cudom.'/'.$cuname.'/'; + + my $prefix='../'x($#urlparts); + foreach my $subdir (@urlparts) { + $currentpath.=$subdir.'/'; + $scrout.=&metaread($logfile,$currentpath.'default.meta',$prefix); + $prefix=~s|^\.\./||; + } - map { - $currentpath.=$_.'/'; - $scrout.=&metaread($logfile,$currentpath.'default.meta'); - } @urlparts; +# ----------------------------------------------------------- Parse file itself +# read %metadatafields from file itself + + $allmeta=&parseformeta($source,$style); # ------------------- Clear out parameters and stores (there should not be any) - map { + foreach (keys %metadatafields) { if (($_=~/^parameter/) || ($_=~/^stores/)) { delete $metadatafields{$_}; } - } keys %metadatafields; + } } else { # ---------------------- Read previous metafile, remember parameters and stores $scrout.=&metaread($logfile,$source.'.meta'); - map { + foreach (keys %metadatafields) { if (($_=~/^parameter/) || ($_=~/^stores/)) { $oldparmstores{$_}=1; delete $metadatafields{$_}; } - } keys %metadatafields; - - } + } +# ------------------------------------------------------------- Save some stuff + my %savemeta=(); + foreach ('title') { + if ($metadatafields{$_}) { $savemeta{$_}=$metadatafields{$_}; } + } +# ------------------------------------------ See if anything new in file itself + + $allmeta=&parseformeta($source,$style); +# ----------------------------------------------------------- Restore the stuff + foreach (keys %savemeta) { + $metadatafields{$_}=$savemeta{$_}; + } + } -# -------------------------------------------------- Parse content for metadata - if ($style eq 'ssi') { - $allmeta=Apache::lonxml::xmlparse('meta',$content); + +# ---------------- Find and document discrepancies in the parameters and stores - &metaeval($allmeta); + my $chparms=''; + foreach (sort keys %metadatafields) { + if (($_=~/^parameter/) || ($_=~/^stores/)) { + unless ($_=~/\.\w+$/) { + unless ($oldparmstores{$_}) { + my $disp_key = $_; + $disp_key =~ tr/\0/_/; + print $logfile ('New: '.$disp_key."\n"); + $chparms .= $disp_key.' '; + } + } + } + } + if ($chparms) { + $scrout.='

'.&mt('New parameters or saved values'). + ': '.$chparms.'

'; } -# ---------------- Find and document discrepancies in the parameters and stores - my $chparms=''; - map { - if (($_=~/^parameter/) || ($_=~/^stores/)) { - unless ($_=~/\.\w+$/) { - unless ($oldparmstores{$_}) { - print $logfile 'New: '.$_."\n"; - $chparms.=$_.' '; - } - } + $chparms=''; + foreach (sort keys %oldparmstores) { + if (($_=~/^parameter/) || ($_=~/^stores/)) { + unless (($metadatafields{$_.'.name'}) || + ($metadatafields{$_.'.package'}) || ($_=~/\.\w+$/)) { + my $disp_key = $_; + $disp_key =~ tr/\0/_/; + print $logfile ('Obsolete: '.$disp_key."\n"); + $chparms.=$disp_key.' '; + } + } + } + if ($chparms) { + $scrout.='

'.&mt('Obsolete parameters or saved values').': ' + .$chparms.'

' + .'

'.&mt('Warning!').'
' + .&mt('If this resource is in active use, student performance data from the previous version may become inaccessible.') + .'

'; + } + if ($metadatafields{'copyright'} eq 'priv') { + $scrout.='

'.&mt('Warning!').'
' + .&mt('Copyright/distribution option "Private" is no longer supported. Select another option from below. Consider "Custom Rights" for maximum control over the usage of your resource.') + .'

'; + } + +# ------------------------------------------------------- Now have all metadata + + my %keywords=(); + + if (length($content)<500000) { + my $textonly=$content; + $textonly=~s/\//g; + $textonly=~s/\[^\<]+\<\/m\>//g; + $textonly=~s/\<[^\>]*\>//g; + + #this is a work simplification for german authors for present + $textonly=HTML::Entities::decode($textonly); #decode HTML-character + $textonly=Encode::Encoder::encode('utf8', $textonly); #encode to perl internal unicode + $textonly=~tr/A-ZÜÄÖ/a-züäö/; #add lowercase rule for german "Umlaute" + $textonly=~s/[\$\&][a-z]\w*//g; + $textonly=~s/[^a-z^ü^ä^ö^ß\s]//g; #dont delete german "Umlaute" + + foreach ($textonly=~m/[^\s]+/g) { #match all but whitespaces + unless ($nokey{$_}) { + $keywords{$_}=1; } - } sort keys %metadatafields; - if ($chparms) { - $scrout.='

New parameters or stored values: '. - $chparms; } - my $chparms=''; - map { - if (($_=~/^parameter/) || ($_=~/^stores/)) { - unless (($metadatafields{$_.'.name'}) || - ($metadatafields{$_.'.package'}) || ($_=~/\.\w+$/)) { - print $logfile 'Obsolete: '.$_."\n"; - $chparms.=$_.' '; - } + + } + + foreach my $addkey (split(/[\"\'\,\;]/,$metadatafields{'keywords'})) { + $addkey=~s/\s+/ /g; + $addkey=~s/^\s//; + $addkey=~s/\s$//; + if ($addkey=~/\w/) { + $keywords{$addkey}=1; + } + } +# --------------------------------------------------- Now we also have keywords +# ============================================================================= +# interactive mode html goes into $intr_scrout +# batch mode throws away this HTML +# additionally all of the field functions have a by product of setting +# $env{'from.'..} so that it can be used by the phase two handler in +# batch mode + + my $intr_scrout.='
' + .'

'; + unless ($env{'form.makeobsolete'}) { + $intr_scrout.='

' + .&mt('Searching for your resource will be based on the following metadata. Please provide as much data as possible.') + .'

' + .'

'.&mt('Cancel').'

'; + } + $intr_scrout.=&Apache::lonhtmlcommon::start_pick_box(); + $intr_scrout.= + &hiddenfield('phase','two'). + &hiddenfield('filename',$env{'form.filename'}). + &hiddenfield('allmeta',&escape($allmeta)). + &hiddenfield('dependencies',join(',',keys %allow)); + unless ($env{'form.makeobsolete'}) { + $intr_scrout.= + &textfield('Title','title',$metadatafields{'title'}). + &textfield('Author(s)','author',$metadatafields{'author'}). + &textfield('Subject','subject',$metadatafields{'subject'}); + # --------------------------------------------------- Scan content for keywords + + my $keywords_help = &Apache::loncommon::help_open_topic("Publishing_Keywords"); + my $keywordout=<<"END"; + +END + $keywordout.="\n".&Apache::lonhtmlcommon::row_title(&mt('Keywords')) + .$keywords_help + .'' + .'' + .'


' + .&Apache::loncommon::start_data_table(); + my $cols_per_row = 10; + my $colcount=0; + my $wordcount=0; + my $numkeywords = scalar(keys(%keywords)); + + foreach my $word (sort(keys(%keywords))) { + if ($colcount == 0) { + $keywordout .= &Apache::loncommon::start_data_table_row(); + } + $colcount++; + $wordcount++; + if (($wordcount == $numkeywords) && ($colcount < $cols_per_row)) { + my $colspan = 1+$cols_per_row-$colcount; + $keywordout .= ''; + } else { + $keywordout .= ''; + } + $keywordout.='