From e9f36cc22793869111b9c80449d7f701e511edf2 Mon Sep 17 00:00:00 2001 From: Galen Charlton Date: Fri, 23 Nov 2007 14:44:55 -0600 Subject: [PATCH] MARC import: part 3 of large file support * Introduced C4::UploadedFile to handle management and progress tracking of uploaded files. * Modified stage-marc-import.pl to handle new upload mechanism Signed-off-by: Joshua Ferraro --- C4/UploadedFile.pm | 317 ++++++++++++++++++++++++++++++++++ tools/stage-marc-import.pl | 32 ++-- tools/upload-file-progress.pl | 22 +-- tools/upload-file.pl | 71 +++----- 4 files changed, 352 insertions(+), 90 deletions(-) create mode 100644 C4/UploadedFile.pm diff --git a/C4/UploadedFile.pm b/C4/UploadedFile.pm new file mode 100644 index 0000000000..6b729913e4 --- /dev/null +++ b/C4/UploadedFile.pm @@ -0,0 +1,317 @@ +package C4::UploadedFile; + +# Copyright (C) 2007 LibLime +# Galen Charlton +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place, +# Suite 330, Boston, MA 02111-1307 USA + +use strict; +use C4::Context; +use C4::Auth qw/get_session/; +use IO::File; + +use vars qw($VERSION); + +# set the version for version checking +$VERSION = 3.00; + +=head1 NAME + +C4::UploadedFile - manage files uploaded by the user +for later processing. + +=head1 SYNOPSIS + +=over 4 + +# create and store data +my $uploaded_file = C4::UploadedFile->new($sessionID); +my $fileID = $uploaded_file->id(); +$uploaded_file->name('c:\temp\file.mrc'); +$uploaded_file->max_size(1024); +while ($have_more_data) { + $uploaded_file->stash($data, $bytes_read); +} +$uploaded_file->done(); + +# check status of current file upload +my $progress = C4::UploadedFile->upload_progress($sessionID); + +# get file handle for reading uploaded file +my $uploaded_file = C4::UploadedFile->fetch($fileID); +my $fh = $uploaded_file->fh(); + +=back + +Stores files uploaded by the user from their web browser. The +uploaded files are temporary and at present are not guaranteed +to survive beyond the life of the user's session. + +This module allows for tracking the progress of the file +currently being uploaded. + +TODO: implement secure persistant storage of uploaded files. + +=cut + +=head1 METHODS + +=cut + +=head2 new + +=over 4 + +my $uploaded_file = C4::UploadedFile->new($sessionID); + +=back + +Creates a new object to represent the uploaded file. Requires +the current session ID. + +=cut + +sub new { + my $class = shift; + my $sessionID = shift; + + my $self = {}; + + $self->{'sessionID'} = $sessionID; + $self->{'fileID'} = Digest::MD5::md5_hex(Digest::MD5::md5_hex(time().{}.rand().{}.$$)); + # FIXME - make staging area configurable + my $TEMPROOT = "/tmp"; + my $OUTPUTDIR = "$TEMPROOT/$sessionID"; + mkdir $OUTPUTDIR; + my $tmp_file_name = "$OUTPUTDIR/$self->{'fileID'}"; + my $fh = new IO::File $tmp_file_name, "w"; + unless (defined $fh) { + return undef; + } + $fh->binmode(); # Windows compatibility + $self->{'fh'} = $fh; + $self->{'tmp_file_name'} = $tmp_file_name; + my $session = get_session($sessionID); + $session->param("$self->{'fileID'}.uploaded_tmpfile", $tmp_file_name); + $session->param('current_upload', $self->{'fileID'}); + $session->flush(); + $self->{'session'} = $session; + $self->{'name'} = ''; + $self->{'max_size'} = 0; + $self->{'progress'} = 0; + + bless $self, $class; + + return $self; + +} + +=head2 id + +=over 4 + +my $fileID = $uploaded_file->id(); + +=back + +=cut + +sub id { + my $self = shift; + return $self->{'fileID'}; +} + +=head2 name + +=over 4 + +my $name = $uploaded_file->name(); +$uploaded_file->name($name); + +=back + +Accessor method for the name by which the file is to be known. + +=cut + +sub name { + my $self = shift; + if (@_) { + $self->{'name'} = shift; + $self->{'session'}->param("$self->{'fileID'}.uploaded_filename", $self->{'name'}); + $self->{'session'}->flush(); + } else { + return $self->{'name'}; + } +} + +=head2 max_size + +=over 4 + +my $max_size = $uploaded_file->max_size(); +$uploaded_file->max_size($max_size); + +=back + +Accessor method for the maximum size of the uploaded file. + +=cut + +sub max_size { + my $self = shift; + @_ ? $self->{'max_size'} = shift : $self->{'max_size'}; +} + +=head2 stash + +=over 4 + +$uploaded_file->stash($dataref, $bytes_read); + +=back + +Write C<$dataref> to the temporary file. C<$bytes_read> represents +the number of bytes (out of C<$max_size>) transmitted so far. + +=cut + +sub stash { + my $self = shift; + my $dataref = shift; + my $bytes_read = shift; + + my $fh = $self->{'fh'}; + print $fh $$dataref; + + my $percentage = int(($bytes_read / $self->{'max_size'}) * 100); + if ($percentage > $self->{'progress'}) { + $self->{'progress'} = $percentage; + $self->{'session'}->param("$self->{'fileID'}.uploadprogress", $self->{'progress'}); + $self->{'session'}->flush(); + } +} + +=head2 done + +=over 4 + +$uploaded_file->done(); + +=back + +Indicates that all of the bytes have been uploaded. + +=cut + +sub done { + my $self = shift; + $self->{'session'}->param("$self->{'fileID'}.uploadprogress", 'done'); + $self->{'session'}->flush(); + $self->{'fh'}->close(); +} + +=head2 upload_progress + +=over 4 + +my $upload_progress = C4::UploadFile->upload_progress($sessionID); + +=back + +Returns (as an integer from 0 to 100) the percentage +progress of the current file upload. + +=cut + +sub upload_progress { + my ($class, $sessionID) = shift; + + my $session = get_session($sessionID); + + my $fileID = $session->param('current_upload'); + + my $reported_progress = 0; + if (defined $fileID and $fileID ne "") { + my $progress = $session->param("$fileID.uploadprogress"); + if (defined $progress) { + if ($progress eq "done") { + $reported_progress = 100; + } else { + $reported_progress = $progress; + } + } + } + return $reported_progress; +} + +=head2 fetch + +=over 4 + + my $uploaded_file = C4::UploadedFile->fetch($sessionID, $fileID); + +=back + +Retrieves an uploaded file object from the current session. + +=cut + +sub fetch { + my $class = shift; + my $sessionID = shift; + my $fileID = shift; + + my $self = {}; + + $self->{'sessionID'} = $sessionID; + $self->{'fileID'} = $fileID; + my $session = get_session($sessionID); + $self->{'session'} = $session; + $self->{'tmp_file_name'} = $session->param("$self->{'fileID'}.uploaded_tmpfile"); + $self->{'name'} = $session->param("$self->{'fileID'}.uploaded_filename"); + my $fh = new IO::File $self->{'tmp_file_name'}, "r"; + $self->{'fh'} = $fh; + + bless $self, $class; + + return $self; +} + +=head2 fh + +=over + +my $fh = $uploaded_file->fh(); + +=back + +Returns an IO::File handle to read the uploaded file. + +=cut + +sub fh { + my $self = shift; + return $self->{'fh'}; +} + +=head1 AUTHOR + +Koha Development Team + +Galen Charlton + +=cut diff --git a/tools/stage-marc-import.pl b/tools/stage-marc-import.pl index 22dff90062..4bea9324a5 100755 --- a/tools/stage-marc-import.pl +++ b/tools/stage-marc-import.pl @@ -28,6 +28,7 @@ use strict; # standard or CPAN modules used use CGI; +use CGI::Cookie; use MARC::File::USMARC; # Koha modules used @@ -38,27 +39,12 @@ use C4::Output; use C4::Biblio; use C4::ImportBatch; use C4::Matcher; - -#------------------ -# Constants - -my $includes = C4::Context->config('includes') || - "/usr/local/www/hdl/htdocs/includes"; - -# HTML colors for alternating lines -my $lc1='#dddddd'; -my $lc2='#ddaaaa'; - -#------------- -#------------- -# Initialize - -my $userid=$ENV{'REMOTE_USER'}; +use C4::UploadedFile; my $input = new CGI; my $dbh = C4::Context->dbh; -my $uploadmarc=$input->param('uploadmarc'); +my $fileID=$input->param('uploadedfileid'); my $matcher_id = $input->param('matcher'); my $parse_items = $input->param('parse_items'); my $comments = $input->param('comments'); @@ -73,15 +59,19 @@ my ($template, $loggedinuser, $cookie) }); $template->param(SCRIPT_NAME => $ENV{'SCRIPT_NAME'}, - uploadmarc => $uploadmarc); -my $filename = $uploadmarc; -if ($uploadmarc && length($uploadmarc)>0) { + uploadmarc => $fileID); + +if ($fileID) { + my %cookies = parse CGI::Cookie($cookie); + my $uploaded_file = C4::UploadedFile->fetch($cookies{'CGISESSID'}->value, $fileID); + my $fh = $uploaded_file->fh(); my $marcrecord=''; - while (<$uploadmarc>) { + while (<$fh>) { $marcrecord.=$_; } # FIXME branch code + my $filename = $uploaded_file->name(); my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($syntax, $marcrecord, $filename, $comments, '', $parse_items, 0); my $num_with_matches = 0; diff --git a/tools/upload-file-progress.pl b/tools/upload-file-progress.pl index c7b1f3ec27..3f740f7eb4 100755 --- a/tools/upload-file-progress.pl +++ b/tools/upload-file-progress.pl @@ -24,10 +24,10 @@ use IO::File; use CGI; use CGI::Session; use C4::Context; -use C4::Auth qw/get_session check_cookie_auth/; +use C4::Auth qw/check_cookie_auth/; +use C4::UploadedFile; use CGI::Cookie; # need to check cookies before # having CGI parse the POST request -use Digest::MD5; my %cookies = fetch CGI::Cookie; my %cookies = fetch CGI::Cookie; @@ -39,23 +39,7 @@ if ($auth_status ne "ok") { exit 0; } -my $session = get_session($sessionID); - -my $query = CGI->new; -my $fileid = $session->param('current_upload'); - -my $reported_progress = 0; -if (defined $fileid and $fileid ne "") { - my $progress = $session->param("$fileid.uploadprogress"); - if (defined $progress) { - if ($progress eq "done") { - $reported_progress = 100; - } else { - $reported_progress = $progress; - } - } -} - +my $reported_progress = C4::UploadedFile->upload_progress($sessionID); my $reply = CGI->new(""); print $reply->header(-type => 'text/html'); diff --git a/tools/upload-file.pl b/tools/upload-file.pl index ad9efc899f..1c11a04fc5 100755 --- a/tools/upload-file.pl +++ b/tools/upload-file.pl @@ -24,20 +24,10 @@ use IO::File; use CGI; use CGI::Session; use C4::Context; -use C4::Auth qw/get_session check_cookie_auth/; +use C4::Auth qw/check_cookie_auth/; use CGI::Cookie; # need to check cookies before # having CGI parse the POST request -use Digest::MD5; - -my %cookies = fetch CGI::Cookie; -my ($auth_status, $sessionID) = check_cookie_auth($cookies{'CGISESSID'}->value, { tools => 1 }); -if ($auth_status ne "ok") { - $auth_status = 'denied' if $auth_status eq 'failed'; - send_reply($auth_status, "", ""); - exit 0; -} - -my $session = get_session($sessionID); +use C4::UploadedFile; # upload-file.pl must authenticate the user # before processing the POST request, @@ -45,68 +35,49 @@ my $session = get_session($sessionID); # not authorized. Consequently, unlike # most of the other CGI scripts, upload-file.pl # requires that the session cookie already -# have been created., $fileid, $tmp_file_name - -my $fileid = Digest::MD5::md5_hex(Digest::MD5::md5_hex(time().{}.rand().{}.$$)); +# have been created. -# FIXME - make staging area configurable -my $TEMPROOT = "/tmp"; -my $OUTPUTDIR = "$TEMPROOT/$sessionID"; -mkdir $OUTPUTDIR; -my $tmp_file_name = "$OUTPUTDIR/$fileid"; +my %cookies = fetch CGI::Cookie; +my ($auth_status, $sessionID) = check_cookie_auth($cookies{'CGISESSID'}->value, { tools => 1 }); +if ($auth_status ne "ok") { + $auth_status = 'denied' if $auth_status eq 'failed'; + send_reply($auth_status, ""); + exit 0; +} -my $fh = new IO::File $tmp_file_name, "w"; -unless (defined $fh) { +my $uploaded_file = C4::UploadedFile->new($sessionID); +unless (defined $uploaded_file) { # FIXME - failed to create file for some reason - send_reply('failed', '', ''); + send_reply('failed', ''); exit 0; } -$fh->binmode(); # for Windows compatibility -$session->param("$fileid.uploaded_tmpfile", $tmp_file_name); -$session->param('current_upload', $fileid); -$session->flush(); +$uploaded_file->max_size($ENV{'CONTENT_LENGTH'}); # may not be the file size, exactly -my $progress = 0; my $first_chunk = 1; -my $max_size = $ENV{'CONTENT_LENGTH'}; # may not be the file size, exactly my $query; -$|++; -$query = new CGI \&upload_hook, $session; -clean_up(); -send_reply('done', $fileid, $tmp_file_name); +$query = new CGI \&upload_hook; +$uploaded_file->done(); +send_reply('done', $uploaded_file->id()); # FIXME - if possible, trap signal caused by user cancelling upload # FIXME - something is wrong during cleanup: \t(in cleanup) Can't call method "commit" on unblessed reference at /usr/local/share/perl/5.8.8/CGI/Session/Driver/DBI.pm line 130 during global destruction. exit 0; -sub clean_up { - $session->param("$fileid.uploadprogress", 'done'); - $session->flush(); -} - sub upload_hook { my ($file_name, $buffer, $bytes_read, $session) = @_; - print $fh $buffer; - # stash received file name + $uploaded_file->stash(\$buffer, $bytes_read); if ($first_chunk) { - $session->param("$fileid.uploaded_filename", $file_name); - $session->flush(); + $uploaded_file->name($file_name); $first_chunk = 0; } - my $percentage = int(($bytes_read / $max_size) * 100); - if ($percentage > $progress) { - $progress = $percentage; - $session->param("$fileid.uploadprogress", $progress); - $session->flush(); - } } sub send_reply { - my ($upload_status, $fileid, $tmp_file_name) = @_; + my ($upload_status, $fileid) = @_; my $reply = CGI->new(""); print $reply->header(-type => 'text/html'); # response will be sent back as JSON - print "{ status: '$upload_status', fileid: '$fileid', tmp_file_name: '$tmp_file_name' }"; + print "{ status: '$upload_status', fileid: '$fileid' }"; } -- 2.39.2