Browse Source

MARC import: part 3 of large file support

* Introduced C4::UploadedFile to handle management and
  progress tracking of uploaded files.
* Modified stage-marc-import.pl to handle new upload
  mechanism

Signed-off-by: Joshua Ferraro <jmf@liblime.com>
3.0.x
Galen Charlton 17 years ago
committed by Joshua Ferraro
parent
commit
e9f36cc227
  1. 317
      C4/UploadedFile.pm
  2. 32
      tools/stage-marc-import.pl
  3. 22
      tools/upload-file-progress.pl
  4. 71
      tools/upload-file.pl

317
C4/UploadedFile.pm

@ -0,0 +1,317 @@
package C4::UploadedFile;
# Copyright (C) 2007 LibLime
# Galen Charlton <galen.charlton@liblime.com>
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
# Suite 330, Boston, MA 02111-1307 USA
use strict;
use C4::Context;
use C4::Auth qw/get_session/;
use IO::File;
use vars qw($VERSION);
# set the version for version checking
$VERSION = 3.00;
=head1 NAME
C4::UploadedFile - manage files uploaded by the user
for later processing.
=head1 SYNOPSIS
=over 4
# create and store data
my $uploaded_file = C4::UploadedFile->new($sessionID);
my $fileID = $uploaded_file->id();
$uploaded_file->name('c:\temp\file.mrc');
$uploaded_file->max_size(1024);
while ($have_more_data) {
$uploaded_file->stash($data, $bytes_read);
}
$uploaded_file->done();
# check status of current file upload
my $progress = C4::UploadedFile->upload_progress($sessionID);
# get file handle for reading uploaded file
my $uploaded_file = C4::UploadedFile->fetch($fileID);
my $fh = $uploaded_file->fh();
=back
Stores files uploaded by the user from their web browser. The
uploaded files are temporary and at present are not guaranteed
to survive beyond the life of the user's session.
This module allows for tracking the progress of the file
currently being uploaded.
TODO: implement secure persistant storage of uploaded files.
=cut
=head1 METHODS
=cut
=head2 new
=over 4
my $uploaded_file = C4::UploadedFile->new($sessionID);
=back
Creates a new object to represent the uploaded file. Requires
the current session ID.
=cut
sub new {
my $class = shift;
my $sessionID = shift;
my $self = {};
$self->{'sessionID'} = $sessionID;
$self->{'fileID'} = Digest::MD5::md5_hex(Digest::MD5::md5_hex(time().{}.rand().{}.$$));
# FIXME - make staging area configurable
my $TEMPROOT = "/tmp";
my $OUTPUTDIR = "$TEMPROOT/$sessionID";
mkdir $OUTPUTDIR;
my $tmp_file_name = "$OUTPUTDIR/$self->{'fileID'}";
my $fh = new IO::File $tmp_file_name, "w";
unless (defined $fh) {
return undef;
}
$fh->binmode(); # Windows compatibility
$self->{'fh'} = $fh;
$self->{'tmp_file_name'} = $tmp_file_name;
my $session = get_session($sessionID);
$session->param("$self->{'fileID'}.uploaded_tmpfile", $tmp_file_name);
$session->param('current_upload', $self->{'fileID'});
$session->flush();
$self->{'session'} = $session;
$self->{'name'} = '';
$self->{'max_size'} = 0;
$self->{'progress'} = 0;
bless $self, $class;
return $self;
}
=head2 id
=over 4
my $fileID = $uploaded_file->id();
=back
=cut
sub id {
my $self = shift;
return $self->{'fileID'};
}
=head2 name
=over 4
my $name = $uploaded_file->name();
$uploaded_file->name($name);
=back
Accessor method for the name by which the file is to be known.
=cut
sub name {
my $self = shift;
if (@_) {
$self->{'name'} = shift;
$self->{'session'}->param("$self->{'fileID'}.uploaded_filename", $self->{'name'});
$self->{'session'}->flush();
} else {
return $self->{'name'};
}
}
=head2 max_size
=over 4
my $max_size = $uploaded_file->max_size();
$uploaded_file->max_size($max_size);
=back
Accessor method for the maximum size of the uploaded file.
=cut
sub max_size {
my $self = shift;
@_ ? $self->{'max_size'} = shift : $self->{'max_size'};
}
=head2 stash
=over 4
$uploaded_file->stash($dataref, $bytes_read);
=back
Write C<$dataref> to the temporary file. C<$bytes_read> represents
the number of bytes (out of C<$max_size>) transmitted so far.
=cut
sub stash {
my $self = shift;
my $dataref = shift;
my $bytes_read = shift;
my $fh = $self->{'fh'};
print $fh $$dataref;
my $percentage = int(($bytes_read / $self->{'max_size'}) * 100);
if ($percentage > $self->{'progress'}) {
$self->{'progress'} = $percentage;
$self->{'session'}->param("$self->{'fileID'}.uploadprogress", $self->{'progress'});
$self->{'session'}->flush();
}
}
=head2 done
=over 4
$uploaded_file->done();
=back
Indicates that all of the bytes have been uploaded.
=cut
sub done {
my $self = shift;
$self->{'session'}->param("$self->{'fileID'}.uploadprogress", 'done');
$self->{'session'}->flush();
$self->{'fh'}->close();
}
=head2 upload_progress
=over 4
my $upload_progress = C4::UploadFile->upload_progress($sessionID);
=back
Returns (as an integer from 0 to 100) the percentage
progress of the current file upload.
=cut
sub upload_progress {
my ($class, $sessionID) = shift;
my $session = get_session($sessionID);
my $fileID = $session->param('current_upload');
my $reported_progress = 0;
if (defined $fileID and $fileID ne "") {
my $progress = $session->param("$fileID.uploadprogress");
if (defined $progress) {
if ($progress eq "done") {
$reported_progress = 100;
} else {
$reported_progress = $progress;
}
}
}
return $reported_progress;
}
=head2 fetch
=over 4
my $uploaded_file = C4::UploadedFile->fetch($sessionID, $fileID);
=back
Retrieves an uploaded file object from the current session.
=cut
sub fetch {
my $class = shift;
my $sessionID = shift;
my $fileID = shift;
my $self = {};
$self->{'sessionID'} = $sessionID;
$self->{'fileID'} = $fileID;
my $session = get_session($sessionID);
$self->{'session'} = $session;
$self->{'tmp_file_name'} = $session->param("$self->{'fileID'}.uploaded_tmpfile");
$self->{'name'} = $session->param("$self->{'fileID'}.uploaded_filename");
my $fh = new IO::File $self->{'tmp_file_name'}, "r";
$self->{'fh'} = $fh;
bless $self, $class;
return $self;
}
=head2 fh
=over
my $fh = $uploaded_file->fh();
=back
Returns an IO::File handle to read the uploaded file.
=cut
sub fh {
my $self = shift;
return $self->{'fh'};
}
=head1 AUTHOR
Koha Development Team <info@koha.org>
Galen Charlton <galen.charlton@liblime.com>
=cut

32
tools/stage-marc-import.pl

@ -28,6 +28,7 @@ use strict;
# standard or CPAN modules used
use CGI;
use CGI::Cookie;
use MARC::File::USMARC;
# Koha modules used
@ -38,27 +39,12 @@ use C4::Output;
use C4::Biblio;
use C4::ImportBatch;
use C4::Matcher;
#------------------
# Constants
my $includes = C4::Context->config('includes') ||
"/usr/local/www/hdl/htdocs/includes";
# HTML colors for alternating lines
my $lc1='#dddddd';
my $lc2='#ddaaaa';
#-------------
#-------------
# Initialize
my $userid=$ENV{'REMOTE_USER'};
use C4::UploadedFile;
my $input = new CGI;
my $dbh = C4::Context->dbh;
my $uploadmarc=$input->param('uploadmarc');
my $fileID=$input->param('uploadedfileid');
my $matcher_id = $input->param('matcher');
my $parse_items = $input->param('parse_items');
my $comments = $input->param('comments');
@ -73,15 +59,19 @@ my ($template, $loggedinuser, $cookie)
});
$template->param(SCRIPT_NAME => $ENV{'SCRIPT_NAME'},
uploadmarc => $uploadmarc);
my $filename = $uploadmarc;
if ($uploadmarc && length($uploadmarc)>0) {
uploadmarc => $fileID);
if ($fileID) {
my %cookies = parse CGI::Cookie($cookie);
my $uploaded_file = C4::UploadedFile->fetch($cookies{'CGISESSID'}->value, $fileID);
my $fh = $uploaded_file->fh();
my $marcrecord='';
while (<$uploadmarc>) {
while (<$fh>) {
$marcrecord.=$_;
}
# FIXME branch code
my $filename = $uploaded_file->name();
my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($syntax, $marcrecord, $filename,
$comments, '', $parse_items, 0);
my $num_with_matches = 0;

22
tools/upload-file-progress.pl

@ -24,10 +24,10 @@ use IO::File;
use CGI;
use CGI::Session;
use C4::Context;
use C4::Auth qw/get_session check_cookie_auth/;
use C4::Auth qw/check_cookie_auth/;
use C4::UploadedFile;
use CGI::Cookie; # need to check cookies before
# having CGI parse the POST request
use Digest::MD5;
my %cookies = fetch CGI::Cookie;
my %cookies = fetch CGI::Cookie;
@ -39,23 +39,7 @@ if ($auth_status ne "ok") {
exit 0;
}
my $session = get_session($sessionID);
my $query = CGI->new;
my $fileid = $session->param('current_upload');
my $reported_progress = 0;
if (defined $fileid and $fileid ne "") {
my $progress = $session->param("$fileid.uploadprogress");
if (defined $progress) {
if ($progress eq "done") {
$reported_progress = 100;
} else {
$reported_progress = $progress;
}
}
}
my $reported_progress = C4::UploadedFile->upload_progress($sessionID);
my $reply = CGI->new("");
print $reply->header(-type => 'text/html');

71
tools/upload-file.pl

@ -24,20 +24,10 @@ use IO::File;
use CGI;
use CGI::Session;
use C4::Context;
use C4::Auth qw/get_session check_cookie_auth/;
use C4::Auth qw/check_cookie_auth/;
use CGI::Cookie; # need to check cookies before
# having CGI parse the POST request
use Digest::MD5;
my %cookies = fetch CGI::Cookie;
my ($auth_status, $sessionID) = check_cookie_auth($cookies{'CGISESSID'}->value, { tools => 1 });
if ($auth_status ne "ok") {
$auth_status = 'denied' if $auth_status eq 'failed';
send_reply($auth_status, "", "");
exit 0;
}
my $session = get_session($sessionID);
use C4::UploadedFile;
# upload-file.pl must authenticate the user
# before processing the POST request,
@ -45,68 +35,49 @@ my $session = get_session($sessionID);
# not authorized. Consequently, unlike
# most of the other CGI scripts, upload-file.pl
# requires that the session cookie already
# have been created., $fileid, $tmp_file_name
my $fileid = Digest::MD5::md5_hex(Digest::MD5::md5_hex(time().{}.rand().{}.$$));
# have been created.
# FIXME - make staging area configurable
my $TEMPROOT = "/tmp";
my $OUTPUTDIR = "$TEMPROOT/$sessionID";
mkdir $OUTPUTDIR;
my $tmp_file_name = "$OUTPUTDIR/$fileid";
my %cookies = fetch CGI::Cookie;
my ($auth_status, $sessionID) = check_cookie_auth($cookies{'CGISESSID'}->value, { tools => 1 });
if ($auth_status ne "ok") {
$auth_status = 'denied' if $auth_status eq 'failed';
send_reply($auth_status, "");
exit 0;
}
my $fh = new IO::File $tmp_file_name, "w";
unless (defined $fh) {
my $uploaded_file = C4::UploadedFile->new($sessionID);
unless (defined $uploaded_file) {
# FIXME - failed to create file for some reason
send_reply('failed', '', '');
send_reply('failed', '');
exit 0;
}
$fh->binmode(); # for Windows compatibility
$session->param("$fileid.uploaded_tmpfile", $tmp_file_name);
$session->param('current_upload', $fileid);
$session->flush();
$uploaded_file->max_size($ENV{'CONTENT_LENGTH'}); # may not be the file size, exactly
my $progress = 0;
my $first_chunk = 1;
my $max_size = $ENV{'CONTENT_LENGTH'}; # may not be the file size, exactly
my $query;
$|++;
$query = new CGI \&upload_hook, $session;
clean_up();
send_reply('done', $fileid, $tmp_file_name);
$query = new CGI \&upload_hook;
$uploaded_file->done();
send_reply('done', $uploaded_file->id());
# FIXME - if possible, trap signal caused by user cancelling upload
# FIXME - something is wrong during cleanup: \t(in cleanup) Can't call method "commit" on unblessed reference at /usr/local/share/perl/5.8.8/CGI/Session/Driver/DBI.pm line 130 during global destruction.
exit 0;
sub clean_up {
$session->param("$fileid.uploadprogress", 'done');
$session->flush();
}
sub upload_hook {
my ($file_name, $buffer, $bytes_read, $session) = @_;
print $fh $buffer;
# stash received file name
$uploaded_file->stash(\$buffer, $bytes_read);
if ($first_chunk) {
$session->param("$fileid.uploaded_filename", $file_name);
$session->flush();
$uploaded_file->name($file_name);
$first_chunk = 0;
}
my $percentage = int(($bytes_read / $max_size) * 100);
if ($percentage > $progress) {
$progress = $percentage;
$session->param("$fileid.uploadprogress", $progress);
$session->flush();
}
}
sub send_reply {
my ($upload_status, $fileid, $tmp_file_name) = @_;
my ($upload_status, $fileid) = @_;
my $reply = CGI->new("");
print $reply->header(-type => 'text/html');
# response will be sent back as JSON
print "{ status: '$upload_status', fileid: '$fileid', tmp_file_name: '$tmp_file_name' }";
print "{ status: '$upload_status', fileid: '$fileid' }";
}

Loading…
Cancel
Save