From d24568b67256fab79b42ead54302c76d1c9653d7 Mon Sep 17 00:00:00 2001 From: Marcel de Rooy Date: Mon, 7 Aug 2017 21:24:48 +0200 Subject: [PATCH] Bug 19049: Fix regression on stage-marc-import with to_marc plugin Bug 12412 added the use of to_marc plugins allowing arbitrary file formats in stage-marc-import (as long as the plugins can handle them). The feature was not very visible in the code, and when bug 10407 added the marcxml format, it made some changes that broke the use of to_marc. This patch restores the functionality by: [1] Adding a sub RecordsFromMarcPlugin to ImportBatch.pm, specifically addressing the conversion from arbitrary formats to MARC::Record. The original to_marc interface is used: pass it the file contents, and it returns a string consisting of a number of MARC blobs separated by \x1D. Consequently, the call of to_marc is removed from routine BatchStageMarcRecords where it did not belong. The to_marc_plugin parameter is removed and two calls are adjusted accordingly. [2] Instead of a separate combo with plugins, the format combo contains MARC, MARCXML and optionally some plugin formats. [3] The code in stage-marc-import.pl now clearly shows the three main format types: MARC, MARCXML or plugin based. Note: This patch restores more or less the situation after bug 12412, but I would actually recommend to have the to_marc plugins return MARC::Record objects instead of large text strings. In the second example I added a to_marc plugin that actually converts MARC record objects to string format, while RecordsFromMarcPlugin reconverts them to MARC::Records. Test plan: See second patch. Signed-off-by: Marcel de Rooy Signed-off-by: Katrin Fischer Signed-off-by: Kyle M Hall Signed-off-by: Jonathan Druart --- C4/ImportBatch.pm | 50 +++++++++++++++---- .../en/modules/tools/stage-marc-import.tt | 20 ++------ misc/stage_file.pl | 2 +- tools/stage-marc-import.pl | 12 +++-- 4 files changed, 52 insertions(+), 32 deletions(-) diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index c4550eb4ab..82d6af2bfc 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -352,8 +352,8 @@ sub ModAuthInBatch { ( $batch_id, $num_records, $num_items, @invalid_records ) = BatchStageMarcRecords( - $encoding, $marc_records, - $file_name, $to_marc_plugin, + $record_type, $encoding, + $marc_records, $file_name, $marc_modification_template, $comments, $branch_code, $parse_items, $leave_as_staging, $progress_interval, @@ -367,7 +367,6 @@ sub BatchStageMarcRecords { my $encoding = shift; my $marc_records = shift; my $file_name = shift; - my $to_marc_plugin = shift; my $marc_modification_template = shift; my $comments = shift; my $branch_code = shift; @@ -399,13 +398,6 @@ sub BatchStageMarcRecords { SetImportBatchItemAction($batch_id, 'ignore'); } - $marc_records = Koha::Plugins::Handler->run( - { - class => $to_marc_plugin, - method => 'to_marc', - params => { data => $marc_records } - } - ) if $to_marc_plugin && @$marc_records; my $marc_type = C4::Context->preference('marcflavour'); $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth'); @@ -1556,6 +1548,44 @@ sub RecordsFromMARCXMLFile { return (\@errors, \@marcRecords); } +=head2 RecordsFromMarcPlugin + + Converts text of input_file into array of MARC records with to_marc plugin + +=cut + +sub RecordsFromMarcPlugin { + my ($input_file, $plugin_class, $encoding) = @_; + + # Read input file + open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n"; + my ( $text, $marc, @return ); + $/ = "\035"; + while () { + s/^\s+//; + s/\s+$//; + next unless $_; + $text .= $_; + } + close IN; + + # Convert to large MARC blob with plugin + $text = Koha::Plugins::Handler->run({ + class => $plugin_class, + method => 'to_marc', + params => { data => $text }, + }); + + # Convert to array of MARC records + my $marc_type = C4::Context->preference('marcflavour'); + foreach my $blob ( split(/\x1D/, $text) ) { + next if $blob =~ /^\s*$/; + my ($marcrecord) = MarcToUTF8Record($blob, $marc_type, $encoding); + push @return, $marcrecord; + } + return \@return; +} + # internal functions sub _create_import_record { diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt index 2383451977..4f23ce7ea5 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt @@ -198,27 +198,13 @@ function cbUpload( status, fileid ) { - [% IF plugins %] -
- Transform file to MARC: -
    -
  1. - - -
    2. -
-
- [% END %] - [% IF MarcModificationTemplatesLoop %]
Use MARC Modification Template: diff --git a/misc/stage_file.pl b/misc/stage_file.pl index ad0b7267ab..1f11b6adac 100755 --- a/misc/stage_file.pl +++ b/misc/stage_file.pl @@ -116,7 +116,7 @@ sub process_batch { print "... staging MARC records -- please wait\n"; #FIXME: We should really allow the use of marc modification frameworks and to_marc plugins here if possible my ($batch_id, $num_valid_records, $num_items, @import_errors) = - BatchStageMarcRecords($record_type, $params->{encoding}, $marc_records, $params->{input_file}, undef, undef, $params->{batch_comment}, '', $params->{add_items}, 0, + BatchStageMarcRecords($record_type, $params->{encoding}, $marc_records, $params->{input_file}, undef, $params->{batch_comment}, '', $params->{add_items}, 0, 100, \&print_progress_and_commit); print "... finished staging MARC records\n"; diff --git a/tools/stage-marc-import.pl b/tools/stage-marc-import.pl index 63dd4e5130..0ad0128958 100755 --- a/tools/stage-marc-import.pl +++ b/tools/stage-marc-import.pl @@ -57,8 +57,7 @@ my $item_action = $input->param('item_action'); my $comments = $input->param('comments'); my $record_type = $input->param('record_type'); my $encoding = $input->param('encoding') || 'UTF-8'; -my $format = $input->param('format') || 'ISO2709'; -my $to_marc_plugin = $input->param('to_marc_plugin'); +my $format = $input->param('format'); my $marc_modification_template = $input->param('marc_modification_template_id'); my ( $template, $loggedinuser, $cookie ) = get_template_and_user( @@ -92,8 +91,13 @@ if ($completedJobID) { my ( $errors, $marcrecords ); if( $format eq 'MARCXML' ) { ( $errors, $marcrecords ) = C4::ImportBatch::RecordsFromMARCXMLFile( $file, $encoding); - } else { + } elsif( $format eq 'MARC' ) { ( $errors, $marcrecords ) = C4::ImportBatch::RecordsFromISO2709File( $file, $record_type, $encoding ); + } elsif( $format ) { # plugin + $errors = []; + $marcrecords = C4::ImportBatch::RecordsFromMarcPlugin( $file, $format, $encoding ); + } else { + die "No format specified"; } warn "$filename: " . ( join ',', @$errors ) if @$errors; # no need to exit if we have no records (or only errors) here @@ -141,7 +145,7 @@ if ($completedJobID) { BatchStageMarcRecords( $record_type, $encoding, $marcrecords, $filename, - $to_marc_plugin, $marc_modification_template, + $marc_modification_template, $comments, '', $parse_items, 0, 50, staging_progress_callback( $job, $dbh ) -- 2.39.5