From 8497ed67b7238ec751714b8f4d6a11e7eb1cc715 Mon Sep 17 00:00:00 2001 From: Jonathan Druart Date: Mon, 13 Jun 2022 14:29:45 +0200 Subject: [PATCH] Bug 27421: Use Background job for staging MARC records for import Signed-off-by: Nick Clemens Signed-off-by: Marcel de Rooy Signed-off-by: Tomas Cohen Arazi --- C4/ImportBatch.pm | 15 ++ Koha/BackgroundJob.pm | 1 + Koha/BackgroundJob/StageMARCForImport.pm | 198 ++++++++++++++++++ .../apache-shared-intranet-plack.conf | 1 - .../background_jobs/stage_marc_for_import.inc | 46 ++++ .../prog/en/modules/admin/background_jobs.tt | 2 + .../en/modules/tools/stage-marc-import.tt | 97 ++++----- tools/stage-marc-import.pl | 189 ++++------------- 8 files changed, 339 insertions(+), 210 deletions(-) create mode 100644 Koha/BackgroundJob/StageMARCForImport.pm create mode 100644 koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/stage_marc_for_import.inc diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index 3f469f6a44..ddf10e666d 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -390,6 +390,10 @@ sub BatchStageMarcRecords { } # FIXME branch_code, number of bibs, number of items _update_batch_record_counts($batch_id); + if ($progress_interval){ + &$progress_callback($rec_num); + } + return ($batch_id, $num_valid, $num_items, @invalid_records); } @@ -494,6 +498,11 @@ sub BatchFindDuplicates { SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'no_match'); } } + + if ($progress_interval){ + &$progress_callback($rec_num); + } + $sth->finish(); return $num_with_matches; } @@ -690,7 +699,13 @@ sub BatchCommitRecords { SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored'); } } + + if ($progress_interval){ + &$progress_callback($rec_num); + } + $schema->txn_commit; # Commit final records that may not have hit callback threshold + $sth->finish(); if ( @biblio_ids ) { diff --git a/Koha/BackgroundJob.pm b/Koha/BackgroundJob.pm index e4aede3fe3..9bfd33a5eb 100644 --- a/Koha/BackgroundJob.pm +++ b/Koha/BackgroundJob.pm @@ -421,6 +421,7 @@ sub core_types_to_classes { batch_hold_cancel => 'Koha::BackgroundJob::BatchCancelHold', update_elastic_index => 'Koha::BackgroundJob::UpdateElasticIndex', update_holds_queue_for_biblios => 'Koha::BackgroundJob::BatchUpdateBiblioHoldsQueue', + stage_marc_for_import => 'Koha::BackgroundJob::StageMARCForImport', }; } diff --git a/Koha/BackgroundJob/StageMARCForImport.pm b/Koha/BackgroundJob/StageMARCForImport.pm new file mode 100644 index 0000000000..a2c12a78fd --- /dev/null +++ b/Koha/BackgroundJob/StageMARCForImport.pm @@ -0,0 +1,198 @@ +package Koha::BackgroundJob::StageMARCForImport; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; +use Try::Tiny; + +use base 'Koha::BackgroundJob'; + +use C4::Matcher; +use C4::ImportBatch qw( + RecordsFromMARCXMLFile + RecordsFromISO2709File + RecordsFromMarcPlugin + BatchStageMarcRecords + BatchFindDuplicates + SetImportBatchMatcher + SetImportBatchOverlayAction + SetImportBatchNoMatchAction + SetImportBatchItemAction +); + +=head1 NAME + +Koha::BackgroundJob::StageMARCForImport - Stage MARC records for import + +This is a subclass of Koha::BackgroundJob. + +=head1 API + +=head2 Class methods + +=head3 job_type + +Define the job type of this job: stage_marc_for_import + +=cut + +sub job_type { + return 'stage_marc_for_import'; +} + +=head3 process + +Stage the MARC records for import. + +=cut + +sub process { + my ( $self, $args ) = @_; + + $self->start; + + my $record_type = $args->{record_type}; + my $encoding = $args->{encoding}; + my $format = $args->{format}; + my $filepath = $args->{filepath}; + my $filename = $args->{filename}; + my $marc_modification_template = $args->{marc_modification_template}; + my $comments = $args->{comments}; + my $parse_items = $args->{parse_items}; + my $matcher_id = $args->{matcher_id}; + my $overlay_action = $args->{overlay_action}; + my $nomatch_action = $args->{nomatch_action}; + my $item_action = $args->{item_action}; + my $vendor_id = $args->{vendor_id}; + my $basket_id = $args->{basket_id}; + my $profile_id = $args->{profile_id}; + + my @messages; + my ( $batch_id, $num_valid, $num_items, @import_errors ); + my $num_with_matches = 0; + my $checked_matches = 0; + my $matcher_failed = 0; + my $matcher_code = ""; + + try { + my $schema = Koha::Database->new->schema; + $schema->storage->txn_begin; + + my ( $errors, $marcrecords ); + if ( $format eq 'MARCXML' ) { + ( $errors, $marcrecords ) = + C4::ImportBatch::RecordsFromMARCXMLFile( $filepath, $encoding ); + } + elsif ( $format eq 'ISO2709' ) { + ( $errors, $marcrecords ) = + C4::ImportBatch::RecordsFromISO2709File( $filepath, $record_type, + $encoding ); + } + else { # plugin based + $errors = []; + $marcrecords = + C4::ImportBatch::RecordsFromMarcPlugin( $filepath, $format, + $encoding ); + } + + $self->size(scalar @$marcrecords)->store; + + ( $batch_id, $num_valid, $num_items, @import_errors ) = + BatchStageMarcRecords( + $record_type, $encoding, + $marcrecords, $filename, + $marc_modification_template, $comments, + '', $parse_items, + 0, 50, + sub { + my $job_progress = shift; + if ($matcher_id) { + $job_progress /= 2; + } + $self->progress( int($job_progress) )->store; + } + ); + + if ($profile_id) { + my $ibatch = Koha::ImportBatches->find($batch_id); + $ibatch->set( { profile_id => $profile_id } )->store; + } + + if ($matcher_id) { + my $matcher = C4::Matcher->fetch($matcher_id); + if ( defined $matcher ) { + $checked_matches = 1; + $matcher_code = $matcher->code(); + $num_with_matches = + BatchFindDuplicates( $batch_id, $matcher, 10, 50, + sub { my $job_progress = shift; $self->progress( $self->progress + $job_progress )->store } ); + SetImportBatchMatcher( $batch_id, $matcher_id ); + SetImportBatchOverlayAction( $batch_id, $overlay_action ); + SetImportBatchNoMatchAction( $batch_id, $nomatch_action ); + SetImportBatchItemAction( $batch_id, $item_action ); + $schema->storage->txn_commit; + } + else { + $matcher_failed = 1; + $schema->storage->txn_rollback; + } + } else { + $schema->storage->txn_commit; + } + } + catch { + warn $_; + die "Something terrible has happened!" + if ( $_ =~ /Rollback failed/ ); # Rollback failed + }; + + my $report = { + staged => $num_valid, + matched => $num_with_matches, + num_items => $num_items, + import_errors => scalar(@import_errors), + total => $num_valid + scalar(@import_errors), + checked_matches => $checked_matches, + matcher_failed => $matcher_failed, + matcher_code => $matcher_code, + import_batch_id => $batch_id, + vendor_id => $vendor_id, + basket_id => $basket_id, + }; + + my $data = $self->decoded_data; + $data->{messages} = \@messages; + $data->{report} = $report; + + $self->finish($data); +} + +=head3 enqueue + +Enqueue the new job + +=cut + +sub enqueue { + my ( $self, $args) = @_; + + $self->SUPER::enqueue({ + job_size => 0, # unknown for now + job_args => $args + }); +} + +1; diff --git a/debian/templates/apache-shared-intranet-plack.conf b/debian/templates/apache-shared-intranet-plack.conf index 311b5d7f37..8428e03a50 100644 --- a/debian/templates/apache-shared-intranet-plack.conf +++ b/debian/templates/apache-shared-intranet-plack.conf @@ -15,7 +15,6 @@ ProxyPass "/cgi-bin/koha/tools/background-job-progress.pl" "!" ProxyPass "/cgi-bin/koha/tools/export.pl" "!" ProxyPass "/cgi-bin/koha/tools/manage-marc-import.pl" "!" - ProxyPass "/cgi-bin/koha/tools/stage-marc-import.pl" "!" ProxyPass "/cgi-bin/koha/tools/upload-cover-image.pl" "!" ProxyPass "/cgi-bin/koha/svc/cataloguing/metasearch" "!" diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/stage_marc_for_import.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/stage_marc_for_import.inc new file mode 100644 index 0000000000..11cde88c1f --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/stage_marc_for_import.inc @@ -0,0 +1,46 @@ +[% USE Koha %] + +[% BLOCK report %] + [% SET report = job.report %] + [% IF report %] +

MARC staging results

+ [% SWITCH (record_type) %] + [% CASE 'biblio' %] +

Processing bibliographic records

+ [% CASE 'auth' %] +

Processing authority records

+ [% END %] +
    +
  • [% report.total | html %] records in file
    • +
  • [% report.import_errors | html %] records not staged because of MARC error
    • +
  • [% report.staged | html %] records staged
    • + [% IF ( report.checked_matches ) %] +
  • [% report.matched | html %] records with at least one match in catalog per matching rule + "[% report.matcher_code | html %]"
    • + [% ELSE %] + [% IF ( report.matcher_failed ) %] +
  • Record matching failed -- unable to retrieve selected matching rule.
    • + [% ELSE %] +
  • Did not check for matches with existing records in catalog
    • + [% END %] + [% END %] + [% IF report.record_type == 'biblio' %] +
  • [% report.num_items | html %] item records found and staged
    • + [% END %] + [% IF ( report.label_batch ) %] +
  • New label batch created: # [% report.label_batch | html %]
    • + [% END %] +
+ [% IF report.basketno && report.booksellerid %] +

+ Add staged files to basket +

+ [% END %] + [% END %] +[% END %] + +[% BLOCK detail %] +[% END %] + +[% BLOCK js %] +[% END %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt index 0fafb30da4..2821892765 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt @@ -41,6 +41,8 @@ Update Elasticsearch index [% CASE 'update_holds_queue_for_biblios' %] Holds queue update + [% CASE 'stage_marc_for_import' %] + Staged MARC records for import [% CASE %]Unknown job type '[% job_type | html %]' [% END %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt index 3ab3419d85..5049140329 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tt @@ -48,54 +48,41 @@
-[% IF ( uploadmarc ) %] - -[% END %] + [% FOREACH message IN messages %] + [% IF message.type == 'success' %] +
+ [% ELSIF message.type == 'warning' %] +
+ [% ELSIF message.type == 'error' %] +
+ [% END %] + [% IF message.code == 'cannot_enqueue_job' %] + Cannot enqueue this job. + [% END %] + [% IF message.error %] + (The error was: [% message.error | html %], see the Koha log file for more information). + [% END %] +
+ [% END %] -[% IF ( uploadmarc ) %] -

MARC staging results

-
    - [% SWITCH (record_type) %] - [% CASE 'biblio' %] -
  • Processing bibliographic records
    • - [% CASE 'auth' %] -
  • Processing authority records
    • - [% END %] -
  • [% total | html %] records in file
    • -
  • [% import_errors | html %] records not staged because of MARC error
    • -
  • [% staged | html %] records staged
    • - [% IF ( checked_matches ) %] -
  • [% matched | html %] records with at least one match in catalog per matching rule - "[% matcher_code | html %]"
    • - [% ELSE %] - [% IF ( matcher_failed ) %] -
  • Record matching failed -- unable to retrieve selected matching rule.
    • + [% IF job_enqueued %] + + +

    MARC staging

    +
    +

    The job has been enqueued! It will be processed as soon as possible.

    +

    View detail of the enqueued job +

    [% ELSE %] -
  • Did not check for matches with existing records in catalog
    • - [% END %] - [% END %] - [% IF record_type == 'biblio' %] -
  • [% num_items | html %] item records found and staged
    • - [% END %] - [% IF ( label_batch ) %] -
  • New label batch created: # [% label_batch | html %]
    • - [% END %] -
-[% IF basketno && booksellerid %] -

- Add staged files to basket -

-[% END %] -[% ELSE %]

Stage MARC records for import

  • Select a MARC file to stage in the import reservoir. It will be parsed, and each valid record staged for later import into the catalog.
  • You can enter a name for this import. It may be useful, when creating a record, to remember where the suggested MARC data comes from!
-
+
Upload a file to stage
    @@ -136,7 +123,7 @@
- + [% IF basketno && booksellerid %] @@ -253,10 +240,7 @@
- -
Job progress:
0%
-
- +
[% END %] @@ -273,7 +257,6 @@ [% MACRO jsinclude BLOCK %] [% Asset.js("js/tools-menu.js") | $raw %] [% Asset.js("lib/jquery/plugins/humanmsg.js") | $raw %] - [% Asset.js("js/background-job-progressbar.js") | $raw %] [% Asset.js("js/file-upload.js") | $raw %]