From 809b6e701691118bbd1589428abc6fd1bada4b8c Mon Sep 17 00:00:00 2001 From: Martin Renvoize Date: Fri, 20 May 2022 16:59:23 +0100 Subject: [PATCH] Bug 30822: Make BatchCommitRecords update the index in one request MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit When committing staged marc imports to the catalogue we will often be importing a batch of records. We don't want to send one index request per biblio affected, we want to index them all after the records have been modified otherwise we will end up with multiple tasks per record (when items are also affected). Test plan: 1) Use the stage marc record tool to stage and commit a set of records and confirm the behaviour remains correct. 2) If using Elastic, check that only one indexing job is queued to take place resulting from the committed import. Signed-off-by: Joonas Kylmälä Signed-off-by: Marcel de Rooy Signed-off-by: Tomas Cohen Arazi --- C4/Biblio.pm | 34 ++++++++++++++++------------------ C4/ImportBatch.pm | 32 +++++++++++++++++++++++++------- 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/C4/Biblio.pm b/C4/Biblio.pm index ef094ab375..41346511e7 100644 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -186,31 +186,29 @@ The first argument is a C object containing the bib to add, while the second argument is the desired MARC framework code. -This function also accepts a third, optional argument: a hashref -to additional options. The only defined option is C, -which if present and mapped to a true value, causes C -to omit the call to save the MARC in C -This option is provided B -for the use of scripts such as C that may need -to do some manipulation of the MARC record for item parsing before -saving it and which cannot afford the performance hit of saving -the MARC record twice. Consequently, do not use that option -unless you can guarantee that C will be called. +The C<$options> argument is a hashref with additional parameters: + +=over 4 + +=item B: used when ModBiblioMarc is handled by the caller + +=item B: used when the indexing schedulling will be handled by the caller + +=back =cut sub AddBiblio { - my $record = shift; - my $frameworkcode = shift; - my $options = @_ ? shift : undef; - my $defer_marc_save = 0; + my ( $record, $frameworkcode, $options ) = @_; + + $options //= {}; + my $skip_record_index = $options->{skip_record_index} || 0; + my $defer_marc_save = $options->{defer_marc_save} || 0; + if (!$record) { carp('AddBiblio called with undefined record'); return; } - if ( defined $options and exists $options->{'defer_marc_save'} and $options->{'defer_marc_save'} ) { - $defer_marc_save = 1; - } my $schema = Koha::Database->schema; my ( $biblionumber, $biblioitemnumber ); @@ -291,7 +289,7 @@ sub AddBiblio { } # now add the record - ModBiblioMarc( $record, $biblionumber ) unless $defer_marc_save; + ModBiblioMarc( $record, $biblionumber, { skip_record_index => $skip_record_index } ) unless $defer_marc_save; # update OAI-PMH sets if(C4::Context->preference("OAI-PMH:AutoUpdateSets")) { diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index 03baee5080..5603f741fe 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -35,6 +35,8 @@ use C4::Charset qw( MarcToUTF8Record SetUTF8Flag StripNonXmlChars ); use C4::AuthoritiesMarc qw( AddAuthority GuessAuthTypeCode GetAuthorityXML ModAuthority DelAuthority ); use C4::MarcModificationTemplates qw( ModifyRecordWithTemplate ); use Koha::Items; +use Koha::SearchEngine; +use Koha::SearchEngine::Indexer; use Koha::Plugins::Handler; use Koha::Logger; @@ -578,6 +580,7 @@ sub BatchCommitRecords { my $logged_in_patron = Koha::Patrons->find( $userenv->{number} ); my $rec_num = 0; + my @biblio_ids; while (my $rowref = $sth->fetchrow_hashref) { $record_type = $rowref->{'record_type'}; $rec_num++; @@ -617,7 +620,8 @@ sub BatchCommitRecords { $num_added++; if ($record_type eq 'biblio') { my $biblioitemnumber; - ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework); + ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework, { skip_record_index => 1 }); + push @biblio_ids, $recordid; $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; # FIXME call SetMatchedBiblionumber instead if ($item_result eq 'create_new' || $item_result eq 'replace') { my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result, $biblioitemnumber); @@ -656,9 +660,16 @@ sub BatchCommitRecords { $context->{userid} = $logged_in_patron->userid; } - ModBiblio($marc_record, $recordid, $oldbiblio->frameworkcode, { - overlay_context => $context - }); + ModBiblio( + $marc_record, + $recordid, + $oldbiblio->frameworkcode, + { + overlay_context => $context, + skip_record_index => 1 + } + ); + push @biblio_ids, $recordid; $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; # FIXME call SetMatchedBiblionumber instead if ($item_result eq 'create_new' || $item_result eq 'replace') { @@ -686,6 +697,7 @@ sub BatchCommitRecords { $num_ignored++; if ($record_type eq 'biblio' and defined $recordid and ( $item_result eq 'create_new' || $item_result eq 'replace' ) ) { my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result); + push @biblio_ids, $recordid if $bib_items_added || $bib_items_replaced; $num_items_added += $bib_items_added; $num_items_replaced += $bib_items_replaced; $num_items_errored += $bib_items_errored; @@ -699,6 +711,12 @@ sub BatchCommitRecords { } } $sth->finish(); + + if ( @biblio_ids ) { + my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX }); + $indexer->index_records( \@biblio_ids, "specialUpdate", "biblioserver" ); + } + SetImportBatchStatus($batch_id, 'imported'); return ($num_added, $num_updated, $num_items_added, $num_items_replaced, $num_items_errored, $num_ignored); } @@ -744,7 +762,7 @@ sub BatchCommitItems { my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, itemnumber = ?, import_error = ? WHERE import_items_id = ?"); if ( $action eq "replace" && $duplicate_itemnumber ) { # Duplicate itemnumbers have precedence, that way we can update barcodes by overlaying - ModItemFromMarc( $item_marc, $biblionumber, $item->{itemnumber} ); + ModItemFromMarc( $item_marc, $biblionumber, $item->{itemnumber}, { skip_record_index => 1 } ); $updsth->bind_param( 1, 'imported' ); $updsth->bind_param( 2, $item->{itemnumber} ); $updsth->bind_param( 3, undef ); @@ -754,7 +772,7 @@ sub BatchCommitItems { $num_items_replaced++; } elsif ( $action eq "replace" && $duplicate_barcode ) { my $itemnumber = $duplicate_barcode->itemnumber; - ModItemFromMarc( $item_marc, $biblionumber, $itemnumber ); + ModItemFromMarc( $item_marc, $biblionumber, $itemnumber, { skip_record_index => 1 } ); $updsth->bind_param( 1, 'imported' ); $updsth->bind_param( 2, $item->{itemnumber} ); $updsth->bind_param( 3, undef ); @@ -774,7 +792,7 @@ sub BatchCommitItems { my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField( "items.itemnumber" ); $item_marc->field($itemtag)->delete_subfield( code => $itemsubfield ); - my ( $item_biblionumber, $biblioitemnumber, $itemnumber ) = AddItemFromMarc( $item_marc, $biblionumber, {biblioitemnumber => $biblioitemnumber} ); + my ( $item_biblionumber, $biblioitemnumber, $itemnumber ) = AddItemFromMarc( $item_marc, $biblionumber, { biblioitemnumber => $biblioitemnumber, skip_record_index => 1 } ); if( $itemnumber ) { $updsth->bind_param( 1, 'imported' ); $updsth->bind_param( 2, $itemnumber ); -- 2.39.5