From ace2e944c483db7ac2a83a294b9bb43d63410b71 Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Thu, 2 May 2019 11:27:27 +0000 Subject: [PATCH] Bug 22828: Elasticsearch - display errors encountered during indexing on the command line MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit To test: 1 - Use the Koha sample data, or insert a blank 245$b into a record (easiest way is using advanced cataloging editor 2 - Reindex elasticsearch 3 - Check the ES count on the about page 4 - Check the count in the DB (SELECT count(*) FROM biblio) 5 - They don't match! 6 - perl misc/search_tools/rebuild_elastic_search.pl -v -v 7 - No errors indicated 8 - Apply patch 9 - perl misc/search_tools/rebuild_elastic_search.pl -v 10 - You should be notified of an error 11 - perl misc/search_tools/rebuild_elastic_search.pl -v -v 12 - You should be notified of the specific biblio with an error and a (somewhat) readable reason 13 - perl misc/search_tools/rebuild_elastic_search.pl 14 - No output Signed-off-by: Ere Maijala Signed-off-by: Séverine QUEUNE Signed-off-by: Bouzid Fergani Signed-off-by: Jonathan Druart Signed-off-by: Martin Renvoize Signed-off-by: Joy Nelson (cherry picked from commit 5a0a350ed7d9e1fa5e93500b2eb92ae33589de4a) Signed-off-by: Lucas Gass --- Koha/SearchEngine/Elasticsearch/Indexer.pm | 6 ++--- misc/search_tools/rebuild_elasticsearch.pl | 30 ++++++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/Koha/SearchEngine/Elasticsearch/Indexer.pm b/Koha/SearchEngine/Elasticsearch/Indexer.pm index 504ccc5856..86fe3da66d 100644 --- a/Koha/SearchEngine/Elasticsearch/Indexer.pm +++ b/Koha/SearchEngine/Elasticsearch/Indexer.pm @@ -117,15 +117,15 @@ sub update_index { }; push @body, $document; } + my $response; if (@body) { - my $response = $elasticsearch->bulk( + $response = $elasticsearch->bulk( index => $conf->{index_name}, type => 'data', # is just hard coded in Indexer.pm? body => \@body ); } - # TODO: handle response - return 1; + return $response; } =head2 set_index_status_ok diff --git a/misc/search_tools/rebuild_elasticsearch.pl b/misc/search_tools/rebuild_elasticsearch.pl index 1199b63922..124cff2206 100755 --- a/misc/search_tools/rebuild_elasticsearch.pl +++ b/misc/search_tools/rebuild_elasticsearch.pl @@ -201,6 +201,8 @@ if ($slice_index == 0) { } } +=head1 INTERNAL METHODS + =head2 _verify_index_state _verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, 1); @@ -236,6 +238,7 @@ sub _verify_index_state { _do_reindex($callback, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX); Does the actual reindexing. $callback is a function that always returns the next record. +For each index we iterate through the records, committing at specified count =cut @@ -261,7 +264,8 @@ sub _do_reindex { push @commit_buffer, $record; if ( !( --$commit_count ) ) { _log( 1, "Committing $commit records...\n" ); - $indexer->update_index( \@id_buffer, \@commit_buffer ); + my $response = $indexer->update_index( \@id_buffer, \@commit_buffer ); + _handle_response($response); $commit_count = $commit; @id_buffer = (); @commit_buffer = (); @@ -271,7 +275,8 @@ sub _do_reindex { # There are probably uncommitted records _log( 1, "Committing final records...\n" ); - $indexer->update_index( \@id_buffer, \@commit_buffer ); + my $response = $indexer->update_index( \@id_buffer, \@commit_buffer ); + _handle_response($response); _log( 1, "Total $count records indexed\n" ); } @@ -289,6 +294,27 @@ sub _sanity_check { die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf ); } +=head2 _handle_response + +Parse the return from update_index and display errors depending on verbosity of the script + +=cut + +sub _handle_response { + my ($response) = @_; + if( $response->{errors} eq 'true' ){ + _log( 1, "There were errors during indexing\n" ); + if ( $verbose > 1 ){ + foreach my $item (@{$response->{items}}){ + next unless defined $item->{index}->{error}; + print "Record #" . $item->{index}->{_id} . " " . + $item->{index}->{error}->{reason} . " (" . $item->{index}->{error}->{type} . ") : " . + $item->{index}->{error}->{caused_by}->{type} . " (" . $item->{index}->{error}->{caused_by}->{reason} . ")\n"; + } + } + } +} + =head2 _log _log($level, "Message\n"); -- 2.39.5