From e42b4088ed83eb093389f3f5a841095ecc8f036f Mon Sep 17 00:00:00 2001 From: Nick Clemens Date: Fri, 7 Aug 2020 13:15:36 +0000 Subject: [PATCH] Bug 26180: Add descending option to rebuild_elasticsearch.pl MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit While the ES index is incremental and provides results as it commits, we currently index from the oldest records to the newest. This patch provides the option to go the other direction To test: 1 - Have ES setup and running for Koha 2 - perl misc/search_tools/rebuild_elasticsearch.pl -v -v -b 3 - Note the biblios index from number 1 the end 4 - perl misc/search_tools/rebuild_elasticsearch.pl -v -v -a 5 - Notice the same 6 - Apply patch 7 - perl misc/search_tools/rebuild_elasticsearch.pl -v -v -b 8 - Still in ascending order 9 - perl misc/search_tools/rebuild_elasticsearch.pl -v -v -b --desc 10 - Now records index in descending order 11 - perl misc/search_tools/rebuild_elasticsearch.pl -v -v -a 12 - Still ascending 13 - perl misc/search_tools/rebuild_elasticsearch.pl -v -v -a --desc 14 - Now descending Signed-off-by: Séverine QUEUNE Signed-off-by: Martin Renvoize JD amended patch: fix typo "inde" vs "index" and add commit body Signed-off-by: Jonathan Druart --- Koha/BiblioUtils.pm | 7 ++++++- Koha/MetadataRecord/Authority.pm | 7 ++++++- misc/search_tools/rebuild_elasticsearch.pl | 17 +++++++++++++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/Koha/BiblioUtils.pm b/Koha/BiblioUtils.pm index 02c10f58aa..9be2e6168a 100644 --- a/Koha/BiblioUtils.pm +++ b/Koha/BiblioUtils.pm @@ -134,12 +134,17 @@ sub get_all_biblios_iterator { $search_terms = \[ 'mod(biblionumber, ?) = ?', $slice_count, $slice_modulo ]; } + my $search_options = { columns => [qw/ biblionumber /] }; + if ( $options{desc} ){ + $search_options->{order_by} = { -desc => 'biblionumber' }; + } + my $database = Koha::Database->new(); my $schema = $database->schema(); my $rs = $schema->resultset('Biblio')->search( $search_terms, - { columns => [qw/ biblionumber /] } ); + $search_options ); my $next_func = sub { # Warn and skip bad records, otherwise we break the loop while (1) { diff --git a/Koha/MetadataRecord/Authority.pm b/Koha/MetadataRecord/Authority.pm index c3e1cf8edd..47f2172976 100644 --- a/Koha/MetadataRecord/Authority.pm +++ b/Koha/MetadataRecord/Authority.pm @@ -191,12 +191,17 @@ sub get_all_authorities_iterator { }; } + my $search_options->{columns} = [qw/ authid authtypecode marcxml /]; + if ($options{desc}) { + $search_options->{order_by} = { -desc => 'authid' }; + } + my $database = Koha::Database->new(); my $schema = $database->schema(); my $rs = $schema->resultset('AuthHeader')->search( $search_terms, - { columns => [qw/ authid authtypecode marcxml /] } ); + $search_options); my $next_func = sub { my $row = $rs->next(); return if !$row; diff --git a/misc/search_tools/rebuild_elasticsearch.pl b/misc/search_tools/rebuild_elasticsearch.pl index 6cff2eae2e..ff83562f25 100755 --- a/misc/search_tools/rebuild_elasticsearch.pl +++ b/misc/search_tools/rebuild_elasticsearch.pl @@ -31,6 +31,7 @@ B [B<-r|--reset>] [B<-a|--authorities>] [B<-b|--biblios>] +[B<--desc>] [B<-bn|--bnumber>] [B<-ai|--authid>] [B<-p|--processes>] @@ -70,6 +71,12 @@ specifying neither and so both get indexed. Index the biblios only. Combining this with B<-a> is the same as specifying neither and so both get indexed. +=item B<--desc> + +Index the records in descending id order. Intended to index newer record before older records. +Default is to index in ascending order. +Does not work with --bnumber or --authid + =item B<-bn|--bnumber> Only index the supplied biblionumber, mostly for testing purposes. May be @@ -122,6 +129,7 @@ my $commit = 5000; my ($delete, $reset, $help, $man, $processes); my ($index_biblios, $index_authorities); my (@biblionumbers,@authids); +my $desc; $|=1; # flushes output @@ -131,8 +139,9 @@ GetOptions( 'r|reset' => \$reset, 'a|authorities' => \$index_authorities, 'b|biblios' => \$index_biblios, - 'bn|bnumber=i' => \@biblionumbers, - 'ai|authid=i' => \@authids, + 'desc' => \$desc, + 'bn|bnumber=i' => \@biblionumbers, + 'ai|authid=i' => \@authids, 'p|processes=i' => \$processes, 'v|verbose+' => \$verbose, 'h|help' => \$help, @@ -183,6 +192,10 @@ if ($slice_count > 1) { $iterator_options{slice} = { index => $slice_index, count => $slice_count }; } +if( $desc ){ + $iterator_options{desc} = 1; +} + my $next; if ($index_biblios) { _log(1, "Indexing biblios\n"); -- 2.39.5