Bug 20248: Improve Elasticsearch mappings UI and rebuild_elastic_search.pl.

Improvements:
1) Mappings UI now has button that allows one to reset the mappings.
2) Mappings UI displays the items in alphabetical order.
3) Indexing script drops and recreates the index right away, which
helps prevent ES from autocreating a bad index if someone does something
while the first batch of records is being processed.
4) Indexing script has nicer output.

To test:
1) Change mappings.yaml file
2) Reset mappings in UI in mappings.pl
3) Verify the mappings have been changed in UI
4) The field order is alphabetical
5) Rebuild script has clean output
6) Run test t/db_dependent/Koha_Elasticsearch_Indexer.t

Signed-off-by: Bouzid Fergani <bouzid.fergani@inlibro.com>
Signed-off-by: Julian Maurice <julian.maurice@biblibre.com>

Signed-off-by: Nick Clemens <nick@bywatersolutions.com>

Koha/SearchEngine/Elasticsearch/Indexer.pm

@@ -139,6 +139,28 @@ sub delete_index_background {
     $self->delete_index(@_);
 }
 
+=head2 $indexer->create_index();
+
+Create an index on the Elasticsearch server.
+
+=cut
+
+sub create_index {
+    my ($self) = @_;
+
+    if (!$self->store) {
+        my $params  = $self->get_elasticsearch_params();
+        $self->store(
+            Catmandu::Store::ElasticSearch->new(
+                %$params,
+                index_settings => $self->get_elasticsearch_settings(),
+                index_mappings => $self->get_elasticsearch_mappings(),
+            )
+        );
+    }
+    $self->store->bag->commit;
+}
+
 =head2 $indexer->drop_index();
 
 Drops the index from the elasticsearch server. Calling C<update_index>
@@ -161,8 +183,9 @@ sub drop_index {
             )
         );
     }
-    $self->store->drop();
+    my $store = $self->store;
     $self->store(undef);
+    $store->drop();
 }
 
 sub _sanitise_records {

admin/searchengine/elasticsearch/mappings.pl

@@ -97,15 +97,17 @@ if ( $op eq 'edit' ) {
         $schema->storage->txn_commit;
     }
 }
-elsif( $op eq 'reset' ) {
-    # TODO Move this feature to the interface
-    my $sure = $input->param('i_know_what_i_am_doing');
-    if ( $sure ) {
-        Koha::SearchMarcMaps->search->delete;
-        Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
-    }
+elsif( $op eq 'reset' || $op eq 'reset_confirmed' ) {
+    Koha::SearchMarcMaps->delete;
+    Koha::SearchFields->delete;
+    Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
+    push @messages, { type => 'message', code => 'success_on_reset' };
+}
+elsif( $op eq 'reset_confirm' ) {
+    $template->param( reset_confirm => 1 );
 }
 
+
 my @indexes;
 
 for my $index_name (qw| biblios authorities |) {
@@ -114,6 +116,7 @@ for my $index_name (qw| biblios authorities |) {
         {   join => { search_marc_to_fields => 'search_marc_map' },
             '+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ],
             '+as'     => [ 'facet',                       'suggestible',                       'sort',                       'marc_field' ],
+            order_by => { -asc => [qw/name marc_field/] }
         }
     );
 

koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt

@@ -81,7 +81,9 @@ a.add, a.delete {
           An error occurred when deleting the existing mappings. Nothing has been changed!
           (search field [% m.values.field_name | html %] with mapping [% m.values.marc_field | html %].)
         [% CASE 'success_on_update' %]
-          Mapping updated successfully.
+          Mappings updated successfully.
+        [% CASE 'success_on_reset' %]
+          Mappings have been reset successfully.
         [% CASE %]
           [% m.code | html %]
         [% END %]
@@ -109,6 +111,19 @@ a.add, a.delete {
         </div>
     [% END %]
 
+    [% IF reset_confirm %]
+        <div class="dialog alert">
+            <h3>The current mappings you see on the screen will be erased and replaced by the mappings in the mappings.yaml file.</h3>
+            <form action="?" method="post">
+                <input type="hidden" name="op" value="reset_confirmed" />
+                <button type="submit" class="approve"><i class="fa fa-fw fa-check"></i> Yes, reset mappings</button>
+            </form>
+            <br>
+            <form action="?" method="post">
+                <button type="submit" class="deny"><i class="fa fa-fw fa-remove"></i> No, do not reset mappings</button>
+            </form>
+        </div>
+    [% END %]
     <form method="post">
         <input type="hidden" name="op" value="edit" />
         <div id="tabs" class="toptabs" style="clear:both">
@@ -298,7 +313,10 @@ a.add, a.delete {
                 </div>
             [% END %]
         </div>
-        <p><button class="btn btn-default" type="submit"><i class="fa fa-hdd-o" aria-hidden="true"></i> Save</button></p>
+        <p>
+            <button class="btn btn-default" type="submit"><i class="fa fa-hdd-o" aria-hidden="true"></i> Save</button>
+            <a class="btn btn-default" href="?op=reset_confirm"><i class="fa fa-refresh" aria-hidden="true"></i> Reset Mappings</a>
+        </p>
     </form>
 </div>
 

misc/search_tools/rebuild_elastic_search.pl

@@ -92,14 +92,14 @@ use MARC::Record;
 use Modern::Perl;
 use Pod::Usage;
 
-use Data::Dumper; # TODO remove
-
 my $verbose = 0;
 my $commit = 5000;
 my ($delete, $help, $man);
 my ($index_biblios, $index_authorities);
 my (@biblionumbers);
 
+$|=1; # flushes output
+
 GetOptions(
     'c|commit=i'       => \$commit,
     'd|delete'         => \$delete,
@@ -161,10 +161,8 @@ sub do_reindex {
 
     my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
     if ($delete) {
-
-        # We know it's safe to not recreate the indexer because update_index
-        # hasn't been called yet.
         $indexer->drop_index();
+        $indexer->create_index();
     }
 
     my $count        = 0;
@@ -173,23 +171,29 @@ sub do_reindex {
     while ( my $record = $next->() ) {
         my $id     = $record->id;
         my $record = $record->record;
-        _log( 1, "$id\n" );
         $count++;
+        if ( $verbose == 1 ) {
+            _log( 1, "$count records processed\n" ) if ( $count % 1000 == 0);
+        } else {
+            _log( 2, "$id\n" );
+        }
 
         push @id_buffer,     $id;
         push @commit_buffer, $record;
         if ( !( --$commit_count ) ) {
-            _log( 2, "Committing...\n" );
+            _log( 1, "Committing $commit records..." );
             $indexer->update_index( \@id_buffer, \@commit_buffer );
             $commit_count  = $commit;
             @id_buffer     = ();
             @commit_buffer = ();
+            _log( 1, " done\n" );
         }
     }
 
     # There are probably uncommitted records
+    _log( 1, "Committing final records...\n" );
     $indexer->update_index( \@id_buffer, \@commit_buffer );
-    _log( 1, "$count records indexed.\n" );
+    _log( 1, "Total $count records indexed\n" );
 }
 
 # Checks some basic stuff to ensure that it's sane before we start.

t/db_dependent/Koha_Elasticsearch_Indexer.t

@@ -17,7 +17,7 @@
 
 use Modern::Perl;
 
-use Test::More tests => 6;
+use Test::More tests => 7;
 use Test::MockModule;
 use t::lib::Mocks;
 
@@ -52,12 +52,42 @@ SKIP: {
 
     eval { $indexer->get_elasticsearch_params; };
 
-    skip 'ElasticSeatch configuration not available', 1
+    skip 'Elasticsearch configuration not available', 1
         if $@;
 
     ok( $indexer->update_index(undef,$records), 'Update Index' );
 }
 
+subtest 'create_index() tests' => sub {
+
+    plan tests => 3;
+
+    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
+    $se->mock( 'get_elasticsearch_params', sub {
+        my ($self, $sub ) = @_;
+
+        my $method = $se->original( 'get_elasticsearch_params' );
+        my $params = $method->( $self );
+        $params->{index_name} .= '__test';
+        return $params;
+    });
+
+    my $indexer;
+    ok(
+        $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ 'index' => 'biblios' }),
+        'Creating a new indexer object'
+    );
+    ok(
+        $indexer->create_index(),
+        'Creating an index'
+    );
+    $indexer->drop_index();
+    ok(
+        $indexer->drop_index(),
+        'Dropping the index'
+    );
+};
+
 subtest '_convert_marc_to_json() tests' => sub {
 
     plan tests => 4;