From fee0f29792353d895af9eb6328cd50faef743162 Mon Sep 17 00:00:00 2001 From: Fridolin Somers Date: Wed, 11 May 2022 09:38:23 -1000 Subject: [PATCH] Bug 27667: Display a warning for records missing in Elasticsearch On "About Koha" page are displayed the numbers of indexed biblios and authorities with ES. It could be great if we could have comparison with the number of records in the database to display the number of missing records. In the "System information" tab with the other "Data problems". If Searchengine has any issue (index count will be -1), we do not display missing records. Test plan : 1) Use Elasticsearch searchengine 2) Rebuild all records : koha-elasticsearch --rebuild -d kohadev 3) Go to "About Koha" page and look at System information tab => you see no warning 4) On a biblio record delete the MARCXML in biblio_metadata.metadata 5) Rebuild all records : koha-elasticsearch --rebuild -d kohadev 6) Go to "About Koha" page and look at System information tab => you see warning : 1 record(s) missing on a total of 435 in indice koha_kohadev_biblios. 7) On a authority record delete the MARCXML in auth_header.marcxml 8) Rebuild all records : koha-elasticsearch --rebuild -d kohadev 9) Go to "About Koha" page and look at System information tab => you see warning : 1 record(s) missing on a total of 1705 in indice koha_kohadev_authorities. Signed-off-by: David Nind Signed-off-by: Victor Grousset/tuxayo Signed-off-by: Katrin Fischer Signed-off-by: Tomas Cohen Arazi --- about.pl | 31 +++++++++++++++---- .../intranet-tmpl/prog/en/modules/about.tt | 20 ++++++++++-- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/about.pl b/about.pl index ba3434184f..06f4050e28 100755 --- a/about.pl +++ b/about.pl @@ -41,8 +41,10 @@ use C4::Installer::PerlModules; use Koha; use Koha::DateUtils qw( dt_from_string output_pref ); use Koha::Acquisition::Currencies; +use Koha::Authorities; use Koha::BackgroundJob; use Koha::BiblioFrameworks; +use Koha::Biblios; use Koha::Email; use Koha::Patron::Categories; use Koha::Patrons; @@ -354,6 +356,7 @@ if ( C4::Context->preference('SearchEngine') eq 'Elasticsearch' ) { my $es_status; my $es_config_error; my $es_running = 1; + my $es_has_missing = 0; my $es_conf; try { @@ -378,15 +381,15 @@ if ( C4::Context->preference('SearchEngine') eq 'Elasticsearch' ) { my $es_status->{version} = $es->info->{version}->{number}; foreach my $index ( @indexes ) { - my $count; + my $index_count; try { - $count = $es->indices->stats( index => $index ) + $index_count = $es->indices->stats( index => $index ) ->{_all}{primaries}{docs}{count}; } catch { if ( ref($_) eq 'Search::Elasticsearch::Error::Missing' ) { push @{ $es_status->{errors} }, "Index not found ($index)"; - $count = -1; + $index_count = -1; } elsif ( ref($_) eq 'Search::Elasticsearch::Error::NoNodes' ) { $es_running = 0; @@ -397,15 +400,31 @@ if ( C4::Context->preference('SearchEngine') eq 'Elasticsearch' ) { } }; + my $db_count = -1; + my $missing_count = 0; + if ( $index eq $biblios_index_name ) { + $db_count = Koha::Biblios->search->count; + } elsif ( $index eq $authorities_index_name ) { + $db_count = Koha::Authorities->search->count; + } + if ( $db_count != -1 && $index_count != -1 ) { + $missing_count = $db_count - $index_count; + $es_has_missing = 1 if $missing_count > 0; + } push @{ $es_status->{indexes} }, { - index_name => $index, - count => $count + index_name => $index, + index_count => $index_count, + db_count => $db_count, + missing_count => $missing_count, }; } $es_status->{running} = $es_running; - $template->param( elasticsearch_status => $es_status ); + $template->param( + elasticsearch_status => $es_status, + elasticsearch_has_missing => $es_has_missing, + ); } } diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/about.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/about.tt index 928ac2c476..ba996b2af8 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/about.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/about.tt @@ -80,7 +80,7 @@ | Indices: [% FOREACH index IN elasticsearch_status.indexes %] - [% index.index_name | html %] (count: [% index.count | html %])[% UNLESS loop.last %], [% END %] + [% index.index_name | html %] (count: [% index.index_count | html %])[% UNLESS loop.last %], [% END %] [% END %] [% ELSE %] not running @@ -216,7 +216,7 @@
- [% IF warnPrefBiblioAddsAuthorities || warnPrefEasyAnalyticalRecords || warnPrefAnonymousPatronOPACPrivacy || warnPrefAnonymousPatronAnonSuggestions || warnPrefAnonymousPatronOPACPrivacy_PatronDoesNotExist || warnPrefAnonymousPatronAnonSuggestions_PatronDoesNotExist || warnPrefKohaAdminEmailAddress || warnPrefOpacHiddenItems || invalid_yesno.count || warnNoActiveCurrency || warnIsRootUser || xml_config_warnings.size || AutoSelfCheckPatronDoesNotHaveSelfCheckPerm || AutoSelfCheckPatronHasTooManyPerm || warnStatisticsFieldsError || warnNoTemplateCaching || warnILLConfiguration || has_ai_issues || oauth2_missing_deps || bad_yaml_prefs || warnRelationships || log4perl_errors || config_bcrypt_settings_no_set || warnHiddenBiblionumbers.size || warnConnectBroker %] + [% IF warnPrefBiblioAddsAuthorities || warnPrefEasyAnalyticalRecords || warnPrefAnonymousPatronOPACPrivacy || warnPrefAnonymousPatronAnonSuggestions || warnPrefAnonymousPatronOPACPrivacy_PatronDoesNotExist || warnPrefAnonymousPatronAnonSuggestions_PatronDoesNotExist || warnPrefKohaAdminEmailAddress || warnPrefOpacHiddenItems || invalid_yesno.count || warnNoActiveCurrency || warnIsRootUser || xml_config_warnings.size || AutoSelfCheckPatronDoesNotHaveSelfCheckPerm || AutoSelfCheckPatronHasTooManyPerm || warnStatisticsFieldsError || warnNoTemplateCaching || warnILLConfiguration || has_ai_issues || oauth2_missing_deps || bad_yaml_prefs || warnRelationships || log4perl_errors || config_bcrypt_settings_no_set || warnHiddenBiblionumbers.size || warnConnectBroker || elasticsearch_has_missing %] [% IF (warnIsRootUser) %]

Warning regarding current user

You are logged in as the database administrative user. This is not recommended because some parts of Koha will not function as expected when using this account.

@@ -511,6 +511,22 @@ [% END %] + [% IF elasticsearch_has_missing %] + + + [% FOREACH index IN elasticsearch_status.indexes %] + [% IF index.missing_count %] + + + + + [% END %] + [% END %] +
Records are missing in Elasticsearch search engine
Warning + [% index.missing_count | html %] record(s) missing on a total of [% index.index_count | html %] in indice [% index.index_name | html %]. +
+ [% END %] + [% ELSE %]

No warnings.

[% END %] -- 2.39.5