1 package Koha::SearchEngine::Elasticsearch;
3 # Copyright 2015 Catalyst IT
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 3 of the License, or (at your option) any later
12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License along
17 # with Koha; if not, write to the Free Software Foundation, Inc.,
18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 use base qw(Class::Accessor);
25 use Koha::Exceptions::Config;
26 use Koha::SearchFields;
27 use Koha::SearchMarcMaps;
33 use Search::Elasticsearch;
37 __PACKAGE__->mk_ro_accessors(qw( index ));
38 __PACKAGE__->mk_accessors(qw( sort_fields ));
40 # Constants to refer to the standard index names
41 Readonly our $BIBLIOS_INDEX => 'biblios';
42 Readonly our $AUTHORITIES_INDEX => 'authorities';
46 Koha::SearchEngine::Elasticsearch - Base module for things using elasticsearch
54 The name of the index to use, generally 'biblios' or 'authorities'.
64 my $self = $class->SUPER::new(@_);
65 # Check for a valid index
66 croak('No index name provided') unless $self->index;
70 =head2 get_elasticsearch_params
72 my $params = $self->get_elasticsearch_params();
74 This provides a hashref that contains the parameters for connecting to the
75 ElasicSearch servers, in the form:
78 'nodes' => ['127.0.0.1:9200', 'anotherserver:9200'],
79 'index_name' => 'koha_instance_index',
82 This is configured by the following in the C<config> block in koha-conf.xml:
85 <server>127.0.0.1:9200</server>
86 <server>anotherserver:9200</server>
87 <index_name>koha_instance</index_name>
92 sub get_elasticsearch_params {
95 # Copy the hash so that we're not modifying the original
96 my $conf = C4::Context->config('elasticsearch');
97 die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
98 my $es = { %{ $conf } };
100 # Helpfully, the multiple server lines end up in an array for us anyway
101 # if there are multiple ones, but not if there's only one.
102 my $server = $es->{server};
103 delete $es->{server};
104 if ( ref($server) eq 'ARRAY' ) {
106 # store it called 'nodes' (which is used by newer Search::Elasticsearch)
107 $es->{nodes} = $server;
110 $es->{nodes} = [$server];
113 die "No elasticsearch servers were specified in koha-conf.xml.\n";
115 die "No elasticserver index_name was specified in koha-conf.xml.\n"
116 if ( !$es->{index_name} );
117 # Append the name of this particular index to our namespace
118 $es->{index_name} .= '_' . $self->index;
120 $es->{key_prefix} = 'es_';
124 =head2 get_elasticsearch_settings
126 my $settings = $self->get_elasticsearch_settings();
128 This provides the settings provided to elasticsearch when an index is created.
129 These can do things like define tokenisation methods.
131 A hashref containing the settings is returned.
135 sub get_elasticsearch_settings {
138 # Use state to speed up repeated calls
139 state $settings = undef;
140 if (!defined $settings) {
141 my $config_file = C4::Context->config('elasticsearch_index_config');
142 $config_file ||= C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/index_config.yaml';
143 $settings = LoadFile( $config_file );
149 =head2 get_elasticsearch_mappings
151 my $mappings = $self->get_elasticsearch_mappings();
153 This provides the mappings that get passed to elasticsearch when an index is
158 sub get_elasticsearch_mappings {
161 # Use state to speed up repeated calls
165 if (!defined $all_mappings{$self->index}) {
166 $sort_fields{$self->index} = {};
168 data => _get_elasticsearch_mapping('general', '')
170 my $marcflavour = lc C4::Context->preference('marcflavour');
171 $self->_foreach_mapping(
173 my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
174 return if $marc_type ne $marcflavour;
175 # TODO if this gets any sort of complexity to it, it should
176 # be broken out into its own function.
178 # TODO be aware of date formats, but this requires pre-parsing
179 # as ES will simply reject anything with an invalid date.
180 my $es_type = 'text';
181 if ($type eq 'boolean') {
182 $es_type = 'boolean';
183 } elsif ($type eq 'number' || $type eq 'sum') {
184 $es_type = 'integer';
185 } elsif ($type eq 'isbn' || $type eq 'stdno') {
189 $mappings->{data}{properties}{$name} = _get_elasticsearch_mapping('search', $es_type);
192 $mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_mapping('facet', $es_type);
195 $mappings->{data}{properties}{ $name . '__suggestion' } = _get_elasticsearch_mapping('suggestible', $es_type);
197 # Sort is a bit special as it can be true, false, undef.
198 # We care about "true" or "undef",
199 # "undef" means to do the default thing, which is make it sortable.
200 if (!defined $sort || $sort) {
201 $mappings->{data}{properties}{ $name . '__sort' } = _get_elasticsearch_mapping('sort', $es_type);
202 $sort_fields{$self->index}{$name} = 1;
206 $all_mappings{$self->index} = $mappings;
208 $self->sort_fields(\%{$sort_fields{$self->index}});
210 return $all_mappings{$self->index};
213 =head2 _get_elasticsearch_mapping
215 Get the ES mappings for the given purpose and data type
217 $mapping = _get_elasticsearch_mapping('search', 'text');
221 sub _get_elasticsearch_mapping {
223 my ( $purpose, $type ) = @_;
225 # Use state to speed up repeated calls
226 state $settings = undef;
227 if (!defined $settings) {
228 my $config_file = C4::Context->config('elasticsearch_field_config');
229 $config_file ||= C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/field_config.yaml';
230 $settings = LoadFile( $config_file );
233 if (!defined $settings->{$purpose}) {
234 die "Field purpose $purpose not defined in field config";
237 return $settings->{$purpose};
239 if (defined $settings->{$purpose}{$type}) {
240 return $settings->{$purpose}{$type};
242 if (defined $settings->{$purpose}{'default'}) {
243 return $settings->{$purpose}{'default'};
248 sub reset_elasticsearch_mappings {
249 my ( $reset_fields ) = @_;
250 my $mappings_yaml = C4::Context->config('elasticsearch_index_mappings');
251 $mappings_yaml ||= C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml';
252 my $indexes = LoadFile( $mappings_yaml );
254 while ( my ( $index_name, $fields ) = each %$indexes ) {
255 while ( my ( $field_name, $data ) = each %$fields ) {
256 my $field_type = $data->{type};
257 my $field_label = $data->{label};
258 my $mappings = $data->{mappings};
259 my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' });
260 for my $mapping ( @$mappings ) {
261 my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} });
262 $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort} } );
268 # This overrides the accessor provided by Class::Accessor so that if
269 # sort_fields isn't set, then it'll generate it.
273 $self->_sort_fields_accessor(@_);
276 my $val = $self->_sort_fields_accessor();
279 # This will populate the accessor as a side effect
280 $self->get_elasticsearch_mappings();
281 return $self->_sort_fields_accessor();
284 # Provides the rules for data conversion.
285 sub get_fixer_rules {
288 my $marcflavour = lc C4::Context->preference('marcflavour');
291 $self->_foreach_mapping(
293 my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
294 return if $marc_type ne $marcflavour;
297 push @rules, "marc_map('$marc_field','${name}.\$append', $options)";
299 push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)";
303 #"marc_map('$marc_field','${name}__suggestion.input.\$append', '')"; #must not have nested data structures in .input
304 "marc_map('$marc_field','${name}__suggestion.input.\$append')";
306 if ( $type eq 'boolean' ) {
308 # boolean gets special handling, basically if it doesn't exist,
309 # it's added and set to false. Otherwise we can't query it.
311 "unless exists('$name') add_field('$name', 0) end";
313 if ($type eq 'sum' ) {
314 push @rules, "sum('$name')";
316 if ($self->sort_fields()->{$name}) {
317 if ($sort || !defined $sort) {
318 push @rules, "marc_map('$marc_field','${name}__sort.\$append', $options)";
324 push @rules, "move_field(_id,es_id)"; #Also you must set the Catmandu::Store::ElasticSearch->new(key_prefix: 'es_');
328 =head2 _foreach_mapping
330 $self->_foreach_mapping(
332 my ( $name, $type, $facet, $suggestible, $sort, $marc_type,
335 return unless $marc_type eq 'marc21';
336 print "Data comes from: " . $marc_field . "\n";
340 This allows you to apply a function to each entry in the elasticsearch mappings
341 table, in order to build the mappings for whatever is needed.
343 In the provided function, the files are:
349 The field name for elasticsearch (corresponds to the 'mapping' column in the
354 The type for this value, e.g. 'string'.
358 True if this value should be facetised. This only really makes sense if the
359 field is understood by the facet processing code anyway.
363 True if this is a field that a) needs special sort handling, and b) if it
364 should be sorted on. False if a) but not b). Undef if not a). This allows,
365 for example, author to be sorted on but not everything marked with "author"
366 to be included in that sort.
370 A string that indicates the MARC type that this mapping is for, e.g. 'marc21',
371 'unimarc', 'normarc'.
375 A string that describes the MARC field that contains the data to extract.
376 These are of a form suited to Catmandu's MARC fixers.
382 sub _foreach_mapping {
383 my ( $self, $sub ) = @_;
385 # TODO use a caching framework here
386 my $search_fields = Koha::Database->schema->resultset('SearchField')->search(
388 'search_marc_map.index_name' => $self->index,
390 { join => { search_marc_to_fields => 'search_marc_map' },
392 'search_marc_to_fields.facet',
393 'search_marc_to_fields.suggestible',
394 'search_marc_to_fields.sort',
395 'search_marc_map.marc_type',
396 'search_marc_map.marc_field',
408 while ( my $search_field = $search_fields->next ) {
412 $search_field->get_column('facet'),
413 $search_field->get_column('suggestible'),
414 $search_field->get_column('sort'),
415 $search_field->get_column('marc_type'),
416 $search_field->get_column('marc_field'),
423 die process_error($@);
425 This parses an Elasticsearch error message and produces a human-readable
426 result from it. This result is probably missing all the useful information
427 that you might want in diagnosing an issue, so the warning is also logged.
429 Note that currently the resulting message is not internationalised. This
430 will happen eventually by some method or other.
435 my ($self, $msg) = @_;
437 warn $msg; # simple logging
439 # This is super-primitive
440 return "Unable to understand your search query, please rephrase and try again.\n" if $msg =~ /ParseException/;
442 return "Unable to perform your search. Please try again.\n";
445 =head2 _read_configuration
447 my $conf = _read_configuration();
449 Reads the I<configuration file> and returns a hash structure with the
450 configuration information. It raises an exception if mandatory entries
453 The hashref structure has the following form:
456 'nodes' => ['127.0.0.1:9200', 'anotherserver:9200'],
457 'index_name' => 'koha_instance',
460 This is configured by the following in the C<config> block in koha-conf.xml:
463 <server>127.0.0.1:9200</server>
464 <server>anotherserver:9200</server>
465 <index_name>koha_instance</index_name>
470 sub _read_configuration {
474 my $conf = C4::Context->config('elasticsearch');
475 Koha::Exceptions::Config::MissingEntry->throw(
476 "Missing 'elasticsearch' block in config file")
477 unless defined $conf;
479 if ( $conf && $conf->{server} ) {
480 my $nodes = $conf->{server};
481 if ( ref($nodes) eq 'ARRAY' ) {
482 $configuration->{nodes} = $nodes;
485 $configuration->{nodes} = [$nodes];
489 Koha::Exceptions::Config::MissingEntry->throw(
490 "Missing 'server' entry in config file for elasticsearch");
493 if ( defined $conf->{index_name} ) {
494 $configuration->{index_name} = $conf->{index_name};
497 Koha::Exceptions::Config::MissingEntry->throw(
498 "Missing 'index_name' entry in config file for elasticsearch");
501 return $configuration;
512 =item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
514 =item Robin Sheat C<< <robin@catalyst.net.nz> >>
516 =item Jonathan Druart C<< <jonathan.druart@bugs.koha-community.org> >>