1 package Koha::SearchEngine::Elasticsearch;
3 # Copyright 2015 Catalyst IT
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 3 of the License, or (at your option) any later
12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License along
17 # with Koha; if not, write to the Free Software Foundation, Inc.,
18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 use base qw(Class::Accessor);
25 use Koha::Exceptions::Config;
26 use Koha::SearchFields;
27 use Koha::SearchMarcMaps;
33 use Search::Elasticsearch;
37 use List::Util qw( sum0 reduce );
38 use Search::Elasticsearch;
41 use Encode qw(encode);
43 __PACKAGE__->mk_ro_accessors(qw( index ));
44 __PACKAGE__->mk_accessors(qw( sort_fields ));
46 # Constants to refer to the standard index names
47 Readonly our $BIBLIOS_INDEX => 'biblios';
48 Readonly our $AUTHORITIES_INDEX => 'authorities';
52 Koha::SearchEngine::Elasticsearch - Base module for things using elasticsearch
60 The name of the index to use, generally 'biblios' or 'authorities'.
70 my $self = $class->SUPER::new(@_);
71 # Check for a valid index
72 croak('No index name provided') unless $self->index;
76 sub get_elasticsearch {
78 unless (defined $self->{elasticsearch}) {
79 my $conf = $self->get_elasticsearch_params();
80 $self->{elasticsearch} = Search::Elasticsearch->new(
81 client => "5_0::Direct",
82 nodes => $conf->{nodes},
86 return $self->{elasticsearch};
89 =head2 get_elasticsearch_params
91 my $params = $self->get_elasticsearch_params();
93 This provides a hashref that contains the parameters for connecting to the
94 ElasicSearch servers, in the form:
97 'nodes' => ['127.0.0.1:9200', 'anotherserver:9200'],
98 'index_name' => 'koha_instance_index',
101 This is configured by the following in the C<config> block in koha-conf.xml:
104 <server>127.0.0.1:9200</server>
105 <server>anotherserver:9200</server>
106 <index_name>koha_instance</index_name>
111 sub get_elasticsearch_params {
114 # Copy the hash so that we're not modifying the original
115 my $conf = C4::Context->config('elasticsearch');
116 die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
117 my $es = { %{ $conf } };
119 # Helpfully, the multiple server lines end up in an array for us anyway
120 # if there are multiple ones, but not if there's only one.
121 my $server = $es->{server};
122 delete $es->{server};
123 if ( ref($server) eq 'ARRAY' ) {
125 # store it called 'nodes' (which is used by newer Search::Elasticsearch)
126 $es->{nodes} = $server;
129 $es->{nodes} = [$server];
132 die "No elasticsearch servers were specified in koha-conf.xml.\n";
134 die "No elasticserver index_name was specified in koha-conf.xml.\n"
135 if ( !$es->{index_name} );
136 # Append the name of this particular index to our namespace
137 $es->{index_name} .= '_' . $self->index;
139 $es->{key_prefix} = 'es_';
143 =head2 get_elasticsearch_settings
145 my $settings = $self->get_elasticsearch_settings();
147 This provides the settings provided to elasticsearch when an index is created.
148 These can do things like define tokenisation methods.
150 A hashref containing the settings is returned.
154 sub get_elasticsearch_settings {
157 # Use state to speed up repeated calls
158 state $settings = undef;
159 if (!defined $settings) {
160 my $config_file = C4::Context->config('elasticsearch_index_config');
161 $config_file ||= C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/index_config.yaml';
162 $settings = LoadFile( $config_file );
168 =head2 get_elasticsearch_mappings
170 my $mappings = $self->get_elasticsearch_mappings();
172 This provides the mappings that get passed to elasticsearch when an index is
177 sub get_elasticsearch_mappings {
180 # Use state to speed up repeated calls
184 if (!defined $all_mappings{$self->index}) {
185 $sort_fields{$self->index} = {};
187 data => scalar _get_elasticsearch_mapping('general', '')
189 my $marcflavour = lc C4::Context->preference('marcflavour');
190 $self->_foreach_mapping(
192 my ( $name, $type, $facet, $suggestible, $sort, $marc_type ) = @_;
193 return if $marc_type ne $marcflavour;
194 # TODO if this gets any sort of complexity to it, it should
195 # be broken out into its own function.
197 # TODO be aware of date formats, but this requires pre-parsing
198 # as ES will simply reject anything with an invalid date.
199 my $es_type = 'text';
200 if ($type eq 'boolean') {
201 $es_type = 'boolean';
202 } elsif ($type eq 'number' || $type eq 'sum') {
203 $es_type = 'integer';
204 } elsif ($type eq 'isbn' || $type eq 'stdno') {
208 $mappings->{data}{properties}{$name} = _get_elasticsearch_mapping('search', $es_type);
211 $mappings->{data}{properties}{ $name . '__facet' } = _get_elasticsearch_mapping('facet', $es_type);
214 $mappings->{data}{properties}{ $name . '__suggestion' } = _get_elasticsearch_mapping('suggestible', $es_type);
216 # Sort is a bit special as it can be true, false, undef.
217 # We care about "true" or "undef",
218 # "undef" means to do the default thing, which is make it sortable.
219 if (!defined $sort || $sort) {
220 $mappings->{data}{properties}{ $name . '__sort' } = _get_elasticsearch_mapping('sort', $es_type);
221 $sort_fields{$self->index}{$name} = 1;
225 $all_mappings{$self->index} = $mappings;
227 $self->sort_fields(\%{$sort_fields{$self->index}});
229 return $all_mappings{$self->index};
232 =head2 _get_elasticsearch_mapping
234 Get the ES mappings for the given purpose and data type
236 $mapping = _get_elasticsearch_mapping('search', 'text');
240 sub _get_elasticsearch_mapping {
242 my ( $purpose, $type ) = @_;
244 # Use state to speed up repeated calls
245 state $settings = undef;
246 if (!defined $settings) {
247 my $config_file = C4::Context->config('elasticsearch_field_config');
248 $config_file ||= C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/field_config.yaml';
249 $settings = LoadFile( $config_file );
252 if (!defined $settings->{$purpose}) {
253 die "Field purpose $purpose not defined in field config";
256 return $settings->{$purpose};
258 if (defined $settings->{$purpose}{$type}) {
259 return $settings->{$purpose}{$type};
261 if (defined $settings->{$purpose}{'default'}) {
262 return $settings->{$purpose}{'default'};
267 sub reset_elasticsearch_mappings {
268 my ( $reset_fields ) = @_;
269 my $mappings_yaml = C4::Context->config('elasticsearch_index_mappings');
270 $mappings_yaml ||= C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml';
271 my $indexes = LoadFile( $mappings_yaml );
273 while ( my ( $index_name, $fields ) = each %$indexes ) {
274 while ( my ( $field_name, $data ) = each %$fields ) {
275 my $field_type = $data->{type};
276 my $field_label = $data->{label};
277 my $mappings = $data->{mappings};
278 my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' });
279 for my $mapping ( @$mappings ) {
280 my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} });
281 $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet} || 0, suggestible => $mapping->{suggestible} || 0, sort => $mapping->{sort} } );
287 # This overrides the accessor provided by Class::Accessor so that if
288 # sort_fields isn't set, then it'll generate it.
292 $self->_sort_fields_accessor(@_);
295 my $val = $self->_sort_fields_accessor();
298 # This will populate the accessor as a side effect
299 $self->get_elasticsearch_mappings();
300 return $self->_sort_fields_accessor();
303 sub marc_records_to_documents {
304 my ($self, $records) = @_;
305 my $rules = $self->get_marc_mapping_rules();
306 my $control_fields_rules = $rules->{control_fields};
307 my $data_fields_rules = $rules->{data_fields};
308 my $marcflavour = lc C4::Context->preference('marcflavour');
309 my $serialization_format = C4::Context->preference('ElasticsearchMARCSerializationFormat');
311 my @record_documents;
313 sub _process_mappings {
314 my ($mappings, $data, $record_document) = @_;
315 foreach my $mapping (@{$mappings}) {
316 my ($target, $options) = @{$mapping};
317 # Copy (scalar) data since can have multiple targets
318 # with differing options for (possibly) mutating data
319 # so need a different copy for each
321 $record_document->{$target} //= [];
322 if (defined $options->{substr}) {
323 my ($start, $length) = @{$options->{substr}};
324 $_data = length($data) > $start ? substr $data, $start, $length : '';
326 if (defined $options->{value_callbacks}) {
327 $_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}});
329 if (defined $options->{property}) {
331 $options->{property} => $_data
334 push @{$record_document->{$target}}, $_data;
337 foreach my $record (@{$records}) {
338 my $record_document = {};
339 my $mappings = $rules->{leader};
341 _process_mappings($mappings, $record->leader(), $record_document);
343 foreach my $field ($record->fields()) {
344 if($field->is_control_field()) {
345 my $mappings = $control_fields_rules->{$field->tag()};
347 _process_mappings($mappings, $field->data(), $record_document);
351 my $subfields_mappings = $data_fields_rules->{$field->tag()};
352 if ($subfields_mappings) {
353 my $wildcard_mappings = $subfields_mappings->{'*'};
354 foreach my $subfield ($field->subfields()) {
355 my ($code, $data) = @{$subfield};
356 my $mappings = $subfields_mappings->{$code} // [];
357 if ($wildcard_mappings) {
358 $mappings = [@{$mappings}, @{$wildcard_mappings}];
361 _process_mappings($mappings, $data, $record_document);
367 foreach my $field (keys %{$rules->{defaults}}) {
368 unless (defined $record_document->{$field}) {
369 $record_document->{$field} = $rules->{defaults}->{$field};
372 foreach my $field (@{$rules->{sum}}) {
373 if (defined $record_document->{$field}) {
374 # TODO: validate numeric? filter?
375 # TODO: Or should only accept fields without nested values?
376 # TODO: Quick and dirty, improve if needed
377 $record_document->{$field} = sum0(grep { !ref($_) && m/\d+(\.\d+)?/} @{$record_document->{$field}});
380 # TODO: Perhaps should check if $records_document non empty, but really should never be the case
381 $record->encoding('UTF-8');
384 # Temporarily intercept all warn signals (MARC::Record carps when record length > 99999)
385 local $SIG{__WARN__} = sub {
386 push @warnings, $_[0];
388 $record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
391 # Suppress warnings if record length exceeded
392 unless (substr($record->leader(), 0, 5) eq '99999') {
393 foreach my $warning (@warnings) {
397 $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
398 $record_document->{'marc_format'} = 'MARCXML';
401 $record_document->{'marc_format'} = 'base64ISO2709';
403 my $id = $record->subfield('999', 'c');
404 push @record_documents, [$id, $record_document];
406 return \@record_documents;
409 # Provides the rules for marc to Elasticsearch JSON document conversion.
410 sub get_marc_mapping_rules {
413 my $marcflavour = lc C4::Context->preference('marcflavour');
416 sub _field_mappings {
417 my ($facet, $suggestible, $sort, $target_name, $target_type, $range) = @_;
418 my %mapping_defaults = ();
421 my $substr_args = undef;
423 # TODO: use value_callback instead?
424 my ($start, $end) = map(int, split /-/, $range, 2);
425 $substr_args = [$start];
426 push @{$substr_args}, (defined $end ? $end - $start + 1 : 1);
428 my $default_options = {};
430 $default_options->{substr} = $substr_args;
433 # TODO: Should probably have per type value callback/hook
434 # but hard code for now
435 if ($target_type eq 'boolean') {
436 $default_options->{value_callbacks} //= [];
437 push @{$default_options->{value_callbacks}}, sub {
439 # Trim whitespace at both ends
440 $value =~ s/^\s+|\s+$//g;
441 return $value ? 'true' : 'false';
445 my $mapping = [$target_name, $default_options];
446 push @mappings, $mapping;
449 push @suffixes, 'facet' if $facet;
450 push @suffixes, 'suggestion' if $suggestible;
451 push @suffixes, 'sort' if !defined $sort || $sort;
453 foreach my $suffix (@suffixes) {
454 my $mapping = ["${target_name}__$suffix"];
455 # Hack, fix later in less hideous manner
456 if ($suffix eq 'suggestion') {
457 push @{$mapping}, {%{$default_options}, property => 'input'};
460 push @{$mapping}, $default_options;
462 push @mappings, $mapping;
466 my $field_spec_regexp = qr/^([0-9]{3})([0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/;
467 my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/;
470 'control_fields' => {},
476 $self->_foreach_mapping(sub {
477 my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
478 return if $marc_type ne $marcflavour;
480 if ($type eq 'sum') {
481 push @{$rules->{sum}}, $name;
483 elsif($type eq 'boolean') {
484 # boolean gets special handling, if value doesn't exist for a field,
486 $rules->{defaults}->{$name} = 'false';
489 if ($marc_field =~ $field_spec_regexp) {
491 my $subfields = defined $2 ? $2 : '*';
492 my $range = defined $3 ? $3 : undef;
493 if ($field_tag < 10) {
494 $rules->{control_fields}->{$field_tag} //= [];
495 my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
496 push @{$rules->{control_fields}->{$field_tag}}, @mappings;
499 $rules->{data_fields}->{$field_tag} //= {};
500 foreach my $subfield (split //, $subfields) {
501 $rules->{data_fields}->{$field_tag}->{$subfield} //= [];
502 my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
503 push @{$rules->{data_fields}->{$field_tag}->{$subfield}}, @mappings;
507 elsif ($marc_field =~ $leader_regexp) {
508 my $range = defined $1 ? $1 : undef;
509 my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
510 push @{$rules->{leader}}, @mappings;
513 die("Invalid marc field: $marc_field");
519 =head2 _foreach_mapping
521 $self->_foreach_mapping(
523 my ( $name, $type, $facet, $suggestible, $sort, $marc_type,
526 return unless $marc_type eq 'marc21';
527 print "Data comes from: " . $marc_field . "\n";
531 This allows you to apply a function to each entry in the elasticsearch mappings
532 table, in order to build the mappings for whatever is needed.
534 In the provided function, the files are:
540 The field name for elasticsearch (corresponds to the 'mapping' column in the
545 The type for this value, e.g. 'string'.
549 True if this value should be facetised. This only really makes sense if the
550 field is understood by the facet processing code anyway.
554 True if this is a field that a) needs special sort handling, and b) if it
555 should be sorted on. False if a) but not b). Undef if not a). This allows,
556 for example, author to be sorted on but not everything marked with "author"
557 to be included in that sort.
561 A string that indicates the MARC type that this mapping is for, e.g. 'marc21',
562 'unimarc', 'normarc'.
566 A string that describes the MARC field that contains the data to extract.
567 These are of a form suited to Catmandu's MARC fixers.
573 sub _foreach_mapping {
574 my ( $self, $sub ) = @_;
576 # TODO use a caching framework here
577 my $search_fields = Koha::Database->schema->resultset('SearchField')->search(
579 'search_marc_map.index_name' => $self->index,
581 { join => { search_marc_to_fields => 'search_marc_map' },
583 'search_marc_to_fields.facet',
584 'search_marc_to_fields.suggestible',
585 'search_marc_to_fields.sort',
586 'search_marc_map.marc_type',
587 'search_marc_map.marc_field',
599 while ( my $search_field = $search_fields->next ) {
603 $search_field->get_column('facet'),
604 $search_field->get_column('suggestible'),
605 $search_field->get_column('sort'),
606 $search_field->get_column('marc_type'),
607 $search_field->get_column('marc_field'),
614 die process_error($@);
616 This parses an Elasticsearch error message and produces a human-readable
617 result from it. This result is probably missing all the useful information
618 that you might want in diagnosing an issue, so the warning is also logged.
620 Note that currently the resulting message is not internationalised. This
621 will happen eventually by some method or other.
626 my ($self, $msg) = @_;
628 warn $msg; # simple logging
630 # This is super-primitive
631 return "Unable to understand your search query, please rephrase and try again.\n" if $msg =~ /ParseException/;
633 return "Unable to perform your search. Please try again.\n";
636 =head2 _read_configuration
638 my $conf = _read_configuration();
640 Reads the I<configuration file> and returns a hash structure with the
641 configuration information. It raises an exception if mandatory entries
644 The hashref structure has the following form:
647 'nodes' => ['127.0.0.1:9200', 'anotherserver:9200'],
648 'index_name' => 'koha_instance',
651 This is configured by the following in the C<config> block in koha-conf.xml:
654 <server>127.0.0.1:9200</server>
655 <server>anotherserver:9200</server>
656 <index_name>koha_instance</index_name>
661 sub _read_configuration {
665 my $conf = C4::Context->config('elasticsearch');
666 Koha::Exceptions::Config::MissingEntry->throw(
667 "Missing 'elasticsearch' block in config file")
668 unless defined $conf;
670 if ( $conf && $conf->{server} ) {
671 my $nodes = $conf->{server};
672 if ( ref($nodes) eq 'ARRAY' ) {
673 $configuration->{nodes} = $nodes;
676 $configuration->{nodes} = [$nodes];
680 Koha::Exceptions::Config::MissingEntry->throw(
681 "Missing 'server' entry in config file for elasticsearch");
684 if ( defined $conf->{index_name} ) {
685 $configuration->{index_name} = $conf->{index_name};
688 Koha::Exceptions::Config::MissingEntry->throw(
689 "Missing 'index_name' entry in config file for elasticsearch");
692 return $configuration;
703 =item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
705 =item Robin Sheat C<< <robin@catalyst.net.nz> >>
707 =item Jonathan Druart C<< <jonathan.druart@bugs.koha-community.org> >>