1 package Koha::SearchEngine::Elasticsearch::Search;
3 # Copyright 2014 Catalyst IT
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it
8 # under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 3 of the License, or
10 # (at your option) any later version.
12 # Koha is distributed in the hope that it will be useful, but
13 # WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with Koha; if not, see <http://www.gnu.org/licenses>.
22 Koha::SearchEngine::Elasticsearch::Search - search functions for Elasticsearch
27 Koha::SearchEngine::Elasticsearch::Search->new( { index => $index } );
28 my $builder = Koha::SearchEngine::Elasticsearch::QueryBuilder->new(
29 { index => $index } );
30 my $query = $builder->build_query('perl');
31 my $results = $searcher->search($query);
32 print "There were " . $results->total . " results.\n";
43 use base qw(Koha::SearchEngine::Elasticsearch);
45 use C4::AuthoritiesMarc;
47 use Koha::AuthorisedValues;
48 use Koha::SearchEngine::QueryBuilder;
49 use Koha::SearchEngine::Search;
50 use Koha::Exceptions::Elasticsearch;
53 use Data::Dumper; #TODO remove
57 Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
61 my $results = $searcher->search($query, $page, $count, %options);
63 Run a search using the query. It'll return C<$count> results, starting at page
64 C<$page> (C<$page> counts from 1, anything less that, or C<undef> becomes 1.)
65 C<$count> is also the number of entries on a page.
67 C<%options> is a hash containing extra options:
73 If provided, this overrides the C<$page> value, and specifies the record as
74 an offset (i.e. the number of the record to start with), rather than a page.
83 my ($self, $query, $page, $count, %options) = @_;
85 # 20 is the default number of results per page
86 $query->{size} = $count // 20;
87 # ES doesn't want pages, it wants a record to start from.
88 if (exists $options{offset}) {
89 $query->{from} = $options{offset};
91 $page = (!defined($page) || ($page <= 0)) ? 0 : $page - 1;
92 $query->{from} = $page * $query->{size};
94 my $elasticsearch = $self->get_elasticsearch();
96 $elasticsearch->search(
97 index => $self->index_name,
102 die $self->process_error($@);
109 my $count = $searcher->count($query);
111 This mimics a search request, but just gets the result count instead. That's
112 faster than pulling all the data in, usually.
117 my ( $self, $query ) = @_;
118 my $elasticsearch = $self->get_elasticsearch();
120 # TODO: Probably possible to exclude results
121 # and just return number of hits
122 my $result = $elasticsearch->search(
123 index => $self->index_name,
127 return $result->{hits}->{total};
132 my ( $error, $results, $facets ) = $search->search_compat(
133 $query, $simple_query, \@sort_by, \@servers,
134 $results_per_page, $offset, undef, $item_types,
138 A search interface somewhat compatible with L<C4::Search->getRecords>. Anything
139 that is returned in the query created by build_query_compat will probably
140 get ignored here, along with some other things (like C<@servers>.)
146 $self, $query, $simple_query, $sort_by,
147 $servers, $results_per_page, $offset, $branches,
148 $item_types, $query_type, $scan
152 return $self->_aggregation_scan( $query, $results_per_page, $offset );
156 if ( !defined $offset or $offset < 0 ) {
159 $options{offset} = $offset;
160 my $results = $self->search($query, undef, $results_per_page, %options);
162 # Convert each result into a MARC::Record
164 # opac-search expects results to be put in the
165 # right place in the array, according to $offset
167 my $hits = $results->{'hits'};
168 foreach my $es_record (@{$hits->{'hits'}}) {
169 $records[$index++] = $self->decode_record_from_result($es_record->{'_source'});
172 # consumers of this expect a name-spaced result, we provide the default
175 $result{biblioserver}{hits} = $hits->{'total'};
176 $result{biblioserver}{RECORDS} = \@records;
177 return (undef, \%result, $self->_convert_facets($results->{aggregations}));
180 =head2 search_auth_compat
182 my ( $results, $total ) =
183 $searcher->search_auth_compat( $query, $offset, $count, $skipmetadata, %options );
185 This has a similar calling convention to L<search>, however it returns its
186 results in a form the same as L<C4::AuthoritiesMarc::SearchAuthorities>.
190 sub search_auth_compat {
191 my ($self, $query, $offset, $count, $skipmetadata, %options) = @_;
193 if ( !defined $offset or $offset <= 0 ) {
196 # Uh, authority search uses 1-based offset..
197 $options{offset} = $offset - 1;
198 my $database = Koha::Database->new();
199 my $schema = $database->schema();
200 my $res = $self->search($query, undef, $count, %options);
202 my $bib_searcher = Koha::SearchEngine::Elasticsearch::Search->new({index => 'biblios'});
204 my $hits = $res->{'hits'};
205 foreach my $es_record (@{$hits->{'hits'}}) {
206 my $record = $es_record->{'_source'};
209 # We are using the authid to create links, we should honor the authid as stored in the db, not
210 # the 001 which, in some circumstances, can contain other data
211 my $authid = $es_record->{_id};
214 $result{authid} = $authid;
216 if (!defined $skipmetadata || !$skipmetadata) {
217 # TODO put all this info into the record at index time so we
218 # don't have to go and sort it all out now.
219 my $authtypecode = $record->{authtype};
220 my $rs = $schema->resultset('AuthType')
221 ->search( { authtypecode => $authtypecode } );
223 # FIXME there's an assumption here that we will get a result.
224 # the original code also makes an assumption that some provided
225 # authtypecode may sometimes be used instead of the one stored
226 # with the record. It's not documented why this is the case, so
227 # it's not reproduced here yet.
228 my $authtype = $rs->single;
229 my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
230 my $marc = $self->decode_record_from_result($record);
231 my $mainentry = $marc->field($auth_tag_to_report);
234 foreach ( $mainentry->subfields() ) {
235 $reported_tag .= '$' . $_->[0] . $_->[1];
238 # Turn the resultset into a hash
239 $result{authtype} = $authtype ? $authtype->authtypetext : $authtypecode;
240 $result{reported_tag} = $reported_tag;
242 # Reimplementing BuildSummary is out of scope because it'll be hard
244 C4::AuthoritiesMarc::BuildSummary( $marc, $result{authid},
246 $result{used} = $self->count_auth_use($bib_searcher, $authid);
248 push @records, \%result;
250 return ( \@records, $hits->{'total'} );
253 =head2 count_auth_use
255 my $count = $auth_searcher->count_auth_use($bib_searcher, $authid);
257 This runs a search to determine the number of records that reference the
258 specified authid. C<$bib_searcher> must be something compatible with
259 elasticsearch, as the query is built in this function.
264 my ($self, $bib_searcher, $authid) = @_;
269 # query => { match_all => {} },
270 filter => { term => { 'koha-auth-number' => $authid } }
274 $bib_searcher->count($query);
277 =head2 simple_search_compat
279 my ( $error, $marcresults, $total_hits ) =
280 $searcher->simple_search( $query, $offset, $max_results, %options );
282 This is a simpler interface to the searching, intended to be similar enough to
283 L<C4::Search::SimpleSearch>.
291 A thing to search for. It could be a simple string, or something constructed
292 with the appropriate QueryBuilder module.
296 How many results to skip from the start of the results.
298 =item C<$max_results>
300 The max number of results to return. The default is 100 (because unlimited
301 is a pretty terrible thing to do.)
305 These options are unused by Elasticsearch
315 if something went wrong, this'll contain some kind of error
318 =item C<$marcresults>
320 an arrayref of MARC::Records (note that this is different from the
321 L<C4::Search> version which will return plain XML, but too bad.)
325 the total number of results that this search could have returned.
331 sub simple_search_compat {
332 my ($self, $query, $offset, $max_results) = @_;
334 return ('No query entered', undef, undef) unless $query;
337 $offset = 0 if not defined $offset or $offset < 0;
338 $options{offset} = $offset;
339 $max_results //= 100;
341 unless (ref $query) {
342 # We'll push it through the query builder to sanitise everything.
343 my $qb = Koha::SearchEngine::QueryBuilder->new({index => $self->index});
344 (undef,$query) = $qb->build_query_compat(undef, [$query]);
346 my $results = $self->search($query, undef, $max_results, %options);
348 my $hits = $results->{'hits'};
349 foreach my $es_record (@{$hits->{'hits'}}) {
350 push @records, $self->decode_record_from_result($es_record->{'_source'});
352 return (undef, \@records, $hits->{'total'});
355 =head2 extract_biblionumber
357 my $biblionumber = $searcher->extract_biblionumber( $searchresult );
359 $searchresult comes from simple_search_compat.
361 Returns the biblionumber from the search result record.
365 sub extract_biblionumber {
366 my ( $self, $searchresultrecord ) = @_;
367 return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
370 =head2 decode_record_from_result
371 my $marc_record = $self->decode_record_from_result(@result);
373 Extracts marc data from Elasticsearch result and decodes to MARC::Record object
377 sub decode_record_from_result {
378 # Result is passed in as array, will get flattened
379 # and first element will be $result
380 my ( $self, $result ) = @_;
381 if ($result->{marc_format} eq 'base64ISO2709') {
382 return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
384 elsif ($result->{marc_format} eq 'MARCXML') {
385 return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
387 elsif ($result->{marc_format} eq 'ARRAY') {
388 return $self->_array_to_marc($result->{marc_data_array});
391 Koha::Exceptions::Elasticsearch->throw("Missing marc_format field in Elasticsearch result");
395 =head2 max_result_window
397 Returns the maximum number of results that can be fetched
399 This directly requests Elasticsearch for the setting index.max_result_window (or
400 the default value for this setting in case it is not set)
404 sub max_result_window {
407 my $elasticsearch = $self->get_elasticsearch();
409 my $response = $elasticsearch->indices->get_settings(
410 index => $self->index_name,
411 flat_settings => 'true',
412 include_defaults => 'true'
415 my $max_result_window = $response->{$self->index_name}->{settings}->{'index.max_result_window'};
416 $max_result_window //= $response->{$self->index_name}->{defaults}->{'index.max_result_window'};
418 return $max_result_window;
421 =head2 _convert_facets
423 my $koha_facets = _convert_facets($es_facets);
425 Converts elasticsearch facets types to the form that Koha expects.
426 It expects the ES facet name to match the Koha type, for example C<itype>,
427 C<au>, C<su-to>, etc.
431 sub _convert_facets {
432 my ( $self, $es, $exp_facet ) = @_;
436 # These should correspond to the ES field names, as opposed to the CCL
437 # things that zebra uses.
441 itype => 'ItemTypes',
442 location => 'Location',
443 'su-geo' => 'Places',
444 'title-series' => 'Series',
446 ccode => 'CollectionCodes',
447 holdingbranch => 'HoldingLibrary',
448 homebranch => 'HomeLibrary',
451 my @facetable_fields =
452 Koha::SearchEngine::Elasticsearch->get_facetable_fields;
453 for my $f (@facetable_fields) {
454 next unless defined $f->facet_order;
455 $type_to_label{ $f->name } =
456 { order => $f->facet_order, label => $label{ $f->name } };
459 # We also have some special cases, e.g. itypes that need to show the
460 # value rather than the code.
461 my @itypes = Koha::ItemTypes->search;
462 my @libraries = Koha::Libraries->search;
463 my $library_names = { map { $_->branchcode => $_->branchname } @libraries };
464 my @locations = Koha::AuthorisedValues->search( { category => 'LOC' } );
465 my $opac = C4::Context->interface eq 'opac' ;
467 itype => { map { $_->itemtype => $_->description } @itypes },
468 location => { map { $_->authorised_value => ( $opac ? ( $_->lib_opac || $_->lib ) : $_->lib ) } @locations },
469 holdingbranch => $library_names,
470 homebranch => $library_names
474 while ( my ( $type, $data ) = each %$es ) {
475 next if !exists( $type_to_label{$type} );
477 # We restrict to the most popular $limit !results
478 my $limit = C4::Context->preference('FacetMaxCount');
480 type_id => $type . '_id',
481 "type_label_$type_to_label{$type}{label}" => 1,
482 type_link_value => $type,
483 order => $type_to_label{$type}{order},
485 $limit = @{ $data->{buckets} } if ( $limit > @{ $data->{buckets} } );
486 foreach my $term ( @{ $data->{buckets} }[ 0 .. $limit - 1 ] ) {
487 my $t = $term->{key};
488 my $c = $term->{doc_count};
490 if ( exists( $special{$type} ) ) {
491 $label = $special{$type}->{$t} // $t;
496 push @{ $facet->{facets} }, {
498 facet_link_value => $t,
499 facet_title_value => $t . " ($c)",
500 facet_label_value => $label, # TODO either truncate this,
501 # or make the template do it like it should anyway
502 type_link_value => $type,
505 push @facets, $facet if exists $facet->{facets};
508 @facets = sort { $a->{order} <=> $b->{order} } @facets;
512 =head2 _aggregation_scan
514 my $result = $self->_aggregration_scan($query, 10, 0);
516 Perform an aggregation request for scan purposes.
520 sub _aggregation_scan {
521 my ($self, $query, $results_per_page, $offset) = @_;
523 if (!scalar(keys %{$query->{aggregations}})) {
530 return (undef, \%result, undef);
532 my ($field) = keys %{$query->{aggregations}};
533 $query->{aggregations}{$field}{terms}{size} = 1000;
534 my $results = $self->search($query, 1, 0);
536 # Convert each result into a MARC::Record
537 my (@records, $index);
538 # opac-search expects results to be put in the
539 # right place in the array, according to $offset
540 $index = $offset - 1;
542 my $count = scalar(@{$results->{aggregations}{$field}{buckets}});
543 for (my $index = $offset; $index - $offset < $results_per_page && $index < $count; $index++) {
544 my $bucket = $results->{aggregations}{$field}{buckets}->[$index];
545 # Scan values are expressed as:
546 # - MARC21: 100a (count) and 245a (term)
547 # - UNIMARC: 200f (count) and 200a (term)
548 my $marc = MARC::Record->new;
549 $marc->encoding('UTF-8');
550 if (C4::Context->preference('marcflavour') eq 'UNIMARC') {
551 $marc->append_fields(
552 MARC::Field->new((200, ' ', ' ', 'f' => $bucket->{doc_count}))
554 $marc->append_fields(
555 MARC::Field->new((200, ' ', ' ', 'a' => $bucket->{key}))
558 $marc->append_fields(
559 MARC::Field->new((100, ' ', ' ', 'a' => $bucket->{doc_count}))
561 $marc->append_fields(
562 MARC::Field->new((245, ' ', ' ', 'a' => $bucket->{key}))
565 $records[$index] = $marc->as_usmarc();
567 # consumers of this expect a namespaced result, we provide the default
570 $result{biblioserver}{hits} = $count;
571 $result{biblioserver}{RECORDS} = \@records;
572 return (undef, \%result, undef);