3 # This file is part of Koha.
5 # Koha is free software; you can redistribute it and/or modify it
6 # under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # Koha is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with Koha; if not, see <http://www.gnu.org/licenses>.
20 use Test::More tests => 6;
24 use t::lib::TestBuilder;
30 use List::Util qw( any );
32 use Koha::SearchEngine::Elasticsearch;
33 use Koha::SearchEngine::Elasticsearch::Search;
35 my $schema = Koha::Database->new->schema;
36 $schema->storage->txn_begin;
38 subtest '_read_configuration() tests' => sub {
43 t::lib::Mocks::mock_config( 'elasticsearch', undef );
45 # 'elasticsearch' missing in configuration
47 $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
49 'Koha::Exceptions::Config::MissingEntry',
50 'Configuration problem, exception thrown';
53 "Missing <elasticsearch> entry in koha-conf.xml",
54 'Exception message is correct'
57 # 'elasticsearch' present but no 'server' entry
58 t::lib::Mocks::mock_config( 'elasticsearch', {} );
60 $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
62 'Koha::Exceptions::Config::MissingEntry',
63 'Configuration problem, exception thrown';
66 "Missing <elasticsearch>/<server> entry in koha-conf.xml",
67 'Exception message is correct'
70 # 'elasticsearch' and 'server' entries present, but no 'index_name'
71 t::lib::Mocks::mock_config( 'elasticsearch', { server => 'a_server' } );
73 $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
75 'Koha::Exceptions::Config::MissingEntry',
76 'Configuration problem, exception thrown';
79 "Missing <elasticsearch>/<index_name> entry in koha-conf.xml",
80 'Exception message is correct'
83 # Correct configuration, only one server
84 t::lib::Mocks::mock_config( 'elasticsearch', { server => 'a_server', index_name => 'index' } );
86 $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
87 is( $configuration->{index_name}, 'index', 'Index configuration parsed correctly' );
88 is_deeply( $configuration->{nodes}, ['a_server'], 'Server configuration parsed correctly' );
90 # Correct configuration, two servers
91 my @servers = ('a_server', 'another_server');
92 t::lib::Mocks::mock_config( 'elasticsearch', { server => \@servers, index_name => 'index' } );
94 $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
95 is( $configuration->{index_name}, 'index', 'Index configuration parsed correctly' );
96 is( $configuration->{cxn_pool}, 'Static', 'cxn_pool configuration set correctly to Static if not specified' );
97 is_deeply( $configuration->{nodes}, \@servers , 'Server configuration parsed correctly' );
99 t::lib::Mocks::mock_config( 'elasticsearch', { server => \@servers, index_name => 'index', cxn_pool => 'Fluid' } );
101 $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
102 is( $configuration->{cxn_pool}, 'Fluid', 'cxn_pool configuration parsed correctly' );
104 my $params = Koha::SearchEngine::Elasticsearch::get_elasticsearch_params;
105 is_deeply( $configuration->{nodes}, \@servers , 'get_elasticsearch_params is just a wrapper for _read_configuration' );
109 subtest 'get_elasticsearch_settings() tests' => sub {
115 # test reading index settings
116 my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );
117 $settings = $es->get_elasticsearch_settings();
118 is( $settings->{index}{analysis}{analyzer}{analyzer_phrase}{tokenizer}, 'keyword', 'Index settings parsed correctly' );
121 subtest 'get_elasticsearch_mappings() tests' => sub {
127 # test reading mappings
128 my $es = Koha::SearchEngine::Elasticsearch->new( {index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX} );
129 $mappings = $es->get_elasticsearch_mappings();
130 is( $mappings->{data}{properties}{isbn__sort}{index}, 'false', 'Field mappings parsed correctly' );
133 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
137 t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
138 t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
142 name => 'control_number',
148 marc_type => 'marc21',
158 marc_type => 'marc21',
159 marc_field => '020a',
168 marc_type => 'marc21',
169 marc_field => '100a',
178 marc_type => 'marc21',
179 marc_field => '110a',
188 marc_type => 'marc21',
189 marc_field => '245(ab)ab',
192 name => 'unimarc_title',
198 marc_type => 'unimarc',
199 marc_field => '245a',
205 suggestible => undef,
208 marc_type => 'marc21',
212 name => 'uniform_title',
218 marc_type => 'marc21',
219 marc_field => '240a',
222 name => 'title_wildcard',
228 marc_type => 'marc21',
232 name => 'sum_item_price',
238 marc_type => 'marc21',
239 marc_field => '952g',
242 name => 'items_withdrawn_status',
248 marc_type => 'marc21',
249 marc_field => '9520',
252 name => 'local_classification',
258 marc_type => 'marc21',
259 marc_field => '952o',
262 name => 'type_of_record',
268 marc_type => 'marc21',
269 marc_field => 'leader_/6',
272 name => 'type_of_record_and_bib_level',
278 marc_type => 'marc21',
279 marc_field => 'leader_/6-7',
288 marc_type => 'marc21',
289 marc_field => '007_/0',
298 marc_type => 'marc21',
299 marc_field => '952l',
308 marc_type => 'marc21',
309 marc_field => '260c',
312 name => 'date-of-publication',
318 marc_type => 'marc21',
319 marc_field => '008_/7-10',
324 my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
325 $se->mock('_foreach_mapping', sub {
326 my ($self, $sub) = @_;
328 foreach my $map (@mappings) {
342 my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
344 my $callno = 'ABC123';
345 my $callno2 = 'ABC456';
346 my $long_callno = '1234567890' x 30;
348 my $marc_record_1 = MARC::Record->new();
349 $marc_record_1->leader(' cam 22 a 4500');
350 $marc_record_1->append_fields(
351 MARC::Field->new('001', '123'),
352 MARC::Field->new('007', 'ku'),
353 MARC::Field->new('008', '901111s1962 xxk|||| |00| ||eng c'),
354 MARC::Field->new('020', '', '', a => '1-56619-909-3'),
355 MARC::Field->new('100', '', '', a => 'Author 1'),
356 MARC::Field->new('110', '', '', a => 'Corp Author'),
357 MARC::Field->new('210', '', '', a => 'Title 1'),
358 MARC::Field->new('240', '', '4', a => 'The uniform title with nonfiling indicator'),
359 MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
360 MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', c => 'c1962'),
361 MARC::Field->new('999', '', '', c => '1234567'),
362 # ' ' for testing trimming of white space in boolean value callback:
363 MARC::Field->new('952', '', '', 0 => ' ', g => '123.30', o => $callno, l => 3),
364 MARC::Field->new('952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2),
365 MARC::Field->new('952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1),
367 my $marc_record_2 = MARC::Record->new();
368 $marc_record_2->leader(' cam 22 a 4500');
369 $marc_record_2->append_fields(
370 MARC::Field->new('008', '901111s19uu xxk|||| |00| ||eng c'),
371 MARC::Field->new('100', '', '', a => 'Author 2'),
372 # MARC::Field->new('210', '', '', a => 'Title 2'),
373 # MARC::Field->new('245', '', '', a => 'Title: second record'),
374 MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', c => '1963-2003'),
375 MARC::Field->new('999', '', '', c => '1234568'),
376 MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno),
378 my $records = [ $marc_record_1, $marc_record_2 ];
380 $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
382 my $docs = $see->marc_records_to_documents($records);
385 is(scalar @{$docs}, 2, 'Two records converted to documents');
387 is_deeply($docs->[0]->{control_number}, ['123'], 'First record control number should be set correctly');
389 is_deeply($docs->[0]->{'ff7-00'}, ['k'], 'First record ff7-00 should be set correctly');
391 is(scalar @{$docs->[0]->{author}}, 2, 'First document author field should contain two values');
392 is_deeply($docs->[0]->{author}, ['Author 1', 'Corp Author'], 'First document author field should be set correctly');
394 is(scalar @{$docs->[0]->{author__sort}}, 1, 'First document author__sort field should have a single value');
395 is_deeply($docs->[0]->{author__sort}, ['Author 1 Corp Author'], 'First document author__sort field should be set correctly');
397 is(scalar @{$docs->[0]->{title__sort}}, 1, 'First document title__sort field should have a single');
398 is_deeply($docs->[0]->{title__sort}, ['Title: first record Title: first record'], 'First document title__sort field should be set correctly');
400 is($docs->[0]->{issues}, 6, 'Issues field should be sum of the issues for each item');
401 is($docs->[0]->{issues__sort}, 6, 'Issues sort field should also be a sum of the issues');
403 is(scalar @{$docs->[0]->{title_wildcard}}, 2, 'First document title_wildcard field should have two values');
404 is_deeply($docs->[0]->{title_wildcard}, ['Title:', 'first record'], 'First document title_wildcard field should be set correctly');
407 is(scalar @{$docs->[0]->{author__suggestion}}, 2, 'First document author__suggestion field should contain two values');
409 $docs->[0]->{author__suggestion},
412 'input' => 'Author 1'
415 'input' => 'Corp Author'
418 'First document author__suggestion field should be set correctly'
421 is(scalar @{$docs->[0]->{title__suggestion}}, 3, 'First document title__suggestion field should contain three values');
423 $docs->[0]->{title__suggestion},
425 { 'input' => 'Title:' },
426 { 'input' => 'first record' },
427 { 'input' => 'Title: first record' }
429 'First document title__suggestion field should be set correctly'
432 ok(!(defined $docs->[0]->{title__facet}), 'First document should have no title__facet field');
434 is(scalar @{$docs->[0]->{author__facet}}, 2, 'First document author__facet field should have two values');
436 $docs->[0]->{author__facet},
437 ['Author 1', 'Corp Author'],
438 'First document author__facet field should be set correctly'
441 is(scalar @{$docs->[0]->{items_withdrawn_status}}, 2, 'First document items_withdrawn_status field should have two values');
443 $docs->[0]->{items_withdrawn_status},
445 'First document items_withdrawn_status field should be set correctly'
449 $docs->[0]->{sum_item_price},
451 'First document sum_item_price field should be set correctly'
454 ok(defined $docs->[0]->{marc_data}, 'First document marc_data field should be set');
455 ok(defined $docs->[0]->{marc_format}, 'First document marc_format field should be set');
456 is($docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
458 my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
460 ok($decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result");
461 is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded base64ISO2709 record has same data as original record");
463 is(scalar @{$docs->[0]->{type_of_record}}, 1, 'First document type_of_record field should have one value');
465 $docs->[0]->{type_of_record},
467 'First document type_of_record field should be set correctly'
470 is(scalar @{$docs->[0]->{type_of_record_and_bib_level}}, 1, 'First document type_of_record_and_bib_level field should have one value');
472 $docs->[0]->{type_of_record_and_bib_level},
474 'First document type_of_record_and_bib_level field should be set correctly'
477 is(scalar @{$docs->[0]->{isbn}}, 4, 'First document isbn field should contain four values');
478 is_deeply($docs->[0]->{isbn}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');
481 $docs->[0]->{'local_classification'},
482 [$callno, $callno2, $long_callno],
483 'First document local_classification field should be set correctly'
486 # Nonfiling characters for sort fields
488 $docs->[0]->{uniform_title},
489 ['The uniform title with nonfiling indicator'],
490 'First document uniform_title field should contain the title verbatim'
493 $docs->[0]->{uniform_title__sort},
494 ['uniform title with nonfiling indicator'],
495 'First document uniform_title__sort field should contain the title with the first four initial characters removed'
498 # Tests for 'year' type
499 is(scalar @{$docs->[0]->{'date-of-publication'}}, 1, 'First document date-of-publication field should contain one value');
500 is_deeply($docs->[0]->{'date-of-publication'}, ['1962'], 'First document date-of-publication field should be set correctly');
503 $docs->[0]->{'copydate'},
505 'First document copydate field should be set correctly'
510 is(scalar @{$docs->[1]->{author}}, 1, 'Second document author field should contain one value');
511 is_deeply($docs->[1]->{author}, ['Author 2'], 'Second document author field should be set correctly');
513 is(scalar @{$docs->[1]->{items_withdrawn_status}}, 1, 'Second document items_withdrawn_status field should have one value');
515 $docs->[1]->{items_withdrawn_status},
517 'Second document items_withdrawn_status field should be set correctly'
521 $docs->[1]->{sum_item_price},
523 'Second document sum_item_price field should be set correctly'
527 $docs->[1]->{local_classification__sort},
528 [substr($long_callno, 0, 255)],
529 'Second document local_classification__sort field should be set correctly'
532 # Tests for 'year' type
534 $docs->[1]->{'copydate'},
536 'Second document copydate field should be set correctly'
539 $docs->[1]->{'date-of-publication'},
541 'Second document date-of-publication field should be set correctly'
544 # Mappings marc_type:
546 ok(!(defined $docs->[0]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
548 # Marc serialization format fallback for records exceeding ISO2709 max record size
550 my $large_marc_record = MARC::Record->new();
551 $large_marc_record->leader(' cam 22 a 4500');
553 $large_marc_record->append_fields(
554 MARC::Field->new('100', '', '', a => 'Author 1'),
555 MARC::Field->new('110', '', '', a => 'Corp Author'),
556 MARC::Field->new('210', '', '', a => 'Title 1'),
557 MARC::Field->new('245', '', '', a => 'Title:', b => 'large record'),
558 MARC::Field->new('999', '', '', c => '1234567'),
561 my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');
562 my $items_count = 1638;
563 while(--$items_count) {
564 $large_marc_record->append_fields($item_field);
567 $docs = $see->marc_records_to_documents([$large_marc_record]);
569 is($docs->[0]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
571 $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
573 ok($decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result");
574 is($decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(), "Decoded MARCXML record has same data as original record");
582 marc_type => 'marc21',
583 marc_field => '245((ab)ab',
586 my $exception = try {
587 $see->marc_records_to_documents($records);
593 ok(defined $exception, "Exception has been thrown when processing mapping with unmatched opening parenthesis");
594 ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
595 ok($exception->message =~ /Unmatched opening parenthesis/, "Exception has the correct message");
604 marc_type => 'marc21',
605 marc_field => '245(ab))ab',
609 $see->marc_records_to_documents($records);
615 ok(defined $exception, "Exception has been thrown when processing mapping with unmatched closing parenthesis");
616 ok($exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class");
617 ok($exception->message =~ /Unmatched closing parenthesis/, "Exception has the correct message");
620 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () tests' => sub {
624 t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
625 t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ARRAY');
629 name => 'control_number',
635 marc_type => 'marc21',
640 my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
641 $se->mock('_foreach_mapping', sub {
642 my ($self, $sub) = @_;
644 foreach my $map (@mappings) {
658 my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
660 my $marc_record_1 = MARC::Record->new();
661 $marc_record_1->leader(' cam 22 a 4500');
662 $marc_record_1->append_fields(
663 MARC::Field->new('001', '123'),
664 MARC::Field->new('020', '', '', a => '1-56619-909-3'),
665 MARC::Field->new('100', '', '', a => 'Author 1'),
666 MARC::Field->new('110', '', '', a => 'Corp Author'),
667 MARC::Field->new('210', '', '', a => 'Title 1'),
668 MARC::Field->new('245', '', '', a => 'Title:', b => 'first record'),
669 MARC::Field->new('999', '', '', c => '1234567'),
671 my $marc_record_2 = MARC::Record->new();
672 $marc_record_2->leader(' cam 22 a 4500');
673 $marc_record_2->append_fields(
674 MARC::Field->new('100', '', '', a => 'Author 2'),
675 # MARC::Field->new('210', '', '', a => 'Title 2'),
676 # MARC::Field->new('245', '', '', a => 'Title: second record'),
677 MARC::Field->new('999', '', '', c => '1234568'),
678 MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
680 my $records = [ $marc_record_1, $marc_record_2 ];
682 $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
684 my $docs = $see->marc_records_to_documents($records);
687 is(scalar @{$docs}, 2, 'Two records converted to documents');
689 is_deeply($docs->[0]->{control_number}, ['123'], 'First record control number should be set correctly');
691 is($docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly');
693 my $decoded_marc_record = $see->decode_record_from_result($docs->[0]);
695 ok($decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result");
696 is($decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(), "Decoded ARRAY record has same data as original record");
699 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authority tests' => sub {
703 t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
704 t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
706 my $builder = t::lib::TestBuilder->new;
707 my $auth_type = $builder->build_object({ class => 'Koha::Authority::Types', value =>{
708 auth_tag_to_report => '150'
720 marc_type => 'marc21',
721 marc_field => '150(ae)',
725 my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
726 $se->mock('_foreach_mapping', sub {
727 my ($self, $sub) = @_;
729 foreach my $map (@mappings) {
743 my $see = Koha::SearchEngine::Elasticsearch::Search->new({ index => $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX });
744 my $marc_record_1 = MARC::Record->new();
745 $marc_record_1->append_fields(
746 MARC::Field->new('001', '123'),
747 MARC::Field->new('007', 'ku'),
748 MARC::Field->new('020', '', '', a => '1-56619-909-3'),
749 MARC::Field->new('150', '', '', a => 'Subject', v => 'Genresubdiv', x => 'Generalsubdiv', z => 'Geosubdiv'),
751 my $marc_record_2 = MARC::Record->new();
752 $marc_record_2->append_fields(
753 MARC::Field->new('150', '', '', a => 'Subject', v => 'Genresubdiv', z => 'Geosubdiv', x => 'Generalsubdiv', e => 'wrongsubdiv' ),
755 my $records = [ $marc_record_1, $marc_record_2 ];
757 $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
759 my $docs = $see->marc_records_to_documents($records);
762 any { $_ eq "Subject formsubdiv Genresubdiv generalsubdiv Generalsubdiv geographicsubdiv Geosubdiv" }
763 @{$docs->[0]->{'match-heading'}},
764 "First record match-heading should contain the correctly formatted heading"
767 any { $_ eq "Subject formsubdiv Genresubdiv geographicsubdiv Geosubdiv generalsubdiv Generalsubdiv" }
768 @{$docs->[1]->{'match-heading'}},
769 "Second record match-heading should contain the correctly formatted heading without wrong subfield"
773 $schema->storage->txn_rollback;