From f24dd95e5d4b3c1b7b9adf5618790f516b5889c5 Mon Sep 17 00:00:00 2001 From: Brendan Gallagher Date: Tue, 26 Apr 2016 20:55:25 +0000 Subject: [PATCH] DBRev for Bug 12478 - Elasticsearch support for Koha Signed-off-by: Brendan Gallagher --- Koha.pm | 2 +- .../mysql/atomicupdate/elastic_search.perl | 88 ------------------- installer/data/mysql/updatedatabase.pl | 87 ++++++++++++++++++ 3 files changed, 88 insertions(+), 89 deletions(-) delete mode 100644 installer/data/mysql/atomicupdate/elastic_search.perl diff --git a/Koha.pm b/Koha.pm index f80b7115da..884ceaf498 100644 --- a/Koha.pm +++ b/Koha.pm @@ -29,7 +29,7 @@ use vars qw{ $VERSION }; # - #4 : the developer version. The 4th number is the database subversion. # used by developers when the database changes. updatedatabase take care of the changes itself # and is automatically called by Auth.pm when needed. -$VERSION = "3.23.00.049"; +$VERSION = "3.23.00.050"; sub version { return $VERSION; diff --git a/installer/data/mysql/atomicupdate/elastic_search.perl b/installer/data/mysql/atomicupdate/elastic_search.perl deleted file mode 100644 index 62e72dde95..0000000000 --- a/installer/data/mysql/atomicupdate/elastic_search.perl +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/perl - -use Modern::Perl; - -use C4::Context; -use YAML::Syck; - -use Koha::SearchMarcMaps; -use Koha::SearchFields; - -my $dbh = C4::Context->dbh; - - -$dbh->do(q|INSERT IGNORE INTO systempreferences (variable,value,explanation,options,type) - VALUES('SearchEngine','Zebra','Choose format to display postal addresses','','Choice')|); - - -$dbh->do(q|DROP TABLE IF EXISTS search_marc_to_field|); -$dbh->do(q|DROP TABLE IF EXISTS search_marc_map|); -$dbh->do(q|DROP TABLE IF EXISTS search_field|); - -# This specifies the fields that will be stored in the search engine. -$dbh->do(q| - CREATE TABLE `search_field` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `name` varchar(255) NOT NULL COMMENT 'the name of the field as it will be stored in the search engine', - `label` varchar(255) NOT NULL COMMENT 'the human readable name of the field, for display', - `type` ENUM('string', 'date', 'number', 'boolean', 'sum') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine', - PRIMARY KEY (`id`), - UNIQUE KEY (`name`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci -|); - -# This contains a MARC field specifier for a given index, marc type, and marc -# field. -$dbh->do(q| - CREATE TABLE `search_marc_map` ( - id int(11) NOT NULL AUTO_INCREMENT, - index_name ENUM('biblios','authorities') NOT NULL COMMENT 'what storage index this map is for', - marc_type ENUM('marc21', 'unimarc', 'normarc') NOT NULL COMMENT 'what MARC type this map is for', - marc_field VARCHAR(255) NOT NULL COMMENT 'the MARC specifier for this field', - PRIMARY KEY(`id`), - unique key( index_name, marc_field, marc_type), - INDEX (`index_name`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci -|); - -# This joins the two search tables together. We can have any combination: -# one marc field could have many search fields (maybe you want one value -# to go to 'author' and 'corporate-author) and many marc fields could go -# to one search field (e.g. all the various author fields going into -# 'author'.) -# -# a note about the sort field: -# * if all the entries for a mapping are 'null', nothing special is done with that mapping. -# * if any of the entries are not null, then a __sort field is created in ES for this mapping. In this case: -# * any mapping with sort == false WILL NOT get copied into a __sort field -# * any mapping with sort == true or is null WILL get copied into a __sort field -# * any sorts on the field name will be applied to $fieldname.'__sort' instead. -# this means that we can have search for author that includes 1xx, 245$c, and 7xx, but the sort only applies to 1xx. -$dbh->do(q| - CREATE TABLE `search_marc_to_field` ( - search_marc_map_id int(11) NOT NULL, - search_field_id int(11) NOT NULL, - facet boolean DEFAULT FALSE COMMENT 'true if a facet field should be generated for this', - suggestible boolean DEFAULT FALSE COMMENT 'true if this field can be used to generate suggestions for browse', - sort boolean DEFAULT NULL COMMENT 'true/false creates special sort handling, null doesn''t', - PRIMARY KEY(search_marc_map_id, search_field_id), - FOREIGN KEY(search_marc_map_id) REFERENCES search_marc_map(id) ON DELETE CASCADE ON UPDATE CASCADE, - FOREIGN KEY(search_field_id) REFERENCES search_field(id) ON DELETE CASCADE ON UPDATE CASCADE - ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci -|); - -my $mappings_yaml = C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml'; -my $indexes = LoadFile( $mappings_yaml ); - -while ( my ( $index_name, $fields ) = each %$indexes ) { - while ( my ( $field_name, $data ) = each %$fields ) { - my $field_type = $data->{type}; - my $field_label = $data->{label}; - my $mappings = $data->{mappings}; - my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' }); - for my $mapping ( @$mappings ) { - my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} }); - $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet}, suggestible => $mapping->{suggestible}, sort => $mapping->{sort} } ); - } - } -} diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl index 30b4ce6d85..a5f8ca34c2 100755 --- a/installer/data/mysql/updatedatabase.pl +++ b/installer/data/mysql/updatedatabase.pl @@ -12255,6 +12255,93 @@ if ( $column_has_been_used ) { SetVersion($DBversion); } + +$DBversion = "3.23.00.050"; +if ( CheckVersion($DBversion) ) { + use YAML::Syck; + use Koha::SearchMarcMaps; + use Koha::SearchFields; + + $dbh->do(q|INSERT IGNORE INTO systempreferences (variable,value,explanation,options,type) + VALUES('SearchEngine','Zebra','Choose Search Engine','','Choice')|); + + + $dbh->do(q|DROP TABLE IF EXISTS search_marc_to_field|); + $dbh->do(q|DROP TABLE IF EXISTS search_marc_map|); + $dbh->do(q|DROP TABLE IF EXISTS search_field|); + +# This specifies the fields that will be stored in the search engine. + $dbh->do(q| + CREATE TABLE `search_field` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) NOT NULL COMMENT 'the name of the field as it will be stored in the search engine', + `label` varchar(255) NOT NULL COMMENT 'the human readable name of the field, for display', + `type` ENUM('string', 'date', 'number', 'boolean', 'sum') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine', + PRIMARY KEY (`id`), + UNIQUE KEY (`name`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci + |); +# This contains a MARC field specifier for a given index, marc type, and marc +# field. +$dbh->do(q| + CREATE TABLE `search_marc_map` ( + id int(11) NOT NULL AUTO_INCREMENT, + index_name ENUM('biblios','authorities') NOT NULL COMMENT 'what storage index this map is for', + marc_type ENUM('marc21', 'unimarc', 'normarc') NOT NULL COMMENT 'what MARC type this map is for', + marc_field VARCHAR(255) NOT NULL COMMENT 'the MARC specifier for this field', + PRIMARY KEY(`id`), + unique key( index_name, marc_field, marc_type), + INDEX (`index_name`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci + |); + +# This joins the two search tables together. We can have any combination: +# one marc field could have many search fields (maybe you want one value +# to go to 'author' and 'corporate-author) and many marc fields could go +# to one search field (e.g. all the various author fields going into +# 'author'.) +# +# a note about the sort field: +# * if all the entries for a mapping are 'null', nothing special is done with that mapping. +# * if any of the entries are not null, then a __sort field is created in ES for this mapping. In this case: +# * any mapping with sort == false WILL NOT get copied into a __sort field +# * any mapping with sort == true or is null WILL get copied into a __sort field +# * any sorts on the field name will be applied to $fieldname.'__sort' instead. +# this means that we can have search for author that includes 1xx, 245$c, and 7xx, but the sort only applies to 1xx. + +dbh->do(q| + CREATE TABLE `search_marc_to_field` ( + search_marc_map_id int(11) NOT NULL, + search_field_id int(11) NOT NULL, + facet boolean DEFAULT FALSE COMMENT 'true if a facet field should be generated for this', + suggestible boolean DEFAULT FALSE COMMENT 'true if this field can be used to generate suggestions for browse', + sort boolean DEFAULT NULL COMMENT 'true/false creates special sort handling, null doesn''t', + PRIMARY KEY(search_marc_map_id, search_field_id), + FOREIGN KEY(search_marc_map_id) REFERENCES search_marc_map(id) ON DELETE CASCADE ON UPDATE CASCADE, + FOREIGN KEY(search_field_id) REFERENCES search_field(id) ON DELETE CASCADE ON UPDATE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci + |); + +my $mappings_yaml = C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml'; +my $indexes = LoadFile( $mappings_yaml ); + +while ( my ( $index_name, $fields ) = each %$indexes ) { + while ( my ( $field_name, $data ) = each %$fields ) { + my $field_type = $data->{type}; + my $field_label = $data->{label}; + my $mappings = $data->{mappings}; + my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' }); + for my $mapping ( @$mappings ) { + my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} }); + $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet}, suggestible => $mapping->{suggestible}, sort => $mapping->{sort} } ); + } + } +} + +print "Upgrade to $DBversion done (Bug 12478 - Elasticsearch support for Koha)\n"; + SetVersion($DBversion); + } + # DEVELOPER PROCESS, search for anything to execute in the db_update directory # SEE bug 13068 # if there is anything in the atomicupdate, read and execute it. -- 2.39.5