From 623f3a2c84fea04e4ad6203db49f6fdd6cfc62cd Mon Sep 17 00:00:00 2001 From: Jonathan Druart Date: Wed, 13 Jun 2012 11:48:22 +0200 Subject: [PATCH] Bug 8233 : SearchEngine: Add a Koha::SearchEngine module First draft introducing solr into Koha :-) List of files : $ tree t/searchengine/ t/searchengine |-- 000_conn | `-- conn.t |-- 001_search | `-- search_base.t |-- 002_index | `-- index_base.t |-- 003_query | `-- buildquery.t |-- 004_config | `-- load_config.t `-- indexes.yaml just do `prove -r t/searchengine/**/*.t` t/lib |-- Mocks | `-- Context.pm `-- Mocks.pm provide a mock to SearchEngine syspref (set_zebra and set_solr). $ tree Koha/SearchEngine Koha/SearchEngine |-- Config.pm |-- ConfigRole.pm |-- FacetsBuilder.pm |-- FacetsBuilderRole.pm |-- Index.pm |-- IndexRole.pm |-- QueryBuilder.pm |-- QueryBuilderRole.pm |-- Search.pm |-- SearchRole.pm |-- Solr | |-- Config.pm | |-- FacetsBuilder.pm | |-- Index.pm | |-- QueryBuilder.pm | `-- Search.pm |-- Solr.pm |-- Zebra | |-- QueryBuilder.pm | `-- Search.pm `-- Zebra.pm How to install and configure Solr ? See the wiki page: http://wiki.koha-community.org/wiki/SearchEngine_Layer_RFC http://bugs.koha-community.org/show_bug.cgi?id=8233 Signed-off-by: Chris Cormack --- Koha/SearchEngine.pm | 29 ++ Koha/SearchEngine/Config.pm | 12 + Koha/SearchEngine/ConfigRole.pm | 7 + Koha/SearchEngine/FacetsBuilder.pm | 12 + Koha/SearchEngine/FacetsBuilderRole.pm | 7 + Koha/SearchEngine/Index.pm | 11 + Koha/SearchEngine/IndexRole.pm | 6 + Koha/SearchEngine/QueryBuilder.pm | 12 + Koha/SearchEngine/QueryBuilderRole.pm | 7 + Koha/SearchEngine/Search.pm | 12 + Koha/SearchEngine/SearchRole.pm | 7 + Koha/SearchEngine/Solr.pm | 45 +++ Koha/SearchEngine/Solr/Config.pm | 115 ++++++ Koha/SearchEngine/Solr/FacetsBuilder.pm | 41 ++ Koha/SearchEngine/Solr/Index.pm | 100 +++++ Koha/SearchEngine/Solr/QueryBuilder.pm | 146 +++++++ Koha/SearchEngine/Solr/Search.pm | 108 +++++ Koha/SearchEngine/Zebra.pm | 14 + Koha/SearchEngine/Zebra/QueryBuilder.pm | 14 + Koha/SearchEngine/Zebra/Search.pm | 40 ++ admin/searchengine/solr/indexes.pl | 103 +++++ etc/searchengine/solr/config.yaml | 1 + etc/searchengine/solr/indexes.yaml | 45 +++ etc/searchengine/solr/indexes.yaml.bak | 33 ++ etc/solr/indexes.yaml | 33 ++ installer/data/mysql/updatedatabase.pl | 7 + .../prog/en/css/staff-global.css | 4 + .../plugins/jquery.textarea-expander.js | 95 +++++ .../prog/en/lib/jquery/plugins/tablednd.js | 382 ++++++++++++++++++ .../prog/en/modules/admin/admin-home.tt | 2 + .../en/modules/admin/preferences/admin.pref | 8 + .../admin/searchengine/solr/indexes.tt | 190 +++++++++ .../prog/en/includes/search/facets.inc | 56 +++ .../prog/en/includes/search/page-numbers.inc | 17 + .../prog/en/includes/search/resort_form.inc | 23 ++ .../prog/en/modules/search/results.tt | 108 +++++ kohaversion.pl | 2 +- misc/migration_tools/rebuild_solr.pl | 179 ++++++++ opac/opac-search.pl | 16 +- opac/search.pl | 172 ++++++++ t/lib/Mocks.pm | 23 ++ t/lib/Mocks/Context.pm | 13 + t/searchengine/000_conn/conn.t | 23 ++ t/searchengine/001_search/search_base.t | 12 + t/searchengine/002_index/index_base.t | 15 + t/searchengine/003_query/buildquery.t | 45 +++ t/searchengine/004_config/load_config.t | 54 +++ t/searchengine/indexes.yaml | 33 ++ 48 files changed, 2436 insertions(+), 3 deletions(-) create mode 100644 Koha/SearchEngine.pm create mode 100644 Koha/SearchEngine/Config.pm create mode 100644 Koha/SearchEngine/ConfigRole.pm create mode 100644 Koha/SearchEngine/FacetsBuilder.pm create mode 100644 Koha/SearchEngine/FacetsBuilderRole.pm create mode 100644 Koha/SearchEngine/Index.pm create mode 100644 Koha/SearchEngine/IndexRole.pm create mode 100644 Koha/SearchEngine/QueryBuilder.pm create mode 100644 Koha/SearchEngine/QueryBuilderRole.pm create mode 100644 Koha/SearchEngine/Search.pm create mode 100644 Koha/SearchEngine/SearchRole.pm create mode 100644 Koha/SearchEngine/Solr.pm create mode 100644 Koha/SearchEngine/Solr/Config.pm create mode 100644 Koha/SearchEngine/Solr/FacetsBuilder.pm create mode 100644 Koha/SearchEngine/Solr/Index.pm create mode 100644 Koha/SearchEngine/Solr/QueryBuilder.pm create mode 100644 Koha/SearchEngine/Solr/Search.pm create mode 100644 Koha/SearchEngine/Zebra.pm create mode 100644 Koha/SearchEngine/Zebra/QueryBuilder.pm create mode 100644 Koha/SearchEngine/Zebra/Search.pm create mode 100755 admin/searchengine/solr/indexes.pl create mode 100644 etc/searchengine/solr/config.yaml create mode 100644 etc/searchengine/solr/indexes.yaml create mode 100644 etc/searchengine/solr/indexes.yaml.bak create mode 100644 etc/solr/indexes.yaml create mode 100644 koha-tmpl/intranet-tmpl/prog/en/lib/jquery/plugins/jquery.textarea-expander.js create mode 100644 koha-tmpl/intranet-tmpl/prog/en/lib/jquery/plugins/tablednd.js create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/solr/indexes.tt create mode 100644 koha-tmpl/opac-tmpl/prog/en/includes/search/facets.inc create mode 100644 koha-tmpl/opac-tmpl/prog/en/includes/search/page-numbers.inc create mode 100644 koha-tmpl/opac-tmpl/prog/en/includes/search/resort_form.inc create mode 100644 koha-tmpl/opac-tmpl/prog/en/modules/search/results.tt create mode 100755 misc/migration_tools/rebuild_solr.pl create mode 100755 opac/search.pl create mode 100644 t/lib/Mocks.pm create mode 100644 t/lib/Mocks/Context.pm create mode 100644 t/searchengine/000_conn/conn.t create mode 100755 t/searchengine/001_search/search_base.t create mode 100755 t/searchengine/002_index/index_base.t create mode 100644 t/searchengine/003_query/buildquery.t create mode 100644 t/searchengine/004_config/load_config.t create mode 100644 t/searchengine/indexes.yaml diff --git a/Koha/SearchEngine.pm b/Koha/SearchEngine.pm new file mode 100644 index 0000000000..407156f229 --- /dev/null +++ b/Koha/SearchEngine.pm @@ -0,0 +1,29 @@ +package Koha::SearchEngine; + +use Moose; +use C4::Context; +use Koha::SearchEngine::Config; + +has 'name' => ( + is => 'ro', + default => sub { + C4::Context->preference('SearchEngine'); + } +); + +has config => ( + is => 'rw', + lazy => 1, + default => sub { + Koha::SearchEngine::Config->new; + } +# lazy => 1, +# builder => '_build_config', +); + +#sub _build_config { +# my ( $self ) = @_; +# Koha::SearchEngine::Config->new( $self->name ); +#); + +1; diff --git a/Koha/SearchEngine/Config.pm b/Koha/SearchEngine/Config.pm new file mode 100644 index 0000000000..089bac68ee --- /dev/null +++ b/Koha/SearchEngine/Config.pm @@ -0,0 +1,12 @@ +package Koha::SearchEngine::Config; + +use Moose; + +use Moose::Util qw( apply_all_roles ); + +sub BUILD { + my $self = shift; + my $syspref = C4::Context->preference("SearchEngine"); + apply_all_roles( $self, "Koha::SearchEngine::${syspref}::Config" ); +}; +1; diff --git a/Koha/SearchEngine/ConfigRole.pm b/Koha/SearchEngine/ConfigRole.pm new file mode 100644 index 0000000000..9ea224f24a --- /dev/null +++ b/Koha/SearchEngine/ConfigRole.pm @@ -0,0 +1,7 @@ +package Koha::SearchEngine::ConfigRole; + +use Moose::Role; + +requires 'indexes', 'index', 'ressource_types'; + +1; diff --git a/Koha/SearchEngine/FacetsBuilder.pm b/Koha/SearchEngine/FacetsBuilder.pm new file mode 100644 index 0000000000..2c974b1de4 --- /dev/null +++ b/Koha/SearchEngine/FacetsBuilder.pm @@ -0,0 +1,12 @@ +package Koha::SearchEngine::FacetsBuilder; + +use Moose; + +use Moose::Util qw( apply_all_roles ); + +sub BUILD { + my $self = shift; + my $syspref = C4::Context->preference("SearchEngine"); + apply_all_roles( $self, "Koha::SearchEngine::${syspref}::FacetsBuilder" ); +}; +1; diff --git a/Koha/SearchEngine/FacetsBuilderRole.pm b/Koha/SearchEngine/FacetsBuilderRole.pm new file mode 100644 index 0000000000..f0886e0344 --- /dev/null +++ b/Koha/SearchEngine/FacetsBuilderRole.pm @@ -0,0 +1,7 @@ +package Koha::SearchEngine::FacetsBuilderRole; + +use Moose::Role; + +requires 'build_facets'; + +1; diff --git a/Koha/SearchEngine/Index.pm b/Koha/SearchEngine/Index.pm new file mode 100644 index 0000000000..2807fbf98d --- /dev/null +++ b/Koha/SearchEngine/Index.pm @@ -0,0 +1,11 @@ +package Koha::SearchEngine::Index; +use Moose; + +use Moose::Util qw( apply_all_roles ); + +sub BUILD { + my $self = shift; + my $syspref = 'Solr'; + apply_all_roles( $self, "Koha::SearchEngine::${syspref}::Index" ); +}; +1; diff --git a/Koha/SearchEngine/IndexRole.pm b/Koha/SearchEngine/IndexRole.pm new file mode 100644 index 0000000000..a52f852065 --- /dev/null +++ b/Koha/SearchEngine/IndexRole.pm @@ -0,0 +1,6 @@ +package Koha::SearchEngine::IndexRole; +use Moose::Role; + +requires 'index_record'; + +1; diff --git a/Koha/SearchEngine/QueryBuilder.pm b/Koha/SearchEngine/QueryBuilder.pm new file mode 100644 index 0000000000..7c2f793de0 --- /dev/null +++ b/Koha/SearchEngine/QueryBuilder.pm @@ -0,0 +1,12 @@ +package Koha::SearchEngine::QueryBuilder; + +use Moose; + +use Moose::Util qw( apply_all_roles ); + +sub BUILD { + my $self = shift; + my $syspref = C4::Context->preference("SearchEngine"); + apply_all_roles( $self, "Koha::SearchEngine::${syspref}::QueryBuilder" ); +}; +1; diff --git a/Koha/SearchEngine/QueryBuilderRole.pm b/Koha/SearchEngine/QueryBuilderRole.pm new file mode 100644 index 0000000000..b768035922 --- /dev/null +++ b/Koha/SearchEngine/QueryBuilderRole.pm @@ -0,0 +1,7 @@ +package Koha::SearchEngine::QueryBuilderRole; + +use Moose::Role; + +requires 'build_query'; + +1; diff --git a/Koha/SearchEngine/Search.pm b/Koha/SearchEngine/Search.pm new file mode 100644 index 0000000000..8d9b469305 --- /dev/null +++ b/Koha/SearchEngine/Search.pm @@ -0,0 +1,12 @@ +package Koha::SearchEngine::Search; +use Moose; +use C4::Context; + +use Moose::Util qw( apply_all_roles ); + +sub BUILD { + my $self = shift; + my $syspref = C4::Context->preference("SearchEngine"); + apply_all_roles( $self, "Koha::SearchEngine::${syspref}::Search" ); +}; +1; diff --git a/Koha/SearchEngine/SearchRole.pm b/Koha/SearchEngine/SearchRole.pm new file mode 100644 index 0000000000..40a5c5bfd9 --- /dev/null +++ b/Koha/SearchEngine/SearchRole.pm @@ -0,0 +1,7 @@ +package Koha::SearchEngine::SearchRole; +use Moose::Role; + +requires 'search'; +requires 'dosmth'; + +1; diff --git a/Koha/SearchEngine/Solr.pm b/Koha/SearchEngine/Solr.pm new file mode 100644 index 0000000000..3d7041d992 --- /dev/null +++ b/Koha/SearchEngine/Solr.pm @@ -0,0 +1,45 @@ +package Koha::SearchEngine::Solr; +use Moose; +use Koha::SearchEngine::Config; + +extends 'Koha::SearchEngine', 'Data::SearchEngine::Solr'; + +has '+url' => ( + is => 'ro', + isa => 'Str', +# default => sub { +# C4::Context->preference('SolrAPI'); +# }, + lazy => 1, + builder => '_build_url', + required => 1 +); + +sub _build_url { + my ( $self ) = @_; + $self->config->SolrAPI; +} + +has '+options' => ( + is => 'ro', + isa => 'HashRef', + default => sub { + { + wt => 'json', + fl => '*,score', + fq => 'recordtype:biblio', + facets => 'true' + } + } + +); + +has indexes => ( + is => 'ro', + lazy => 1, + default => sub { +# my $dbh => ...; + }, +); + +1; diff --git a/Koha/SearchEngine/Solr/Config.pm b/Koha/SearchEngine/Solr/Config.pm new file mode 100644 index 0000000000..016208239b --- /dev/null +++ b/Koha/SearchEngine/Solr/Config.pm @@ -0,0 +1,115 @@ +package Koha::SearchEngine::Solr::Config; + +use Modern::Perl; +use Moose::Role; +use YAML; + +with 'Koha::SearchEngine::ConfigRole'; + +has index_config => ( + is => 'rw', + lazy => 1, + builder => '_load_index_config_file', +); + +has solr_config => ( + is => 'rw', + lazy => 1, + builder => '_load_solr_config_file', +); + +has index_filename => ( + is => 'rw', + lazy => 1, + default => C4::Context->config("installdir") . qq{/etc/searchengine/solr/indexes.yaml}, +); +has solr_filename => ( + is => 'rw', + lazy => 1, + default => C4::Context->config("installdir") . qq{/etc/searchengine/solr/config.yaml}, +); + +sub _load_index_config_file { + my ( $self, $filename ) = @_; + $self->index_filename( $filename ) if defined $filename; + die "The config index file (" . $self->index_filename . ") for Solr is not exist" if not -e $self->index_filename; + + return YAML::LoadFile($self->index_filename); +} + +sub _load_solr_config_file { + my ( $self ) = @_; + die "The solr config index file (" . $self->solr_filename . ") for Solr is not exist" if not -e $self->solr_filename; + + return YAML::LoadFile($self->solr_filename); +} + +sub set_config_filename { + my ( $self, $filename ) = @_; + $self->index_config( $self->_load_index_config_file( $filename ) ); +} + +sub SolrAPI { + my ( $self ) = @_; + return $self->solr_config->{SolrAPI}; +} +sub indexes { # FIXME Return index list if param not an hashref (string ressource_type) + my ( $self, $indexes ) = @_; + return $self->write( { indexes => $indexes } ) if defined $indexes; + return $self->index_config->{indexes}; +} + +sub index { + my ( $self, $code ) = @_; + my @index = map { ( $_->{code} eq $code ) ? $_ : () } @{$self->index_config->{indexes}}; + return $index[0]; +} + +sub ressource_types { + my ( $self ) = @_; + my $config = $self->index_config; + return $config->{ressource_types}; +} + +sub sortable_indexes { + my ( $self ) = @_; + my @sortable_indexes = map { $_->{sortable} ? $_ : () } @{ $self->index_config->{indexes} }; + return \@sortable_indexes; +} + +sub facetable_indexes { + my ( $self ) = @_; + my @facetable_indexes = map { $_->{facetable} ? $_ : () } @{ $self->index_config->{indexes} }; + return \@facetable_indexes; +} + +sub reload { + my ( $self ) = @_; + $self->index_config( $self->_load_index_config_file ); +} +sub write { + my ( $self, $values ) = @_; + my $r; + while ( my ( $k, $v ) = each %$values ) { + $r->{$k} = $v; + } + + if ( not grep /^ressource_type$/, keys %$values ) { + $r->{ressource_types} = $self->ressource_types; + } + + if ( not grep /^indexes$/, keys %$values ) { + $r->{indexes} = $self->indexes; + } + + eval { + YAML::DumpFile( $self->index_filename, $r ); + }; + if ( $@ ) { + die "Failed to dump the index config into the specified file ($@)"; + } + + $self->reload; +} + +1; diff --git a/Koha/SearchEngine/Solr/FacetsBuilder.pm b/Koha/SearchEngine/Solr/FacetsBuilder.pm new file mode 100644 index 0000000000..e8478a0032 --- /dev/null +++ b/Koha/SearchEngine/Solr/FacetsBuilder.pm @@ -0,0 +1,41 @@ +package Koha::SearchEngine::Solr::FacetsBuilder; + +use Modern::Perl; +use Moose::Role; + +with 'Koha::SearchEngine::FacetsBuilderRole'; + +sub build_facets { + my ( $self, $results, $facetable_indexes, $filters ) = @_; + my @facets_loop; + for my $index ( @$facetable_indexes ) { + my $index_name = $index->{type} . '_' . $index->{code}; + my $facets = $results->facets->{'str_' . $index->{code}}; + if ( @$facets > 1 ) { + my @values; + $index =~ m/^([^_]*)_(.*)$/; + for ( my $i = 0 ; $i < scalar(@$facets) ; $i++ ) { + my $value = $facets->[$i++]; + my $count = $facets->[$i]; + utf8::encode($value); + my $lib =$value; + push @values, { + 'lib' => $lib, + 'value' => $value, + 'count' => $count, + 'active' => ( $filters->{$index_name} and scalar( grep /"?\Q$value\E"?/, @{ $filters->{$index_name} } ) ) ? 1 : 0, + }; + } + + push @facets_loop, { + 'indexname' => $index_name, + 'label' => $index->{label}, + 'values' => \@values, + 'size' => scalar(@values), + }; + } + } + return @facets_loop; +} + +1; diff --git a/Koha/SearchEngine/Solr/Index.pm b/Koha/SearchEngine/Solr/Index.pm new file mode 100644 index 0000000000..fd7c3d78d0 --- /dev/null +++ b/Koha/SearchEngine/Solr/Index.pm @@ -0,0 +1,100 @@ +package Koha::SearchEngine::Solr::Index; +use Moose::Role; +with 'Koha::SearchEngine::IndexRole'; + +use Data::SearchEngine::Solr; +use Data::Dump qw(dump); +use List::MoreUtils qw(uniq); + +use Koha::SearchEngine::Solr; +use C4::AuthoritiesMarc; +use C4::Biblio; + +has searchengine => ( + is => 'rw', + isa => 'Koha::SearchEngine::Solr', + default => sub { Koha::SearchEngine::Solr->new }, + lazy => 1 +); + +sub optimize { + my ( $self ) = @_; + return $self->searchengine->_solr->optimize; +} + +sub index_record { + my ($self, $recordtype, $recordids) = @_; + + my $indexes = $self->searchengine->config->indexes; + my @records; + + my $recordids_str = ref($recordids) eq 'ARRAY' + ? join " ", @$recordids + : $recordids; + warn "IndexRecord called with $recordtype $recordids_str"; + + for my $id ( @$recordids ) { + my $record; + + $record = GetAuthority( $id ) if $recordtype eq "authority"; + $record = GetMarcBiblio( $id ) if $recordtype eq "biblio"; + + next unless ( $record ); + + my $index_values = { + recordid => $id, + recordtype => $recordtype, + }; + + warn "Indexing $recordtype $id"; + + for my $index ( @$indexes ) { + next if $index->{ressource_type} ne $recordtype; + my @values; + eval { + my $mappings = $index->{mappings}; + for my $tag_subf_code ( sort @$mappings ) { + my ( $f, $sf ) = split /\$/, $tag_subf_code; + for my $field ( $record->field( $f ) ) { + if ( $field->is_control_field ) { + push @values, $field->data; + } else { + my @sfvals = $sf eq '*' + ? map { $_->[1] } $field->subfields + : map { $_ } $field->subfield( $sf ); + + for ( @sfvals ) { + $_ = NormalizeDate( $_ ) if $index->{type} eq 'date'; + push @values, $_ if $_; + } + } + } + } + @values = uniq (@values); #Removes duplicates + + $index_values->{$index->{type}."_".$index->{code}} = \@values; + if ( $index->{sortable} ){ + $index_values->{"srt_" . $index->{type} . "_".$index->{code}} = $values[0]; + } + # Add index str for facets if it's not exist + if ( $index->{facetable} and @values > 0 and $index->{type} ne 'str' ) { + $index_values->{"str_" . $index->{code}} = $values[0]; + } + }; + if ( $@ ) { + chomp $@; + warn "Error during indexation : recordid $id, index $index->{code} ( $@ )"; + } + } + + my $solrrecord = Data::SearchEngine::Item->new( + id => "${recordtype}_$id", + score => 1, + values => $index_values, + ); + push @records, $solrrecord; + } + $self->searchengine->add( \@records ); +} + +1; diff --git a/Koha/SearchEngine/Solr/QueryBuilder.pm b/Koha/SearchEngine/Solr/QueryBuilder.pm new file mode 100644 index 0000000000..ba653b32b5 --- /dev/null +++ b/Koha/SearchEngine/Solr/QueryBuilder.pm @@ -0,0 +1,146 @@ +package Koha::SearchEngine::Solr::QueryBuilder; + +use Modern::Perl; +use Moose::Role; + +with 'Koha::SearchEngine::QueryBuilderRole'; + +sub build_advanced_query { + my ($class, $indexes, $operands, $operators) = @_; + + my $q = ''; + my $i = 0; + my $index_name; + + @$operands or return "*:*"; #push @$operands, "[* TO *]"; + + # Foreach operands + for my $kw (@$operands){ + $kw =~ s/(\w*\*)/\L$1\E/g; # Lower case on words with right truncation + $kw =~ s/(\s*\w*\?+\w*\s*)/\L$1\E/g; # Lower case on words contain wildcard ? + $kw =~ s/([^\\]):/$1\\:/g; # escape colons if not already escaped + # First element + if ($i == 0){ + if ( (my @x = eval {@$indexes} ) == 0 ){ + # There is no index, then query is in first operand + $q = @$operands[0]; + last; + } + + # Catch index name if it's not 'all_fields' + if ( @$indexes[$i] ne 'all_fields' ) { + $index_name = @$indexes[$i]; + }else{ + $index_name = ''; + } + + # Generate index:operand + $q .= BuildTokenString($index_name, $kw); + $i = $i + 1; + + next; + } + # And others + $index_name = @$indexes[$i] if @$indexes[$i]; + my $operator = defined @$operators[$i-1] ? @$operators[$i-1] : 'AND'; + given ( uc ( $operator ) ) { + when ('OR'){ + $q .= BuildTokenString($index_name, $kw, 'OR'); + } + when ('NOT'){ + $q .= BuildTokenString($index_name, $kw, 'NOT'); + } + default { + $q .= BuildTokenString($index_name, $kw, 'AND'); + } + } + $i = $i + 1; + } + + return $q; + +} + +sub BuildTokenString { + my ($index, $string, $operator) = @_; + my $r; + + if ($index ne 'all_fields' && $index ne ''){ + # Operand can contains an expression in brackets + if ( + $string =~ / / + and not ( $string =~ /^\(.*\)$/ ) + and not $string =~ /\[.*TO.*\]/ ) { + my @dqs; #double-quoted string + while ( $string =~ /"(?:[^"\\]++|\\.)*+"/g ) { + push @dqs, $&; + $string =~ s/\ *\Q$&\E\ *//; # Remove useless space before and after + } + + my @words = defined $string ? split ' ', $string : undef; + my $join = join qq{ AND } , map { + my $value = $_; + if ( $index =~ /^date_/ ) { + #$value = C4::Search::Engine::Solr::buildDateOperand( $value ); TODO + } + ( $value =~ /^"/ and $value ne '""' + and $index ne "emallfields" + and $index =~ /(txt_|ste_)/ ) + ? qq{em$index:$value} + : qq{$index:$value}; + } (@dqs, @words); + $r .= qq{($join)}; + } else { + if ( $index =~ /^date_/ ) { + #$string = C4::Search::Engine::Solr::buildDateOperand( $string ); TODO + } + + $r = "$index:$string"; + } + }else{ + $r = $string; + } + + return " $operator $r" if $operator; + return $r; +} + +sub build_query { + my ($class, $query) = @_; + + return "*:*" if not defined $query; + + # Particular *:* query + if ($query eq '*:*'){ + return $query; + } + + $query =~ s/(\w*\*)/\L$1\E/g; # Lower case on words with right truncation + $query =~ s/(\s*\w*\?+\w*\s*)/\L$1\E/g; # Lower case on words contain wildcard ? + + my @quotes; # Process colons in quotes + while ( $query =~ /'(?:[^'\\]++|\\.)*+'/g ) { + push @quotes, $&; + } + + for ( @quotes ) { + my $replacement = $_; + $replacement =~ s/[^\\]\K:/\\:/g; + $query =~ s/$_/$replacement/; + } + + $query =~ s/ : / \\: /g; # escape colons if " : " + + my $new_query = $query;#C4::Search::Query::splitToken($query); TODO + + $new_query =~ s/all_fields://g; + + # Upper case for operators + $new_query =~ s/ or / OR /g; + $new_query =~ s/ and / AND /g; + $new_query =~ s/ not / NOT /g; + + return $new_query; +} + +1; diff --git a/Koha/SearchEngine/Solr/Search.pm b/Koha/SearchEngine/Solr/Search.pm new file mode 100644 index 0000000000..5c626ba8c8 --- /dev/null +++ b/Koha/SearchEngine/Solr/Search.pm @@ -0,0 +1,108 @@ +package Koha::SearchEngine::Solr::Search; +use Moose::Role; +with 'Koha::SearchEngine::SearchRole'; + +use Data::Dump qw(dump); +use XML::Simple; + +use Data::SearchEngine::Solr; +use Data::Pagination; +use Data::SearchEngine::Query; +use Koha::SearchEngine::Solr; + +has searchengine => ( + is => 'rw', + isa => 'Koha::SearchEngine::Solr', + default => sub { Koha::SearchEngine::Solr->new }, + lazy => 1 +); + +sub search { + my ( $self, $q, $filters, $params ) = @_; + + $q ||= '*:*'; + $filters ||= {}; + my $page = defined $params->{page} ? $params->{page} : 1; + my $count = defined $params->{count} ? $params->{count} : 999999999; + my $sort = defined $params->{sort} ? $params->{sort} : 'score desc'; + my $facets = defined $params->{facets} ? $params->{facets} : 0; + + # Construct fl from $params->{fl} + # If "recordid" or "id" not exist, we push them + my $fl = join ",", + defined $params->{fl} + ? ( + @{$params->{fl}}, + grep ( /^recordid$/, @{$params->{fl}} ) ? () : "recordid", + grep ( /^id$/, @{$params->{fl}} ) ? () : "id" + ) + : ( "recordid", "id" ); + + my $recordtype = ref($filters->{recordtype}) eq 'ARRAY' + ? $filters->{recordtype}[0] + : $filters->{recordtype} + if defined $filters && defined $filters->{recordtype}; + + if ( $facets ) { + $self->searchengine->options->{"facet"} = 'true'; + $self->searchengine->options->{"facet.mincount"} = 1; + $self->searchengine->options->{"facet.limit"} = 10; # TODO create a new systempreference C4::Context->preference("numFacetsDisplay") + my @facetable_indexes = map { 'str_' . $_->{code} } @{$self->searchengine->config->facetable_indexes}; + $self->searchengine->options->{"facet.field"} = \@facetable_indexes; + } + $self->searchengine->options->{sort} = $sort; + $self->searchengine->options->{fl} = $fl; + + # Construct filters + $self->searchengine->options->{fq} = [ + map { + my $idx = $_; + ref($filters->{$idx}) eq 'ARRAY' + ? + '(' + . join( ' AND ', + map { + my $filter_str = $_; + utf8::decode($filter_str); + my $quotes_existed = ( $filter_str =~ m/^".*"$/ ); + $filter_str =~ s/^"(.*)"$/$1/; #remove quote around value if exist + $filter_str =~ s/[^\\]\K"/\\"/g; + $filter_str = qq{"$filter_str"} # Add quote around value if not exist + if not $filter_str =~ /^".*"$/ + and $quotes_existed; + qq{$idx:$filter_str}; + } @{ $filters->{$idx} } ) + . ')' + : "$idx:$filters->{$idx}"; + } keys %$filters + ]; + + my $sq = Data::SearchEngine::Query->new( + page => $page, + count => $count, + query => $q, + ); + + # Get results + my $results = eval { $self->searchengine->search( $sq ) }; + + # Get error if exists + if ( $@ ) { + my $err = $@; + + $err =~ s#^[^\n]*\n##; # Delete first line + if ( $err =~ "400 URL must be absolute" ) { + $err = "Your system preference 'SolrAPI' is not set correctly"; + } + elsif ( not $err =~ 'Connection refused' ) { + my $document = XMLin( $err ); + $err = "$$document{body}{h2} : $$document{body}{pre}"; + } + $results->{error} = $err; + } + return $results; +} + +sub dosmth {'bou' } + +1; diff --git a/Koha/SearchEngine/Zebra.pm b/Koha/SearchEngine/Zebra.pm new file mode 100644 index 0000000000..9071071842 --- /dev/null +++ b/Koha/SearchEngine/Zebra.pm @@ -0,0 +1,14 @@ +package Koha::SearchEngine::Zebra; +use Moose; + +extends 'Data::SearchEngine::Zebra'; + +# the configuration file is retrieved from KOHA_CONF by default, provide it from there² +has '+conf_file' => ( + is => 'ro', + isa => 'Str', + default => $ENV{KOHA_CONF}, + required => 1 +); + +1; diff --git a/Koha/SearchEngine/Zebra/QueryBuilder.pm b/Koha/SearchEngine/Zebra/QueryBuilder.pm new file mode 100644 index 0000000000..50792a0494 --- /dev/null +++ b/Koha/SearchEngine/Zebra/QueryBuilder.pm @@ -0,0 +1,14 @@ +package Koha::SearchEngine::Zebra::QueryBuilder; + +use Modern::Perl; +use Moose::Role; +use C4::Search; + +with 'Koha::SearchEngine::QueryBuilderRole'; + +sub build_query { + shift; + C4::Search::buildQuery @_; +} + +1; diff --git a/Koha/SearchEngine/Zebra/Search.pm b/Koha/SearchEngine/Zebra/Search.pm new file mode 100644 index 0000000000..0531ab92dd --- /dev/null +++ b/Koha/SearchEngine/Zebra/Search.pm @@ -0,0 +1,40 @@ +package Koha::SearchEngine::Zebra::Search; +use Moose::Role; +with 'Koha::SearchEngine::SearchRole'; + +use Data::SearchEngine::Zebra; +use Data::SearchEngine::Query; +use Koha::SearchEngine::Zebra; +use Data::Dump qw(dump); + +has searchengine => ( + is => 'rw', + isa => 'Koha::SearchEngine::Zebra', + default => sub { Koha::SearchEngine::Zebra->new }, + lazy => 1 +); + +sub search { + my ($self,$query_string) = @_; + + my $query = Data::SearchEngine::Query->new( + count => 10, + page => 1, + query => $query_string, + ); + + warn "search for $query_string"; + + my $results = $self->searchengine->search($query); + + foreach my $item (@{ $results->items }) { + my $title = $item->get_value('ste_title'); + #utf8::encode($title); + print "$title\n"; + warn dump $title; + } +} + +sub dosmth {'bou' } + +1; diff --git a/admin/searchengine/solr/indexes.pl b/admin/searchengine/solr/indexes.pl new file mode 100755 index 0000000000..39a5276984 --- /dev/null +++ b/admin/searchengine/solr/indexes.pl @@ -0,0 +1,103 @@ +#!/usr/bin/perl + +# Copyright 2012 BibLibre SARL +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use CGI; +use C4::Koha; +use C4::Output; +use C4::Auth; +use Koha::SearchEngine; + +my $input = new CGI; +my ( $template, $borrowernumber, $cookie ) = get_template_and_user( + { + template_name => 'admin/searchengine/solr/indexes.tt', + query => $input, + type => 'intranet', +# authnotrequired => 0, +# flagsrequired => { reserveforothers => "place_holds" }, #TODO + } +); + +my $ressource_type = $input->param('ressource_type') || 'biblio'; +my $se = Koha::SearchEngine->new; +my $se_config = $se->config; + +my $indexes; +if ( $input->param('op') and $input->param('op') eq 'edit' ) { + my @code = $input->param('code'); + my @label = $input->param('label'); + my @type = $input->param('type'); + my @sortable = $input->param('sortable'); + my @facetable = $input->param('facetable'); + my @mandatory = $input->param('mandatory'); + my @ressource_type = $input->param('ressource_type'); + my @mappings = $input->param('mappings'); + my @indexes; + my @errors; + for ( 0 .. @code-1 ) { + my $icode = $code[$_]; + my @current_mappings = split /\r\n/, $mappings[$_]; + if ( not @current_mappings ) { + @current_mappings = split /\n/, $mappings[$_]; + } + if ( not @current_mappings ) { + push @errors, { type => 'no_mapping', value => $icode}; + } + + push @indexes, { + code => $icode, + label => $label[$_], + type => $type[$_], + sortable => scalar(grep(/^$icode$/, @sortable)), + facetable => scalar(grep(/^$icode$/, @facetable)), + mandatory => $mandatory[$_] eq '1' ? '1' : '0', + ressource_type => $ressource_type[$_], + mappings => \@current_mappings, + }; + for my $m ( @current_mappings ) { + push @errors, {type => 'malformed_mapping', value => $m} + if not $m =~ /^\d(\d|\*|\.){2}\$.$/; + } + } + $indexes = \@indexes if @errors; + $template->param( errors => \@errors ); + + $se_config->indexes(\@indexes) if not @errors; +} + +my $ressource_types = $se_config->ressource_types; +$indexes //= $se_config->indexes; + +my $indexloop; +for my $rt ( @$ressource_types ) { + my @indexes = map { + $_->{ressource_type} eq $rt ? $_ : (); + } @$indexes; + push @$indexloop, { + ressource_type => $rt, + indexes => \@indexes, + } +} + +$template->param( + indexloop => $indexloop, +); + +output_html_with_http_headers $input, $cookie, $template->output; diff --git a/etc/searchengine/solr/config.yaml b/etc/searchengine/solr/config.yaml new file mode 100644 index 0000000000..26c219b71f --- /dev/null +++ b/etc/searchengine/solr/config.yaml @@ -0,0 +1 @@ +SolrAPI: 'http://localhost:8983/solr/solr' diff --git a/etc/searchengine/solr/indexes.yaml b/etc/searchengine/solr/indexes.yaml new file mode 100644 index 0000000000..ca598f8ea4 --- /dev/null +++ b/etc/searchengine/solr/indexes.yaml @@ -0,0 +1,45 @@ +--- +indexes: + - code: title + facetable: 1 + label: Title + mandatory: 1 + mappings: + - 200$a + - 210$a + - 4..$t + ressource_type: biblio + sortable: 1 + type: ste + - code: author + facetable: 1 + label: Author + mandatory: 0 + mappings: + - 700$* + - 710$* + ressource_type: biblio + sortable: 1 + type: str + - code: subject + facetable: 0 + label: Subject + mandatory: 0 + mappings: + - 600$a + - 601$a + ressource_type: biblio + sortable: 0 + type: str + - code: biblionumber + facetable: 0 + label: Biblionumber + mandatory: 1 + mappings: + - 001$@ + ressource_type: biblio + sortable: 0 + type: int +ressource_types: + - biblio + - authority diff --git a/etc/searchengine/solr/indexes.yaml.bak b/etc/searchengine/solr/indexes.yaml.bak new file mode 100644 index 0000000000..b369160e99 --- /dev/null +++ b/etc/searchengine/solr/indexes.yaml.bak @@ -0,0 +1,33 @@ +ressource_types: + - biblio + - authority + +indexes: + - code: title + label: Title + type: ste + ressource_type: biblio + sortable: 1 + mandatory: 1 + mappings: + - 200$a + - 210$a + - 4..$t + - code: author + label: Author + type: str + ressource_type: biblio + sortable: 1 + mandatory: 0 + mappings: + - 700$* + - 710$* + - code: subject + label: Subject + type: str + ressource_type: biblio + sortable: 0 + mandatory: 0 + mappings: + - 600$a + - 601$a diff --git a/etc/solr/indexes.yaml b/etc/solr/indexes.yaml new file mode 100644 index 0000000000..b369160e99 --- /dev/null +++ b/etc/solr/indexes.yaml @@ -0,0 +1,33 @@ +ressource_types: + - biblio + - authority + +indexes: + - code: title + label: Title + type: ste + ressource_type: biblio + sortable: 1 + mandatory: 1 + mappings: + - 200$a + - 210$a + - 4..$t + - code: author + label: Author + type: str + ressource_type: biblio + sortable: 1 + mandatory: 0 + mappings: + - 700$* + - 710$* + - code: subject + label: Subject + type: str + ressource_type: biblio + sortable: 0 + mandatory: 0 + mappings: + - 600$a + - 601$a diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl index 97a722eb56..15f7969fc4 100755 --- a/installer/data/mysql/updatedatabase.pl +++ b/installer/data/mysql/updatedatabase.pl @@ -5459,6 +5459,13 @@ if (C4::Context->preference("Version") < TransformToNum($DBversion)) { SetVersion($DBversion); } +$DBversion = "3.07.00.023"; +if ( C4::Context->preference("Version") < TransformToNum($DBversion) ) { + $dbh->do("INSERT IGNORE INTO systempreferences (variable,value,options,explanation,type) VALUES('SearchEngine','Zebra','Solr|Zebra','Search Engine','Choice')"); + print "Upgrade to $DBversion done (Add system preference SearchEngine )\n"; + SetVersion($DBversion); +} + =head1 FUNCTIONS =head2 TableExists($table) diff --git a/koha-tmpl/intranet-tmpl/prog/en/css/staff-global.css b/koha-tmpl/intranet-tmpl/prog/en/css/staff-global.css index 8179f15946..0cfcdf763b 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/css/staff-global.css +++ b/koha-tmpl/intranet-tmpl/prog/en/css/staff-global.css @@ -285,6 +285,10 @@ td { vertical-align : top; } +table.indexes td { + vertical-align : middle; +} + td.borderless { border-collapse : separate; border : 0 none; diff --git a/koha-tmpl/intranet-tmpl/prog/en/lib/jquery/plugins/jquery.textarea-expander.js b/koha-tmpl/intranet-tmpl/prog/en/lib/jquery/plugins/jquery.textarea-expander.js new file mode 100644 index 0000000000..06e35d897a --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/lib/jquery/plugins/jquery.textarea-expander.js @@ -0,0 +1,95 @@ +/** + * TextAreaExpander plugin for jQuery + * v1.0 + * Expands or contracts a textarea height depending on the + * quatity of content entered by the user in the box. + * + * By Craig Buckler, Optimalworks.net + * + * As featured on SitePoint.com: + * http://www.sitepoint.com/blogs/2009/07/29/build-auto-expanding-textarea-1/ + * + * Please use as you wish at your own risk. + */ + +/** + * Usage: + * + * From JavaScript, use: + * $().TextAreaExpander(, ); + * where: + * is the DOM node selector, e.g. "textarea" + * is the minimum textarea height in pixels (optional) + * is the maximum textarea height in pixels (optional) + * + * Alternatively, in you HTML: + * Assign a class of "expand" to any + * + * Or assign a class of "expandMIN-MAX" to set the + * The textarea will use an appropriate height between 50 and 200 pixels. + */ + +(function($) { + + // jQuery plugin definition + $.fn.TextAreaExpander = function(minHeight, maxHeight) { + + var hCheck = !($.browser.msie || $.browser.opera); + + // resize a textarea + function ResizeTextarea(e) { + + // event or initialize element? + e = e.target || e; + + // find content length and box width + var vlen = e.value.length, ewidth = e.offsetWidth; + if (vlen != e.valLength || ewidth != e.boxWidth) { + + if (hCheck && (vlen < e.valLength || ewidth != e.boxWidth)) e.style.height = "0px"; + var h = Math.max(e.expandMin, Math.min(e.scrollHeight, e.expandMax)); + + e.style.overflow = (e.scrollHeight > h ? "auto" : "hidden"); + e.style.height = h + "px"; + + e.valLength = vlen; + e.boxWidth = ewidth; + } + + return true; + }; + + // initialize + this.each(function() { + + // is a textarea? + if (this.nodeName.toLowerCase() != "textarea") return; + + // set height restrictions + var p = this.className.match(/expand(\d+)\-*(\d+)*/i); + this.expandMin = minHeight || (p ? parseInt('0'+p[1], 10) : 0); + this.expandMax = maxHeight || (p ? parseInt('0'+p[2], 10) : 99999); + + // initial resize + ResizeTextarea(this); + + // zero vertical padding and add events + if (!this.Initialized) { + this.Initialized = true; + $(this).css("padding-top", 0).css("padding-bottom", 0); + $(this).bind("keyup", ResizeTextarea).bind("focus", ResizeTextarea); + } + }); + + return this; + }; + +})(jQuery); + + +// initialize all expanding textareas +jQuery(document).ready(function() { + jQuery("textarea[class*=expand]").TextAreaExpander(); +}); diff --git a/koha-tmpl/intranet-tmpl/prog/en/lib/jquery/plugins/tablednd.js b/koha-tmpl/intranet-tmpl/prog/en/lib/jquery/plugins/tablednd.js new file mode 100644 index 0000000000..6786ca3501 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/lib/jquery/plugins/tablednd.js @@ -0,0 +1,382 @@ +/** + * TableDnD plug-in for JQuery, allows you to drag and drop table rows + * You can set up various options to control how the system will work + * Copyright (c) Denis Howlett + * Licensed like jQuery, see http://docs.jquery.com/License. + * + * Configuration options: + * + * onDragStyle + * This is the style that is assigned to the row during drag. There are limitations to the styles that can be + * associated with a row (such as you can't assign a border--well you can, but it won't be + * displayed). (So instead consider using onDragClass.) The CSS style to apply is specified as + * a map (as used in the jQuery css(...) function). + * onDropStyle + * This is the style that is assigned to the row when it is dropped. As for onDragStyle, there are limitations + * to what you can do. Also this replaces the original style, so again consider using onDragClass which + * is simply added and then removed on drop. + * onDragClass + * This class is added for the duration of the drag and then removed when the row is dropped. It is more + * flexible than using onDragStyle since it can be inherited by the row cells and other content. The default + * is class is tDnD_whileDrag. So to use the default, simply customise this CSS class in your + * stylesheet. + * onDrop + * Pass a function that will be called when the row is dropped. The function takes 2 parameters: the table + * and the row that was dropped. You can work out the new order of the rows by using + * table.rows. + * onDragStart + * Pass a function that will be called when the user starts dragging. The function takes 2 parameters: the + * table and the row which the user has started to drag. + * onAllowDrop + * Pass a function that will be called as a row is over another row. If the function returns true, allow + * dropping on that row, otherwise not. The function takes 2 parameters: the dragged row and the row under + * the cursor. It returns a boolean: true allows the drop, false doesn't allow it. + * scrollAmount + * This is the number of pixels to scroll if the user moves the mouse cursor to the top or bottom of the + * window. The page should automatically scroll up or down as appropriate (tested in IE6, IE7, Safari, FF2, + * FF3 beta + * dragHandle + * This is the name of a class that you assign to one or more cells in each row that is draggable. If you + * specify this class, then you are responsible for setting cursor: move in the CSS and only these cells + * will have the drag behaviour. If you do not specify a dragHandle, then you get the old behaviour where + * the whole row is draggable. + * + * Other ways to control behaviour: + * + * Add class="nodrop" to any rows for which you don't want to allow dropping, and class="nodrag" to any rows + * that you don't want to be draggable. + * + * Inside the onDrop method you can also call $.tableDnD.serialize() this returns a string of the form + * []=&[]= so that you can send this back to the server. The table must have + * an ID as must all the rows. + * + * Other methods: + * + * $("...").tableDnDUpdate() + * Will update all the matching tables, that is it will reapply the mousedown method to the rows (or handle cells). + * This is useful if you have updated the table rows using Ajax and you want to make the table draggable again. + * The table maintains the original configuration (so you don't have to specify it again). + * + * $("...").tableDnDSerialize() + * Will serialize and return the serialized string as above, but for each of the matching tables--so it can be + * called from anywhere and isn't dependent on the currentTable being set up correctly before calling + * + * Known problems: + * - Auto-scoll has some problems with IE7 (it scrolls even when it shouldn't), work-around: set scrollAmount to 0 + * + * Version 0.2: 2008-02-20 First public version + * Version 0.3: 2008-02-07 Added onDragStart option + * Made the scroll amount configurable (default is 5 as before) + * Version 0.4: 2008-03-15 Changed the noDrag/noDrop attributes to nodrag/nodrop classes + * Added onAllowDrop to control dropping + * Fixed a bug which meant that you couldn't set the scroll amount in both directions + * Added serialize method + * Version 0.5: 2008-05-16 Changed so that if you specify a dragHandle class it doesn't make the whole row + * draggable + * Improved the serialize method to use a default (and settable) regular expression. + * Added tableDnDupate() and tableDnDSerialize() to be called when you are outside the table + */ +jQuery.tableDnD = { + /** Keep hold of the current table being dragged */ + currentTable : null, + /** Keep hold of the current drag object if any */ + dragObject: null, + /** The current mouse offset */ + mouseOffset: null, + /** Remember the old value of Y so that we don't do too much processing */ + oldY: 0, + + /** Actually build the structure */ + build: function(options) { + // Set up the defaults if any + + this.each(function() { + // This is bound to each matching table, set up the defaults and override with user options + this.tableDnDConfig = jQuery.extend({ + onDragStyle: null, + onDropStyle: null, + // Add in the default class for whileDragging + onDragClass: "tDnD_whileDrag", + onDrop: null, + onDragStart: null, + scrollAmount: 5, + serializeRegexp: /[^\-]*$/, // The regular expression to use to trim row IDs + serializeParamName: null, // If you want to specify another parameter name instead of the table ID + dragHandle: null // If you give the name of a class here, then only Cells with this class will be draggable + }, options || {}); + // Now make the rows draggable + jQuery.tableDnD.makeDraggable(this); + }); + + // Now we need to capture the mouse up and mouse move event + // We can use bind so that we don't interfere with other event handlers + jQuery(document) + .bind('mousemove', jQuery.tableDnD.mousemove) + .bind('mouseup', jQuery.tableDnD.mouseup); + + // Don't break the chain + return this; + }, + + /** This function makes all the rows on the table draggable apart from those marked as "NoDrag" */ + makeDraggable: function(table) { + var config = table.tableDnDConfig; + if (table.tableDnDConfig.dragHandle) { + // We only need to add the event to the specified cells + var cells = jQuery("td."+table.tableDnDConfig.dragHandle, table); + cells.each(function() { + // The cell is bound to "this" + jQuery(this).mousedown(function(ev) { + jQuery.tableDnD.dragObject = this.parentNode; + jQuery.tableDnD.currentTable = table; + jQuery.tableDnD.mouseOffset = jQuery.tableDnD.getMouseOffset(this, ev); + if (config.onDragStart) { + // Call the onDrop method if there is one + config.onDragStart(table, this); + } + return false; + }); + }) + } else { + // For backwards compatibility, we add the event to the whole row + var rows = jQuery("tr", table); // get all the rows as a wrapped set + rows.each(function() { + // Iterate through each row, the row is bound to "this" + var row = jQuery(this); + if (! row.hasClass("nodrag")) { + row.mousedown(function(ev) { + if (ev.target.tagName == "TD") { + jQuery.tableDnD.dragObject = this; + jQuery.tableDnD.currentTable = table; + jQuery.tableDnD.mouseOffset = jQuery.tableDnD.getMouseOffset(this, ev); + if (config.onDragStart) { + // Call the onDrop method if there is one + config.onDragStart(table, this); + } + return false; + } + }).css("cursor", "move"); // Store the tableDnD object + } + }); + } + }, + + updateTables: function() { + this.each(function() { + // this is now bound to each matching table + if (this.tableDnDConfig) { + jQuery.tableDnD.makeDraggable(this); + } + }) + }, + + /** Get the mouse coordinates from the event (allowing for browser differences) */ + mouseCoords: function(ev){ + if(ev.pageX || ev.pageY){ + return {x:ev.pageX, y:ev.pageY}; + } + return { + x:ev.clientX + document.body.scrollLeft - document.body.clientLeft, + y:ev.clientY + document.body.scrollTop - document.body.clientTop + }; + }, + + /** Given a target element and a mouse event, get the mouse offset from that element. + To do this we need the element's position and the mouse position */ + getMouseOffset: function(target, ev) { + ev = ev || window.event; + + var docPos = this.getPosition(target); + var mousePos = this.mouseCoords(ev); + return {x:mousePos.x - docPos.x, y:mousePos.y - docPos.y}; + }, + + /** Get the position of an element by going up the DOM tree and adding up all the offsets */ + getPosition: function(e){ + var left = 0; + var top = 0; + /** Safari fix -- thanks to Luis Chato for this! */ + if (e.offsetHeight == 0) { + /** Safari 2 doesn't correctly grab the offsetTop of a table row + this is detailed here: + http://jacob.peargrove.com/blog/2006/technical/table-row-offsettop-bug-in-safari/ + the solution is likewise noted there, grab the offset of a table cell in the row - the firstChild. + note that firefox will return a text node as a first child, so designing a more thorough + solution may need to take that into account, for now this seems to work in firefox, safari, ie */ + e = e.firstChild; // a table cell + } + + while (e.offsetParent){ + left += e.offsetLeft; + top += e.offsetTop; + e = e.offsetParent; + } + + left += e.offsetLeft; + top += e.offsetTop; + + return {x:left, y:top}; + }, + + mousemove: function(ev) { + if (jQuery.tableDnD.dragObject == null) { + return; + } + + var dragObj = jQuery(jQuery.tableDnD.dragObject); + var config = jQuery.tableDnD.currentTable.tableDnDConfig; + var mousePos = jQuery.tableDnD.mouseCoords(ev); + var y = mousePos.y - jQuery.tableDnD.mouseOffset.y; + //auto scroll the window + var yOffset = window.pageYOffset; + if (document.all) { + // Windows version + //yOffset=document.body.scrollTop; + if (typeof document.compatMode != 'undefined' && + document.compatMode != 'BackCompat') { + yOffset = document.documentElement.scrollTop; + } + else if (typeof document.body != 'undefined') { + yOffset=document.body.scrollTop; + } + + } + + if (mousePos.y-yOffset < config.scrollAmount) { + window.scrollBy(0, -config.scrollAmount); + } else { + var windowHeight = window.innerHeight ? window.innerHeight + : document.documentElement.clientHeight ? document.documentElement.clientHeight : document.body.clientHeight; + if (windowHeight-(mousePos.y-yOffset) < config.scrollAmount) { + window.scrollBy(0, config.scrollAmount); + } + } + + + if (y != jQuery.tableDnD.oldY) { + // work out if we're going up or down... + var movingDown = y > jQuery.tableDnD.oldY; + // update the old value + jQuery.tableDnD.oldY = y; + // update the style to show we're dragging + if (config.onDragClass) { + dragObj.addClass(config.onDragClass); + } else { + dragObj.css(config.onDragStyle); + } + // If we're over a row then move the dragged row to there so that the user sees the + // effect dynamically + var currentRow = jQuery.tableDnD.findDropTargetRow(dragObj, y); + if (currentRow) { + // TODO worry about what happens when there are multiple TBODIES + if (movingDown && jQuery.tableDnD.dragObject != currentRow) { + jQuery.tableDnD.dragObject.parentNode.insertBefore(jQuery.tableDnD.dragObject, currentRow.nextSibling); + } else if (! movingDown && jQuery.tableDnD.dragObject != currentRow) { + jQuery.tableDnD.dragObject.parentNode.insertBefore(jQuery.tableDnD.dragObject, currentRow); + } + } + } + + return false; + }, + + /** We're only worried about the y position really, because we can only move rows up and down */ + findDropTargetRow: function(draggedRow, y) { + var rows = jQuery.tableDnD.currentTable.rows; + for (var i=0; i rowY - rowHeight) && (y < (rowY + rowHeight))) { + // that's the row we're over + // If it's the same as the current row, ignore it + if (row == draggedRow) {return null;} + var config = jQuery.tableDnD.currentTable.tableDnDConfig; + if (config.onAllowDrop) { + if (config.onAllowDrop(draggedRow, row)) { + return row; + } else { + return null; + } + } else { + // If a row has nodrop class, then don't allow dropping (inspired by John Tarr and Famic) + var nodrop = jQuery(row).hasClass("nodrop"); + if (! nodrop) { + return row; + } else { + return null; + } + } + return row; + } + } + return null; + }, + + mouseup: function(e) { + if (jQuery.tableDnD.currentTable && jQuery.tableDnD.dragObject) { + var droppedRow = jQuery.tableDnD.dragObject; + var config = jQuery.tableDnD.currentTable.tableDnDConfig; + // If we have a dragObject, then we need to release it, + // The row will already have been moved to the right place so we just reset stuff + if (config.onDragClass) { + jQuery(droppedRow).removeClass(config.onDragClass); + } else { + jQuery(droppedRow).css(config.onDropStyle); + } + jQuery.tableDnD.dragObject = null; + if (config.onDrop) { + // Call the onDrop method if there is one + config.onDrop(jQuery.tableDnD.currentTable, droppedRow); + } + jQuery.tableDnD.currentTable = null; // let go of the table too + } + }, + + serialize: function() { + if (jQuery.tableDnD.currentTable) { + return jQuery.tableDnD.serializeTable(jQuery.tableDnD.currentTable); + } else { + return "Error: No Table id set, you need to set an id on your table and every row"; + } + }, + + serializeTable: function(table) { + var result = ""; + var tableId = table.id; + var rows = table.rows; + for (var i=0; i 0) result += "&"; + var rowId = rows[i].id; + if (rowId && rowId && table.tableDnDConfig && table.tableDnDConfig.serializeRegexp) { + rowId = rowId.match(table.tableDnDConfig.serializeRegexp)[0]; + } + + result += tableId + '[]=' + rowId; + } + return result; + }, + + serializeTables: function() { + var result = ""; + this.each(function() { + // this is now bound to each matching table + result += jQuery.tableDnD.serializeTable(this); + }); + return result; + } + +} + +jQuery.fn.extend( + { + tableDnD : jQuery.tableDnD.build, + tableDnDUpdate : jQuery.tableDnD.updateTables, + tableDnDSerialize: jQuery.tableDnD.serializeTables + } +); diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt index ef46ede474..6e19132983 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt @@ -76,6 +76,8 @@
Manage rules for automatically matching MARC records during record imports.
OAI sets configuration
Manage OAI Sets
+
Search engine configuration
+
Manage indexes, facets, and their mappings to MARC fields and subfields.

Acquisition parameters

diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref index 08eb954fb6..4e907e102b 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref @@ -104,3 +104,11 @@ Administration: Common Name: the Common Name emailAddress: the emailAddress - field for SSL client certificate authentication + Search Engine: + - + - pref: SearchEngine + default: Zebra + choices: + Solr: Solr + Zebra: Zebra + - is the search engine used. diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/solr/indexes.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/solr/indexes.tt new file mode 100644 index 0000000000..2ccbe5a376 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/solr/indexes.tt @@ -0,0 +1,190 @@ +[% INCLUDE 'doc-head-open.inc' %] +Koha › Administration › Solr config +[% INCLUDE 'doc-head-close.inc' %] + + + + + +[% INCLUDE 'header.inc' %] +[% INCLUDE 'cat-search.inc' %] + + + +
+ +
+
+
+

Search engine configuration

+
+ Warning: Any modification in these configurations will need a total reindexation to be fully taken into account ! +
+ [% IF ( errors ) %] +
+ Errors occurred, Modifications does not apply. Please check following values: +
    + [% FOREACH e IN errors %] +
  • + [% IF ( e.type == "malformed_mapping" ) %] + The value "[% e.value %]" is not supported for mappings + [% ELSIF ( e.type == "no_mapping" ) %] + There is no mapping for the index [% e.value %] + [% END %] +
    • + [% END %] +
+
+ [% END %] + +
+ + [% FOREACH rt IN indexloop %] +

[% rt.ressource_type %]

+ [% IF ( rt.ressource_type == 'authority' ) %] + This part is not yet implemented + [% END %] + + + + + + + + + + + + + + [% FOREACH index IN rt.indexes %] + + + + + + + + + + [% END %] + + + + + + + + + + + + +
CodeLabelTypeSortableFacetableMapping
+ [% IF ( index.mandatory ) %] + + + [% ELSE %] + + [% END %] + + + + [% IF ( index.mandatory ) %] + + [% END %] + + + + + + + + [% UNLESS ( index.mandatory ) %]Delete[% END %]
+ + + + + + + + Add
+ [% END %] +

+
+
+ +
+
+[% INCLUDE 'admin-menu.inc' %] +
+
+[% INCLUDE 'intranet-bottom.inc' %] diff --git a/koha-tmpl/opac-tmpl/prog/en/includes/search/facets.inc b/koha-tmpl/opac-tmpl/prog/en/includes/search/facets.inc new file mode 100644 index 0000000000..93c8b0f591 --- /dev/null +++ b/koha-tmpl/opac-tmpl/prog/en/includes/search/facets.inc @@ -0,0 +1,56 @@ +[% IF facets_loop %] +
+

Refine your search

+
    + [% FOR facets IN facets_loop %] +
  • + [% facets.label %] +
      + [% FOR value IN facets.values %] +
    • + [% IF ( value.active ) %] + [% value.lib %] ([% value.count %]) + [% SET url = "/cgi-bin/koha/opac-search.pl?" %] + [% SET first = 1 %] + [% FOR p IN follower_params %] + [% IF p.var != 'filters' %] + [% UNLESS first %] + [% SET url = url _ '&' %] + [% END %] + [% SET first = 0 %] + [% SET url = url _ p.var _ '=' _ p.val %] + [% END %] + [% END %] + [% FOR f IN filters %] + [% UNLESS f.var == facets.indexname && f.val == value.value %] + [% SET url = url _ '&filters=' _ f.var _ ':"' _ f.val _ '"' %] + [% END %] + [% END %] + [x] + [% ELSE %] + [% SET url = "/cgi-bin/koha/opac-search.pl?" %] + [% SET first = 1 %] + [% FOR p IN follower_params %] + [% IF p.var != 'filters' %] + [% UNLESS first %] + [% SET url = url _ '&' %] + [% END %] + [% SET first = 0 %] + [% SET url = url _ p.var _ '=' _ p.val %] + [% END %] + [% END %] + [% FOR f IN filters %] + [% SET url = url _ '&filters=' _ f.var _ ':"' _ f.val _ '"' %] + [% END %] + [% SET url = url _ '&filters=' _ facets.indexname _ ':"' _ value.value _ '"' %] + + [% value.lib %] ([% value.count %]) + [% END %] +
      • + [% END %] +
    +
    • + [% END %] +
+
+[% END %] diff --git a/koha-tmpl/opac-tmpl/prog/en/includes/search/page-numbers.inc b/koha-tmpl/opac-tmpl/prog/en/includes/search/page-numbers.inc new file mode 100644 index 0000000000..2de511fa55 --- /dev/null +++ b/koha-tmpl/opac-tmpl/prog/en/includes/search/page-numbers.inc @@ -0,0 +1,17 @@ +[% IF ( PAGE_NUMBERS ) %] +
+ [% IF ( previous_page ) %] + << Previous + [% END %] + [% FOREACH PAGE_NUMBER IN PAGE_NUMBERS %] + [% IF ( PAGE_NUMBER.current ) %] + [% PAGE_NUMBER.page %] + [% ELSE %] + [% PAGE_NUMBER.page %] + [% END %] + [% END %] + [% IF ( next_page ) %] + Next >> + [% END %] +
+[% END %] diff --git a/koha-tmpl/opac-tmpl/prog/en/includes/search/resort_form.inc b/koha-tmpl/opac-tmpl/prog/en/includes/search/resort_form.inc new file mode 100644 index 0000000000..88c9f66c06 --- /dev/null +++ b/koha-tmpl/opac-tmpl/prog/en/includes/search/resort_form.inc @@ -0,0 +1,23 @@ +[% IF sort_by == "score asc" %] + +[% ELSE %] + +[% END %] +[% IF sort_by == "score desc" %] + +[% ELSE %] + +[% END %] + +[% FOREACH ind IN sortable_indexes %] + [% IF sort_by == "$ind.code asc" %] + + [% ELSE %] + + [% END %] + [% IF sort_by == "$ind.code desc" %] + + [% ELSE %] + + [% END %] +[% END %] diff --git a/koha-tmpl/opac-tmpl/prog/en/modules/search/results.tt b/koha-tmpl/opac-tmpl/prog/en/modules/search/results.tt new file mode 100644 index 0000000000..ab2ea12589 --- /dev/null +++ b/koha-tmpl/opac-tmpl/prog/en/modules/search/results.tt @@ -0,0 +1,108 @@ +[% INCLUDE 'doc-head-open.inc' %] +[% IF ( LibraryNameTitle ) %][% LibraryNameTitle %][% ELSE %]Koha online[% END %] catalog › +[% IF ( searchdesc ) %] + Results of search [% IF ( query_desc ) %]for '[% query_desc | html%]'[% END %][% IF ( limit_desc ) %] with limit(s): '[% limit_desc | html %]'[% END %] +[% ELSE %] + You did not specify any search criteria. +[% END %] +[% INCLUDE 'doc-head-close.inc' %] + + + + + + + + + + +
+
+ +[% INCLUDE 'masthead.inc' %] + + +
+
+
+ +[% IF ( query_error ) %] +
+

Error:

+ [% query_error %] +
+[% END %] + + +[% IF ( total ) %] +
+ We have [% total %] results for your search +
+
+
+ + + + + + + + + + [% FOREACH SEARCH_RESULT IN SEARCH_RESULTS %] + + + + + [% END %] + +
+
+ + [% FOREACH param IN follower_params %] + [% UNLESS param.var == 'sort_by' %] + + [% END %] + [% END %] + + + + +
+
+ + + +
+
+ + + [% SEARCH_RESULT.title |html %] + by [% SEARCH_RESULT.author %] +
+ +
+ [% INCLUDE 'search/page-numbers.inc' %] +[% END %] +
+
+
+ +
+
+ [% INCLUDE 'search/facets.inc' %] +
+
+ +
+ +[% INCLUDE 'opac-bottom.inc' %] diff --git a/kohaversion.pl b/kohaversion.pl index c64ad3d11f..73bedf8637 100644 --- a/kohaversion.pl +++ b/kohaversion.pl @@ -16,7 +16,7 @@ the kohaversion is divided in 4 parts : use strict; sub kohaversion { - our $VERSION = '3.09.00.022'; + our $VERSION = '3.09.00.023'; # version needs to be set this way # so that it can be picked up by Makefile.PL # during install diff --git a/misc/migration_tools/rebuild_solr.pl b/misc/migration_tools/rebuild_solr.pl new file mode 100755 index 0000000000..b72b10e9ba --- /dev/null +++ b/misc/migration_tools/rebuild_solr.pl @@ -0,0 +1,179 @@ +#!/usr/bin/perl + +# Copyright 2012 BibLibre SARL +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Data::Dumper; +use Getopt::Long; +use LWP::Simple; +use XML::Simple; + +use C4::Context; +use C4::Search; +use Koha::SearchEngine::Index; + +$|=1; # flushes output + +if ( C4::Context->preference("SearchEngine") ne 'Solr' ) { + warn "System preference 'SearchEngine' not equal 'Solr'."; + warn "We can not indexing"; + exit(1); +} + +#Setup + +my ( $reset, $number, $recordtype, $biblionumbers, $optimize, $info, $want_help ); +GetOptions( + 'r' => \$reset, + 'n:s' => \$number, + 't:s' => \$recordtype, + 'w:s' => \$biblionumbers, + 'o' => \$optimize, + 'i' => \$info, + 'h|help' => \$want_help, +); +my $debug = C4::Context->preference("DebugLevel"); +my $index_service = Koha::SearchEngine::Index->new; +my $solrurl = $index_service->searchengine->config->SolrAPI; + +my $ping = &ping_command; +if (!defined $ping) { + print "SolrAPI = $solrurl\n"; + print "Solr is Down\n"; + exit(1); +} + +#Script + +&print_help if ($want_help); +&print_info if ($info); +if ($reset){ + if ($recordtype){ + &reset_index("recordtype:".$recordtype); + } else { + &reset_index("*:*"); + } +} + +if (defined $biblionumbers){ + if (not defined $recordtype) { print "You must specify a recordtype\n"; exit 1;} + &index_biblio($_) for split ',', $biblionumbers; +} elsif (defined $recordtype) { + &index_data; + &optimise_index; +} + +if ($optimize) { + &optimise_index; +} + +#Functions + +sub index_biblio { + my ($biblionumber) = @_; + $index_service->index_record($recordtype, [ $biblionumber ] ); +} + +sub index_data { + my $dbh = C4::Context->dbh; + $dbh->do('SET NAMES UTF8;'); + + my $query; + if ( $recordtype eq 'biblio' ) { + $query = "SELECT biblionumber FROM biblio ORDER BY biblionumber"; + } elsif ( $recordtype eq 'authority' ) { + $query = "SELECT authid FROM auth_header ORDER BY authid"; + } + $query .= " LIMIT $number" if $number; + + my $sth = $dbh->prepare( $query ); + $sth->execute(); + + $index_service->index_record($recordtype, [ map { $_->[0] } @{ $sth->fetchall_arrayref } ] ); + + $sth->finish; +} + +sub reset_index { + &reset_command; + &commit_command; + $debug eq '2' && &count_all_docs eq 0 && warn "Index cleaned!" +} + +sub commit_command { + my $commiturl = "/update?stream.body=%3Ccommit/%3E"; + my $urlreturns = get $solrurl.$commiturl; +} + +sub ping_command { + my $pingurl = "/admin/ping"; + my $urlreturns = get $solrurl.$pingurl; +} + +sub reset_command { + my ($query) = @_; + my $deleteurl = "/update?stream.body=%3Cdelete%3E%3Cquery%3E".$query."%3C/query%3E%3C/delete%3E"; + my $urlreturns = get $solrurl.$deleteurl; +} + +sub optimise_index { + $index_service->optimize; +} + +sub count_all_docs { + my $queryurl = "/select/?q=*:*"; + my $urlreturns = get $solrurl.$queryurl; + my $xmlsimple = XML::Simple->new(); + my $data = $xmlsimple->XMLin($urlreturns); + return $data->{result}->{numFound}; +} + +sub print_info { + my $count = &count_all_docs; + print <<_USAGE_; +SolrAPI = $solrurl +How many indexed documents = $count; +_USAGE_ +} + +sub print_help { + print <<_USAGE_; +$0: reindex biblios and/or authorities in Solr. + +Use this batch job to reindex all biblio or authority records in your Koha database. This job is useful only if you are using Solr search engine. + +Parameters: + -t biblio index bibliographic records + + -t authority index authority records + + -r clear Solr index before adding records to index - use this option carefully! + + -n 100 index 100 first records + + -n "100,2" index 2 records after 100th (101 and 102) + + -w 101 index biblio with biblionumber equals 101 + + -o launch optimize command at the end of indexing + + -i gives solr install information: SolrAPI value and count all documents indexed + + --help or -h show this message. +_USAGE_ +} diff --git a/opac/opac-search.pl b/opac/opac-search.pl index 404b48d4fc..8028c0a595 100755 --- a/opac/opac-search.pl +++ b/opac/opac-search.pl @@ -21,14 +21,26 @@ # Script to perform searching # Mostly copied from search.pl, see POD there -use strict; # always use -use warnings; +use Modern::Perl; ## STEP 1. Load things that are used in both search page and # results page and decide which template to load, operations # to perform, etc. ## load Koha modules use C4::Context; + +my $searchengine = C4::Context->preference("SearchEngine"); +given ( $searchengine ) { + when ( /^Solr$/ ) { + warn "We use Solr"; + require 'opac/search.pl'; + exit; + } + when ( /^Zebra$/ ) { + + } +} + use C4::Output; use C4::Auth qw(:DEFAULT get_session); use C4::Languages qw(getAllLanguages); diff --git a/opac/search.pl b/opac/search.pl new file mode 100755 index 0000000000..1a8aa415e5 --- /dev/null +++ b/opac/search.pl @@ -0,0 +1,172 @@ +#!/usr/bin/perl + +# Copyright 2012 BibLibre +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use C4::Context; +use CGI; +use C4::Auth; +use C4::Koha; +use C4::Output; +use Koha::SearchEngine::Search; +use Koha::SearchEngine::QueryBuilder; +use Koha::SearchEngine::FacetsBuilder; + +my $cgi = new CGI; + +my $template_name; +my $template_type = "basic"; +if ( $cgi->param("idx") or $cgi->param("q") ) { + $template_name = 'search/results.tt'; +} else { + $template_name = 'search/advsearch.tt'; + $template_type = 'advsearch'; +} + +# load the template +my ( $template, $borrowernumber, $cookie ) = get_template_and_user( + { template_name => $template_name, + query => $cgi, + type => "opac", + authnotrequired => 1, + } +); + +my $format = $cgi->param("format") || 'html'; + + + + +# load the Type stuff +my $itemtypes = GetItemTypes; + +my $page = $cgi->param("page") || 1; +my $count = $cgi->param('count') || C4::Context->preference('OPACnumSearchResults') || 20; +$count = 5; +my $q = $cgi->param("q"); +my $builder = Koha::SearchEngine::QueryBuilder->new; +$q = $builder->build_query( $q ); +my $search_service = Koha::SearchEngine::Search->new; + +# load the sorting stuff +my $sort_by = $cgi->param('sort_by') + || C4::Context->preference('OPACdefaultSortField') . ' ' . C4::Context->preference('OPACdefaultSortOrder'); + +my $search_engine_config = Koha::SearchEngine->new->config; +my $sortable_indexes = $search_engine_config->sortable_indexes; +my ( $sort_indexname, $sort_order ); +( $sort_indexname, $sort_order ) = ($1, $2) if ( $sort_by =~ m/^(.*) (asc|desc)$/ ); +my $sort_by_indexname = eval { + [ + map { + $_->{code} eq $sort_indexname + ? 'srt_' . $_->{type} . '_' . $_->{code} . ' ' . $sort_order + : () + } @$sortable_indexes + ]->[0] +}; + +# This array is used to build facets GUI +my %filters; +my @tplfilters; +for my $filter ( $cgi->param('filters') ) { + next if not $filter; + my ($k, @v) = $filter =~ /(?: \\. | [^:] )+/xg; + my $v = join ':', @v; + push @{$filters{$k}}, $v; + $v =~ s/^"(.*)"$/$1/; # Remove quotes around + push @tplfilters, { + 'var' => $k, + 'val' => $v, + }; +} +push @{$filters{recordtype}}, 'biblio'; + +my $results = $search_service->search( + $q, + \%filters, + { + page => $page, + count => $count, + sort => $sort_by_indexname, + facets => 1, + fl => ["ste_title", "str_author", 'int_biblionumber'], + } +); + +if ($results->{error}){ + $template->param(query_error => $results->{error}); + output_with_http_headers $cgi, $cookie, $template->output, 'html'; + exit; +} + + +# populate results with records +my @r; +for my $searchresult ( @{ $results->items } ) { + my $biblionumber = $searchresult->{values}->{recordid}; + + my $nr; + while ( my ($k, $v) = each %{$searchresult->{values}} ) { + my $nk = $k; + $nk =~ s/^[^_]*_(.*)$/$1/; + $nr->{$nk} = ref $v ? shift @{$v} : $v; + } + push( @r, $nr ); +} + +# build facets +my $facets_builder = Koha::SearchEngine::FacetsBuilder->new; +my @facets_loop = $facets_builder->build_facets( $results, $search_engine_config->facetable_indexes, \%filters ); + +my $total = $results->{pager}->{total_entries}; +my $pager = Data::Pagination->new( + $total, + $count, + 20, + $page, +); + +# params we want to pass for all actions require another query (pagination, sort, facets) +my @follower_params = map { { + var => 'filters', + val => $_->{var}.':"'.$_->{val}.'"' +} } @tplfilters; +push @follower_params, { var => 'q', val => $q}; +push @follower_params, { var => 'sort_by', val => $sort_by}; + +# Pager template params +$template->param( + previous_page => $pager->{'prev_page'}, + next_page => $pager->{'next_page'}, + PAGE_NUMBERS => [ map { { page => $_, current => $_ == $page } } @{ $pager->{'numbers_of_set'} } ], + current_page => $page, + follower_params => \@follower_params, + total => $total, + SEARCH_RESULTS => \@r, + query => $q, + count => $count, + sort_by => $sort_by, + sortable_indexes => $sortable_indexes, + facets_loop => \@facets_loop, + filters => \@tplfilters, +); + +my $content_type = ( $format eq 'rss' or $format eq 'atom' ) ? $format : 'html'; +output_with_http_headers $cgi, $cookie, $template->output, $content_type; diff --git a/t/lib/Mocks.pm b/t/lib/Mocks.pm new file mode 100644 index 0000000000..9e2d831a70 --- /dev/null +++ b/t/lib/Mocks.pm @@ -0,0 +1,23 @@ +package t::lib::Mocks; + +use Modern::Perl; +use Test::MockModule; +use t::lib::Mocks::Context; + +our (@ISA,@EXPORT,@EXPORT_OK); +BEGIN { + require Exporter; + @ISA = qw(Exporter); + push @EXPORT, qw( + &set_solr + &set_zebra + ); +} + +my $context = new Test::MockModule('C4::Context'); +sub set_solr { + $context->mock('preference', sub { &t::lib::Mocks::Context::MockPreference( @_, "Solr", $context ) }); +} +sub set_zebra { + $context->mock('preference', sub { &t::lib::Mocks::Context::MockPreference( @_, "Zebra", $context ) }); +} diff --git a/t/lib/Mocks/Context.pm b/t/lib/Mocks/Context.pm new file mode 100644 index 0000000000..185209a549 --- /dev/null +++ b/t/lib/Mocks/Context.pm @@ -0,0 +1,13 @@ +package t::lib::Mocks::Context; +use t::lib::Mocks::Context; +use C4::Context; + +sub MockPreference { + my ( $self, $syspref, $value, $mock_object ) = @_; + return $value if $syspref eq 'SearchEngine'; + $mock_object->unmock("preference"); + my $r = C4::Context->preference($syspref); + $mock_object->mock('preference', sub { &t::lib::Mocks::Context::MockPreference( @_, $value, $mock_object ) }); + return $r; +} +1; diff --git a/t/searchengine/000_conn/conn.t b/t/searchengine/000_conn/conn.t new file mode 100644 index 0000000000..8510269ca0 --- /dev/null +++ b/t/searchengine/000_conn/conn.t @@ -0,0 +1,23 @@ +use Modern::Perl; +use Test::More; +use Koha::SearchEngine::Solr; +use Koha::SearchEngine::Zebra; +use Koha::SearchEngine::Search; +use t::lib::Mocks; + +my $se_index = Koha::SearchEngine::Solr->new; +ok($se_index->isa('Data::SearchEngine::Solr'), 'Solr is a Solr data searchengine'); + +$se_index = Koha::SearchEngine::Zebra->new; +ok($se_index->isa('Data::SearchEngine::Zebra'), 'Zebra search engine'); + +set_solr(); +$se_index = Koha::SearchEngine::Search->new; +ok($se_index->searchengine->isa('Data::SearchEngine::Solr'), 'Solr search engine'); + +set_zebra(); +$se_index = Koha::SearchEngine::Search->new; +ok($se_index->searchengine->isa('Data::SearchEngine::Zebra'), 'Zebra search engine'); + + +done_testing; diff --git a/t/searchengine/001_search/search_base.t b/t/searchengine/001_search/search_base.t new file mode 100755 index 0000000000..8351b1798d --- /dev/null +++ b/t/searchengine/001_search/search_base.t @@ -0,0 +1,12 @@ +use Test::More; + +use t::lib::Mocks; + +set_solr; +use Koha::SearchEngine::Search; +my $search_service = Koha::SearchEngine::Search->new; +isnt (scalar $search_service->search("fort"), 0, 'test search') ; + +#$search_service->search($query_service->build_query(@,@,@)); + +done_testing; diff --git a/t/searchengine/002_index/index_base.t b/t/searchengine/002_index/index_base.t new file mode 100755 index 0000000000..7f86bb8baa --- /dev/null +++ b/t/searchengine/002_index/index_base.t @@ -0,0 +1,15 @@ +use Test::More; +use FindBin qw($Bin); + +use t::lib::::Mocks; + +use Koha::SearchEngine::Index; + +set_solr; +my $index_service = Koha::SearchEngine::Index->new; +system( qq{/bin/cp $FindBin::Bin/../indexes.yaml /tmp/indexes.yaml} ); +$index_service->searchengine->config->set_config_filename( "/tmp/indexes.yaml" ); +is ($index_service->index_record("biblio", [2]), 1, 'test search') ; +is ($index_service->optimize, 1, 'test search') ; + +done_testing; diff --git a/t/searchengine/003_query/buildquery.t b/t/searchengine/003_query/buildquery.t new file mode 100644 index 0000000000..008e1e1013 --- /dev/null +++ b/t/searchengine/003_query/buildquery.t @@ -0,0 +1,45 @@ +use Modern::Perl; +use Test::More; +use C4::Context; + +use Koha::SearchEngine; +use Koha::SearchEngine::QueryBuilder; +use t::lib::Mocks; + +my $titleindex = "title"; +my $authorindex = "author"; +#my $eanindex = "str_ean"; +#my $pubdateindex = "date_pubdate"; + +my ($operands, $indexes, $operators); + + +# === Solr part === +@$operands = ('cup', 'rowling'); +@$indexes = ('ti', 'au'); +@$operators = ('AND'); + +set_solr; +my $qs = Koha::SearchEngine::QueryBuilder->new; + +my $se = Koha::SearchEngine->new; +is( $se->name, "Solr", "Test searchengine name eq Solr" ); + +my $gotsolr = $qs->build_advanced_query($indexes, $operands, $operators); +my $expectedsolr = "ti:cup AND au:rowling"; +is($gotsolr, $expectedsolr, "Test build_query Solr"); + + +# === Zebra part === +set_zebra; +$se = Koha::SearchEngine->new; +is( $se->name, "Zebra", "Test searchengine name eq Zebra" ); +$qs = Koha::SearchEngine::QueryBuilder->new; +my ( $builterror, $builtquery, $simple_query, $query_cgi, $query_desc, $limit, $limit_cgi, $limit_desc, $stopwords_removed, $query_type ) = $qs->build_query($operators, $operands, $indexes); +my $gotzebra = $builtquery; +my $expectedzebra = qq{ti,wrdl= cup AND au,wrdl= rowling }; +is($gotzebra, $expectedzebra, "Test Zebra indexes in 'normal' search"); +# @and @attr 1=title @attr 4=6 "des mots de mon titre" @attr 1=author Jean en PQF + + +done_testing; diff --git a/t/searchengine/004_config/load_config.t b/t/searchengine/004_config/load_config.t new file mode 100644 index 0000000000..4e3665d2aa --- /dev/null +++ b/t/searchengine/004_config/load_config.t @@ -0,0 +1,54 @@ +use Modern::Perl; +use Test::More; +use FindBin qw($Bin); + +use C4::Context; +use Koha::SearchEngine; +use t::lib::Mocks; + +set_solr; + +my $se = Koha::SearchEngine->new; +is( $se->name, "Solr", "Test searchengine name eq Solr" ); + +my $config = $se->config; +$config->set_config_filename( "$FindBin::Bin/../indexes.yaml" ); +my $ressource_types = $config->ressource_types; +is ( grep ( /^biblio$/, @$ressource_types ), 1, "Ressource type biblio must to be defined" ); +is ( grep ( /^authority$/, @$ressource_types ), 1, "Ressource type authority must to be defined" ); + +my $indexes = $config->indexes; +is ( scalar(@$indexes), 3, "There are 3 indexes configured" ); + +my $index1 = @$indexes[0]; +is ( $index1->{code}, 'title', "My index first have code=title"); +is ( $index1->{label}, 'Title', "My index first have label=Title"); +is ( $index1->{type}, 'ste', "My index first have type=ste"); +is ( $index1->{ressource_type}, 'biblio', "My index first have ressource_type=biblio"); +is ( $index1->{sortable}, '1', "My index first have sortable=1"); +is ( $index1->{mandatory}, '1', "My index first have mandatory=1"); +eq_array ( $index1->{mappings}, ["200\$a", "4..\$t"], "My first index have mappings=[200\$a,4..\$t]"); + +system( qq{/bin/cp $FindBin::Bin/../indexes.yaml /tmp/indexes.yaml} ); +$config->set_config_filename( "/tmp/indexes.yaml" ); +$indexes = $config->indexes; +my $new_index = { + code => 'isbn', + label => 'ISBN', + type => 'str', + ressource_type => 'biblio', + sortable => 0, + mandatory => 0 +}; +push @$indexes, $new_index; +$config->indexes( $indexes ); + +$indexes = $config->indexes; + +my $isbn_index = $config->index( 'isbn' ); +is( $isbn_index->{code}, 'isbn', 'Index isbn has been written' ); + +my $sortable_indexes = $config->sortable_indexes; +is ( @$sortable_indexes, 2, "There are 2 sortable indexes" ); + +done_testing; diff --git a/t/searchengine/indexes.yaml b/t/searchengine/indexes.yaml new file mode 100644 index 0000000000..b369160e99 --- /dev/null +++ b/t/searchengine/indexes.yaml @@ -0,0 +1,33 @@ +ressource_types: + - biblio + - authority + +indexes: + - code: title + label: Title + type: ste + ressource_type: biblio + sortable: 1 + mandatory: 1 + mappings: + - 200$a + - 210$a + - 4..$t + - code: author + label: Author + type: str + ressource_type: biblio + sortable: 1 + mandatory: 0 + mappings: + - 700$* + - 710$* + - code: subject + label: Subject + type: str + ressource_type: biblio + sortable: 0 + mandatory: 0 + mappings: + - 600$a + - 601$a -- 2.39.5