2 # Script to perform searching
3 # For documentation try 'perldoc /path/to/search'
7 # Copyright 2006 LibLime
9 # This file is part of Koha
11 # Koha is free software; you can redistribute it and/or modify it under the
12 # terms of the GNU General Public License as published by the Free Software
13 # Foundation; either version 2 of the License, or (at your option) any later
16 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
17 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
18 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
20 # You should have received a copy of the GNU General Public License along with
21 # Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
22 # Suite 330, Boston, MA 02111-1307 USA
26 search - a search script for finding records in a Koha system (Version 3)
30 This script utilizes a new search API for Koha 3. It is designed to be
31 simple to use and configure, yet capable of performing feats like stemming,
32 field weighting, relevance ranking, support for multiple query language
33 formats (CCL, CQL, PQF), full support for the bib1 attribute set, extended
34 attribute sets defined in Zebra profiles, access to the full range of Z39.50
35 and SRU query options, federated searches on Z39.50/SRU targets, etc.
37 The API as represented in this script is mostly sound, even if the individual
38 functions in Search.pm and Koha.pm need to be cleaned up. Of course, you are
41 I will attempt to describe what is happening at each part of this script.
42 -- Joshua Ferraro <jmf AT liblime DOT com>
46 This script performs two functions:
50 =item 1. interacts with Koha to retrieve and display the results of a search
52 =item 2. loads the advanced search page
56 These two functions share many of the same variables and modules, so the first
57 task is to load what they have in common and determine which template to use.
58 Once determined, proceed to only load the variables and procedures necessary
61 =head2 LOADING ADVANCED SEARCH PAGE
63 This is fairly straightforward, and I won't go into detail ;-)
65 =head2 PERFORMING A SEARCH
67 If we're performing a search, this script performs three primary
72 =item 1. builds query strings (yes, plural)
74 =item 2. perform the search and return the results array
76 =item 3. build the HTML for output to the template
80 There are several additional secondary functions performed that I will
83 =head3 1. Building Query Strings
85 There are several types of queries needed in the process of search and retrieve:
89 =item 1 $query - the fully-built query passed to zebra
91 This is the most complex query that needs to be built. The original design goal
92 was to use a custom CCL2PQF query parser to translate an incoming CCL query into
93 a multi-leaf query to pass to Zebra. It needs to be multi-leaf to allow field
94 weighting, koha-specific relevance ranking, and stemming. When I have a chance
95 I'll try to flesh out this section to better explain.
97 This query incorporates query profiles that aren't compatible with most non-Zebra
98 Z39.50 targets to acomplish the field weighting and relevance ranking.
100 =item 2 $simple_query - a simple query that doesn't contain the field weighting,
101 stemming, etc., suitable to pass off to other search targets
103 This query is just the user's query expressed in CCL CQL, or PQF for passing to a
104 non-zebra Z39.50 target (one that doesn't support the extended profile that Zebra does).
106 =item 3 $query_cgi - passed to the template / saved for future refinements of
109 This is a simple string that completely expresses the query as a CGI string that
110 can be used for future refinements of the query or as a part of a history feature.
112 =item 4 $query_desc - Human search description - what the user sees in search
115 This is a simple string that is human readable. It will contain '=', ',', etc.
119 =head3 2. Perform the Search
121 This section takes the query strings and performs searches on the named servers,
122 including the Koha Zebra server, stores the results in a deeply nested object,
123 builds 'faceted results', and returns these objects.
127 The final major section of this script takes the objects collected thusfar and
128 builds the HTML for output to the template and user.
130 =head3 Additional Notes
136 use strict; # always use
138 ## STEP 1. Load things that are used in both search page and
139 # results page and decide which template to load, operations
147 use C4::Languages qw(getAllLanguages);
149 use POSIX qw(ceil floor);
150 use C4::Branch; # GetBranches
152 # create a new CGI object
153 # FIXME: no_undef_params needs to be tested
154 use CGI qw('-no_undef_params');
157 my ($template,$borrowernumber,$cookie);
159 # decide which template to use
162 my @params = $cgi->param("limit");
163 if ((@params>=1) || ($cgi->param("q")) || ($cgi->param('multibranchlimit')) || ($cgi->param('limit-yr')) ) {
164 $template_name = 'catalogue/results.tmpl';
167 $template_name = 'catalogue/advsearch.tmpl';
168 $template_type = 'advsearch';
171 ($template, $borrowernumber, $cookie) = get_template_and_user({
172 template_name => $template_name,
175 authnotrequired => 0,
176 flagsrequired => { catalogue => 1 },
179 if (C4::Context->preference("marcflavour") eq "UNIMARC" ) {
180 $template->param('UNIMARC' => 1);
184 # Deprecated, but preserved because it's interesting :-)
185 # The same thing can be accomplished with mod_rewrite in
189 #my $uri = $cgi->url(-base => 1);
190 #my $relative_url = $cgi->url(-relative=>1);
191 #$uri.="/".$relative_url."?";
193 #my @cgi_params_list = $cgi->param();
194 #my $url_params = $cgi->Vars;
196 #for my $each_param_set (@cgi_params_list) {
197 # $uri.= join "", map "\&$each_param_set=".$_, split("\0",$url_params->{$each_param_set}) if $url_params->{$each_param_set};
199 #warn "New URI:$uri";
200 # Only re-write a URI if there are params or if it already hasn't been re-written
201 #unless (($cgi->param('r')) || (!$cgi->param()) ) {
202 # print $cgi->redirect( -uri=>$uri."&r=1",
203 # -cookie => $cookie);
208 my $branches = GetBranches();
211 for my $branch_hash (sort { $branches->{$a}->{branchname} cmp $branches->{$b}->{branchname} } keys %$branches) {
212 push @branch_loop, {value => "$branch_hash" , branchname => $branches->{$branch_hash}->{'branchname'}, };
215 my $categories = GetBranchCategories(undef,'searchdomain');
217 $template->param(branchloop => \@branch_loop, searchdomainloop => $categories);
219 # load the Type stuff
220 # load the Type stuff
221 my $itemtypes = GetItemTypes;
222 # the index parameter is different for item-level itemtypes
223 my $itype_or_itemtype = (C4::Context->preference("item-level_itypes"))?'itype':'itemtype';
227 my $advanced_search_types = C4::Context->preference("AdvancedSearchTypes");
229 if (!$advanced_search_types or $advanced_search_types eq 'itemtypes') { foreach my $thisitemtype ( sort {$itemtypes->{$a}->{'description'} cmp $itemtypes->{$b}->{'description'} } keys %$itemtypes ) {
230 my %row =( number=>$cnt++,
231 ccl => $itype_or_itemtype,
232 code => $thisitemtype,
233 selected => $selected,
234 description => $itemtypes->{$thisitemtype}->{'description'},
236 imageurl=> getitemtypeimagelocation( 'intranet', $itemtypes->{$thisitemtype}->{'imageurl'} ),
238 $selected = 0 if ($selected) ;
239 push @itemtypesloop, \%row;
241 $template->param(itemtypeloop => \@itemtypesloop);
243 my $advsearchtypes = GetAuthorisedValues($advanced_search_types);
244 for my $thisitemtype (@$advsearchtypes) {
247 ccl => $advanced_search_types,
248 code => $thisitemtype->{authorised_value},
249 selected => $selected,
250 description => $thisitemtype->{'lib'},
252 imageurl=> getitemtypeimagelocation( 'intranet', $thisitemtype->{'imageurl'} ),
254 push @itemtypesloop, \%row;
256 $template->param(itemtypeloop => \@itemtypesloop);
259 # The following should only be loaded if we're bringing up the advanced search template
260 if ( $template_type eq 'advsearch' ) {
262 # load the servers (used for searching -- to do federated searching, etc.)
263 my $primary_servers_loop;# = displayPrimaryServers();
264 $template->param(outer_servers_loop => $primary_servers_loop,);
266 my $secondary_servers_loop;# = displaySecondaryServers();
267 $template->param(outer_sup_servers_loop => $secondary_servers_loop,);
269 # set the default sorting
270 my $default_sort_by = C4::Context->preference('defaultSortField')."_".C4::Context->preference('defaultSortOrder')
271 if (C4::Context->preference('OPACdefaultSortField') && C4::Context->preference('OPACdefaultSortOrder'));
272 $template->param($default_sort_by => 1);
274 # determine what to display next to the search boxes (ie, boolean option
275 # shouldn't appear on the first one, scan indexes should, adding a new
276 # box should only appear on the last, etc.
277 my @search_boxes_array;
278 my $search_boxes_count = C4::Context->preference("OPACAdvSearchInputCount") | 3; # FIXME: should be a syspref
279 for (my $i=1;$i<=$search_boxes_count;$i++) {
280 # if it's the first one, don't display boolean option, but show scan indexes
282 push @search_boxes_array,
288 # if it's the last one, show the 'add field' box
289 elsif ($i==$search_boxes_count) {
290 push @search_boxes_array,
297 push @search_boxes_array,
304 $template->param(uc(C4::Context->preference("marcflavour")) => 1,
305 search_boxes_loop => \@search_boxes_array);
307 # load the language limits (for search)
308 my $languages_limit_loop = getAllLanguages();
309 $template->param(search_languages_loop => $languages_limit_loop,);
311 # use the global setting by default
312 if ( C4::Context->preference("expandedSearchOption") == 1) {
313 $template->param( expanded_options => C4::Context->preference("expandedSearchOption") );
315 # but let the user override it
316 if ( ($cgi->param('expanded_options') == 0) || ($cgi->param('expanded_options') == 1 ) ) {
317 $template->param( expanded_options => $cgi->param('expanded_options'));
320 output_html_with_http_headers $cgi, $cookie, $template->output;
324 ### OK, if we're this far, we're performing a search, not just loading the advanced search page
326 # Fetch the paramater list as a hash in scalar context:
327 # * returns paramater list as tied hash ref
328 # * we can edit the values by changing the key
329 # * multivalued CGI paramaters are returned as a packaged string separated by "\0" (null)
330 my $params = $cgi->Vars;
332 # Params that can have more than one value
333 # sort by is used to sort the query
334 # in theory can have more than one but generally there's just one
336 my $default_sort_by = C4::Context->preference('defaultSortField')."_".C4::Context->preference('defaultSortOrder')
337 if (C4::Context->preference('defaultSortField') && C4::Context->preference('defaultSortOrder'));
339 @sort_by = split("\0",$params->{'sort_by'}) if $params->{'sort_by'};
340 $sort_by[0] = $default_sort_by unless $sort_by[0];
341 foreach my $sort (@sort_by) {
342 $template->param($sort => 1);
344 $template->param('sort_by' => $sort_by[0]);
346 # Use the servers defined, or just search our local catalog(default)
348 @servers = split("\0",$params->{'server'}) if $params->{'server'};
350 #FIXME: this should be handled using Context.pm
351 @servers = ("biblioserver");
352 # @servers = C4::Context->config("biblioserver");
354 # operators include boolean and proximity operators and are used
355 # to evaluate multiple operands
357 @operators = split("\0",$params->{'op'}) if $params->{'op'};
359 # indexes are query qualifiers, like 'title', 'author', etc. They
360 # can be single or multiple parameters separated by comma: kw,right-Truncation
362 @indexes = split("\0",$params->{'idx'});
364 # if a simple index (only one) display the index used in the top search box
365 if ($indexes[0] && !$indexes[1]) {
366 $template->param("ms_".$indexes[0] => 1);}
369 # an operand can be a single term, a phrase, or a complete ccl query
371 @operands = split("\0",$params->{'q'}) if $params->{'q'};
373 # limits are use to limit to results to a pre-defined category such as branch or language
375 @limits = split("\0",$params->{'limit'}) if $params->{'limit'};
377 if($params->{'multibranchlimit'}) {
378 push @limits, join(" or ", map { "branch: $_ "} @{GetBranchesInCategory($params->{'multibranchlimit'})}) ;
382 foreach my $limit(@limits) {
383 if ($limit =~/available/) {
387 $template->param(available => $available);
389 # append year limits if they exist
392 if ($params->{'limit-yr'}) {
393 if ($params->{'limit-yr'} =~ /\d{4}-\d{4}/) {
394 my ($yr1,$yr2) = split(/-/, $params->{'limit-yr'});
395 $limit_yr = "yr,st-numeric,ge=$yr1 and yr,st-numeric,le=$yr2";
396 $limit_yr_value = "$yr1-$yr2";
398 elsif ($params->{'limit-yr'} =~ /\d{4}/) {
399 $limit_yr = "yr,st-numeric=$params->{'limit-yr'}";
400 $limit_yr_value = $params->{'limit-yr'};
402 push @limits,$limit_yr;
403 #FIXME: Should return a error to the user, incorect date format specified
406 # Params that can only have one value
407 my $scan = $params->{'scan'};
408 my $count = C4::Context->preference('numSearchResults') || 20;
409 my $results_per_page = $params->{'count'} || $count;
410 my $offset = $params->{'offset'} || 0;
411 my $page = $cgi->param('page') || 1;
412 #my $offset = ($page-1)*$results_per_page;
414 my $expanded_facet = $params->{'expand'};
416 # Define some global variables
417 my ( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$stopwords_removed,$query_type);
421 ## I. BUILD THE QUERY
422 ( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$stopwords_removed,$query_type) = buildQuery(\@operators,\@operands,\@indexes,\@limits,\@sort_by,$scan);
424 ## parse the query_cgi string and put it into a form suitable for <input>s
426 my $scan_index_to_use;
428 for my $this_cgi ( split('&',$query_cgi) ) {
429 next unless $this_cgi;
430 $this_cgi =~ m/(.*=)(.*)/;
432 my $input_value = $2;
433 $input_name =~ s/=$//;
434 push @query_inputs, { input_name => $input_name, input_value => $input_value };
435 if ($input_name eq 'idx') {
436 $scan_index_to_use = $input_value; # unless $scan_index_to_use;
439 $template->param ( QUERY_INPUTS => \@query_inputs,
440 scan_index_to_use => $scan_index_to_use );
442 ## parse the limit_cgi string and put it into a form suitable for <input>s
444 for my $this_cgi ( split('&',$limit_cgi) ) {
445 next unless $this_cgi;
446 # handle special case limit-yr
447 if ($this_cgi =~ /yr,st-numeric/) {
448 push @limit_inputs, { input_name => 'limit-yr', input_value => $limit_yr_value };
451 $this_cgi =~ m/(.*=)(.*)/;
453 my $input_value = $2;
454 $input_name =~ s/=$//;
455 push @limit_inputs, { input_name => $input_name, input_value => $input_value };
457 $template->param ( LIMIT_INPUTS => \@limit_inputs );
459 ## II. DO THE SEARCH AND GET THE RESULTS
460 my $total; # the total results for the whole set
461 my $facets; # this object stores the faceted results that display on the left-hand of the results page
465 if (C4::Context->preference('NoZebra')) {
466 $query=~s/yr(:|=)\s*([\d]{1,4})-([\d]{1,4})/(yr>=$2 and yr<=$3)/g;
467 $simple_query=~s/yr\s*(:|=)([\d]{1,4})-([\d]{1,4})/(yr>=$2 and yr<=$3)/g;
470 ($error, $results_hashref, $facets) = NZgetRecords($query,$simple_query,\@sort_by,\@servers,$results_per_page,$offset,$expanded_facet,$branches,$query_type,$scan);
474 ($error, $results_hashref, $facets) = getRecords($query,$simple_query,\@sort_by,\@servers,$results_per_page,$offset,$expanded_facet,$branches,$query_type,$scan);
478 $template->param(query_error => $error.$@);
480 output_html_with_http_headers $cgi, $cookie, $template->output;
484 # FIXME: This belongs in tools/ not in the primary search results page
485 my $op=$cgi->param("operation");
486 if ($op eq "bulkedit"){
487 my ($countchanged,$listunchanged)=
488 ModBiblios($results_hashref->{'biblioserver'}->{"RECORDS"},
489 $params->{"tagsubfield"},
490 $params->{"inputvalue"},
491 $params->{"targetvalue"},
494 $template->param(bulkeditresults=>1,
495 tagsubfield=>$params->{"tagsubfield"},
496 inputvalue=>$params->{"inputvalue"},
497 targetvalue=>$params->{"targetvalue"},
498 countchanged=>$countchanged,
499 countunchanged=>scalar(@$listunchanged),
500 listunchanged=>$listunchanged);
502 if (C4::Context->userenv->{'flags'}==1 ||(C4::Context->userenv->{'flags'} & ( 2**9 ) )){
503 #Edit Catalogue Permissions
504 my $editable_subfields = GetManagedTagSubfields();
505 # change '--' to '—' to avoid escaping issues
506 for (my $i = 0; $i <= $#{$editable_subfields}; $i++) {
507 $editable_subfields->[$i]->{subfielddesc} =~ s/--/—/g;
508 $editable_subfields->[$i]->{tagdesc} =~ s/--/—/g;
510 $template->param(bulkedit => 1);
511 $template->param(tagsubfields=>$editable_subfields);
515 # At this point, each server has given us a result set
516 # now we build that set for template display
517 my @sup_results_array;
518 for (my $i=0;$i<@servers;$i++) {
519 my $server = $servers[$i];
520 if ($server =~/biblioserver/) { # this is the local bibliographic server
521 $hits = $results_hashref->{$server}->{"hits"};
522 my $page = $cgi->param('page') || 0;
523 my @newresults = searchResults( $query_desc,$hits,$results_per_page,$offset,$scan,@{$results_hashref->{$server}->{"RECORDS"}});
524 $total = $total + $results_hashref->{$server}->{"hits"};
525 ## If there's just one result, redirect to the detail page
527 my $biblionumber=@newresults[0]->{biblionumber};
528 if (C4::Context->preference('IntranetBiblioDefaultView') eq 'isbd') {
529 print $cgi->redirect("/cgi-bin/koha/catalogue/ISBDdetail.pl?biblionumber=$biblionumber");
530 } elsif (C4::Context->preference('IntranetBiblioDefaultView') eq 'marc') {
531 print $cgi->redirect("/cgi-bin/koha/catalogue/MARCdetail.pl?biblionumber=$biblionumber");
533 print $cgi->redirect("/cgi-bin/koha/catalogue/detail.pl?biblionumber=$biblionumber");
541 $template->param(total => $hits);
542 my $limit_cgi_not_availablity = $limit_cgi;
543 $limit_cgi_not_availablity =~ s/&limit=available//g;
544 $template->param(limit_cgi_not_availablity => $limit_cgi_not_availablity);
545 $template->param(limit_cgi => $limit_cgi);
546 $template->param(query_cgi => $query_cgi);
547 $template->param(query_desc => $query_desc);
548 $template->param(limit_desc => $limit_desc);
549 if ($query_desc || $limit_desc) {
550 $template->param(searchdesc => 1);
552 $template->param(stopwords_removed => "@$stopwords_removed") if $stopwords_removed;
553 $template->param(results_per_page => $results_per_page);
554 $template->param(SEARCH_RESULTS => \@newresults);
556 ## FIXME: add a global function for this, it's better than the current global one
557 ## Build the page numbers on the bottom of the page
559 # total number of pages there will be
560 my $pages = ceil($hits / $results_per_page);
561 # default page number
562 my $current_page_number = 1;
563 $current_page_number = ($offset / $results_per_page + 1) if $offset;
564 my $previous_page_offset = $offset - $results_per_page unless ($offset - $results_per_page <0);
565 my $next_page_offset = $offset + $results_per_page;
566 # If we're within the first 10 pages, keep it simple
567 #warn "current page:".$current_page_number;
568 if ($current_page_number < 10) {
569 # just show the first 10 pages
570 # Loop through the pages
571 my $pages_to_show = 10;
572 $pages_to_show = $pages if $pages<10;
573 for (my $i=1; $i<=$pages_to_show;$i++) {
574 # the offset for this page
575 my $this_offset = (($i*$results_per_page)-$results_per_page);
576 # the page number for this page
577 my $this_page_number = $i;
578 # it should only be highlighted if it's the current page
579 my $highlight = 1 if ($this_page_number == $current_page_number);
580 # put it in the array
581 push @page_numbers, { offset => $this_offset, pg => $this_page_number, highlight => $highlight, sort_by => join " ",@sort_by };
589 # now, show twenty pages, with the current one smack in the middle
591 for (my $i=$current_page_number; $i<=($current_page_number + 20 );$i++) {
592 my $this_offset = ((($i-9)*$results_per_page)-$results_per_page);
593 my $this_page_number = $i-9;
594 my $highlight = 1 if ($this_page_number == $current_page_number);
595 if ($this_page_number <= $pages) {
596 push @page_numbers, { offset => $this_offset, pg => $this_page_number, highlight => $highlight, sort_by => join " ",@sort_by };
601 # FIXME: no previous_page_offset when pages < 2
602 $template->param( PAGE_NUMBERS => \@page_numbers,
603 previous_page_offset => $previous_page_offset) unless $pages < 2;
604 $template->param( next_page_offset => $next_page_offset) unless $pages eq $current_page_number;
613 $template->param(searchdesc => 1,query_desc => $query_desc,limit_desc => $limit_desc);
620 } # end of the if local
622 # asynchronously search the authority server
623 elsif ($server =~/authorityserver/) { # this is the local authority server
624 my @inner_sup_results_array;
625 for my $sup_record ( @{$results_hashref->{$server}->{"RECORDS"}} ) {
626 my $marc_record_object = MARC::Record->new_from_usmarc($sup_record);
627 # warn "Authority Found: ".$marc_record_object->as_formatted();
628 push @inner_sup_results_array, {
629 'title' => $marc_record_object->field(100)->subfield('a'),
630 'link' => "&idx=an&q=".$marc_record_object->field('001')->as_string(),
633 my $servername = $server;
634 push @sup_results_array, { servername => $servername,
635 inner_sup_results_loop => \@inner_sup_results_array} if @inner_sup_results_array;
637 # FIXME: can add support for other targets as needed here
638 $template->param( outer_sup_results_loop => \@sup_results_array);
639 } #/end of the for loop
640 #$template->param(FEDERATED_RESULTS => \@results_array);
644 #classlist => $classlist,
647 facets_loop => $facets,
649 search_error => $error,
652 if ($query_desc || $limit_desc) {
653 $template->param(searchdesc => 1);
656 ## Now let's find out if we have any supplemental data to show the user
657 # and in the meantime, save the current query for statistical purposes, etc.
658 my $koha_spsuggest; # a flag to tell if we've got suggestions coming from Koha
659 my @koha_spsuggest; # place we store the suggestions to be returned to the template as LOOP
660 my $phrases = $query_desc;
663 if ( C4::Context->preference("kohaspsuggest") ) {
664 my ($suggest_host, $suggest_dbname, $suggest_user, $suggest_pwd) = split(':', C4::Context->preference("kohaspsuggest"));
666 my $koha_spsuggest_dbh;
667 # FIXME: this needs to be moved to Context.pm
669 $koha_spsuggest_dbh=DBI->connect("DBI:mysql:$suggest_dbname:$suggest_host","$suggest_user","$suggest_pwd");
672 warn "can't connect to spsuggest db";
675 my $koha_spsuggest_insert = "INSERT INTO phrase_log(phr_phrase,phr_resultcount,phr_ip) VALUES(?,?,?)";
676 my $koha_spsuggest_query = "SELECT display FROM distincts WHERE strcmp(soundex(suggestion), soundex(?)) = 0 order by soundex(suggestion) limit 0,5";
677 my $koha_spsuggest_sth = $koha_spsuggest_dbh->prepare($koha_spsuggest_query);
678 $koha_spsuggest_sth->execute($phrases);
679 while (my $spsuggestion = $koha_spsuggest_sth->fetchrow_array) {
680 $spsuggestion =~ s/(:|\/)//g;
682 $line{spsuggestion} = $spsuggestion;
683 push @koha_spsuggest,\%line;
687 # Now save the current query
688 $koha_spsuggest_sth=$koha_spsuggest_dbh->prepare($koha_spsuggest_insert);
689 #$koha_spsuggest_sth->execute($phrases,$results_per_page,$ipaddress);
690 $koha_spsuggest_sth->finish;
692 $template->param( koha_spsuggest => $koha_spsuggest ) unless $hits;
693 $template->param( SPELL_SUGGEST => \@koha_spsuggest,
698 warn "Kohaspsuggest failure:".$@;
702 # VI. BUILD THE TEMPLATE
703 output_html_with_http_headers $cgi, $cookie, $template->output;