3 # Copyright (C) 2007 LibLime
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 2 of the License, or (at your option) any later
12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License along with
17 # Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
18 # Suite 330, Boston, MA 02111-1307 USA
26 use vars qw($VERSION);
28 # set the version for version checking
33 C4::Matcher - find MARC records matching another one
39 my @matchers = C4::Matcher::GetMatcherList();
41 my $matcher = C4::Matcher->new($record_type);
42 $matcher->threshold($threshold);
43 $matcher->code($code);
44 $matcher->description($description);
46 $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, '');
47 $matcher->add_simple_matchpoint('Date', 1000, '008', '', 7, 4, '');
48 $matcher->add_matchpoint('isbn', 1000, [ { tag => '020', subfields => 'a', norms => [] } ]);
50 $matcher->add_simple_required_check('245', 'a', -1, 0, '', '245', 'a', -1, 0, '');
51 $matcher->add_required_check([ { tag => '245', subfields => 'a', norms => [] } ],
52 [ { tag => '245', subfields => 'a', norms => [] } ]);
54 my @matches = $matcher->get_matches($marc_record, $max_matches);
56 foreach $match (@matches) {
58 # matches already sorted in order of
60 print "record ID: $match->{'record_id'};
61 print "score: $match->{'score'};
65 my $matcher_description = $matcher->dump();
77 my @matchers = C4::Matcher::GetMatcherList();
81 Returns an array of hashrefs list all matchers
82 present in the database. Each hashref includes:
91 my $dbh = C4::Context->dbh;
93 my $sth = $dbh->prepare_cached("SELECT matcher_id, code, description FROM marc_matchers ORDER BY matcher_id");
96 while (my $row = $sth->fetchrow_hashref) {
110 my $matcher = C4::Matcher->new($record_type, $threshold);
114 Creates a new Matcher. C<$record_type> indicates which search
115 database to use, e.g., 'biblio' or 'authority' and defaults to
116 'biblio', while C<$threshold> is the minimum score required for a match
117 and defaults to 1000.
125 $self->{'id'} = undef;
128 $self->{'record_type'} = shift;
130 $self->{'record_type'} = 'biblio';
134 $self->{'threshold'} = shift;
136 $self->{'threshold'} = 1000;
139 $self->{'code'} = '';
140 $self->{'description'} = '';
142 $self->{'matchpoints'} = [];
143 $self->{'required_checks'} = [];
153 my $matcher = C4::Matcher->fetch($id);
157 Creates a matcher object from the version stored
158 in the database. If a matcher with the given
159 id does not exist, returns undef.
166 my $dbh = C4::Context->dbh();
168 my $sth = $dbh->prepare_cached("SELECT * FROM marc_matchers WHERE matcher_id = ?");
170 my $row = $sth->fetchrow_hashref;
172 return undef unless defined $row;
175 $self->{'id'} = $row->{'matcher_id'};
176 $self->{'record_type'} = $row->{'record_type'};
177 $self->{'code'} = $row->{'code'};
178 $self->{'description'} = $row->{'description'};
179 $self->{'threshold'} = int($row->{'threshold'});
183 $self->{'matchpoints'} = [];
184 $sth = $dbh->prepare_cached("SELECT * FROM matcher_matchpoints WHERE matcher_id = ? ORDER BY matchpoint_id");
185 $sth->execute($self->{'id'});
186 while (my $row = $sth->fetchrow_hashref) {
187 my $matchpoint = $self->_fetch_matchpoint($row->{'matchpoint_id'});
188 push @{ $self->{'matchpoints'} }, $matchpoint;
192 $self->{'required_checks'} = [];
193 $sth = $dbh->prepare_cached("SELECT * FROM matchchecks WHERE matcher_id = ? ORDER BY matchcheck_id");
194 $sth->execute($self->{'id'});
195 while (my $row = $sth->fetchrow_hashref) {
196 my $source_matchpoint = $self->_fetch_matchpoint($row->{'source_matchpoint_id'});
197 my $target_matchpoint = $self->_fetch_matchpoint($row->{'target_matchpoint_id'});
199 $matchcheck->{'source_matchpoint'} = $source_matchpoint;
200 $matchcheck->{'target_matchpoint'} = $target_matchpoint;
201 push @{ $self->{'required_checks'} }, $matchcheck;
207 sub _fetch_matchpoint {
209 my $matchpoint_id = shift;
211 my $dbh = C4::Context->dbh;
212 my $sth = $dbh->prepare_cached("SELECT * FROM matchpoints WHERE matchpoint_id = ?");
213 $sth->execute($matchpoint_id);
214 my $row = $sth->fetchrow_hashref;
216 $matchpoint->{'index'} = $row->{'search_index'};
217 $matchpoint->{'score'} = int($row->{'score'});
220 $matchpoint->{'components'} = [];
221 $sth = $dbh->prepare_cached("SELECT * FROM matchpoint_components WHERE matchpoint_id = ? ORDER BY sequence");
222 $sth->execute($matchpoint_id);
223 while ($row = $sth->fetchrow_hashref) {
225 $component->{'tag'} = $row->{'tag'};
226 $component->{'subfields'} = { map { $_ => 1 } split(//, $row->{'subfields'}) };
227 $component->{'offset'} = int($row->{'offset'});
228 $component->{'length'} = int($row->{'length'});
229 $component->{'norms'} = [];
230 my $sth2 = $dbh->prepare_cached("SELECT *
231 FROM matchpoint_component_norms
232 WHERE matchpoint_component_id = ? ORDER BY sequence");
233 $sth2->execute($row->{'matchpoint_component_id'});
234 while (my $row2 = $sth2->fetchrow_hashref) {
235 push @{ $component->{'norms'} }, $row2->{'norm_routine'};
237 push @{ $matchpoint->{'components'} }, $component;
246 my $id = $matcher->store();
250 Stores matcher in database. The return value is the ID
251 of the marc_matchers row. If the matcher was
252 previously retrieved from the database via the fetch()
253 method, the DB representation of the matcher
261 if (defined $self->{'id'}) {
263 $self->_del_matcher_components();
264 $self->_update_marc_matchers();
267 $self->_new_marc_matchers();
269 $self->_store_matcher_components();
270 return $self->{'id'};
273 sub _del_matcher_components {
276 my $dbh = C4::Context->dbh();
277 my $sth = $dbh->prepare_cached("DELETE FROM matchpoints WHERE matcher_id = ?");
278 $sth->execute($self->{'id'});
279 $sth = $dbh->prepare_cached("DELETE FROM matchchecks WHERE matcher_id = ?");
280 $sth->execute($self->{'id'});
281 # foreign key delete cascades take care of deleting relevant rows
282 # from matcher_matchpoints, matchpoint_components, and
283 # matchpoint_component_norms
286 sub _update_marc_matchers {
289 my $dbh = C4::Context->dbh();
290 my $sth = $dbh->prepare_cached("UPDATE marc_matchers
295 WHERE matcher_id = ?");
296 $sth->execute($self->{'code'}, $self->{'description'}, $self->{'record_type'}, $self->{'threshold'}, $self->{'id'});
299 sub _new_marc_matchers {
302 my $dbh = C4::Context->dbh();
303 my $sth = $dbh->prepare_cached("INSERT INTO marc_matchers
304 (code, description, record_type, threshold)
305 VALUES (?, ?, ?, ?)");
306 $sth->execute($self->{'code'}, $self->{'description'}, $self->{'record_type'}, $self->{'threshold'});
307 $self->{'id'} = $dbh->{'mysql_insertid'};
310 sub _store_matcher_components {
313 my $dbh = C4::Context->dbh();
315 my $matcher_id = $self->{'id'};
316 foreach my $matchpoint (@{ $self->{'matchpoints'}}) {
317 my $matchpoint_id = $self->_store_matchpoint($matchpoint);
318 $sth = $dbh->prepare_cached("INSERT INTO matcher_matchpoints (matcher_id, matchpoint_id)
320 $sth->execute($matcher_id, $matchpoint_id);
322 foreach my $matchcheck (@{ $self->{'required_checks'} }) {
323 my $source_matchpoint_id = $self->_store_matchpoint($matchcheck->{'source_matchpoint'});
324 my $target_matchpoint_id = $self->_store_matchpoint($matchcheck->{'target_matchpoint'});
325 $sth = $dbh->prepare_cached("INSERT INTO matchchecks
326 (matcher_id, source_matchpoint_id, target_matchpoint_id)
328 $sth->execute($matcher_id, $source_matchpoint_id, $target_matchpoint_id);
333 sub _store_matchpoint {
335 my $matchpoint = shift;
337 my $dbh = C4::Context->dbh();
339 my $matcher_id = $self->{'id'};
340 $sth = $dbh->prepare_cached("INSERT INTO matchpoints (matcher_id, search_index, score)
342 $sth->execute($matcher_id, $matchpoint->{'index'}, $matchpoint->{'score'});
343 my $matchpoint_id = $dbh->{'mysql_insertid'};
345 foreach my $component (@{ $matchpoint->{'components'} }) {
347 $sth = $dbh->prepare_cached("INSERT INTO matchpoint_components
348 (matchpoint_id, sequence, tag, subfields, offset, length)
349 VALUES (?, ?, ?, ?, ?, ?)");
350 $sth->bind_param(1, $matchpoint_id);
351 $sth->bind_param(2, $seqnum);
352 $sth->bind_param(3, $component->{'tag'});
353 $sth->bind_param(4, join "", sort keys %{ $component->{'subfields'} });
354 $sth->bind_param(5, $component->{'offset'});
355 $sth->bind_param(6, $component->{'length'});
357 my $matchpoint_component_id = $dbh->{'mysql_insertid'};
359 foreach my $norm (@{ $component->{'norms'} }) {
361 $sth = $dbh->prepare_cached("INSERT INTO matchpoint_component_norms
362 (matchpoint_component_id, sequence, norm_routine)
364 $sth->execute($matchpoint_component_id, $normseq, $norm);
367 return $matchpoint_id;
375 C4::Matcher->delete($id);
379 Deletes the matcher of the specified ID
386 my $matcher_id = shift;
388 my $dbh = C4::Context->dbh;
389 my $sth = $dbh->prepare("DELETE FROM marc_matchers WHERE matcher_id = ?");
390 $sth->execute($matcher_id); # relying on cascading deletes to clean up everything
397 $matcher->threshold(1000);
398 my $threshold = $matcher->threshold();
408 @_ ? $self->{'threshold'} = shift : $self->{'threshold'};
416 my $id = $matcher->_id();
420 Accessor method. Note that using this method
421 to set the DB ID of the matcher should not be
422 done outside of the editing CGI.
428 @_ ? $self->{'id'} = shift : $self->{'id'};
435 $matcher->code('ISBN');
436 my $code = $matcher->code();
446 @_ ? $self->{'code'} = shift : $self->{'code'};
453 $matcher->description('match on ISBN');
454 my $description = $matcher->description();
464 @_ ? $self->{'description'} = shift : $self->{'description'};
467 =head2 add_matchpoint
471 $matcher->add_matchpoint($index, $score, $matchcomponents);
475 Adds a matchpoint that may include multiple components. The $index
476 parameter identifies the index that will be searched, while $score
477 is the weight that will be added if a match is found.
479 $matchcomponents should be a reference to an array of matchpoint
480 compoents, each of which should be a hash containing the following
488 The normalization_rules value should in turn be a reference to an
489 array, each element of which should be a reference to a
490 normalization subroutine (under C4::Normalize) to be applied
491 to the source string.
497 my ($index, $score, $matchcomponents) = @_;
500 $matchpoint->{'index'} = $index;
501 $matchpoint->{'score'} = $score;
502 $matchpoint->{'components'} = [];
503 foreach my $input_component (@{ $matchcomponents }) {
504 push @{ $matchpoint->{'components'} }, _parse_match_component($input_component);
506 push @{ $self->{'matchpoints'} }, $matchpoint;
509 =head2 add_simple_matchpoint
513 $matcher->add_simple_matchpoint($index, $score, $source_tag, $source_subfields,
514 $source_offset, $source_length,
519 Adds a simple matchpoint rule -- after composing a key based on the source tag and subfields,
520 normalized per the normalization fuction, search the index. All records retrieved
521 will receive the assigned score.
525 sub add_simple_matchpoint {
527 my ($index, $score, $source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer) = @_;
529 $self->add_matchpoint($index, $score, [
530 { tag => $source_tag, subfields => $source_subfields,
531 offset => $source_offset, 'length' => $source_length,
532 norms => [ $source_normalizer ]
537 =head2 add_required_check
541 $match->add_required_check($source_matchpoint, $target_matchpoint);
545 Adds a required check definition. A required check means that in
546 order for a match to be considered valid, the key derived from the
547 source (incoming) record must match the key derived from the target
548 (already in DB) record.
550 Unlike a regular matchpoint, only the first repeat of each tag
551 in the source and target match criteria are considered.
553 A typical example of a required check would be verifying that the
554 titles and publication dates match.
556 $source_matchpoint and $target_matchpoint are each a reference to
557 an array of hashes, where each hash follows the same definition
558 as the matchpoint component specification in add_matchpoint, i.e.,
566 The normalization_rules value should in turn be a reference to an
567 array, each element of which should be a reference to a
568 normalization subroutine (under C4::Normalize) to be applied
569 to the source string.
573 sub add_required_check {
575 my ($source_matchpoint, $target_matchpoint) = @_;
578 $matchcheck->{'source_matchpoint'}->{'index'} = '';
579 $matchcheck->{'source_matchpoint'}->{'score'} = 0;
580 $matchcheck->{'source_matchpoint'}->{'components'} = [];
581 $matchcheck->{'target_matchpoint'}->{'index'} = '';
582 $matchcheck->{'target_matchpoint'}->{'score'} = 0;
583 $matchcheck->{'target_matchpoint'}->{'components'} = [];
584 foreach my $input_component (@{ $source_matchpoint }) {
585 push @{ $matchcheck->{'source_matchpoint'}->{'components'} }, _parse_match_component($input_component);
587 foreach my $input_component (@{ $target_matchpoint }) {
588 push @{ $matchcheck->{'target_matchpoint'}->{'components'} }, _parse_match_component($input_component);
590 push @{ $self->{'required_checks'} }, $matchcheck;
593 =head2 add_simple_required_check
595 $matcher->add_simple_required_check($source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer,
596 $target_tag, $target_subfields, $target_offset, $target_length, $target_normalizer);
600 Adds a required check, which requires that the normalized keys made from the source and targets
601 must match for a match to be considered valid.
607 sub add_simple_required_check {
609 my ($source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer,
610 $target_tag, $target_subfields, $target_offset, $target_length, $target_normalizer) = @_;
612 $self->add_required_check(
613 [ { tag => $source_tag, subfields => $source_subfields, offset => $source_offset, 'length' => $source_length,
614 norms => [ $source_normalizer ] } ],
615 [ { tag => $target_tag, subfields => $target_subfields, offset => $target_offset, 'length' => $target_length,
616 norms => [ $target_normalizer ] } ]
624 my @matches = $matcher->get_matches($marc_record, $max_matches);
625 foreach $match (@matches) {
626 # matches already sorted in order of
628 print "record ID: $match->{'record_id'};
629 print "score: $match->{'score'};
634 Identifies all of the records matching the given MARC record. For a record already
635 in the database to be considered a match, it must meet the following criteria:
639 =item 1. Total score from its matching field must exceed the supplied threshold.
641 =item 2. It must pass all required checks.
645 Only the top $max_matches matches are returned. The returned array is sorted
646 in order of decreasing score, i.e., the best match is first.
652 my ($source_record, $max_matches) = @_;
656 foreach my $matchpoint (@{ $self->{'matchpoints'} }) {
657 my @source_keys = _get_match_keys($source_record, $matchpoint);
658 next if scalar(@source_keys) == 0;
660 my $query = join(" or ", map { "$matchpoint->{'index'}=$_" } @source_keys);
661 # FIXME only searching biblio index at the moment
662 my ($error, $searchresults) = SimpleSearch($query);
664 warn "search failed ($query) $error" if $error;
665 foreach my $matched (@$searchresults) {
666 $matches{$matched} += $matchpoint->{'score'};
670 # get rid of any that don't meet the threshold
671 %matches = map { ($matches{$_} >= $self->{'threshold'}) ? ($_ => $matches{$_}) : () } keys %matches;
673 # get rid of any that don't meet the required checks
674 %matches = map { _passes_required_checks($source_record, $_, $self->{'required_checks'}) ? ($_ => $matches{$_}) : () }
678 foreach my $marcblob (keys %matches) {
679 my $target_record = MARC::Record->new_from_usmarc($marcblob);
680 my $result = TransformMarcToKoha(C4::Context->dbh, $target_record, '');
681 # FIXME - again, bibliospecific
682 # also, can search engine be induced to give just the number in the first place?
683 my $record_number = $result->{'biblionumber'};
684 push @results, { 'record_id' => $record_number, 'score' => $matches{$marcblob} };
686 @results = sort { $b->{'score'} cmp $a->{'score'} } @results;
687 if (scalar(@results) > $max_matches) {
688 @results = @results[0..$max_matches-1];
698 $description = $matcher->dump();
702 Returns a reference to a structure containing all of the information
703 in the matcher object. This is mainly a convenience method to
704 aid setting up a HTML editing form.
713 $result->{'matcher_id'} = $self->{'id'};
714 $result->{'code'} = $self->{'code'};
715 $result->{'description'} = $self->{'description'};
717 $result->{'matchpoints'} = [];
718 foreach my $matchpoint (@{ $self->{'matchpoints'} }) {
719 push @{ $result->{'matchpoints'} }, $matchpoint;
721 $result->{'matchchecks'} = [];
722 foreach my $matchcheck (@{ $self->{'required_checks'} }) {
723 push @{ $result->{'matchchecks'} }, $matchcheck;
729 sub _passes_required_checks {
730 my ($source_record, $target_blob, $matchchecks) = @_;
731 my $target_record = MARC::Record->new_from_usmarc($target_blob); # FIXME -- need to avoid parsing record twice
733 # no checks supplied == automatic pass
734 return 1 if $#{ $matchchecks } == -1;
736 foreach my $matchcheck (@{ $matchchecks }) {
737 my $source_key = join "", _get_match_keys($source_record, $matchcheck->{'source_matchpoint'});
738 my $target_key = join "", _get_match_keys($target_record, $matchcheck->{'target_matchpoint'});
739 return 0 unless $source_key eq $target_key;
744 sub _get_match_keys {
745 my $source_record = shift;
746 my $matchpoint = shift;
747 my $check_only_first_repeat = @_ ? shift : 0;
749 # If there is more than one component to the matchpoint (e.g.,
750 # matchpoint includes both 003 and 001), any repeats
751 # of the first component's tag are identified; repeats
752 # of the subsequent components' tags are appended to
753 # each parallel key dervied from the first component,
754 # up to the number of repeats of the first component's tag.
756 # For example, if the record has one 003 and two 001s, only
757 # one key is retrieved because there is only one 003. The key
758 # will consist of the contents of the first 003 and first 001.
760 # If there are two 003s and two 001s, there will be two keys:
761 # first 003 + first 001
762 # second 003 + second 001
765 for (my $i = 0; $i <= $#{ $matchpoint->{'components'} }; $i++) {
766 my $component = $matchpoint->{'components'}->[$i];
768 FIELD: foreach my $field ($source_record->field($component->{'tag'})) {
770 last FIELD if $j > 0 and $check_only_first_repeat;
771 last FIELD if $i > 0 and $j > $#keys;
773 if ($field->is_control_field()) {
774 if ($component->{'length'}) {
775 $key = _normalize(substr($field->data(), $component->{'offset'}, $component->{'length'}))
776 # FIXME normalize, substr
778 $key = _normalize($field->data());
781 foreach my $subfield ($field->subfields()) {
782 if (exists $component->{'subfields'}->{$subfield->[0]}) {
783 $key .= " " . $subfield->[1];
786 $key = _normalize($key);
789 push @keys, $key if $key;
791 $keys[$j] .= " $key" if $key;
800 sub _parse_match_component {
801 my $input_component = shift;
804 $component->{'tag'} = $input_component->{'tag'};
805 $component->{'subfields'} = { map { $_ => 1 } split(//, $input_component->{'subfields'}) };
806 $component->{'offset'} = exists($input_component->{'offset'}) ? $input_component->{'offset'} : -1;
807 $component->{'length'} = $input_component->{'length'} ? $input_component->{'length'} : 0;
808 $component->{'norms'} = $input_component->{'norms'} ? $input_component->{'norms'} : [];
813 # FIXME - default normalizer
815 my $value = uc shift;
819 $value =~ s/[.;,\]\[\)\(\/"']//g;
827 Koha Development Team <info@koha.org>
829 Galen Charlton <galen.charlton@liblime.com>