bibliographic matching enhancements
* serialization of matching rules to database * added two default matching rules for MARC21 * update import staging CGI to choose matching rule Signed-off-by: Chris Cormack <crc@liblime.com> Signed-off-by: Joshua Ferraro <jmf@liblime.com>
This commit is contained in:
Galen Charlton
Joshua Ferraro
parent
27d85a6a5a
commit
75bd4b2dac
6 changed files with 590 additions and 64 deletions
556
C4/Matcher.pm
556
C4/Matcher.pm
|
|
@ -36,12 +36,20 @@ C4::Matcher - find MARC records matching another one
|
|||
|
||||
=over 4
|
||||
|
||||
my @matchers = C4::Matcher::GetMatcherList();
|
||||
|
||||
my $matcher = C4::Matcher->new($record_type);
|
||||
$matcher->threshold($threshold);
|
||||
$matcher->add_matchpoint($source_tag, $source_subfields, $source_normalizer,
|
||||
$index, $score);
|
||||
$matcher->add_required_check($check_name, $source_tag, $source_subfields, $source_normalizer,
|
||||
$target_tag, $target_subfields, $target_normalizer);
|
||||
$matcher->code($code);
|
||||
$matcher->description($description);
|
||||
|
||||
$matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, '');
|
||||
$matcher->add_simple_matchpoint('Date', 1000, '008', '', 7, 4, '');
|
||||
$matcher->add_matchpoint('isbn', 1000, [ { tag => '020', subfields => 'a', norms => [] } ]);
|
||||
|
||||
$matcher->add_simple_required_check('245', 'a', -1, 0, '', '245', 'a', -1, 0, '');
|
||||
$matcher->add_required_check([ { tag => '245', subfields => 'a', norms => [] } ],
|
||||
[ { tag => '245', subfields => 'a', norms => [] } ]);
|
||||
|
||||
my @matches = $matcher->get_matches($marc_record, $max_matches);
|
||||
|
||||
|
|
@ -56,6 +64,39 @@ foreach $match (@matches) {
|
|||
|
||||
=back
|
||||
|
||||
=head1 FUNCTIONS
|
||||
|
||||
=cut
|
||||
|
||||
=head2 GetMatcherList
|
||||
|
||||
=over 4
|
||||
|
||||
my @matchers = C4::Matcher::GetMatcherList();
|
||||
|
||||
=back
|
||||
|
||||
Returns an array of hashrefs list all matchers
|
||||
present in the database. Each hashref includes:
|
||||
|
||||
matcher_id
|
||||
code
|
||||
description
|
||||
|
||||
=cut
|
||||
|
||||
sub GetMatcherList {
|
||||
my $dbh = C4::Context->dbh;
|
||||
|
||||
my $sth = $dbh->prepare_cached("SELECT matcher_id, code, description FROM marc_matchers ORDER BY matcher_id");
|
||||
$sth->execute();
|
||||
my @results = ();
|
||||
while (my $row = $sth->fetchrow_hashref) {
|
||||
push @results, $row;
|
||||
}
|
||||
return @results;
|
||||
}
|
||||
|
||||
=head1 METHODS
|
||||
|
||||
=cut
|
||||
|
|
@ -79,6 +120,8 @@ sub new {
|
|||
my $class = shift;
|
||||
my $self = {};
|
||||
|
||||
$self->{'id'} = undef;
|
||||
|
||||
if ($#_ > -1) {
|
||||
$self->{'record_type'} = shift;
|
||||
} else {
|
||||
|
|
@ -91,6 +134,9 @@ sub new {
|
|||
$self->{'threshold'} = 1000;
|
||||
}
|
||||
|
||||
$self->{'code'} = '';
|
||||
$self->{'description'} = '';
|
||||
|
||||
$self->{'matchpoints'} = [];
|
||||
$self->{'required_checks'} = [];
|
||||
|
||||
|
|
@ -98,12 +144,232 @@ sub new {
|
|||
return $self;
|
||||
}
|
||||
|
||||
=head2 fetch
|
||||
|
||||
=over 4
|
||||
|
||||
my $matcher = C4::Matcher->fetch($id);
|
||||
|
||||
=back
|
||||
|
||||
Creates a matcher object from the version stored
|
||||
in the database. If a matcher with the given
|
||||
id does not exist, returns undef.
|
||||
|
||||
=cut
|
||||
|
||||
sub fetch {
|
||||
my $class = shift;
|
||||
my $id = shift;
|
||||
my $dbh = C4::Context->dbh();
|
||||
|
||||
my $sth = $dbh->prepare_cached("SELECT * FROM marc_matchers WHERE matcher_id = ?");
|
||||
$sth->execute($id);
|
||||
my $row = $sth->fetchrow_hashref;
|
||||
return undef unless defined $row;
|
||||
|
||||
my $self = {};
|
||||
$self->{'id'} = $row->{'matcher_id'};
|
||||
$self->{'record_type'} = $row->{'record_type'};
|
||||
$self->{'code'} = $row->{'code'};
|
||||
$self->{'description'} = $row->{'description'};
|
||||
$self->{'threshold'} = int($row->{'threshold'});
|
||||
bless $self, $class;
|
||||
|
||||
# matchpoints
|
||||
$self->{'matchpoints'} = [];
|
||||
$sth = $dbh->prepare_cached("SELECT * FROM matcher_matchpoints WHERE matcher_id = ? ORDER BY matchpoint_id");
|
||||
$sth->execute($self->{'id'});
|
||||
while (my $row = $sth->fetchrow_hashref) {
|
||||
my $matchpoint = $self->_fetch_matchpoint($row->{'matchpoint_id'});
|
||||
push @{ $self->{'matchpoints'} }, $matchpoint;
|
||||
}
|
||||
|
||||
# required checks
|
||||
$self->{'required_checks'} = [];
|
||||
$sth = $dbh->prepare_cached("SELECT * FROM matchchecks WHERE matcher_id = ? ORDER BY matchcheck_id");
|
||||
$sth->execute($self->{'id'});
|
||||
while (my $row = $sth->fetchrow_hashref) {
|
||||
my $source_matchpoint = $self->_fetch_matchpoint($row->{'source_matchpoint_id'});
|
||||
my $target_matchpoint = $self->_fetch_matchpoint($row->{'target_matchpoint_id'});
|
||||
my $matchcheck = {};
|
||||
$matchcheck->{'source_matchpoint'} = $source_matchpoint;
|
||||
$matchcheck->{'target_matchpoint'} = $target_matchpoint;
|
||||
push @{ $self->{'required_checks'} }, $matchcheck;
|
||||
}
|
||||
|
||||
return $self;
|
||||
}
|
||||
|
||||
sub _fetch_matchpoint {
|
||||
my $self = shift;
|
||||
my $matchpoint_id = shift;
|
||||
|
||||
my $dbh = C4::Context->dbh;
|
||||
my $sth = $dbh->prepare_cached("SELECT * FROM matchpoints WHERE matchpoint_id = ?");
|
||||
$sth->execute($matchpoint_id);
|
||||
my $row = $sth->fetchrow_hashref;
|
||||
my $matchpoint = {};
|
||||
$matchpoint->{'index'} = $row->{'search_index'};
|
||||
$matchpoint->{'score'} = int($row->{'score'});
|
||||
$sth->finish();
|
||||
|
||||
$matchpoint->{'components'} = [];
|
||||
$sth = $dbh->prepare_cached("SELECT * FROM matchpoint_components WHERE matchpoint_id = ? ORDER BY sequence");
|
||||
$sth->execute($matchpoint_id);
|
||||
while ($row = $sth->fetchrow_hashref) {
|
||||
my $component = {};
|
||||
$component->{'tag'} = $row->{'tag'};
|
||||
$component->{'subfields'} = { map { $_ => 1 } split(//, $row->{'subfields'}) };
|
||||
$component->{'offset'} = int($row->{'offset'});
|
||||
$component->{'length'} = int($row->{'length'});
|
||||
$component->{'norms'} = [];
|
||||
my $sth2 = $dbh->prepare_cached("SELECT *
|
||||
FROM matchpoint_component_norms
|
||||
WHERE matchpoint_component_id = ? ORDER BY sequence");
|
||||
$sth2->execute($row->{'matchpoint_component_id'});
|
||||
while (my $row2 = $sth2->fetchrow_hashref) {
|
||||
push @{ $component->{'norms'} }, $row2->{'norm_routine'};
|
||||
}
|
||||
push @{ $matchpoint->{'components'} }, $component;
|
||||
}
|
||||
return $matchpoint;
|
||||
}
|
||||
|
||||
=head2 store
|
||||
|
||||
=over 4
|
||||
|
||||
my $id = $matcher->store();
|
||||
|
||||
=back
|
||||
|
||||
Stores matcher in database. The return value is the ID
|
||||
of the marc_matchers row. If the matcher was
|
||||
previously retrieved from the database via the fetch()
|
||||
method, the DB representation of the matcher
|
||||
is replaced.
|
||||
|
||||
=cut
|
||||
|
||||
sub store {
|
||||
my $self = shift;
|
||||
|
||||
if (defined $self->{'id'}) {
|
||||
# update
|
||||
$self->_del_matcher_components();
|
||||
$self->_update_marc_matchers();
|
||||
} else {
|
||||
# create new
|
||||
$self->_new_marc_matchers();
|
||||
}
|
||||
$self->_store_matcher_components();
|
||||
return $self->{'id'};
|
||||
}
|
||||
|
||||
sub _del_matcher_components {
|
||||
my $self = shift;
|
||||
|
||||
my $dbh = C4::Context->dbh();
|
||||
my $sth = $dbh->prepare_cached("DELETE FROM matchpoints WHERE matcher_id = ?");
|
||||
$sth->execute($self->{'id'});
|
||||
$sth = $dbh->prepare_cached("DELETE FROM matchchecks WHERE matcher_id = ?");
|
||||
$sth->execute($self->{'id'});
|
||||
# foreign key delete cascades take care of deleting relevant rows
|
||||
# from matcher_matchpoints, matchpoint_components, and
|
||||
# matchpoint_component_norms
|
||||
}
|
||||
|
||||
sub _update_marc_matchers {
|
||||
my $self = shift;
|
||||
|
||||
my $dbh = C4::Context->dbh();
|
||||
my $sth = $dbh->prepare_cached("UPDATE marc_matchers
|
||||
SET code = ?,
|
||||
description = ?,
|
||||
record_type = ?,
|
||||
threshold = ?
|
||||
WHERE matcher_id = ?");
|
||||
$sth->execute($self->{'code'}, $self->{'description'}, $self->{'record_type'}, $self->{'threshold'}, $self->{'id'});
|
||||
}
|
||||
|
||||
sub _new_marc_matchers {
|
||||
my $self = shift;
|
||||
|
||||
my $dbh = C4::Context->dbh();
|
||||
my $sth = $dbh->prepare_cached("INSERT INTO marc_matchers
|
||||
(code, description, record_type, threshold)
|
||||
VALUES (?, ?, ?, ?)");
|
||||
$sth->execute($self->{'code'}, $self->{'description'}, $self->{'record_type'}, $self->{'threshold'});
|
||||
$self->{'id'} = $dbh->{'mysql_insertid'};
|
||||
}
|
||||
|
||||
sub _store_matcher_components {
|
||||
my $self = shift;
|
||||
|
||||
my $dbh = C4::Context->dbh();
|
||||
my $sth;
|
||||
my $matcher_id = $self->{'id'};
|
||||
foreach my $matchpoint (@{ $self->{'matchpoints'}}) {
|
||||
my $matchpoint_id = $self->_store_matchpoint($matchpoint);
|
||||
$sth = $dbh->prepare_cached("INSERT INTO matcher_matchpoints (matcher_id, matchpoint_id)
|
||||
VALUES (?, ?)");
|
||||
$sth->execute($matcher_id, $matchpoint_id);
|
||||
}
|
||||
foreach my $matchcheck (@{ $self->{'required_checks'} }) {
|
||||
my $source_matchpoint_id = $self->_store_matchpoint($matchcheck->{'source_matchpoint'});
|
||||
my $target_matchpoint_id = $self->_store_matchpoint($matchcheck->{'target_matchpoint'});
|
||||
$sth = $dbh->prepare_cached("INSERT INTO matchchecks
|
||||
(matcher_id, source_matchpoint_id, target_matchpoint_id)
|
||||
VALUES (?, ?, ?)");
|
||||
$sth->execute($matcher_id, $source_matchpoint_id, $target_matchpoint_id);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
sub _store_matchpoint {
|
||||
my $self = shift;
|
||||
my $matchpoint = shift;
|
||||
|
||||
my $dbh = C4::Context->dbh();
|
||||
my $sth;
|
||||
my $matcher_id = $self->{'id'};
|
||||
$sth = $dbh->prepare_cached("INSERT INTO matchpoints (matcher_id, search_index, score)
|
||||
VALUES (?, ?, ?)");
|
||||
$sth->execute($matcher_id, $matchpoint->{'index'}, $matchpoint->{'score'});
|
||||
my $matchpoint_id = $dbh->{'mysql_insertid'};
|
||||
my $seqnum = 0;
|
||||
foreach my $component (@{ $matchpoint->{'components'} }) {
|
||||
$seqnum++;
|
||||
$sth = $dbh->prepare_cached("INSERT INTO matchpoint_components
|
||||
(matchpoint_id, sequence, tag, subfields, offset, length)
|
||||
VALUES (?, ?, ?, ?, ?, ?)");
|
||||
$sth->bind_param(1, $matchpoint_id);
|
||||
$sth->bind_param(2, $seqnum);
|
||||
$sth->bind_param(3, $component->{'tag'});
|
||||
$sth->bind_param(4, join "", sort keys %{ $component->{'subfields'} });
|
||||
$sth->bind_param(5, $component->{'offset'});
|
||||
$sth->bind_param(6, $component->{'length'});
|
||||
$sth->execute();
|
||||
my $matchpoint_component_id = $dbh->{'mysql_insertid'};
|
||||
my $normseq = 0;
|
||||
foreach my $norm (@{ $component->{'norms'} }) {
|
||||
$normseq++;
|
||||
$sth = $dbh->prepare_cached("INSERT INTO matchpoint_component_norms
|
||||
(matchpoint_component_id, sequence, norm_routine)
|
||||
VALUES (?, ?, ?)");
|
||||
$sth->execute($matchpoint_component_id, $normseq, $norm);
|
||||
}
|
||||
}
|
||||
return $matchpoint_id;
|
||||
}
|
||||
|
||||
=head2 threshold
|
||||
|
||||
=over 4
|
||||
|
||||
$matcher->threshold(1000);
|
||||
my $threshhold = $matcher->threshhold();
|
||||
my $threshold = $matcher->threshold();
|
||||
|
||||
=back
|
||||
|
||||
|
|
@ -116,40 +382,172 @@ sub threshold {
|
|||
@_ ? $self->{'threshold'} = shift : $self->{'threshold'};
|
||||
}
|
||||
|
||||
=head2 code
|
||||
|
||||
=over 4
|
||||
|
||||
$matcher->code('ISBN');
|
||||
my $code = $matcher->code();
|
||||
|
||||
=back
|
||||
|
||||
Accessor method.
|
||||
|
||||
=cut
|
||||
|
||||
sub code {
|
||||
my $self = shift;
|
||||
@_ ? $self->{'code'} = shift : $self->{'code'};
|
||||
}
|
||||
|
||||
=head2 description
|
||||
|
||||
=over 4
|
||||
|
||||
$matcher->description('match on ISBN');
|
||||
my $description = $matcher->description();
|
||||
|
||||
=back
|
||||
|
||||
Accessor method.
|
||||
|
||||
=cut
|
||||
|
||||
sub description {
|
||||
my $self = shift;
|
||||
@_ ? $self->{'description'} = shift : $self->{'description'};
|
||||
}
|
||||
|
||||
=head2 add_matchpoint
|
||||
|
||||
=over 4
|
||||
|
||||
$matcher->add_matchpoint($source_tag, $source_subfields, $source_normalizer,
|
||||
$index, $score);
|
||||
$matcher->add_matchpoint($index, $score, $matchcomponents);
|
||||
|
||||
=back
|
||||
|
||||
Adds a matchpoint rule -- after composing a key based on the source tag and subfields,
|
||||
Adds a matchpoint that may include multiple components. The $index
|
||||
parameter identifies the index that will be searched, while $score
|
||||
is the weight that will be added if a match is found.
|
||||
|
||||
$matchcomponents should be a reference to an array of matchpoint
|
||||
compoents, each of which should be a hash containing the following
|
||||
keys:
|
||||
tag
|
||||
subfields
|
||||
offset
|
||||
length
|
||||
norms
|
||||
|
||||
The normalization_rules value should in turn be a reference to an
|
||||
array, each element of which should be a reference to a
|
||||
normalization subroutine (under C4::Normalize) to be applied
|
||||
to the source string.
|
||||
|
||||
=cut
|
||||
|
||||
sub add_matchpoint {
|
||||
my $self = shift;
|
||||
my ($index, $score, $matchcomponents) = @_;
|
||||
|
||||
my $matchpoint = {};
|
||||
$matchpoint->{'index'} = $index;
|
||||
$matchpoint->{'score'} = $score;
|
||||
$matchpoint->{'components'} = [];
|
||||
foreach my $input_component (@{ $matchcomponents }) {
|
||||
push @{ $matchpoint->{'components'} }, _parse_match_component($input_component);
|
||||
}
|
||||
push @{ $self->{'matchpoints'} }, $matchpoint;
|
||||
}
|
||||
|
||||
=head2 add_simple_matchpoint
|
||||
|
||||
=over 4
|
||||
|
||||
$matcher->add_simple_matchpoint($index, $score, $source_tag, $source_subfields,
|
||||
$source_offset, $source_length,
|
||||
$source_normalizer);
|
||||
|
||||
=back
|
||||
|
||||
Adds a simple matchpoint rule -- after composing a key based on the source tag and subfields,
|
||||
normalized per the normalization fuction, search the index. All records retrieved
|
||||
will receive the assigned score.
|
||||
|
||||
=cut
|
||||
|
||||
sub add_matchpoint {
|
||||
sub add_simple_matchpoint {
|
||||
my $self = shift;
|
||||
my ($source_tag, $source_subfields, $source_normalizer, $index, $score) = @_;
|
||||
my ($index, $score, $source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer) = @_;
|
||||
|
||||
# FIXME - $source_normalizer not used yet
|
||||
my $matchpoint = {
|
||||
'source_tag' => $source_tag,
|
||||
'source_subfields' => { map { $_ => 1 } split(//, $source_subfields) },
|
||||
'source_normalizer' => $source_normalizer,
|
||||
'index' => $index,
|
||||
'score' => $score
|
||||
};
|
||||
push @{ $self->{'matchpoints'} }, $matchpoint;
|
||||
$self->add_matchpoint($index, $score, [
|
||||
{ tag => $source_tag, subfields => $source_subfields,
|
||||
offset => $source_offset, length => $source_length,
|
||||
norms => [ $source_normalizer ]
|
||||
}
|
||||
]);
|
||||
}
|
||||
|
||||
=head2 add_required_check
|
||||
|
||||
$matcher->add_required_check($check_name, $source_tag, $source_subfields, $source_normalizer,
|
||||
$target_tag, $target_subfields, $target_normalizer);
|
||||
=over 4
|
||||
|
||||
$match->add_required_check($source_matchpoint, $target_matchpoint);
|
||||
|
||||
=back
|
||||
|
||||
Adds a required check definition. A required check means that in
|
||||
order for a match to be considered valid, the key derived from the
|
||||
source (incoming) record must match the key derived from the target
|
||||
(already in DB) record.
|
||||
|
||||
Unlike a regular matchpoint, only the first repeat of each tag
|
||||
in the source and target match criteria are considered.
|
||||
|
||||
A typical example of a required check would be verifying that the
|
||||
titles and publication dates match.
|
||||
|
||||
$source_matchpoint and $target_matchpoint are each a reference to
|
||||
an array of hashes, where each hash follows the same definition
|
||||
as the matchpoint component specification in add_matchpoint, i.e.,
|
||||
|
||||
tag
|
||||
subfields
|
||||
offset
|
||||
length
|
||||
norms
|
||||
|
||||
The normalization_rules value should in turn be a reference to an
|
||||
array, each element of which should be a reference to a
|
||||
normalization subroutine (under C4::Normalize) to be applied
|
||||
to the source string.
|
||||
|
||||
=cut
|
||||
|
||||
sub add_required_check {
|
||||
my $self = shift;
|
||||
my ($source_matchpoint, $target_matchpoint) = @_;
|
||||
|
||||
my $matchcheck = {};
|
||||
$matchcheck->{'source_matchpoint'}->{'index'} = '';
|
||||
$matchcheck->{'source_matchpoint'}->{'score'} = 0;
|
||||
$matchcheck->{'source_matchpoint'}->{'components'} = [];
|
||||
$matchcheck->{'target_matchpoint'}->{'index'} = '';
|
||||
$matchcheck->{'target_matchpoint'}->{'score'} = 0;
|
||||
$matchcheck->{'target_matchpoint'}->{'components'} = [];
|
||||
foreach my $input_component (@{ $source_matchpoint }) {
|
||||
push @{ $matchcheck->{'source_matchpoint'}->{'components'} }, _parse_match_component($input_component);
|
||||
}
|
||||
foreach my $input_component (@{ $target_matchpoint }) {
|
||||
push @{ $matchcheck->{'target_matchpoint'}->{'components'} }, _parse_match_component($input_component);
|
||||
}
|
||||
push @{ $self->{'required_checks'} }, $matchcheck;
|
||||
}
|
||||
|
||||
=head2 add_simple_required_check
|
||||
|
||||
$matcher->add_simple_required_check($source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer,
|
||||
$target_tag, $target_subfields, $target_offset, $target_length, $target_normalizer);
|
||||
|
||||
=over 4
|
||||
|
||||
|
|
@ -160,25 +558,23 @@ must match for a match to be considered valid.
|
|||
|
||||
=cut
|
||||
|
||||
sub add_required_check {
|
||||
sub add_simple_required_check {
|
||||
my $self = shift;
|
||||
my ($check_name, $source_tag, $source_subfields, $source_normalizer, $target_tag, $target_subfields, $target_normalizer) = @_;
|
||||
my ($source_tag, $source_subfields, $source_offset, $source_length, $source_normalizer,
|
||||
$target_tag, $target_subfields, $target_offset, $target_length, $target_normalizer) = @_;
|
||||
|
||||
my $check = {
|
||||
'check_name' => $check_name,
|
||||
'source_tag' => $source_tag,
|
||||
'source_subfields' => { map { $_ => 1 } split(//, $source_subfields) },
|
||||
'source_normalizer' => $source_normalizer,
|
||||
'target_tag' => $target_tag,
|
||||
'target_subfields' => { map { $_ => 1 } split(//, $target_subfields) },
|
||||
'target_normalizer' => $target_normalizer
|
||||
};
|
||||
|
||||
push @{ $self->{'required_checks'} }, $check;
|
||||
$self->add_required_check(
|
||||
[ { tag => $source_tag, subfields => $source_subfields, offset => $source_offset, length => $source_length,
|
||||
norms => [ $source_normalizer ] } ],
|
||||
[ { tag => $target_tag, subfields => $target_subfields, offset => $target_offset, length => $target_length,
|
||||
norms => [ $target_normalizer ] } ]
|
||||
);
|
||||
}
|
||||
|
||||
=head2 find_matches
|
||||
|
||||
=over 4
|
||||
|
||||
my @matches = $matcher->get_matches($marc_record, $max_matches);
|
||||
foreach $match (@matches) {
|
||||
# matches already sorted in order of
|
||||
|
|
@ -212,8 +608,7 @@ sub get_matches {
|
|||
my %matches = ();
|
||||
|
||||
foreach my $matchpoint (@{ $self->{'matchpoints'} }) {
|
||||
my @source_keys = _get_match_keys($source_record, $matchpoint->{'source_tag'},
|
||||
$matchpoint->{'source_subfields'}, $matchpoint->{'source_normalizer'});
|
||||
my @source_keys = _get_match_keys($source_record, $matchpoint);
|
||||
next if scalar(@source_keys) == 0;
|
||||
# build query
|
||||
my $query = join(" or ", map { "$matchpoint->{'index'}=$_" } @source_keys);
|
||||
|
|
@ -229,7 +624,10 @@ sub get_matches {
|
|||
# get rid of any that don't meet the threshold
|
||||
%matches = map { ($matches{$_} >= $self->{'threshold'}) ? ($_ => $matches{$_}) : () } keys %matches;
|
||||
|
||||
# FIXME - implement record checks
|
||||
# get rid of any that don't meet the required checks
|
||||
%matches = map { _passes_required_checks($source_record, $_, $self->{'required_checks'}) ? ($_ => $matches{$_}) : () }
|
||||
keys %matches;
|
||||
|
||||
my @results = ();
|
||||
foreach my $marcblob (keys %matches) {
|
||||
my $target_record = MARC::Record->new_from_usmarc($marcblob);
|
||||
|
|
@ -247,30 +645,90 @@ sub get_matches {
|
|||
|
||||
}
|
||||
|
||||
sub _passes_required_checks {
|
||||
my ($source_record, $target_blob, $matchchecks) = @_;
|
||||
my $target_record = MARC::Record->new_from_usmarc($target_blob); # FIXME -- need to avoid parsing record twice
|
||||
|
||||
# no checks supplied == automatic pass
|
||||
return 1 if $#{ $matchchecks } == -1;
|
||||
|
||||
foreach my $matchcheck (@{ $matchchecks }) {
|
||||
my $source_key = join "", _get_match_keys($source_record, $matchcheck->{'source_matchpoint'});
|
||||
my $target_key = join "", _get_match_keys($target_record, $matchcheck->{'target_matchpoint'});
|
||||
return 0 unless $source_key eq $target_key;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
sub _get_match_keys {
|
||||
my ($source_record, $source_tag, $source_subfields, $source_normalizer) = @_;
|
||||
my $source_record = shift;
|
||||
my $matchpoint = shift;
|
||||
my $check_only_first_repeat = @_ ? shift : 0;
|
||||
|
||||
use Data::Dumper;
|
||||
# If there is more than one component to the matchpoint (e.g.,
|
||||
# matchpoint includes both 003 and 001), any repeats
|
||||
# of the first component's tag are identified; repeats
|
||||
# of the subsequent components' tags are appended to
|
||||
# each parallel key dervied from the first component,
|
||||
# up to the number of repeats of the first component's tag.
|
||||
#
|
||||
# For example, if the record has one 003 and two 001s, only
|
||||
# one key is retrieved because there is only one 003. The key
|
||||
# will consist of the contents of the first 003 and first 001.
|
||||
#
|
||||
# If there are two 003s and two 001s, there will be two keys:
|
||||
# first 003 + first 001
|
||||
# second 003 + second 001
|
||||
|
||||
my @keys = ();
|
||||
foreach my $field ($source_record->field($source_tag)) {
|
||||
if ($field->is_control_field()) {
|
||||
push @keys, _normalize($field->data());
|
||||
} else {
|
||||
for (my $i = 0; $i <= $#{ $matchpoint->{'components'} }; $i++) {
|
||||
my $component = $matchpoint->{'components'}->[$i];
|
||||
my $j = -1;
|
||||
FIELD: foreach my $field ($source_record->field($component->{'tag'})) {
|
||||
$j++;
|
||||
last FIELD if $j > 0 and $check_only_first_repeat;
|
||||
last FIELD if $i > 0 and $j > $#keys;
|
||||
my $key = "";
|
||||
foreach my $subfield ($field->subfields()) {
|
||||
if (exists $source_subfields->{$subfield->[0]}) {
|
||||
$key .= " " . $subfield->[1];
|
||||
if ($field->is_control_field()) {
|
||||
if ($component->{'length'}) {
|
||||
$key = _normalize(substr($field->data(), $component->{'offset'}, $component->{'length'}))
|
||||
# FIXME normalize, substr
|
||||
} else {
|
||||
$key = _normalize($field->data());
|
||||
}
|
||||
} else {
|
||||
foreach my $subfield ($field->subfields()) {
|
||||
if (exists $component->{'subfields'}->{$subfield->[0]}) {
|
||||
$key .= " " . $subfield->[1];
|
||||
}
|
||||
}
|
||||
$key = _normalize($key);
|
||||
}
|
||||
if ($i == 0) {
|
||||
push @keys, $key if $key;
|
||||
} else {
|
||||
$keys[$j] .= " $key" if $key;
|
||||
}
|
||||
$key = _normalize($key);
|
||||
|
||||
push @keys, $key if $key;
|
||||
}
|
||||
}
|
||||
return @keys;
|
||||
|
||||
}
|
||||
|
||||
|
||||
sub _parse_match_component {
|
||||
my $input_component = shift;
|
||||
|
||||
my $component = {};
|
||||
$component->{'tag'} = $input_component->{'tag'};
|
||||
$component->{'subfields'} = { map { $_ => 1 } split(//, $input_component->{'subfields'}) };
|
||||
$component->{'offset'} = exists($input_component->{'offset'}) ? $input_component->{'offset'} : -1;
|
||||
$component->{'length'} = $input_component->{'length'} ? $input_component->{'length'} : 0;
|
||||
$component->{'norms'} = $input_component->{'norms'} ? $input_component->{'norms'} : [];
|
||||
|
||||
return $component;
|
||||
}
|
||||
|
||||
# FIXME - default normalizer
|
||||
sub _normalize {
|
||||
my $value = uc shift;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
--
|
||||
-- Default MARC matching rules for Koha
|
||||
--
|
||||
-- Copyright (C) 2007 LiblimeA
|
||||
--
|
||||
-- This file is part of Koha.
|
||||
--
|
||||
-- Koha is free software; you can redistribute it and/or modify it under the
|
||||
-- terms of the GNU General Public License as published by the Free Software
|
||||
-- Foundation; either version 2 of the License, or (at your option) any later
|
||||
-- version.
|
||||
--
|
||||
-- Koha is distributed in the hope that it will be useful, but WITHOUT ANY
|
||||
-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
|
||||
-- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
||||
--
|
||||
-- You should have received a copy of the GNU General Public License along with
|
||||
-- Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
|
||||
-- Suite 330, Boston, MA 02111-1307 USA
|
||||
|
||||
INSERT INTO marc_matchers (code, description, record_type, threshold)
|
||||
VALUES ('ISBN', '020$a', 'biblio', 1000);
|
||||
INSERT INTO matchpoints (matcher_id, search_index, score) SELECT MAX(matcher_id), 'isbn', 1000 FROM marc_matchers;
|
||||
INSERT INTO matcher_matchpoints SELECT MAX(matcher_id), MAX(matchpoint_id) FROM matchpoints;
|
||||
INSERT INTO matchpoint_components (matchpoint_id, sequence, tag, subfields)
|
||||
SELECT MAX(matchpoint_id), 1, '020', 'a' FROM matchpoints;
|
||||
INSERT INTO matchpoint_component_norms (matchpoint_component_id, sequence, norm_routine)
|
||||
SELECT MAX(matchpoint_component_id), 1, 'ISBN' FROM matchpoint_components;
|
||||
|
||||
INSERT INTO marc_matchers (code, description, record_type, threshold)
|
||||
VALUES ('ISSN', '022$a', 'biblio', 1000);
|
||||
INSERT INTO matchpoints (matcher_id, search_index, score) SELECT MAX(matcher_id), 'isbn', 1000 FROM marc_matchers;
|
||||
INSERT INTO matcher_matchpoints SELECT MAX(matcher_id), MAX(matchpoint_id) FROM matchpoints;
|
||||
INSERT INTO matchpoint_components (matchpoint_id, sequence, tag, subfields)
|
||||
SELECT MAX(matchpoint_id), 1, '022', 'a' FROM matchpoints;
|
||||
INSERT INTO matchpoint_component_norms (matchpoint_component_id, sequence, norm_routine)
|
||||
SELECT MAX(matchpoint_component_id), 1, 'ISSN' FROM matchpoint_components;
|
||||
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
Selected matching rules for MARC 21 bibliographic
|
||||
records, including:
|
||||
|
||||
ISBN
|
||||
ISSN
|
||||
|
|
@ -23,9 +23,14 @@
|
|||
<li><!-- TMPL_VAR name="import_errors" --> records not staged because of MARC error</li>
|
||||
<li><!-- TMPL_VAR name="staged" --> records staged</li>
|
||||
<!-- TMPL_IF name="checked_matches" -->
|
||||
<li><!-- TMPL_VAR name="matched" --> records with at least one match in catalogue</li>
|
||||
<li><!-- TMPL_VAR name="matched" --> records with at least one match in catalogue per matching rule
|
||||
"<!-- TMPL_VAR name="matcher_code" -->"</li>
|
||||
<!-- TMPL_ELSE -->
|
||||
<li>Did not check for matches with existing records in catalogue</li>
|
||||
<!-- TMPL_IF name="matcher_failed" -->
|
||||
<li>Record matching failed -- unable to retrieve selected matching rule.</li>
|
||||
<!-- TMPL_ELSE -->
|
||||
<li>Did not check for matches with existing records in catalogue</li>
|
||||
<!-- /TMPL_IF -->
|
||||
<!-- /TMPL_IF -->
|
||||
<li><!-- TMPL_VAR name="num_items" --> item records found and staged</li>
|
||||
<li><a href="/cgi-bin/koha/tools/manage-marc-import.pl?import_batch_id=<!-- TMPL_VAR name="import_batch_id" -->">Manage staged records</a></li>
|
||||
|
|
@ -55,12 +60,18 @@
|
|||
|
||||
</li>
|
||||
</ol></fieldset>
|
||||
<fieldset class="rows">
|
||||
<legend>Look for existing records in catalogue?</legend><ol><li class="radio">
|
||||
<input type="radio" id="check_for_matchesno" name="check_for_matches" value="0" checked="checked" /><label for="check_for_matchesno">Do not look for matching records (using ISBN)</label></li>
|
||||
<li class="radio">
|
||||
<input type="radio" id="check_for_matchesyes" name="check_for_matches" value="1" /><label for="check_for_matchesyes">Look for matching records</label>
|
||||
</li></ol></fieldset>
|
||||
<fieldset class="rows">
|
||||
<legend>Look for existing records in catalogue?</legend>
|
||||
<ol><li><label for="matcher">Record matching rule</label>
|
||||
<select name="matcher" id="matcher">
|
||||
<option value="">Do not look for matching records</option>
|
||||
<!-- TMPL_LOOP name="available_matchers" -->
|
||||
<option value="<!-- TMPL_VAR name="matcher_id" -->"><!-- TMPL_VAR name="code" --> (<!-- TMPL_VAR name="description" -->)
|
||||
</option>
|
||||
<!-- /TMPL_LOOP -->
|
||||
</select>
|
||||
</li></ol>
|
||||
</fieldset>
|
||||
<fieldset class="rows">
|
||||
<legend>Check for embedded item record data?</legend>
|
||||
<ol>
|
||||
|
|
|
|||
|
|
@ -62,7 +62,9 @@ sub process_batch {
|
|||
my $num_with_matches = 0;
|
||||
if ($match_bibs) {
|
||||
my $matcher = C4::Matcher->new('biblio');
|
||||
$matcher->add_matchpoint("020", "a", '', 'isbn', 1000);
|
||||
$matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, '');
|
||||
$matcher->add_simple_required_check('245', 'a', -1, 0, '',
|
||||
'245', 'a', -1, 0, '');
|
||||
print "... looking for matches with records already in database\n";
|
||||
$num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 100, \&print_progress);
|
||||
print "... finished looking for matches\n";
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ my $input = new CGI;
|
|||
my $dbh = C4::Context->dbh;
|
||||
|
||||
my $uploadmarc=$input->param('uploadmarc');
|
||||
my $check_for_matches = $input->param('check_for_matches');
|
||||
my $matcher_id = $input->param('matcher');
|
||||
my $parse_items = $input->param('parse_items');
|
||||
my $comments = $input->param('comments');
|
||||
my $syntax = $input->param('syntax');
|
||||
|
|
@ -84,13 +84,19 @@ if ($uploadmarc && length($uploadmarc)>0) {
|
|||
# FIXME branch code
|
||||
my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($syntax, $marcrecord, $filename,
|
||||
$comments, '', $parse_items, 0);
|
||||
my $matcher = C4::Matcher->new('biblio');
|
||||
$matcher->add_matchpoint("020", "a", '', 'isbn', 1000);
|
||||
my $num_with_matches = 0;
|
||||
my $checked_matches = 0;
|
||||
if ($check_for_matches) {
|
||||
$checked_matches = 1;
|
||||
$num_with_matches = BatchFindBibDuplicates($batch_id, $matcher);
|
||||
my $matcher_failed = 0;
|
||||
my $matcher_code = "";
|
||||
if ($matcher_id ne "") {
|
||||
my $matcher = C4::Matcher->fetch($matcher_id);
|
||||
if (defined $matcher) {
|
||||
$checked_matches = 1;
|
||||
$matcher_code = $matcher->code();
|
||||
$num_with_matches = BatchFindBibDuplicates($batch_id, $matcher);
|
||||
} else {
|
||||
$matcher_failed = 1;
|
||||
}
|
||||
}
|
||||
|
||||
$template->param(staged => $num_valid,
|
||||
|
|
@ -99,9 +105,15 @@ if ($uploadmarc && length($uploadmarc)>0) {
|
|||
import_errors => scalar(@import_errors),
|
||||
total => $num_valid + scalar(@import_errors),
|
||||
checked_matches => $checked_matches,
|
||||
matcher_failed => $matcher_failed,
|
||||
matcher_code => $matcher_code,
|
||||
import_batch_id => $batch_id
|
||||
);
|
||||
|
||||
} else {
|
||||
# initial form
|
||||
my @matchers = C4::Matcher::GetMatcherList();
|
||||
$template->param(available_matchers => \@matchers);
|
||||
}
|
||||
|
||||
output_html_with_http_headers $input, $cookie, $template->output;
|
||||
|
|
|
|||
Loading…
Reference in a new issue