From 139e6c30d6a88641148ea9ab7f00d0a0722f9fb5 Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Thu, 2 Feb 2017 13:36:38 +0100 Subject: [PATCH] Bug 14957: Merge rules system for merging of MARC records MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Add a rule based system for merging MARC records to for example prevent field data from being overwritten. To test: 1. Apply this patch. 2. Log in to staff client. 3. Enable new syspref MARCMergeRules. 4. Click the new link "MARC merge rules" in the "Catalog" section of the Koha administration page. 5. Create a new rule: Module: source, Filter: *, Tag: 245, Preset: Protect. 6. Clicking "Edit" should allow you to edit corresponding rule. 7. Clicking "Delete" should remove corresponding rule after confirmation. 8. Selecting one or more rules followed by clicking "Delete selected" should remove all selected rules after confirmation. 9. Try creating a rule with tag set to "**", the other options does not matter. Verify that saving this rule produces an error message complaining about invalid tag regular expression. 10. Try creating a rule with tag set to "008" (or other control field) and set Appended: Append and Removed: Skip, the other options does not matter. Verify that saving this rule produces an error message complaining about invalid combination of actions for control field. 11. With the 245 rule in step 5 in place, edit a bibliographic record, change 245a for example (which should be Title for MARC21) and save. 12. Verify that the changes has not been saved. 13. Create a new rule: Module: source, Filter: intranet, Tag: 245, Preset: Overwrite. 14. Repeat step 12, and verify that the changes has now been saved. 15. Run tests in t/db_dependent/Biblio/MarcMergeRules.t and very that all tests pass. Sponsored-by: Halland County Library Sponsored-by: Catalyst IT Sponsored-by: Gothenburg University Library Signed-off-by: David Nind Signed-off-by: Christian Stelzenmüller Signed-off-by: Martin Renvoize Signed-off-by: Tomas Cohen Arazi Signed-off-by: Jonathan Druart --- C4/Biblio.pm | 101 ++- C4/ImportBatch.pm | 2 +- Koha/BackgroundJob/BatchUpdateBiblio.pm | 8 +- Koha/Exceptions/MarcMergeRule.pm | 55 ++ Koha/MarcMergeRule.pm | 58 ++ Koha/MarcMergeRules.pm | 381 +++++++++ admin/marc-merge-rules.pl | 162 ++++ cataloguing/addbiblio.pl | 12 +- .../bug_14957-marc-merge-rules.perl | 41 + installer/data/mysql/kohastructure.sql | 30 + installer/data/mysql/mandatory/sysprefs.sql | 1 + .../data/mysql/mandatory/userpermissions.sql | 1 + .../prog/en/includes/admin-menu.inc | 3 + .../prog/en/includes/permissions.inc | 5 + .../prog/en/modules/admin/admin-home.tt | 2 + .../prog/en/modules/admin/marc-merge-rules.tt | 519 ++++++++++++ .../admin/preferences/cataloguing.pref | 7 + .../prog/en/modules/cataloguing/addbiblio.tt | 1 + misc/link_bibs_to_authorities.pl | 2 +- misc/migration_tools/bulkmarcimport.pl | 2 +- misc/migration_tools/import_lexile.pl | 2 +- t/db_dependent/Biblio.t | 2 +- t/db_dependent/Biblio/MarcMergeRules.t | 779 ++++++++++++++++++ tools/batch_record_modification.pl | 3 +- 24 files changed, 2164 insertions(+), 15 deletions(-) create mode 100644 Koha/Exceptions/MarcMergeRule.pm create mode 100644 Koha/MarcMergeRule.pm create mode 100644 Koha/MarcMergeRules.pm create mode 100755 admin/marc-merge-rules.pl create mode 100644 installer/data/mysql/atomicupdate/bug_14957-marc-merge-rules.perl create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-merge-rules.tt create mode 100755 t/db_dependent/Biblio/MarcMergeRules.t diff --git a/C4/Biblio.pm b/C4/Biblio.pm index 3e03c52501..dcce8a9b10 100644 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -61,6 +61,7 @@ BEGIN { DelBiblio BiblioAutoLink LinkBibHeadingsToAuthorities + ApplyMarcMergeRules TransformMarcToKoha TransformHtmlToMarc TransformHtmlToXml @@ -114,6 +115,7 @@ use Koha::SearchEngine; use Koha::SearchEngine::Indexer; use Koha::Libraries; use Koha::Util::MARC; +use Koha::MarcMergeRules; =head1 NAME @@ -310,7 +312,7 @@ sub AddBiblio { =head2 ModBiblio - ModBiblio( $record,$biblionumber,$frameworkcode, $disable_autolink); + ModBiblio($record, $biblionumber, $frameworkcode, $options); Replace an existing bib record identified by C<$biblionumber> with one supplied by the MARC::Record object C<$record>. The embedded @@ -326,16 +328,32 @@ in the C and C tables, as well as which fields are used to store embedded item, biblioitem, and biblionumber data for indexing. -Unless C<$disable_autolink> is passed ModBiblio will relink record headings +The C<$options> argument is a hashref with additional parameters: + +=over 4 + +=item C + +This parameter is forwared to L where it is used for +selecting the current rule set if Marc Merge Rules is enabled. +See L for more details. + +=item C + +Unless C is passed ModBiblio will relink record headings to authorities based on settings in the system preferences. This flag allows us to not relink records when the authority linker is saving modifications. +=back + Returns 1 on success 0 on failure =cut sub ModBiblio { - my ( $record, $biblionumber, $frameworkcode, $disable_autolink ) = @_; + my ( $record, $biblionumber, $frameworkcode, $options ) = @_; + $options //= {}; + if (!$record) { carp 'No record passed to ModBiblio'; return 0; @@ -346,7 +364,7 @@ sub ModBiblio { logaction( "CATALOGUING", "MODIFY", $biblionumber, "biblio BEFORE=>" . $newrecord->as_formatted ); } - if ( !$disable_autolink && C4::Context->preference('BiblioAddsAuthorities') ) { + if ( !$options->{disable_autolink} && C4::Context->preference('BiblioAddsAuthorities') ) { BiblioAutoLink( $record, $frameworkcode ); } @@ -367,6 +385,16 @@ sub ModBiblio { _strip_item_fields($record, $frameworkcode); + # apply merge rules + if (C4::Context->preference('MARCMergeRules') && $biblionumber && defined $options && exists $options->{'context'}) { + $record = ApplyMarcMergeRules({ + biblionumber => $biblionumber, + record => $record, + context => $options->{'context'}, + } + ); + } + # update biblionumber and biblioitemnumber in MARC # FIXME - this is assuming a 1 to 1 relationship between # biblios and biblioitems @@ -383,7 +411,7 @@ sub ModBiblio { _koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode ); # update the MARC record (that now contains biblio and items) with the new record data - &ModBiblioMarc( $record, $biblionumber ); + ModBiblioMarc( $record, $biblionumber ); # modify the other koha tables _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode ); @@ -2921,7 +2949,7 @@ sub _koha_delete_biblio_metadata { =head2 ModBiblioMarc - &ModBiblioMarc($newrec,$biblionumber); + ModBiblioMarc($newrec,$biblionumber); Add MARC XML data for a biblio to koha @@ -3233,8 +3261,67 @@ sub RemoveAllNsb { return $record; } -1; +=head2 ApplyMarcMergeRules + + my $record = ApplyMarcMergeRules($params) + +Applies marc merge rules to a record. + +C<$params> is expected to be a hashref with below keys defined. + +=over 4 + +=item C +biblionumber of old record + +=item C +Incoming record that will be merged with old record + +=item C +hashref containing at least one context module and filter value on +the form {module => filter, ...}. + +=back + +Returns: + +=over 4 + +=item C<$record> + +Merged MARC record based with merge rules for C applied. If no old +record for C can be found, C is returned unchanged. +Default action when no matching context is found to return C unchanged. +If no rules are found for a certain field tag the default is to overwrite with +fields with this field tag from C. + +=back + +=cut + +sub ApplyMarcMergeRules { + my ($params) = @_; + my $biblionumber = $params->{biblionumber}; + my $incoming_record = $params->{record}; + if (!$biblionumber) { + carp 'ApplyMarcMergeRules called on undefined biblionumber'; + return; + } + if (!$incoming_record) { + carp 'ApplyMarcMergeRules called on undefined record'; + return; + } + my $old_record = GetMarcBiblio({ biblionumber => $biblionumber }); + + # Skip merge rules if called with no context + if ($old_record && defined $params->{context}) { + return Koha::MarcMergeRules->merge_records($old_record, $incoming_record, $params->{context}); + } + return $incoming_record; +} + +1; =head2 _after_biblio_action_hooks diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index db26883354..daa6b7ba74 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -675,7 +675,7 @@ sub BatchCommitRecords { } $oldxml = $old_marc->as_xml($marc_type); - ModBiblio($marc_record, $recordid, $oldbiblio->frameworkcode); + ModBiblio($marc_record, $recordid, $oldbiblio->frameworkcode, {context => {source => 'batchimport'}}); $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; # FIXME call SetMatchedBiblionumber instead if ($item_result eq 'create_new' || $item_result eq 'replace') { diff --git a/Koha/BackgroundJob/BatchUpdateBiblio.pm b/Koha/BackgroundJob/BatchUpdateBiblio.pm index 34f732978a..028f269a0f 100644 --- a/Koha/BackgroundJob/BatchUpdateBiblio.pm +++ b/Koha/BackgroundJob/BatchUpdateBiblio.pm @@ -91,7 +91,13 @@ sub process { my $record = C4::Biblio::GetMarcBiblio({ biblionumber => $biblionumber }); C4::MarcModificationTemplates::ModifyRecordWithTemplate( $mmtid, $record ); my $frameworkcode = C4::Biblio::GetFrameworkCode( $biblionumber ); - C4::Biblio::ModBiblio( $record, $biblionumber, $frameworkcode ); + C4::Biblio::ModBiblio( $record, $biblionumber, $frameworkcode, + { + source => $args->{source}, + categorycode => $args->{categorycode}, + userid => $args->{userid}, + } + ); }; if ( $error and $error != 1 or $@ ) { # ModBiblio returns 1 if everything as gone well push @messages, { diff --git a/Koha/Exceptions/MarcMergeRule.pm b/Koha/Exceptions/MarcMergeRule.pm new file mode 100644 index 0000000000..a3cd708c45 --- /dev/null +++ b/Koha/Exceptions/MarcMergeRule.pm @@ -0,0 +1,55 @@ +package Koha::Exceptions::MarcMergeRule; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use Exception::Class ( + + 'Koha::Exceptions::MarcMergeRule' => { + description => 'Something went wrong!', + }, + 'Koha::Exceptions::MarcMergeRule::InvalidTagRegExp' => { + isa => 'Koha::Exceptions::MarcMergeRule', + description => 'Invalid regular expression for tag' + }, + 'Koha::Exceptions::MarcMergeRule::InvalidControlFieldActions' => { + isa => 'Koha::Exceptions::MarcMergeRule', + description => 'Invalid control field actions' + } +); + +=head1 NAME + +Koha::Exceptions::MarcMergeRule - Base class for MarcMergeRule exceptions + +=head1 Exceptions + +=head2 Koha::Exceptions::MarcMergeRule + +Generic MarcMergeRule exception + +=head2 Koha::Exceptions::MarcMergeRule::InvalidTagRegExp + +Exception for rule validation when rule tag is an invalid regular expression + +=head2 Koha::Exceptions::MarcMergeRule::InvalidControlFieldActions + +Exception for rule validation for control field rules with invalid combination of actions + +=cut + +1; diff --git a/Koha/MarcMergeRule.pm b/Koha/MarcMergeRule.pm new file mode 100644 index 0000000000..d1bdbbac18 --- /dev/null +++ b/Koha/MarcMergeRule.pm @@ -0,0 +1,58 @@ +package Koha::MarcMergeRule; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use parent qw(Koha::Object); + +my $cache = Koha::Caches->get_instance(); + +=head1 NAME + +Koha::MarcMergeRule - Koha MarcMergeRule Object class + +=cut + +=head2 store + +Override C to clear marc merge rules cache. + +=cut + +sub store { + my $self = shift @_; + $cache->clear_from_cache('marc_merge_rules'); + $self->SUPER::store(@_); +} + +=head2 delete + +Override C to clear marc merge rules cache. + +=cut + +sub delete { + my $self = shift @_; + $cache->clear_from_cache('marc_merge_rules'); + $self->SUPER::delete(@_); +} + +sub _type { + return 'MarcMergeRule'; +} + +1; diff --git a/Koha/MarcMergeRules.pm b/Koha/MarcMergeRules.pm new file mode 100644 index 0000000000..555402ff22 --- /dev/null +++ b/Koha/MarcMergeRules.pm @@ -0,0 +1,381 @@ +package Koha::MarcMergeRules; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use List::Util qw(first); +use Koha::MarcMergeRule; +use Carp; + +use Koha::Exceptions::MarcMergeRule; +use Try::Tiny; +use Scalar::Util qw(looks_like_number); + +use parent qw(Koha::Objects); + +my $cache = Koha::Caches->get_instance(); + +=head1 NAME + +Koha::MarcMergeRules - Koha MarcMergeRules Object set class + +=head1 API + +=head2 Class Methods + +=head3 operations + +Returns a list of all valid operations. + +=cut + +sub operations { + return ('add', 'append', 'remove', 'delete'); +} + +=head3 context_rules + + my $rules = Koha::MarcMergeRules->context_rules($context); + +Gets all MARC merge rules for the supplied C<$context> (hashref with { module => filter, ... } values). + +=cut + +sub context_rules { + my ($self, $context) = @_; + + return unless %{$context}; + + my $rules = $cache->get_from_cache('marc_merge_rules', { unsafe => 1 }); + + if (!$rules) { + $rules = {}; + my @rules_rows = $self->_resultset()->search( + undef, + { + order_by => { -desc => [qw/id/] } + } + ); + foreach my $rule_row (@rules_rows) { + my %rule = $rule_row->get_columns(); + my $operations = {}; + + foreach my $operation ($self->operations) { + $operations->{$operation} = { allow => $rule{$operation}, rule => $rule{id} }; + } + + # TODO: Remove unless check and validate on saving rules? + if ($rule{tag} eq '*') { + unless (exists $rules->{$rule{module}}->{$rule{filter}}->{'*'}) { + $rules->{$rule{module}}->{$rule{filter}}->{'*'} = $operations; + } + } + elsif ($rule{tag} =~ /^(\d{3})$/) { + unless (exists $rules->{$rule{module}}->{$rule{filter}}->{tags}->{$rule{tag}}) { + $rules->{$rule{module}}->{$rule{filter}}->{tags}->{$rule{tag}} = $operations; + } + } + else { + my $regexps = ($rules->{$rule{module}}->{$rule{filter}}->{regexps} //= []); + push @{$regexps}, [$rule{tag}, $operations]; + } + } + $cache->set_in_cache('marc_merge_rules', $rules); + } + + my $context_rules = undef; + foreach my $module_name (keys %{$context}) { + if ( + exists $rules->{$module_name} && + exists $rules->{$module_name}->{$context->{$module_name}} + ) { + $context_rules = $rules->{$module_name}->{$context->{$module_name}}; + last; + } + } + if (!$context_rules) { + # No perms matching specific context conditions found, try wildcard value for each active context + foreach my $module_name (keys %{$context}) { + if (exists $rules->{$module_name}->{'*'}) { + $context_rules = $rules->{$module_name}->{'*'}; + last; + } + } + } + return $context_rules; +} + +=head3 merge_records + + my $merged_record = Koha::MarcMergeRules->merge_records($old_record, $incoming_record, $context); + +Merge C<$old_record> with C<$incoming_record> applying merge rules for C<$context>. +Returns merged record C<$merged_record>. C<$old_record>, C<$incoming_record> and +C<$merged_record> are all MARC::Record objects. + +=cut + +sub merge_records { + my ($self, $old_record, $incoming_record, $context) = @_; + + my $rules = $self->context_rules($context); + + # Default when no rules found is to overwrite with incoming record + return $incoming_record unless $rules; + + my $fields_by_tag = sub { + my ($record) = @_; + my $fields = {}; + foreach my $field ($record->fields()) { + $fields->{$field->tag()} //= []; + push @{$fields->{$field->tag()}}, $field; + } + return $fields; + }; + + my $hash_field_data = sub { + my ($field) = @_; + my $indicators = join("\x1E", map { $field->indicator($_) } (1, 2)); + return $indicators . "\x1E" . join("\x1E", sort map { join "\x1E", @{$_} } $field->subfields()); + }; + + my $diff_by_key = sub { + my ($a, $b) = @_; + my @removed; + my @intersecting; + my @added; + my %keys_index = map { $_ => undef } (keys %{$a}, keys %{$b}); + foreach my $key (keys %keys_index) { + if ($a->{$key} && $b->{$key}) { + push @intersecting, $a->{$key}; + } + elsif ($a->{$key}) { + push @removed, $a->{$key}; + } + else { + push @added, $b->{$key}; + } + } + return (\@removed, \@intersecting, \@added); + }; + + my $tag_rules = $rules->{tags} // {}; + my $default_rule = $rules->{'*'} // { + add => { allow => 1, 'rule' => 0}, + append => { allow => 1, 'rule' => 0}, + delete => { allow => 1, 'rule' => 0}, + remove => { allow => 1, 'rule' => 0}, + }; + + # Precompile regexps + my @regexp_rules = map { { regexp => qr/^$_->[0]$/, actions => $_->[1] } } @{$rules->{regexps} // []}; + + my $get_matching_field_rule = sub { + my ($tag) = @_; + # Exact match takes precedence, then regexp, then wildcard/defaults + return $tag_rules->{$tag} // + %{(first { $tag =~ $_->{regexp} } @regexp_rules) // {}}{actions} // + $default_rule; + }; + + my %merged_record_fields; + + my $current_fields = $fields_by_tag->($old_record); + my $incoming_fields = $fields_by_tag->($incoming_record); + + # First we get all new incoming fields + my @new_field_tags = grep { !(exists $current_fields->{$_}) } keys %{$incoming_fields}; + foreach my $tag (@new_field_tags) { + my $rule = $get_matching_field_rule->($tag); + if ($rule->{add}->{allow}) { + $merged_record_fields{$tag} //= []; + push @{$merged_record_fields{$tag}}, @{$incoming_fields->{$tag}}; + } + } + + # Then we get all fields no longer present in incoming fields + my @deleted_field_tags = grep { !(exists $incoming_fields->{$_}) } keys %{$current_fields}; + foreach my $tag (@deleted_field_tags) { + my $rule = $get_matching_field_rule->($tag); + if (!$rule->{delete}->{allow}) { + $merged_record_fields{$tag} //= []; + push @{$merged_record_fields{$tag}}, @{$current_fields->{$tag}}; + } + } + + # Then we get the intersection of fields, present both in + # current and incoming record (possibly to be overwritten) + my @common_field_tags = grep { exists $incoming_fields->{$_} } keys %{$current_fields}; + foreach my $tag (@common_field_tags) { + my $rule = $get_matching_field_rule->($tag); + + # Special handling for control fields + if ($tag < 10) { + if ( + $rule->{append}->{allow} && + !$rule->{remove}->{allow} + ) { + # This should be highly unlikely since we have input validation to protect against this case + carp "Allowing \"append\" and skipping \"remove\" is not permitted for control fields, falling back to skipping both \"append\" and \"remove\""; + push @{$merged_record_fields{$tag}}, @{$current_fields->{$tag}}; + } + elsif ($rule->{append}->{allow}) { + push @{$merged_record_fields{$tag}}, @{$incoming_fields->{$tag}}; + } + else { + push @{$merged_record_fields{$tag}}, @{$current_fields->{$tag}}; + } + } + else { + # Compute intersection and diff using field data + my $sort_weight = 0; + my %current_fields_by_data = map { $hash_field_data->($_) => [$sort_weight++, $_] } @{$current_fields->{$tag}}; + + # Always put incoming fields after current fields + my %incoming_fields_by_data = map { $hash_field_data->($_) => [$sort_weight++, $_] } @{$incoming_fields->{$tag}}; + + my ($current_fields_only, $common_fields, $incoming_fields_only) = $diff_by_key->(\%current_fields_by_data, \%incoming_fields_by_data); + + my @merged_fields; + + # First add common fields (intersection) + # Unchanged + if (@{$common_fields}) { + push @merged_fields, @{$common_fields}; + } + # Removed + if (@{$current_fields_only}) { + if (!$rule->{remove}->{allow}) { + push @merged_fields, @{$current_fields_only}; + } + } + # Appended + if (@{$incoming_fields_only}) { + if ($rule->{append}->{allow}) { + push @merged_fields, @{$incoming_fields_only}; + } + } + $merged_record_fields{$tag} //= []; + + # Sort ascending according to weight (original order) + push @{$merged_record_fields{$tag}}, map { $_->[1] } sort { $a->[0] <=> $b->[0] } @merged_fields; + } + } + + my $merged_record = MARC::Record->new(); + + # Leader is always overwritten, or kept??? + $merged_record->leader($incoming_record->leader()); + + if (%merged_record_fields) { + foreach my $tag (sort keys %merged_record_fields) { + $merged_record->append_fields(@{$merged_record_fields{$tag}}); + } + } + return $merged_record; +} + +sub _clear_caches { + $cache->clear_from_cache('marc_merge_rules'); +} + +=head2 find_or_create + +Override C to clear marc merge rules cache. + +=cut + +sub find_or_create { + my $self = shift @_; + $self->_clear_caches(); + return $self->SUPER::find_or_create(@_); +} + +=head2 update + +Override C to clear marc merge rules cache. + +=cut + +sub update { + my $self = shift @_; + $self->_clear_caches(); + return $self->SUPER::update(@_); +} + +=head2 delete + +Override C to clear marc merge rules cache. + +=cut + +sub delete { + my $self = shift @_; + $self->_clear_caches(); + return $self->SUPER::delete(@_); +} + +=head2 validate + + Koha::MarcMergeRules->validate($rule_data); + +Validates C<$rule_data>. Throws C +if C<$rule_data->{tag}> contains an invalid regular expression. Throws +C if contains invalid +combination of actions for control fields. Otherwise returns true. + +=cut + +sub validate { + my ($self, $rule_data) = @_; + + if(exists $rule_data->{tag}) { + if ($rule_data->{tag} ne '*') { + eval { qr/$rule_data->{tag}/ }; + if ($@) { + Koha::Exceptions::MarcMergeRule::InvalidTagRegExp->throw( + "Invalid tag regular expression" + ); + } + } + # TODO: Regexp or '*' that match controlfield not currently detected + if ( + looks_like_number($rule_data->{tag}) && + $rule_data->{tag} < 10 && + $rule_data->{append} && + !$rule_data->{remove} + ) { + Koha::Exceptions::MarcMergeRule::InvalidControlFieldActions->throw( + "Combination of allow append and skip remove not permitted for control fields" + ); + } + } + return 1; +} + +sub _type { + return 'MarcMergeRule'; +} + +=head3 object_class + +=cut + +sub object_class { + return 'Koha::MarcMergeRule'; +} + +1; diff --git a/admin/marc-merge-rules.pl b/admin/marc-merge-rules.pl new file mode 100755 index 0000000000..0c9a5433fe --- /dev/null +++ b/admin/marc-merge-rules.pl @@ -0,0 +1,162 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + +# standard or CPAN modules used +use CGI qw ( -utf8 ); +use CGI::Cookie; +use MARC::File::USMARC; +use Try::Tiny; + +# Koha modules used +use C4::Context; +use C4::Koha; +use C4::Auth; +use C4::AuthoritiesMarc; +use C4::Output; +use C4::Biblio; +use C4::ImportBatch; +use C4::Matcher; +use C4::BackgroundJob; +use C4::Labels::Batch; +use Koha::MarcMergeRules; +use Koha::MarcMergeRule; +use Koha::Patron::Categories; # TODO: Required? Try without use + +my $script_name = "/cgi-bin/koha/admin/marc-merge-rules.pl"; + +my $input = new CGI; +my $op = $input->param('op') || ''; +my $errors = []; + +my $rule_from_cgi = sub { + my ($cgi) = @_; + + my %rule = map { $_ => scalar $cgi->param($_) } ( + 'tag', + 'module', + 'filter', + 'add', + 'append', + 'remove', + 'delete' + ); + + my $id = $cgi->param('id'); + if ($id) { + $rule{id} = $id; + } + + return \%rule; +}; + +my ($template, $loggedinuser, $cookie) = get_template_and_user( + { + template_name => "admin/marc-merge-rules.tt", + query => $input, + type => "intranet", + authnotrequired => 0, + flagsrequired => { parameters => 'manage_marc_merge_rules' }, + debug => 1, + } +); + +$template->param(script_name => $script_name); + +my %cookies = parse CGI::Cookie($cookie); +our $sessionID = $cookies{'CGISESSID'}->value; + +my $get_rules = sub { + # TODO: order? + return [map { { $_->get_columns() } } Koha::MarcMergeRules->_resultset->all]; +}; +my $rules; + +if ($op eq 'remove' || $op eq 'doremove') { + my @remove_ids = $input->multi_param('batchremove'); + push @remove_ids, scalar $input->param('id') if $input->param('id'); + if ($op eq 'remove') { + $template->{VARS}->{removeConfirm} = 1; + my %remove_ids = map { $_ => undef } @remove_ids; + $rules = $get_rules->(); + for my $rule (@{$rules}) { + $rule->{'removemarked'} = 1 if exists $remove_ids{$rule->{id}}; + } + } + elsif ($op eq 'doremove') { + my @remove_ids = $input->multi_param('batchremove'); + push @remove_ids, scalar $input->param('id') if $input->param('id'); + Koha::MarcMergeRules->search({ id => { in => \@remove_ids } })->delete(); + $rules = $get_rules->(); + } +} +elsif ($op eq 'edit') { + $template->{VARS}->{edit} = 1; + my $id = $input->param('id'); + $rules = $get_rules->(); + for my $rule(@{$rules}) { + if ($rule->{id} == $id) { + $rule->{'edit'} = 1; + last; + } + } +} +elsif ($op eq 'doedit' || $op eq 'add') { + my $rule_data = $rule_from_cgi->($input); + if (!@{$errors}) { + try { + Koha::MarcMergeRules->validate($rule_data); + } + catch { + die $_ unless blessed $_ && $_->can('rethrow'); + + if ($_->isa('Koha::Exceptions::MarcMergeRule::InvalidTagRegExp')) { + push @{$errors}, { + type => 'error', + code => 'invalid_tag_regexp', + tag => $rule_data->{tag}, + }; + } + elsif ($_->isa('Koha::Exceptions::MarcMergeRule::InvalidControlFieldActions')) { + push @{$errors}, { + type => 'error', + code => 'invalid_control_field_actions', + tag => $rule_data->{tag}, + }; + } + else { + $_->rethrow; + } + }; + if (!@{$errors}) { + my $rule = Koha::MarcMergeRules->find_or_create($rule_data); + # Need to call set and store here in case we have an update + $rule->set($rule_data); + $rule->store(); + } + $rules = $get_rules->(); + } +} +else { + $rules = $get_rules->(); +} + +my $categorycodes = Koha::Patron::Categories->search_limited({}, {order_by => ['description']}); +$template->param( rules => $rules, categorycodes => $categorycodes, messages => $errors ); + +output_html_with_http_headers $input, $cookie, $template->output; diff --git a/cataloguing/addbiblio.pl b/cataloguing/addbiblio.pl index 903b30b26f..9a50ce0aa9 100755 --- a/cataloguing/addbiblio.pl +++ b/cataloguing/addbiblio.pl @@ -51,6 +51,7 @@ use Koha::ItemTypes; use Koha::Libraries; use Koha::BiblioFrameworks; +use Koha::Patrons; use MARC::File::USMARC; use MARC::File::XML; @@ -864,7 +865,15 @@ if ( $op eq "addbiblio" ) { if ( !$duplicatebiblionumber or $confirm_not_duplicate ) { my $oldbibitemnum; if ( $is_a_modif ) { - ModBiblio( $record, $biblionumber, $frameworkcode ); + my $member = Koha::Patrons->find($loggedinuser); + ModBiblio( $record, $biblionumber, $frameworkcode, { + context => { + source => $z3950 ? 'z39.50' : 'intranet', + categorycode => $member->categorycode, + userid => $member->userid + } + } + ); } else { ( $biblionumber, $oldbibitemnum ) = AddBiblio( $record, $frameworkcode ); @@ -957,6 +966,7 @@ elsif ( $op eq "delete" ) { $template->param( biblionumberdata => $biblionumber, op => $op, + z3950 => $z3950 ); if ( $op eq "duplicate" ) { $biblionumber = ""; diff --git a/installer/data/mysql/atomicupdate/bug_14957-marc-merge-rules.perl b/installer/data/mysql/atomicupdate/bug_14957-marc-merge-rules.perl new file mode 100644 index 0000000000..e6f1c69c6c --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_14957-marc-merge-rules.perl @@ -0,0 +1,41 @@ +$DBversion = 'XXX'; # will be replaced by the RM +if( CheckVersion( $DBversion ) ) { + my $sql = q{ + CREATE TABLE IF NOT EXISTS `marc_merge_rules` ( + `id` int(11) NOT NULL auto_increment, + `tag` varchar(255) NOT NULL, + `module` varchar(127) NOT NULL, + `filter` varchar(255) NOT NULL, + `add` tinyint NOT NULL, + `append` tinyint NOT NULL, + `remove` tinyint NOT NULL, + `delete` tinyint NOT NULL, + PRIMARY KEY(`id`) + ); + }; + $dbh->do( $sql ); + + $sql = q{ + INSERT IGNORE INTO systempreferences (`variable`, `value`, `options`, `explanation`, `type`) VALUES ( + 'MARCMergeRules', + '0', + NULL, + 'Use the MARC merge rules system to decide what actions to take for each field when modifying records.', + 'YesNo' + ); + }; + $dbh->do( $sql ); + + $sql = q{ + INSERT IGNORE INTO permissions (module_bit, code, description) VALUES ( + 3, + 'manage_marc_merge_rules', + 'Manage MARC merge rules configuration' + ); + }; + $dbh->do( $sql ); + + # Always end with this (adjust the bug info) + SetVersion( $DBversion ); + print "Upgrade to $DBversion done (Bug 14957 - Write protecting MARC fields based on source of import)\n"; +} diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index 48e5370448..fbe01ea66a 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -3385,6 +3385,36 @@ CREATE TABLE `marc_matchers` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; +-- +-- Table structure for table `marc_merge_rules_modules` +-- + +DROP TABLE IF EXISTS `marc_merge_rules_modules`; +CREATE TABLE `marc_merge_rules_modules` ( + `name` varchar(127) NOT NULL, + `description` varchar(255), + `specificity` int(11) NOT NULL UNIQUE, + PRIMARY KEY(`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- +-- Table structure for table `marc_merge_rules` +-- + +DROP TABLE IF EXISTS `marc_merge_rules`; +CREATE TABLE IF NOT EXISTS `marc_merge_rules` ( + `id` int(11) NOT NULL auto_increment, + `tag` varchar(255) NOT NULL, -- can be regexp, so need > 3 chars + `module` varchar(127) NOT NULL, + `filter` varchar(255) NOT NULL, + `add` tinyint NOT NULL, + `append` tinyint NOT NULL, + `remove` tinyint NOT NULL, + `delete` tinyint NOT NULL, + PRIMARY KEY(`id`), + CONSTRAINT `marc_merge_rules_ibfk1` FOREIGN KEY (`module`) REFERENCES `marc_merge_rules_modules` (`name`) ON DELETE CASCADE ON UPDATE CASCADE +); + -- -- Table structure for table `marc_modification_template_actions` -- diff --git a/installer/data/mysql/mandatory/sysprefs.sql b/installer/data/mysql/mandatory/sysprefs.sql index 8d16bdc566..e15cc15500 100644 --- a/installer/data/mysql/mandatory/sysprefs.sql +++ b/installer/data/mysql/mandatory/sysprefs.sql @@ -331,6 +331,7 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('MarcFieldForModifierName','',NULL,'Where to store the name of the record''s last modifier','Free'), ('MarcFieldsToOrder','',NULL,'Set the mapping values for a new order line created from a MARC record in a staged file. In a YAML format.','textarea'), ('MarcItemFieldsToOrder','',NULL,'Set the mapping values for new item records created from a MARC record in a staged file. In a YAML format.','textarea'), +('MARCMergeRules','0',NULL,'Use the MARC merge rules system to decide what actions to take for each field when modifying records.','YesNo'), ('MarkLostItemsAsReturned','batchmod,moredetail,cronjob,additem,pendingreserves,onpayment','claim_returned|batchmod|moredetail|cronjob|additem|pendingreserves|onpayment','Mark items as returned when flagged as lost','multiple'), ('MARCOrgCode','OSt','','Define MARC Organization Code for MARC21 records - http://www.loc.gov/marc/organizations/orgshome.html','free'), ('MaxFine',NULL,'','Maximum fine a patron can have for all late returns at one moment. Single item caps are specified in the circulation rules matrix.','Integer'), diff --git a/installer/data/mysql/mandatory/userpermissions.sql b/installer/data/mysql/mandatory/userpermissions.sql index f1c2d8883b..581aaf2a5e 100644 --- a/installer/data/mysql/mandatory/userpermissions.sql +++ b/installer/data/mysql/mandatory/userpermissions.sql @@ -25,6 +25,7 @@ INSERT INTO permissions (module_bit, code, description) VALUES ( 3, 'manage_oai_sets', 'Manage OAI sets'), ( 3, 'manage_item_search_fields', 'Manage item search fields'), ( 3, 'manage_search_engine_config', 'Manage search engine configuration'), + ( 3, 'manage_marc_merge_rules', 'Manage MARC merge rules configuration'), ( 3, 'manage_search_targets', 'Manage Z39.50 and SRU server configuration'), ( 3, 'manage_didyoumean', 'Manage Did you mean? configuration'), ( 3, 'manage_column_config', 'Manage column configuration'), diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc index 421f59a158..68a005eecc 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc @@ -100,6 +100,9 @@ [% IF ( CAN_user_parameters_manage_search_engine_config ) %]
  • Search engine configuration (Elasticsearch)
    • [% END %] + [% IF ( CAN_user_parameters_manage_marc_merge_rules ) %] +
  • MARC merge rules
    • + [% END %] [% END %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/permissions.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/permissions.inc index cc253be52e..76f39a1af0 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/permissions.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/permissions.inc @@ -210,6 +210,11 @@ Manage search engine configuration ([% name | html %]) + [%- CASE 'manage_marc_merge_rules' -%] + + Manage MARC merge rules configuration + + ([% name | html %]) [%- CASE 'manage_search_targets' -%] Manage Z39.50 and SRU server configuration diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt index 57ed2854a5..fa3b3fb6f7 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt @@ -192,6 +192,8 @@
    Search engine configuration (Elasticsearch)
    Manage indexes, facets, and their mappings to MARC fields and subfields
    [% END %] +
    MARC merge rules
    +
    Managed MARC field merge rules
    [% END %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-merge-rules.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-merge-rules.tt new file mode 100644 index 0000000000..fe4282f8b5 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-merge-rules.tt @@ -0,0 +1,519 @@ +[% USE Koha %] +[% INCLUDE 'doc-head-open.inc' %] +Koha › Administration › MARC merge rules +[% INCLUDE 'doc-head-close.inc' %] +[% Asset.css("css/datatables.css") %] +[% INCLUDE 'datatables.inc' %] + + + + + + +[% INCLUDE 'header.inc' %] +[% INCLUDE 'cat-search.inc' %] + + + +
    +
    +
    + +

    Manage MARC merge rules

    + +[% FOR m IN messages %] +
    + [% SWITCH m.code %] + [% CASE 'invalid_tag_regexp' %] + Invalid regular expression "[% m.tag | html %]". + [% CASE 'invalid_control_field_actions' %] + Invalid combination of actions for tag [% m.tag | html %]. Control field rules do not allow "Appended: Append" and "Removed: Skip". + [% CASE %] + [% m.code | html %] + [% END %] +
    +[% END %] + +[% UNLESS Koha.Preference( 'MARCMergeRules' ) %] +
    + The MARCMergeRules preference is not set, don't forget to enable it for rules to take effect. +
    +[% END %] +[% IF removeConfirm %] +
    +

    Remove rule?

    +

    Are you sure you want to remove the selected rule(s)?

    + +
    + +
    + +
    +[% END %] + +
    + + + + + + + + + + + + + + + [% UNLESS edit %] + + + + + + + + + + + + + + + + [% END %] + + [% FOREACH rule IN rules %] + + [% IF rule.edit %] + + + + + + + + + + + + [% ELSE %] + + + + + + + + + + + + [% END %] + + [% END %] + +
    RuleModuleFilterTagPresetAddedAppendedRemovedDeletedActions 
      + + + + + + + + + + + +
    [% rule.id %] + + + + + + + + + + + + + + + [% rule.id | html %][% rule.module | html %][% rule.filter | html %][% rule.tag | html %][% IF rule.add %]Add[% ELSE %]Skip[% END %][% IF rule.append %]Append[% ELSE %]Skip[% END %][% IF rule.remove %]Remove[% ELSE %]Skip[% END %][% IF rule.delete %]Delete[% ELSE %]Skip[% END %] + Delete + Edit + + [% IF rule.removemarked %] + + [% ELSE %] + + [% END %] +
    +
    + +
    + +
    + +
    + + +
    + +
    + +
    + + +[% INCLUDE 'intranet-bottom.inc' %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/cataloguing.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/cataloguing.pref index 50f35083ec..725f6b6192 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/cataloguing.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/cataloguing.pref @@ -325,3 +325,10 @@ Cataloging: - "All values of repeating tags and subfields will be printed with the given RIS tag." - "
    " - "Use of TY ( record type ) as a key will replace the default TY with the field value of your choosing." + - + - When importing records + - pref: MARCMergeRules + choices: + yes: "use" + no: "don't use" + - MARC merge rules to decide which action to take for each field. diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbiblio.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbiblio.tt index 2d158edac2..2887123012 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbiblio.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbiblio.tt @@ -925,6 +925,7 @@ function PopupMARCFieldDoc(field) { [% END %] + diff --git a/misc/link_bibs_to_authorities.pl b/misc/link_bibs_to_authorities.pl index 21e4cc31bb..dc3de81816 100755 --- a/misc/link_bibs_to_authorities.pl +++ b/misc/link_bibs_to_authorities.pl @@ -233,7 +233,7 @@ sub process_bib { ); } if ( not $test_only ) { - ModBiblio( $bib, $biblionumber, $frameworkcode, 1 ); + ModBiblio( $bib, $biblionumber, $frameworkcode, { disable_autolink => 1 }); #Last param is to note ModBiblio was called from linking script and bib should not be linked again $num_bibs_modified++; } diff --git a/misc/migration_tools/bulkmarcimport.pl b/misc/migration_tools/bulkmarcimport.pl index 8a9405d727..7a1e800b00 100755 --- a/misc/migration_tools/bulkmarcimport.pl +++ b/misc/migration_tools/bulkmarcimport.pl @@ -450,7 +450,7 @@ RECORD: while ( ) { $biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber; }; if ($update) { - eval { ModBiblio( $record, $biblionumber, $framework ) }; + eval { ModBiblio( $record, $biblionumber, $framework, { context => { source => 'bulkmarcimport' } } ) }; if ($@) { warn "ERROR: Edit biblio $biblionumber failed: $@\n"; printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile); diff --git a/misc/migration_tools/import_lexile.pl b/misc/migration_tools/import_lexile.pl index 265b07a8eb..7c81af971f 100755 --- a/misc/migration_tools/import_lexile.pl +++ b/misc/migration_tools/import_lexile.pl @@ -203,7 +203,7 @@ while ( my $row = $csv->getline_hr($fh) ) { $record->append_fields($field); } - ModBiblio( $record, $biblionumber ) unless ( $test ); + ModBiblio( $record, $biblionumber, undef, { context => { source => 'import_lexile' } } ) unless ( $test ); } } diff --git a/t/db_dependent/Biblio.t b/t/db_dependent/Biblio.t index 642207b580..b3d971d833 100755 --- a/t/db_dependent/Biblio.t +++ b/t/db_dependent/Biblio.t @@ -731,7 +731,7 @@ subtest 'ModBiblio called from linker test' => sub { C4::Biblio::ModBiblio($record,$biblionumber,''); is($called,1,"We called to link bibs because not from linker"); $called = 0; - C4::Biblio::ModBiblio($record,$biblionumber,'',1); + C4::Biblio::ModBiblio($record,$biblionumber,'',{ disable_autolink => 1 }); is($called,0,"We didn't call to link bibs because from linker"); }; diff --git a/t/db_dependent/Biblio/MarcMergeRules.t b/t/db_dependent/Biblio/MarcMergeRules.t new file mode 100755 index 0000000000..cf178063d8 --- /dev/null +++ b/t/db_dependent/Biblio/MarcMergeRules.t @@ -0,0 +1,779 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; +use Try::Tiny; + +use MARC::Record; + +use C4::Context; +use C4::Biblio; +use Koha::Database; #?? + +use Test::More tests => 23; +use Test::MockModule; + +use Koha::MarcMergeRules; + +use t::lib::Mocks; + +my $schema = Koha::Database->schema; +$schema->storage->txn_begin; + +t::lib::Mocks::mock_preference('MARCMergeRules', '1'); + +# Create a record +my $orig_record = MARC::Record->new(); +$orig_record->append_fields ( + MARC::Field->new('250', '','', 'a' => '250 bottles of beer on the wall'), + MARC::Field->new('250', '','', 'a' => '256 bottles of beer on the wall'), + MARC::Field->new('500', '','', 'a' => 'One bottle of beer in the fridge'), +); + +my $incoming_record = MARC::Record->new(); +$incoming_record->append_fields( + MARC::Field->new('250', '', '', 'a' => '256 bottles of beer on the wall'), # Unchanged + MARC::Field->new('250', '', '', 'a' => '251 bottles of beer on the wall'), # Appended + # MARC::Field->new('250', '', '', 'a' => '250 bottles of beer on the wall'), # Removed + # MARC::Field->new('500', '', '', 'a' => 'One bottle of beer in the fridge'), # Deleted + MARC::Field->new('501', '', '', 'a' => 'One cold bottle of beer in the fridge'), # Added + MARC::Field->new('501', '', '', 'a' => 'Two cold bottles of beer in the fridge'), # Added +); + +# Test default behavior when MARCMergeRules is enabled, but no rules defined (overwrite) +subtest 'Record fields has been overwritten when no merge rules are defined' => sub { + plan tests => 4; + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + my @fields = $merged_record->field('500'); + cmp_ok(scalar @fields, '==', 0, '"500" field has been deleted'); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +my $rule = Koha::MarcMergeRules->find_or_create({ + tag => '*', + module => 'source', + filter => '*', + add => 0, + append => 0, + remove => 0, + delete => 0 +}); + +subtest 'Record fields has been protected when matched merge all rule operations are set to "0"' => sub { + plan tests => 3; + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall'], + '"250" fields has retained their original value' + ); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); +}; + +subtest 'Only new fields has been added when add = 1, append = 0, remove = 0, delete = 0' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 0, + 'remove' => 0, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 5, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall'], + '"250" fields retain their original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field retain it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Only appended fields has been added when add = 0, append = 1, remove = 0, delete = 0' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 0, + 'append' => 1, + 'remove' => 0, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"251" field has been appended' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + +}; + +subtest 'Appended and added fields has been added when add = 1, append = 1, remove = 0, delete = 0' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 1, + 'remove' => 0, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 6, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"251" field has been appended' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been only removed when add = 0, append = 0, remove = 1, delete = 0' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 0, + 'append' => 0, + 'remove' => 1, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 2, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall'], + '"250" field has been removed' + ); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); +}; + +subtest 'Record fields has been added and removed when add = 1, append = 0, remove = 1, delete = 0' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 0, + 'remove' => 1, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall'], + '"250" field has been removed' + ); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been appended and removed when add = 0, append = 1, remove = 1, delete = 0' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 0, + 'append' => 1, + 'remove' => 1, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); +}; + +subtest 'Record fields has been added, appended and removed when add = 0, append = 1, remove = 1, delete = 0' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 1, + 'remove' => 1, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 5, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been deleted when add = 0, append = 0, remove = 0, delete = 1' => sub { + plan tests => 2; + + $rule->set( + { + 'add' => 0, + 'append' => 0, + 'remove' => 0, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 2, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall'], + '"250" fields has retained their original value' + ); +}; + +subtest 'Record fields has been added and deleted when add = 1, append = 0, remove = 0, delete = 1' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 1, + 'append' => 0, + 'remove' => 0, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall'], + '"250" fields has retained their original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been appended and deleted when add = 0, append = 1, remove = 0, delete = 1' => sub { + plan tests => 2; + + $rule->set( + { + 'add' => 0, + 'append' => 1, + 'remove' => 0, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" field has been appended' + ); +}; + +subtest 'Record fields has been added, appended and deleted when add = 1, append = 1, remove = 0, delete = 1' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 1, + 'append' => 1, + 'remove' => 0, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 5, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" field has been appended' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been removed and deleted when add = 0, append = 0, remove = 1, delete = 1' => sub { + plan tests => 2; + + $rule->set( + { + 'add' => 0, + 'append' => 0, + 'remove' => 1, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 1, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall'], + '"250" field has been removed' + ); +}; + +subtest 'Record fields has been added, removed and deleted when add = 1, append = 0, remove = 1, delete = 1' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 1, + 'append' => 0, + 'remove' => 1, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall'], + '"250" field has been removed' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been appended, removed and deleted when add = 0, append = 1, remove = 1, delete = 1' => sub { + plan tests => 2; + + $rule->set( + { + 'add' => 0, + 'append' => 1, + 'remove' => 1, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 2, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); +}; + +subtest 'Record fields has been overwritten when add = 1, append = 1, remove = 1, delete = 1' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 1, + 'remove' => 1, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + my @fields = $merged_record->field('500'); + cmp_ok(scalar @fields, '==', 0, '"500" field has been deleted'); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +# Test rule tag specificity + +# Protect field 500 with more specific tag value +my $skip_all_rule = Koha::MarcMergeRules->find_or_create({ + tag => '500', + module => 'source', + filter => '*', + add => 0, + append => 0, + remove => 0, + delete => 0 +}); + +subtest '"500" field has been protected when rule matching on tag "500" is add = 0, append = 0, remove = 0, delete = 0' => sub { + plan tests => 4; + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 5, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +# Test regexp matching +subtest '"5XX" fields has been protected when rule matching on regexp "5\d{2}" is add = 0, append = 0, remove = 0, delete = 0' => sub { + plan tests => 3; + + $skip_all_rule->set( + { + 'tag' => '5\d{2}', + } + ); + $skip_all_rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); +}; + +# Test module specificity, the 0 all rule should no longer be included in set of applied rules +subtest 'Record fields has been overwritten when non wild card rule with filter match is add = 1, append = 1, remove = 1, delete = 1' => sub { + plan tests => 4; + + $rule->set( + { + 'filter' => 'test', + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { 'source' => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + my @fields = $merged_record->field('500'); + cmp_ok(scalar @fields, '==', 0, '"500" field has been deleted'); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'An exception is thrown when append = 1, remove = 0 is set for control field rule' => sub { + plan tests => 2; + my $exception = try { + Koha::MarcMergeRules->validate({ + 'tag' => '008', + 'append' => 1, + 'remove' => 0, + }); + } + catch { + return $_; + }; + ok(defined $exception, "Exception was caught"); + ok($exception->isa('Koha::Exceptions::MarcMergeRule::InvalidControlFieldActions'), "Exception is of correct class"); +}; + +subtest 'An exception is thrown when rule tag is set to invalid regexp' => sub { + plan tests => 2; + + my $exception = try { + Koha::MarcMergeRules->validate({ + 'tag' => '**' + }); + } + catch { + return $_; + }; + ok(defined $exception, "Exception was caught"); + ok($exception->isa('Koha::Exceptions::MarcMergeRule::InvalidTagRegExp'), "Exception is of correct class"); +}; + +$skip_all_rule->delete(); + +subtest 'context option in ModBiblio is handled correctly' => sub { + plan tests => 6; + $rule->set( + { + tag => '250', + module => 'source', + filter => '*', + 'add' => 0, + 'append' => 0, + 'remove' => 0, + 'delete' => 0, + } + ); + $rule->store(); + my ($biblionumber) = AddBiblio($orig_record, ''); + + # Since marc merc rules are not run on save, only update + # saved record should be identical to orig_record + my $saved_record = GetMarcBiblio({ biblionumber => $biblionumber }); + + my @all_fields = $saved_record->fields(); + # Koha also adds 999c field, therefore 4 not 3 + cmp_ok(scalar @all_fields, '==', 4, 'Saved record has the expected number of fields'); + is_deeply( + [map { $_->subfield('a') } $saved_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall'], + 'All "250" fields of saved record are identical to original record passed to AddBiblio' + ); + + is_deeply( + [map { $_->subfield('a') } $saved_record->field('500') ], + ['One bottle of beer in the fridge'], + 'All "500" fields of saved record are identical to original record passed to AddBiblio' + ); + + $saved_record->append_fields( + MARC::Field->new('250', '', '', 'a' => '251 bottles of beer on the wall'), # Appended + MARC::Field->new('500', '', '', 'a' => 'One cold bottle of beer in the fridge'), # Appended + ); + + ModBiblio($saved_record, $biblionumber, '', { context => { 'source' => 'test' } }); + + my $updated_record = GetMarcBiblio({ biblionumber => $biblionumber }); + + @all_fields = $updated_record->fields(); + cmp_ok(scalar @all_fields, '==', 5, 'Updated record has the expected number of fields'); + is_deeply( + [map { $_->subfield('a') } $updated_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall'], + '"250" fields have retained their original values' + ); + + is_deeply( + [map { $_->subfield('a') } $updated_record->field('500') ], + ['One bottle of beer in the fridge', 'One cold bottle of beer in the fridge'], + '"500" field has been appended' + ); + + # To trigger removal from search index etc + DelBiblio($biblionumber); +}; + +# Explicityly delete rule to trigger clearing of cache +$rule->delete(); + +$schema->storage->txn_rollback; + +1; diff --git a/tools/batch_record_modification.pl b/tools/batch_record_modification.pl index 3ffc18d524..6c0b225885 100755 --- a/tools/batch_record_modification.pl +++ b/tools/batch_record_modification.pl @@ -162,9 +162,10 @@ if ( $op eq 'form' ) { record_ids => \@record_ids, }; + my $patron = Koha::Patrons->find( $loggedinuser ); my $job_id = $recordtype eq 'biblio' - ? Koha::BackgroundJob::BatchUpdateBiblio->new->enqueue($params) + ? Koha::BackgroundJob::BatchUpdateBiblio->new->enqueue($params, { source => 'batchmod', categorycode => $patron->categorycode, userid => $patron->userid }) : Koha::BackgroundJob::BatchUpdateAuthority->new->enqueue($params); $template->param( -- 2.39.5