From f3c51f99d164a31e10161fdf5d2b80a989735854 Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Fri, 23 Mar 2018 17:32:30 +0100 Subject: [PATCH] Bug 20486: Add --marc_conditions option to export_records.pl Add --marc_conditions option to export_records.pl script, for excluding records based on conditions applied to MARC-record data. How to test: 1) Run tests in t/db_dependent/Exporter/Record.t 2) All tests should pass Sponsored-by: Gothenburg University Library Signed-off-by: Maksim Sen Signed-off-by: Michal Denar Signed-off-by: Chris Cormack Signed-off-by: Nick Clemens --- Koha/Exporter/Record.pm | 63 ++++++++++++++-- misc/export_records.pl | 65 +++++++++++++++- t/db_dependent/Exporter/Record.t | 125 ++++++++++++++++++++++++++++++- 3 files changed, 245 insertions(+), 8 deletions(-) diff --git a/Koha/Exporter/Record.pm b/Koha/Exporter/Record.pm index 5b145f882a..42ac002595 100644 --- a/Koha/Exporter/Record.pm +++ b/Koha/Exporter/Record.pm @@ -9,11 +9,13 @@ use C4::Biblio; use C4::Record; use Koha::CsvProfiles; use Koha::Logger; +use List::Util qw(all any); sub _get_record_for_export { my ($params) = @_; my $record_type = $params->{record_type}; my $record_id = $params->{record_id}; + my $conditions = $params->{record_conditions}; my $dont_export_fields = $params->{dont_export_fields}; my $clean = $params->{clean}; @@ -25,7 +27,60 @@ sub _get_record_for_export { } else { Koha::Logger->get->warn( "Record_type $record_type not supported." ); } - return unless $record; + if ( !$record ) { + Koha::Logger->get->warn( "Record $record_id could not be exported." ); + return; + } + + # If multiple conditions all are required to match (and) + # For matching against multiple marc targets all are also required to match + my %operators = ( + '=' => sub { + return $_[0] eq $_[1]; + }, + '!=' => sub { + return $_[0] ne $_[1]; + }, + '>' => sub { + return $_[0] gt $_[1]; + }, + '<' => sub { + return $_[0] lt $_[1]; + }, + ); + if ($conditions) { + foreach my $condition (@{$conditions}) { + my ($field_tag, $subfield, $operator, $match_value) = @{$condition}; + my @fields = $record->field($field_tag); + my $no_target = 0; + + if (!@fields) { + $no_target = 1; + } + else { + if ($operator eq '?') { + return unless any { $subfield ? $_->subfield($subfield) : $_->data() } @fields; + } elsif ($operator eq '!?') { + return if any { $subfield ? $_->subfield($subfield) : $_->data() } @fields; + } else { + my $op; + if (exists $operators{$operator}) { + $op = $operators{$operator}; + } else { + die("Invalid operator: $op"); + } + my @target_values = map { $subfield ? $_->subfield($subfield) : ($_->data()) } @fields; + if (!@target_values) { + $no_target = 1; + } + else { + return unless all { $op->($_, $match_value) } @target_values; + } + } + } + return if $no_target && $operator ne '!='; + } + } if ($dont_export_fields) { for my $f ( split / /, $dont_export_fields ) { @@ -119,6 +174,7 @@ sub export { if ( $format eq 'iso2709' ) { for my $record_id (@$record_ids) { my $record = _get_record_for_export( { %$params, record_id => $record_id } ); + next unless $record; my $errorcount_on_decode = eval { scalar( MARC::File::USMARC->decode( $record->as_usmarc )->warnings() ) }; if ( $errorcount_on_decode or $@ ) { my $msg = "Record $record_id could not be exported. " . @@ -137,10 +193,7 @@ sub export { print "\n"; for my $record_id (@$record_ids) { my $record = _get_record_for_export( { %$params, record_id => $record_id } ); - if( !$record ) { - Koha::Logger->get->info( "Record $record_id could not be exported." ); - next; - } + next unless $record; print MARC::File::XML::record($record); print "\n"; } diff --git a/misc/export_records.pl b/misc/export_records.pl index 6e363686a3..a1eecc8eb2 100755 --- a/misc/export_records.pl +++ b/misc/export_records.pl @@ -32,7 +32,30 @@ use Koha::CsvProfiles; use Koha::Exporter::Record; use Koha::DateUtils qw( dt_from_string output_pref ); -my ( $output_format, $timestamp, $dont_export_items, $csv_profile_id, $deleted_barcodes, $clean, $filename, $record_type, $id_list_file, $starting_authid, $ending_authid, $authtype, $starting_biblionumber, $ending_biblionumber, $itemtype, $starting_callnumber, $ending_callnumber, $start_accession, $end_accession, $help ); +my ( + $output_format, + $timestamp, + $dont_export_items, + $csv_profile_id, + $deleted_barcodes, + $clean, + $filename, + $record_type, + $id_list_file, + $starting_authid, + $ending_authid, + $authtype, + $starting_biblionumber, + $ending_biblionumber, + $itemtype, + $starting_callnumber, + $ending_callnumber, + $start_accession, + $end_accession, + $marc_conditions, + $help +); + GetOptions( 'format=s' => \$output_format, 'date=s' => \$timestamp, @@ -53,6 +76,7 @@ GetOptions( 'ending_callnumber=s' => \$ending_callnumber, 'start_accession=s' => \$start_accession, 'end_accession=s' => \$end_accession, + 'marc_conditions=s' => \$marc_conditions, 'h|help|?' => \$help ) || pod2usage(1); @@ -90,6 +114,22 @@ if ( $deleted_barcodes and $record_type ne 'bibs' ) { $start_accession = dt_from_string( $start_accession ) if $start_accession; $end_accession = dt_from_string( $end_accession ) if $end_accession; +# Parse marc conditions +my @marc_conditions; +if ($marc_conditions) { + foreach my $condition (split(/,\s*/, $marc_conditions)) { + if ($condition =~ /^(\d{3})([\w\d]?)(=|(?:!=)|>|<)([^,]+)$/) { + push @marc_conditions, [$1, $2, $3, $4]; + } + elsif ($condition =~ /^(exists|not_exists)\((\d{3})([\w\d]?)\)$/) { + push @marc_conditions, [$2, $3, $1 eq 'exists' ? '?' : '!?']; + } + else { + die("Invalid condititon: $condition"); + } + } +} + my $dbh = C4::Context->dbh; # Redirect stdout @@ -199,6 +239,7 @@ else { Koha::Exporter::Record::export( { record_type => $record_type, record_ids => \@record_ids, + record_conditions => @marc_conditions ? \@marc_conditions : undef, format => $output_format, csv_profile_id => $csv_profile_id, export_items => (not $dont_export_items), @@ -310,6 +351,28 @@ Print a brief help message. --end_accession=DATE Export biblio with an item accessionned after DATE +=item B<--marc_conditions> + + --marc_conditions=CONDITIONS Only include biblios with MARC data matching CONDITIONS. + CONDITIONS is on the format: , + or (). + with multiple conditions separated by commas (,). + For example: --marc_conditions="035a!=(EXAMPLE)123,041a=swe". + Multiple conditions are all required to match. + If has multiple values all values + are also required to match. + Valid operators are: = (equal to), != (not equal to), + > (great than) and < (less than). + + Two unary operations are also supported: + exists() and not_exists(). + For example: --marc_conditions="exists(035a)". + + "exists( exists regardless of target value, and + "exists()" will include marc records where + no exists. + =back =head1 AUTHOR diff --git a/t/db_dependent/Exporter/Record.t b/t/db_dependent/Exporter/Record.t index 3dc25d6afb..68eef06775 100644 --- a/t/db_dependent/Exporter/Record.t +++ b/t/db_dependent/Exporter/Record.t @@ -17,7 +17,7 @@ use Modern::Perl; -use Test::More tests => 5; +use Test::More tests => 6; use Test::Warn; use t::lib::TestBuilder; @@ -281,8 +281,129 @@ subtest '_get_biblio_for_export' => sub { }; +subtest '_get_record_for_export MARC field conditions' => sub { + plan tests => 11; + my $biblio = MARC::Record->new(); + $biblio->leader('00266nam a22001097a 4500'); + $biblio->append_fields( + MARC::Field->new( '100', ' ', ' ', a => 'Thurber, James' ), + MARC::Field->new( '245', ' ', ' ', a => 'The 13 Clocks' ), + MARC::Field->new( '080', ' ', ' ', a => '12345' ), + MARC::Field->new( '035', ' ', ' ', a => '(TEST)123' ), + MARC::Field->new( '035', ' ', ' ', a => '(TEST)1234' ), + ); + my ( $biblionumber ) = AddBiblio( $biblio, '' ); + my $record; + $record = Koha::Exporter::Record::_get_record_for_export( + { + record_id => $biblionumber, + record_conditions => [['080', 'a', '=', '12345']], + record_type => 'bibs', + } + ); + ok( $record, "Record condition \"080a=12345\" should match" ); -$schema->storage->txn_rollback; + $record = Koha::Exporter::Record::_get_record_for_export( + { + record_id => $biblionumber, + record_conditions => [['080', 'a', '!=', '12345']], + record_type => 'bibs', + } + ); + is( $record, undef, "Record condition \"080a!=12345\" should not match" ); + + $record = Koha::Exporter::Record::_get_record_for_export( + { + record_id => $biblionumber, + record_conditions => [['080', 'a', '>', '1234']], + record_type => 'bibs', + } + ); + ok( $record, "Record condition \"080a>1234\" should match" ); + + $record = Koha::Exporter::Record::_get_record_for_export( + { + record_id => $biblionumber, + record_conditions => [['080', 'a', '<', '123456']], + record_type => 'bibs', + } + ); + ok( $record, "Record condition \"080a<123456\" should match" ); + + $record = Koha::Exporter::Record::_get_record_for_export( + { + record_id => $biblionumber, + record_conditions => [['080', 'a', '>', '123456']], + record_type => 'bibs', + } + ); + is( $record, undef, "Record condition \"080a>123456\" should not match" ); + + + ## Multiple subfields + + $record = Koha::Exporter::Record::_get_record_for_export( + { + record_id => $biblionumber, + record_conditions => [['035', 'a', '!=', 'TEST(12345)']], + record_type => 'bibs', + } + ); + ok( $record, "Record condition \"035a!=TEST(12345)\" should match" ); + + $record = Koha::Exporter::Record::_get_record_for_export( + { + record_id => $biblionumber, + record_conditions => [['035', 'a', '=', 'TEST(1234)']], + record_type => 'bibs', + } + ); + is( $record, undef, "Record condition \"035a=TEST(1234)\" should not match" ); # Since matching all subfields required + + + ## Multiple conditions + + $record = Koha::Exporter::Record::_get_record_for_export( + { + record_id => $biblionumber, + record_conditions => [['035', 'a', '!=', 'TEST(12345)'], ['080', 'a', '>', '1234']], + record_type => 'bibs', + } + ); + ok( $record, "Record condition \"035a!=TEST(12345),080a>1234\" should match" ); + + $record = Koha::Exporter::Record::_get_record_for_export( + { + record_id => $biblionumber, + record_conditions => [['035', 'a', '!=', 'TEST(12345)'], ['080', 'a', '<', '1234']], + record_type => 'bibs', + } + ); + is( $record, undef, "Record condition \"035a!=TEST(12345),080a<1234\" should not match" ); + + + ## exists/not_exists + + $record = Koha::Exporter::Record::_get_record_for_export( + { + record_id => $biblionumber, + record_conditions => [['035', 'a', '?']], + record_type => 'bibs', + } + ); + ok( $record, "Record condition \"exists(035a)\" should match" ); + $record = Koha::Exporter::Record::_get_record_for_export( + { + record_id => $biblionumber, + record_conditions => [['035', 'a', '!?']], + record_type => 'bibs', + record_type => 'bibs', + } + ); + is( $record, undef, "Record condition \"not_exists(035a)\" should not match" ); +}; + +$schema->storage->txn_rollback; -- 2.39.5