From cefa7c21e28b88351ee8ae0dfefb80a515323df9 Mon Sep 17 00:00:00 2001 From: =?utf8?q?St=C3=A9phane=20Delaune?= Date: Mon, 23 Jul 2012 10:49:31 +0200 Subject: [PATCH] Bug 5635: bulkmarcimport new parameters & features See the script's documentation for more details New parameters are: - authtypes - filter - insert - update - all Signed-off-by: Pascale Nalon This patch is live in Mines ParisTech since 2012-07-24. Signing off Signed-off-by: Katrin Fischer - Moved the sign-off from bugzilla to the commit message. - All tests and QA script pass. - Amended commit message to list new parameters. - Verified this patch works on a UNIMARC installation. - Verified normal import still works correct on a MARC21 installation. Signed-off-by: Jared Camins-Esakov --- C4/AuthoritiesMarc.pm | 4 +- misc/migration_tools/bulkmarcimport.pl | 230 ++++++++++++++++++------- 2 files changed, 174 insertions(+), 60 deletions(-) diff --git a/C4/AuthoritiesMarc.pm b/C4/AuthoritiesMarc.pm index f316b17af4..9ea3b2dbb9 100644 --- a/C4/AuthoritiesMarc.pm +++ b/C4/AuthoritiesMarc.pm @@ -379,9 +379,9 @@ Get the record and tries to guess the adequate authtypecode from its content. =cut sub GuessAuthTypeCode { - my ($record) = @_; + my ($record, $heading_fields) = @_; return unless defined $record; -my $heading_fields = { + $heading_fields //= { "MARC21"=>{ '100'=>{authtypecode=>'PERSO_NAME'}, '110'=>{authtypecode=>'CORPO_NAME'}, diff --git a/misc/migration_tools/bulkmarcimport.pl b/misc/migration_tools/bulkmarcimport.pl index 682af89508..113aaf273b 100755 --- a/misc/migration_tools/bulkmarcimport.pl +++ b/misc/migration_tools/bulkmarcimport.pl @@ -24,15 +24,18 @@ use C4::Koha; use C4::Debug; use C4::Charset; use C4::Items; +use YAML; use Unicode::Normalize; use Time::HiRes qw(gettimeofday); use Getopt::Long; use IO::File; use Pod::Usage; -binmode STDOUT, ':encoding(UTF-8)'; +use open qw( :std :encoding(UTF-8) ); +binmode( STDOUT, ":encoding(UTF-8)" ); my ( $input_marc_file, $number, $offset) = ('',0,0); my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); +my ( $insert, $filters, $update, $all, $yamlfile, $authtypes ); my $cleanisbn = 1; my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode); my $framework = ''; @@ -46,7 +49,7 @@ GetOptions( 'o|offset:f' => \$offset, 'h' => \$version, 'd' => \$delete, - 't' => \$test_parameter, + 't|test' => \$test_parameter, 's' => \$skip_marc8_conversion, 'c:s' => \$char_encoding, 'v:s' => \$verbose, @@ -56,22 +59,37 @@ GetOptions( 'k|keepids:s' => \$keepids, 'b|biblios' => \$biblios, 'a|authorities' => \$authorities, + 'authtypes:s' => \$authtypes, + 'filter=s@' => \$filters, + 'insert' => \$insert, + 'update' => \$update, + 'all' => \$all, 'match=s@' => \$match, 'i|isbn' => \$isbn_check, 'x:s' => \$sourcetag, 'y:s' => \$sourcesubfield, 'idmap:s' => \$idmapfl, 'cleanisbn!' => \$cleanisbn, + 'yaml:s' => \$yamlfile, 'dedupbarcode' => \$dedup_barcode, 'framework=s' => \$framework, ); -$biblios=!$authorities||$biblios; +$biblios ||= !$authorities; +$insert ||= !$update; + +if ($all) { + $insert = 1; + $update = 1; +} if ($version || ($input_marc_file eq '')) { pod2usage( -verbose => 2 ); exit; } +my $dbh = C4::Context->dbh; +my $heading_fields=get_heading_fields(); + if (defined $idmapfl) { open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n"; } @@ -81,8 +99,6 @@ if ((not defined $sourcesubfield) && (not defined $sourcetag)){ $sourcesubfield="a"; } -my $dbh = C4::Context->dbh; - # save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import & # will create problems in the action_logs table, that can't handle more than 1 entry per second per user. my $CataloguingLog = C4::Context->preference('CataloguingLog'); @@ -142,7 +158,7 @@ $batch->warnings_off(); $batch->strict_off(); my $i=0; my $commitnum = $commit ? $commit : 50; - +my $yamlhash; # Skip file offset if ( $offset ) { @@ -199,6 +215,7 @@ RECORD: while ( ) { next RECORD; } } + SetUTF8Flag($record); my $isbn; # remove trailing - in isbn (only for biblios, of course) if ($biblios && $cleanisbn) { @@ -212,45 +229,76 @@ RECORD: while ( ) { } my $id; # search for duplicates (based on Local-number) - if ($match){ - require C4::Search; - my $query=build_query($match,$record); - my $server=($authorities?'authorityserver':'biblioserver'); - my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] ); - die "unable to search the database for duplicates : $error" if (defined $error); - #warn "$query $server : $totalhits"; - if ( @{$results} == 1 ){ - my $marcrecord = MARC::File::USMARC::decode($results->[0]); - $id=GetRecordId($marcrecord,$tagid,$subfieldid); - } - elsif ( @{$results} > 1){ - $debug && warn "more than one match for $query"; - } - else { - $debug && warn "nomatch for $query"; - } + my $originalid; + $originalid = GetRecordId( $record, $tagid, $subfieldid ); + if ($match) { + require C4::Search; + my $query = build_query( $match, $record ); + my $server = ( $authorities ? 'authorityserver' : 'biblioserver' ); + $debug && warn $query; + my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] ); + die "unable to search the database for duplicates : $error" if ( defined $error ); + $debug && warn "$query $server : $totalhits"; + if ( $results && scalar(@$results) == 1 ) { + my $marcrecord = MARC::File::USMARC::decode( $results->[0] ); + SetUTF8Flag($marcrecord); + $id = GetRecordId( $marcrecord, $tagid, $subfieldid ); + if ( $authorities && $marcFlavour ) { + #Skip if authority in database is the same as the on in database + if ( $marcrecord->field('005')->data >= $record->field('005')->data ) { + if ($yamlfile) { + $yamlhash->{$originalid}->{'authid'} = $id; + + # we recover all subfields of the heading authorities + my @subfields; + foreach my $field ( $marcrecord->field("2..") ) { + push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); + } + $yamlhash->{$originalid}->{'subfields'} = \@subfields; + } + next; + } + } + } elsif ( $results && scalar(@$results) > 1 ) { + $debug && warn "more than one match for $query"; + } else { + $debug && warn "nomatch for $query"; + } + } + if ($keepids && $originalid) { + my $storeidfield; + if ( length($keepids) == 3 ) { + $storeidfield = MARC::Field->new( $keepids, $originalid ); + } else { + $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid ); + } + $record->insert_fields_ordered($storeidfield); + $record->delete_field( $record->field($tagid) ); } - my $originalid; - if ($keepids){ - $originalid=GetRecordId($record,$tagid,$subfieldid); - if ($originalid){ - my $storeidfield; - if (length($keepids)==3){ - $storeidfield=MARC::Field->new($keepids,$originalid); - } - else { - $storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid); - } - $record->insert_fields_ordered($storeidfield); - $record->delete_field($record->field($tagid)); - } + foreach my $stringfilter (@$filters) { + if ( length($stringfilter) == 3 ) { + foreach my $field ( $record->field($stringfilter) ) { + $record->delete_field($field); + $debug && warn "removed : ", $field->as_string; + } + } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) { + my $removetag = $1; + my $removesubfield = $2; + my $removematch = $3; + if ( ( $removetag > "010" ) && $removesubfield ) { + foreach my $field ( $record->field($removetag) ) { + $field->delete_subfield( code => "$removesubfield", match => $removematch ); + $debug && warn "Potentially removed : ", $field->subfield($removesubfield); + } + } + } } unless ($test_parameter) { if ($authorities){ use C4::AuthoritiesMarc; - my $authtypecode=GuessAuthTypeCode($record); + my $authtypecode=GuessAuthTypeCode($record, $heading_fields); my $authid= ($id?$id:GuessAuthId($record)); - if ($authid && GetAuthority($authid)){ + if ($authid && GetAuthority($authid) && $update ){ ## Authority has an id and is in database : Replace eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; if ($@){ @@ -283,6 +331,14 @@ RECORD: while ( ) { printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); } } + if ($yamlfile) { + $yamlhash->{$originalid}->{'authid'} = $authid; + my @subfields; + foreach my $field ( $record->field("2..") ) { + push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); + } + $yamlhash->{$originalid}->{'subfields'} = \@subfields; + } } else { my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); @@ -306,20 +362,33 @@ RECORD: while ( ) { } # create biblio, unless we already have it ( either match or isbn ) if ($biblionumber) { - eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};} - } - else - { - eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, $framework, { defer_marc_save => 1 }) }; + eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}; + if ($update) { + eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) }; + if ($@) { + warn "ERROR: Edit biblio $biblionumber failed: $@\n"; + printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile); + next RECORD; + } else { + printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile); + } + } else { + printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile); + } + } else { + if ($insert) { + eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) }; + if ($@) { + warn "ERROR: Adding biblio $biblionumber failed: $@\n"; + printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile); + next RECORD; + } else { + printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile); + } + } else { + printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile); + } } - if ( $@ ) { - warn "ERROR: Adding biblio $biblionumber failed: $@\n"; - printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile); - next RECORD; - } - else{ - printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile); - } eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; my $error_adding = $@; # Work on a clone so that if there are real errors, we can maybe @@ -384,6 +453,7 @@ RECORD: while ( ) { if ($#{ $errors_ref } > -1) { report_item_errors($biblionumber, $errors_ref); } + $yamlhash->{$originalid} = $biblionumber if ($yamlfile); } $dbh->commit() if (0 == $i % $commitnum); } @@ -407,6 +477,10 @@ if ($logfile){ print $loghandle "$i MARC records done in $timeneeded seconds\n"; $loghandle->close; } +if ($yamlfile) { + open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n"; + print $yamlfileout Dump($yamlhash); +} exit 0; sub GetRecordId{ @@ -445,14 +519,16 @@ sub build_query { sub build_simplequery { my $element=shift; my $record=shift; - my ($index,$recorddata)=split /,/,$element; - my ($tag,$subfields) =($1,$2) if ($recorddata=~/(\d{3})(.*)/); - my @searchstrings; + my @searchstrings; + my ($index,$recorddata)=split /,/,$element; + if ($recorddata=~/(\d{3})(.*)/) { + my ($tag,$subfields) =($1,$2); foreach my $field ($record->field($tag)){ if (length($field->as_string("$subfields"))>0){ push @searchstrings,"$index:\"".$field->as_string("$subfields")."\""; } } + } my $QParser; $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser')); my $op; @@ -478,9 +554,21 @@ sub report_item_errors { } sub printlog{ my $logelements=shift; - print $loghandle join (";",@$logelements{qw}),"\n"; + print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw} ), "\n"; +} +sub get_heading_fields{ + my $headingfields; + if ($authtypes){ + $headingfields=YAML::LoadFile($authtypes); + $headingfields={C4::Context->preference('marcflavour')=>$headingfields}; + $debug && warn YAML::Dump($headingfields); + } + unless ($headingfields){ + $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}}); + $headingfields={C4::Context->preference('marcflavour')=>$headingfields}; + } + return $headingfields; } - =head1 NAME @@ -542,7 +630,7 @@ The I of records to wait before performing a 'commit' operation File logs actions done for each record and their status into file -=item B<-t> +=item B<-t, -test> Test mode: parses the file, saying what he would do, but doing nothing. @@ -565,6 +653,32 @@ biblioitems, items Input file I: I or I (defaults to ISO2709) +=item B<-authtypes> + +file yamlfile with authoritiesTypes and distinguishable record field in order +to store the correct authtype + +=item B<-yaml> + +yaml file format a yaml file with ids + +=item B<-filter> + +list of fields that will not be imported. Can be any from 000 to 999 or field, +subfield and subfield's matching value such as 200avalue + +=item B<-insert> + +if set, only insert when possible + +=item B<-update> + +if set, only updates (any biblio should have a matching record) + +=item B<-all> + +if set, do whatever is required + =item B<-k, -keepids>= Field store ids in I (usefull for authorities, where 001 contains the -- 2.39.5