From d37919eab933175cc0a505b335f6e70ca30ac65c Mon Sep 17 00:00:00 2001 From: Galen Charlton Date: Thu, 25 Oct 2007 13:29:35 -0500 Subject: [PATCH] improved import batches part 2 -- replace use of marc_breeding Signed-off-by: Chris Cormack Signed-off-by: Joshua Ferraro --- C4/Breeding.pm | 64 ++++++---- C4/ImportBatch.pm | 244 ++++++++++++++++++++++++++++++++++++ cataloguing/addbiblio.pl | 14 +-- cataloguing/z3950_search.pl | 2 +- tools/import.pl | 2 +- 5 files changed, 293 insertions(+), 33 deletions(-) create mode 100644 C4/ImportBatch.pm diff --git a/C4/Breeding.pm b/C4/Breeding.pm index ac97ca74e9..d21ae31dc1 100644 --- a/C4/Breeding.pm +++ b/C4/Breeding.pm @@ -21,6 +21,7 @@ use strict; use C4::Biblio; use C4::Koha; use MARC::File::USMARC; +use C4::ImportBatch; require Exporter; use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); @@ -35,12 +36,13 @@ C4::Breeding : script to add a biblio in marc_breeding table. =head1 SYNOPSIS use C4::Scan; - &ImportBreeding($marcrecords,$overwrite_biblio,$filename,$z3950random); + &ImportBreeding($marcrecords,$overwrite_biblio,$filename,$z3950random,$batch_type); C<$marcrecord> => the MARC::Record C<$overwrite_biblio> => if set to 1 a biblio with the same ISBN will be overwritted. if set to 0 a biblio with the same isbn will be ignored (the previous will be kept) - if set to -1 the biblio will be added anyway (more than 1 biblio with the same ISBN possible in the breeding + if set to -1 the biblio will be added anyway (more than 1 biblio with the same ISBN + possible in the breeding C<$encoding> => USMARC or UNIMARC. used for char_decoding. If not present, the parameter marcflavour is used instead @@ -60,22 +62,30 @@ C4::Breeding : script to add a biblio in marc_breeding table. =head2 ImportBreeding - ImportBreeding($marcrecords,$overwrite_biblio,$filename,$encoding,$z3950random); + ImportBreeding($marcrecords,$overwrite_biblio,$filename,$encoding,$z3950random,$batch_type); TODO description =cut sub ImportBreeding { - my ($marcrecords,$overwrite_biblio,$filename,$encoding,$z3950random) = @_; + my ($marcrecords,$overwrite_biblio,$filename,$encoding,$z3950random,$batch_type) = @_; my @marcarray = split /\x1D/, $marcrecords; my $dbh = C4::Context->dbh; + + my $batch_id = 0; + if ($batch_type eq 'z3950') { + $batch_id = GetZ3950BatchId($filename); + } else { + # create a new one + # FIXME - handle comments + $batch_id = AddImportBatch('create_new', 'staging', 'batch', $filename, ''); + } my $searchisbn = $dbh->prepare("select biblioitemnumber from biblioitems where isbn=?"); my $searchissn = $dbh->prepare("select biblioitemnumber from biblioitems where issn=?"); - my $searchbreeding = $dbh->prepare("select id from marc_breeding where isbn=? and title=?"); - my $insertsql = $dbh->prepare("insert into marc_breeding (file,isbn,title,author,marc,encoding,z3950random) values(?,?,?,?,?,?,?)"); - my $replacesql = $dbh->prepare("update marc_breeding set file=?,isbn=?,title=?,author=?,marc=?,encoding=?,z3950random=? where id=?"); + # FIXME -- not sure that this kind of checking is actually needed + my $searchbreeding = $dbh->prepare("select import_record_id from import_biblios where isbn=? and title=?"); $encoding = C4::Context->preference("marcflavour") unless $encoding; # fields used for import results @@ -86,23 +96,20 @@ sub ImportBreeding { my $breedingid; for (my $i=0;$i<=$#marcarray;$i++) { my $marcrecord = FixEncoding($marcarray[$i]."\x1D"); - + + # FIXME - currently this does nothing my @warnings = $marcrecord->warnings(); if (scalar($marcrecord->fields()) == 0) { $notmarcrecord++; } else { my $oldbiblio = TransformMarcToKoha($dbh,$marcrecord,''); - my $isbnlength=10; - if($oldbiblio->{isbn}){ - $isbnlength = length($oldbiblio->{isbn}); - } - # if isbn found and biblio does not exist, add it. If isbn found and biblio exists, overwrite or ignore depending on user choice + # if isbn found and biblio does not exist, add it. If isbn found and biblio exists, + # overwrite or ignore depending on user choice # drop every "special" char : spaces, - ... - $oldbiblio->{isbn} =~ s/ |-|\.//g, - $oldbiblio->{isbn} = substr($oldbiblio->{isbn},0,$isbnlength); - $oldbiblio->{issn} =~ s/ |-|\.//g, - $oldbiblio->{issn} = substr($oldbiblio->{issn},0,10); + $oldbiblio->{isbn} =~ s/\(.*$//; + $oldbiblio->{isbn} =~ tr/ -_//; + $oldbiblio->{isbn} = uc $oldbiblio->{isbn}; # search if biblio exists my $biblioitemnumber; if ($oldbiblio->{isbn}) { @@ -117,6 +124,9 @@ sub ImportBreeding { if ($biblioitemnumber) { $alreadyindb++; } else { + # FIXME - in context of batch load, + # rejecting records because already present in the reservoir + # not correct in every case. # search in breeding farm if ($oldbiblio->{isbn}) { $searchbreeding->execute($oldbiblio->{isbn},$oldbiblio->{title}); @@ -128,13 +138,11 @@ sub ImportBreeding { if ($breedingid && $overwrite_biblio eq '0') { $alreadyinfarm++; } else { - my $recoded; - $recoded = $marcrecord->as_usmarc(); if ($breedingid && $overwrite_biblio eq '1') { - $replacesql ->execute($filename,substr($oldbiblio->{isbn}.$oldbiblio->{issn},0,$isbnlength),$oldbiblio->{title},$oldbiblio->{author},$recoded,$encoding,$z3950random,$breedingid); + ModBiblioInBatch($breedingid, $marcrecord); } else { - $insertsql ->execute($filename,substr($oldbiblio->{isbn}.$oldbiblio->{issn},0,$isbnlength),$oldbiblio->{title},$oldbiblio->{author},$recoded,$encoding,$z3950random); - $breedingid=$dbh->{'mysql_insertid'}; + my $import_id = AddBiblioToBatch($batch_id, $imported, $marcrecord, $encoding, $z3950random); + $breedingid = $import_id; } $imported++; } @@ -165,7 +173,11 @@ sub BreedingSearch { my $sth; my @results; - $query = "Select id,file,isbn,title,author from marc_breeding where "; + $query = "SELECT import_record_id, file_name, isbn, title, author + FROM import_biblios + JOIN import_records USING (import_record_id) + JOIN import_batches USING (import_batch_id) + WHERE "; if ($z3950random) { $query .= "z3950random = ?"; @bind=($z3950random); @@ -187,6 +199,12 @@ sub BreedingSearch { $sth->execute(@bind); while (my $data = $sth->fetchrow_hashref) { $results[$count] = $data; + # FIXME - hack to reflect difference in name + # of columns in old marc_breeding and import_records + # There needs to be more separation between column names and + # field names used in the templates + $data->{'file'} = $data->{'file_name'}; + $data->{'id'} = $data->{'import_record_id'}; $count++; } # while diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm new file mode 100644 index 0000000000..056e06bf21 --- /dev/null +++ b/C4/ImportBatch.pm @@ -0,0 +1,244 @@ +package C4::ImportBatch; + +# Copyright (C) 2007 LibLime +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place, +# Suite 330, Boston, MA 02111-1307 USA + +use strict; +use C4::Context; +use C4::Koha; +use C4::Biblio; +require Exporter; + + +use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); + +# set the version for version checking +$VERSION = 3.00; + +=head1 NAME + +C4::ImportBatch - manage batches of imported MARC records + +=head1 SYNOPSIS + +=over 4 + +use C4::ImportBatch; + +=back + +=head1 FUNCTIONS + +=cut + +@ISA = qw(Exporter); +@EXPORT = qw( + GetZ3950BatchId + GetImportRecordMarc + AddImportBatch + AddBiblioToBatch + ModBiblioInBatch +); + +=head2 GetZ3950BatchId + +=over 4 + +my $batchid = GetZ3950BatchId($z3950server); + +=back + +Retrieves the ID of the import batch for the Z39.50 +reservoir for the given target. If necessary, +creates the import batch. + +=cut + +sub GetZ3950BatchId { + my ($z3950server) = @_; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("SELECT import_batch_id FROM import_batches + WHERE batch_type = 'z3950' + AND file_name = ?"); + $sth->execute($z3950server); + my $rowref = $sth->fetchrow_arrayref(); + $sth->finish(); + if (defined $rowref) { + return $rowref->[0]; + } else { + my $batch_id = AddImportBatch('create_new', 'staged', 'z3950', $z3950server, ''); + return $batch_id; + } + +} + +=head2 GetImportRecordMarc + +=over4 + +my ($marcblob, $encoding) = GetImportRecordMarc($import_record_id); + +=back + +=cut + +sub GetImportRecordMarc { + my ($import_record_id) = @_; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("SELECT marc, encoding FROM import_records WHERE import_record_id = ?"); + $sth->execute($import_record_id); + my ($marc, $encoding) = $sth->fetchrow(); + $sth->finish(); + return $marc; + +} + +=head2 AddImportBatch + +=over 4 + +my $batch_id = AddImportBatch($overlay_action, $import_status, $type, $file_name, $comments); + +=back + +=cut + +sub AddImportBatch { + my ($overlay_action, $import_status, $type, $file_name, $comments) = @_; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("INSERT INTO import_batches (overlay_action, import_status, batch_type, + file_name, comments) + VALUES (?, ?, ?, ?, ?)"); + $sth->execute($overlay_action, $import_status, $type, $file_name, $comments); + my $batch_id = $dbh->{'mysql_insertid'}; + $sth->finish(); + + return $batch_id; + +} + +=head2 AddBiblioToBatch + +=over 4 + +my $import_record_id = AddBiblioToBatch($batch_id, $record_sequence, $marc_record, $encoding, $z3950random); + +=cut + +sub AddBiblioToBatch { + my ($batch_id, $record_sequence, $marc_record, $encoding, $z3950random) = @_; + + my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'bib', $encoding, $z3950random); + _add_biblio_fields($import_record_id, $marc_record); + return $import_record_id; +} + +=head2 ModBiblioInBatch + +=over 4 + +ModBiblioInBatch($import_record_id, $marc_record); + +=cut + +sub ModBiblioInBatch { + my ($import_record_id, $marc_record) = @_; + + _update_import_record_marc($import_record_id, $marc_record); + _update_biblio_fields($import_record_id, $marc_record); + +} + +# internal functions + +sub _create_import_record { + my ($batch_id, $record_sequence, $marc_record, $record_type, $encoding, $z3950random) = @_; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("INSERT INTO import_records (import_batch_id, record_sequence, marc, marcxml, + record_type, encoding, z3950random) + VALUES (?, ?, ?, ?, ?, ?, ?)"); + $sth->execute($batch_id, $record_sequence, $marc_record->as_usmarc(), $marc_record->as_xml(), + $record_type, $encoding, $z3950random); + my $import_record_id = $dbh->{'mysql_insertid'}; + $sth->finish(); + return $import_record_id; +} + +sub _update_import_record_marc { + my ($import_record_id, $marc_record) = @_; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("UPDATE import_records SET marc = ?, marcxml = ? + WHERE import_record_id = ?"); + $sth->execute($marc_record->as_usmarc(), $marc_record->as_xml(), $import_record_id); + $sth->finish(); +} + +sub _add_biblio_fields { + my ($import_record_id, $marc_record) = @_; + + my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record); + my $dbh = C4::Context->dbh; + # FIXME no controlnumber, originalsource + # FIXME 2 - should regularize normalization of ISBN wherever it is done + $isbn =~ s/\(.*$//; + $isbn =~ tr/ -_//; + $isbn = uc $isbn; + my $sth = $dbh->prepare("INSERT INTO import_biblios (import_record_id, title, author, isbn, issn) VALUES (?, ?, ?, ?, ?)"); + $sth->execute($import_record_id, $title, $author, $isbn, $issn); + $sth->finish(); + +} + +sub _update_biblio_fields { + my ($import_record_id, $marc_record) = @_; + + my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record); + my $dbh = C4::Context->dbh; + # FIXME no controlnumber, originalsource + # FIXME 2 - should regularize normalization of ISBN wherever it is done + $isbn =~ s/\(.*$//; + $isbn =~ tr/ -_//; + $isbn = uc $isbn; + my $sth = $dbh->prepare("UPDATE import_biblios SET title = ?, author = ?, isbn = ?, issn = ? + WHERE import_record_id = ?"); + $sth->execute($title, $author, $isbn, $issn, $import_record_id); + $sth->finish(); +} + +sub _parse_biblio_fields { + my ($marc_record) = @_; + + my $dbh = C4::Context->dbh; + my $bibliofields = TransformMarcToKoha($dbh, $marc_record, ''); + return ($bibliofields->{'title'}, $bibliofields->{'author'}, $bibliofields->{'isbn'}, $bibliofields->{'issn'}); + +} + +1; + +=head1 AUTHOR + +Koha Development Team + +Galen Charlton + +=cut diff --git a/cataloguing/addbiblio.pl b/cataloguing/addbiblio.pl index 36195bc5c0..3dfa1aabb2 100755 --- a/cataloguing/addbiblio.pl +++ b/cataloguing/addbiblio.pl @@ -31,6 +31,7 @@ use C4::Log; use C4::Koha; # XXX subfield_is_koha_internal_p use C4::Branch; # XXX subfield_is_koha_internal_p use C4::ClassSource; +use C4::ImportBatch; use Date::Calc qw(Today); use MARC::File::USMARC; @@ -44,9 +45,9 @@ our($tagslib,$authorised_values_sth,$is_a_modif,$usedTagsLib,$mandatory_z3950); =item MARCfindbreeding - $record = MARCfindbreeding($dbh, $breedingid); + $record = MARCfindbreeding($breedingid); -Look up the breeding farm with database handle $dbh, for the +Look up the import record repository for the record with record with id $breedingid. If found, returns the decoded MARC::Record; otherwise, -1 is returned (FIXME). Returns as second parameter the character encoding. @@ -54,11 +55,8 @@ Returns as second parameter the character encoding. =cut sub MARCfindbreeding { - my ( $dbh, $id ) = @_; - my $sth = - $dbh->prepare("select file,marc,encoding from marc_breeding where id=?"); - $sth->execute($id); - my ( $file, $marc, $encoding ) = $sth->fetchrow; + my ( $id ) = @_; + my ($marc, $encoding) = GetImportRecordMarc($id); # remove the - in isbn, koha store isbn without any - if ($marc) { my $record = MARC::Record->new_from_usmarc($marc); @@ -768,7 +766,7 @@ if (($biblionumber) && !($breedingid)){ $record = GetMarcBiblio($biblionumber); } if ($breedingid) { - ( $record, $encoding ) = MARCfindbreeding( $dbh, $breedingid ) ; + ( $record, $encoding ) = MARCfindbreeding( $breedingid ) ; } $is_a_modif = 0; diff --git a/cataloguing/z3950_search.pl b/cataloguing/z3950_search.pl index dfc769d453..159a1f0b1c 100755 --- a/cataloguing/z3950_search.pl +++ b/cataloguing/z3950_search.pl @@ -204,7 +204,7 @@ else { $notmarcrecord, $alreadyindb, $alreadyinfarm, $imported, $breedingid ) - = ImportBreeding( $marcdata, 2, $serverhost[$k], $encoding[$k], $random ); + = ImportBreeding( $marcdata, 2, $serverhost[$k], $encoding[$k], $random, 'z3950' ); my %row_data; if ( $i % 2 ) { diff --git a/tools/import.pl b/tools/import.pl index a908c556f3..07a4b100cc 100755 --- a/tools/import.pl +++ b/tools/import.pl @@ -77,7 +77,7 @@ if ($uploadmarc && length($uploadmarc)>0) { while (<$uploadmarc>) { $marcrecord.=$_; } - my ($notmarcrecord,$alreadyindb,$alreadyinfarm,$imported) = ImportBreeding($marcrecord,$overwrite_biblio,$filename,$syntax,int(rand(99999))); + my ($notmarcrecord,$alreadyindb,$alreadyinfarm,$imported) = ImportBreeding($marcrecord,$overwrite_biblio,$filename,$syntax,int(rand(99999)), 'batch'); $template->param(imported => $imported, alreadyindb => $alreadyindb, -- 2.20.1