1 package C4::ImportBatch;
3 # Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it
8 # under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 3 of the License, or
10 # (at your option) any later version.
12 # Koha is distributed in the hope that it will be useful, but
13 # WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with Koha; if not, see <http://www.gnu.org/licenses>.
24 use C4::Koha qw( GetNormalizedISBN );
33 use C4::Items qw( AddItemFromMarc ModItemFromMarc );
34 use C4::Charset qw( MarcToUTF8Record SetUTF8Flag StripNonXmlChars );
35 use C4::AuthoritiesMarc qw( AddAuthority GuessAuthTypeCode GetAuthorityXML ModAuthority DelAuthority GetAuthorizedHeading );
36 use C4::MarcModificationTemplates qw( ModifyRecordWithTemplate );
37 use Koha::BackgroundJob::BatchUpdateBiblioHoldsQueue;
39 use Koha::SearchEngine;
40 use Koha::SearchEngine::Indexer;
41 use Koha::Plugins::Handler;
44 our (@ISA, @EXPORT_OK);
56 AddItemsToImportBiblio
67 GetStagedWebserviceBatches
68 GetImportBatchRangeDesc
69 GetNumberOfNonZ3950ImportBatches
72 GetItemNumbersFromImportBatch
76 GetImportBatchOverlayAction
77 SetImportBatchOverlayAction
78 GetImportBatchNoMatchAction
79 SetImportBatchNoMatchAction
80 GetImportBatchItemAction
81 SetImportBatchItemAction
84 GetImportRecordOverlayStatus
85 SetImportRecordOverlayStatus
88 SetMatchedBiblionumber
89 GetImportRecordMatches
90 SetImportRecordMatches
92 RecordsFromMARCXMLFile
93 RecordsFromISO2709File
100 C4::ImportBatch - manage batches of imported MARC records
108 =head2 GetZ3950BatchId
110 my $batchid = GetZ3950BatchId($z3950server);
112 Retrieves the ID of the import batch for the Z39.50
113 reservoir for the given target. If necessary,
114 creates the import batch.
118 sub GetZ3950BatchId {
119 my ($z3950server) = @_;
121 my $dbh = C4::Context->dbh;
122 my $sth = $dbh->prepare("SELECT import_batch_id FROM import_batches
123 WHERE batch_type = 'z3950'
125 $sth->execute($z3950server);
126 my $rowref = $sth->fetchrow_arrayref();
128 if (defined $rowref) {
131 my $batch_id = AddImportBatch( {
132 overlay_action => 'create_new',
133 import_status => 'staged',
134 batch_type => 'z3950',
135 file_name => $z3950server,
142 =head2 GetWebserviceBatchId
144 my $batchid = GetWebserviceBatchId();
146 Retrieves the ID of the import batch for webservice.
147 If necessary, creates the import batch.
151 my $WEBSERVICE_BASE_QRY = <<EOQ;
152 SELECT import_batch_id FROM import_batches
153 WHERE batch_type = 'webservice'
154 AND import_status = 'staged'
156 sub GetWebserviceBatchId {
159 my $dbh = C4::Context->dbh;
160 my $sql = $WEBSERVICE_BASE_QRY;
162 foreach my $field (qw(matcher_id overlay_action nomatch_action item_action)) {
163 if (my $val = $params->{$field}) {
164 $sql .= " AND $field = ?";
168 my $id = $dbh->selectrow_array($sql, undef, @args);
171 $params->{batch_type} = 'webservice';
172 $params->{import_status} = 'staged';
173 return AddImportBatch($params);
176 =head2 GetImportRecordMarc
178 my ($marcblob, $encoding) = GetImportRecordMarc($import_record_id);
182 sub GetImportRecordMarc {
183 my ($import_record_id) = @_;
185 my $dbh = C4::Context->dbh;
186 my ( $marc, $encoding ) = $dbh->selectrow_array(q|
187 SELECT marc, encoding
189 WHERE import_record_id = ?
190 |, undef, $import_record_id );
192 return $marc, $encoding;
195 sub EmbedItemsInImportBiblio {
196 my ( $record, $import_record_id ) = @_;
197 my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField( "items.itemnumber" );
198 my $dbh = C4::Context->dbh;
199 my $import_items = $dbh->selectall_arrayref(q|
200 SELECT import_items.marcxml
202 WHERE import_record_id = ?
203 |, { Slice => {} }, $import_record_id );
205 for my $import_item ( @$import_items ) {
206 my $item_marc = MARC::Record::new_from_xml($import_item->{marcxml}, 'UTF-8');
207 push @item_fields, $item_marc->field($itemtag);
209 $record->append_fields(@item_fields);
213 =head2 AddImportBatch
215 my $batch_id = AddImportBatch($params_hash);
223 foreach (qw( matcher_id template_id branchcode
224 overlay_action nomatch_action item_action
225 import_status batch_type file_name comments record_type )) {
226 if (exists $params->{$_}) {
228 push @vals, $params->{$_};
231 my $dbh = C4::Context->dbh;
232 $dbh->do("INSERT INTO import_batches (".join( ',', @fields).")
233 VALUES (".join( ',', map '?', @fields).")",
236 return $dbh->{'mysql_insertid'};
239 =head2 GetImportBatch
241 my $row = GetImportBatch($batch_id);
243 Retrieve a hashref of an import_batches row.
250 my $dbh = C4::Context->dbh;
251 my $sth = $dbh->prepare_cached("SELECT b.*, p.name as profile FROM import_batches b LEFT JOIN import_batch_profiles p ON p.id = b.profile_id WHERE import_batch_id = ?");
252 $sth->bind_param(1, $batch_id);
254 my $result = $sth->fetchrow_hashref;
260 =head2 AddBiblioToBatch
262 my $import_record_id = AddBiblioToBatch($batch_id, $record_sequence,
263 $marc_record, $encoding, $update_counts);
267 sub AddBiblioToBatch {
268 my $batch_id = shift;
269 my $record_sequence = shift;
270 my $marc_record = shift;
271 my $encoding = shift;
272 my $update_counts = @_ ? shift : 1;
274 my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'biblio', $encoding, C4::Context->preference('marcflavour'));
275 _add_biblio_fields($import_record_id, $marc_record);
276 _update_batch_record_counts($batch_id) if $update_counts;
277 return $import_record_id;
280 =head2 AddAuthToBatch
282 my $import_record_id = AddAuthToBatch($batch_id, $record_sequence,
283 $marc_record, $encoding, $update_counts, [$marc_type]);
288 my $batch_id = shift;
289 my $record_sequence = shift;
290 my $marc_record = shift;
291 my $encoding = shift;
292 my $update_counts = @_ ? shift : 1;
293 my $marc_type = shift || C4::Context->preference('marcflavour');
295 $marc_type = 'UNIMARCAUTH' if $marc_type eq 'UNIMARC';
297 my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $marc_type);
298 _add_auth_fields($import_record_id, $marc_record);
299 _update_batch_record_counts($batch_id) if $update_counts;
300 return $import_record_id;
303 =head2 BatchStageMarcRecords
305 ( $batch_id, $num_records, $num_items, @invalid_records ) =
306 BatchStageMarcRecords(
307 $record_type, $encoding,
308 $marc_records, $file_name,
309 $marc_modification_template, $comments,
310 $branch_code, $parse_items,
311 $leave_as_staging, $progress_interval,
317 sub BatchStageMarcRecords {
318 my $record_type = shift;
319 my $encoding = shift;
320 my $marc_records = shift;
321 my $file_name = shift;
322 my $marc_modification_template = shift;
323 my $comments = shift;
324 my $branch_code = shift;
325 my $parse_items = shift;
326 my $leave_as_staging = shift;
328 # optional callback to monitor status
330 my $progress_interval = 0;
331 my $progress_callback = undef;
333 $progress_interval = shift;
334 $progress_callback = shift;
335 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
336 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
339 my $batch_id = AddImportBatch( {
340 overlay_action => 'create_new',
341 import_status => 'staging',
342 batch_type => 'batch',
343 file_name => $file_name,
344 comments => $comments,
345 record_type => $record_type,
348 SetImportBatchItemAction($batch_id, 'always_add');
350 SetImportBatchItemAction($batch_id, 'ignore');
354 my $marc_type = C4::Context->preference('marcflavour');
355 $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth');
356 my @invalid_records = ();
359 # FIXME - for now, we're dealing only with bibs
361 foreach my $marc_record (@$marc_records) {
363 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
364 &$progress_callback($rec_num);
367 ModifyRecordWithTemplate( $marc_modification_template, $marc_record ) if ( $marc_modification_template );
369 my $import_record_id;
370 if (scalar($marc_record->fields()) == 0) {
371 push @invalid_records, $marc_record;
374 # Normalize the record so it doesn't have separated diacritics
375 SetUTF8Flag($marc_record);
378 if ($record_type eq 'biblio') {
379 $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, 0);
381 my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0);
382 $num_items += scalar(@import_items_ids);
384 } elsif ($record_type eq 'auth') {
385 $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, 0, $marc_type);
389 unless ($leave_as_staging) {
390 SetImportBatchStatus($batch_id, 'staged');
392 # FIXME branch_code, number of bibs, number of items
393 _update_batch_record_counts($batch_id);
394 if ($progress_interval){
395 &$progress_callback($rec_num);
398 return ($batch_id, $num_valid, $num_items, @invalid_records);
401 =head2 AddItemsToImportBiblio
403 my @import_items_ids = AddItemsToImportBiblio($batch_id,
404 $import_record_id, $marc_record, $update_counts);
408 sub AddItemsToImportBiblio {
409 my $batch_id = shift;
410 my $import_record_id = shift;
411 my $marc_record = shift;
412 my $update_counts = @_ ? shift : 0;
414 my @import_items_ids = ();
416 my $dbh = C4::Context->dbh;
417 my ($item_tag,$item_subfield) = &GetMarcFromKohaField( "items.itemnumber" );
418 foreach my $item_field ($marc_record->field($item_tag)) {
419 my $item_marc = MARC::Record->new();
420 $item_marc->leader("00000 a "); # must set Leader/09 to 'a'
421 $item_marc->append_fields($item_field);
422 $marc_record->delete_field($item_field);
423 my $sth = $dbh->prepare_cached("INSERT INTO import_items (import_record_id, status, marcxml)
425 $sth->bind_param(1, $import_record_id);
426 $sth->bind_param(2, 'staged');
427 $sth->bind_param(3, $item_marc->as_xml("USMARC"));
429 push @import_items_ids, $dbh->{'mysql_insertid'};
433 if ($#import_items_ids > -1) {
434 _update_batch_record_counts($batch_id) if $update_counts;
436 return @import_items_ids;
439 =head2 BatchFindDuplicates
441 my $num_with_matches = BatchFindDuplicates($batch_id, $matcher,
442 $max_matches, $progress_interval, $progress_callback);
444 Goes through the records loaded in the batch and attempts to
445 find duplicates for each one. Sets the matching status
446 of each record to "no_match" or "auto_match" as appropriate.
448 The $max_matches parameter is optional; if it is not supplied,
451 The $progress_interval and $progress_callback parameters are
452 optional; if both are supplied, the sub referred to by
453 $progress_callback will be invoked every $progress_interval
454 records using the number of records processed as the
459 sub BatchFindDuplicates {
460 my $batch_id = shift;
462 my $max_matches = @_ ? shift : 10;
464 # optional callback to monitor status
466 my $progress_interval = 0;
467 my $progress_callback = undef;
469 $progress_interval = shift;
470 $progress_callback = shift;
471 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
472 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
475 my $dbh = C4::Context->dbh;
477 my $sth = $dbh->prepare("SELECT import_record_id, record_type, marc
479 WHERE import_batch_id = ?");
480 $sth->execute($batch_id);
481 my $num_with_matches = 0;
483 while (my $rowref = $sth->fetchrow_hashref) {
485 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
486 &$progress_callback($rec_num);
488 my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
490 if (defined $matcher) {
491 @matches = $matcher->get_matches($marc_record, $max_matches);
493 if (scalar(@matches) > 0) {
495 SetImportRecordMatches($rowref->{'import_record_id'}, @matches);
496 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'auto_match');
498 SetImportRecordMatches($rowref->{'import_record_id'}, ());
499 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'no_match');
503 if ($progress_interval){
504 &$progress_callback($rec_num);
508 return $num_with_matches;
511 =head2 BatchCommitRecords
513 Takes a hashref containing params for committing the batch - optional parameters 'progress_interval' and
514 'progress_callback' will define code called every X records.
516 my ($num_added, $num_updated, $num_items_added, $num_items_replaced, $num_items_errored, $num_ignored) =
518 batch_id => $batch_id,
519 framework => $framework,
520 overlay_framework => $overlay_framework,
521 progress_interval => $progress_interval,
522 progress_callback => $progress_callback,
523 skip_intermediate_commit => $skip_intermediate_commit
526 Parameter skip_intermediate_commit does what is says.
529 sub BatchCommitRecords {
531 my $batch_id = $params->{batch_id};
532 my $framework = $params->{framework};
533 my $overlay_framework = $params->{overlay_framework};
534 my $skip_intermediate_commit = $params->{skip_intermediate_commit};
535 my $progress_interval = $params->{progress_interval} // 0;
536 my $progress_callback = $params->{progress_callback};
537 $progress_interval = 0 unless $progress_interval && $progress_interval =~ /^\d+$/;
538 $progress_interval = 0 unless ref($progress_callback) eq 'CODE';
540 my $schema = Koha::Database->schema;
545 my $num_items_added = 0;
546 my $num_items_replaced = 0;
547 my $num_items_errored = 0;
549 # commit (i.e., save, all records in the batch)
550 my $overlay_action = GetImportBatchOverlayAction($batch_id);
551 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
552 my $item_action = GetImportBatchItemAction($batch_id);
555 my $dbh = C4::Context->dbh;
556 my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marc, encoding
558 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
559 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
560 WHERE import_batch_id = ?");
561 $sth->execute($batch_id);
562 my $marcflavour = C4::Context->preference('marcflavour');
564 my $userenv = C4::Context->userenv;
565 my $logged_in_patron = Koha::Patrons->find( $userenv->{number} );
570 while (my $rowref = $sth->fetchrow_hashref) {
572 $record_type = $rowref->{'record_type'};
576 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
578 &$progress_callback( $rec_num );
580 if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'imported') {
586 if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') {
587 $marc_type = 'UNIMARCAUTH';
588 } elsif ($marcflavour eq 'UNIMARC') {
589 $marc_type = 'UNIMARC';
591 $marc_type = 'USMARC';
593 my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
595 if ($record_type eq 'biblio') {
596 # remove any item tags - rely on _batchCommitItems
597 ($item_tag,$item_subfield) = &GetMarcFromKohaField( "items.itemnumber" );
598 foreach my $item_field ($marc_record->field($item_tag)) {
599 $marc_record->delete_field($item_field);
601 if(C4::Context->preference('autoControlNumber') eq 'biblionumber'){
602 my @control_num = $marc_record->field('001');
603 $marc_record->delete_fields(@control_num);
607 my ($record_result, $item_result, $record_match) =
608 _get_commit_action($overlay_action, $nomatch_action, $item_action,
609 $rowref->{'overlay_status'}, $rowref->{'import_record_id'}, $record_type);
613 if ($record_result eq 'create_new') {
615 if ($record_type eq 'biblio') {
616 my $biblioitemnumber;
617 ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework, { skip_record_index => 1 });
618 push @biblio_ids, $recordid if $recordid;
619 $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; # FIXME call SetMatchedBiblionumber instead
620 if ($item_result eq 'create_new' || $item_result eq 'replace') {
621 my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = _batchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result, $biblioitemnumber);
622 $num_items_added += $bib_items_added;
623 $num_items_replaced += $bib_items_replaced;
624 $num_items_errored += $bib_items_errored;
627 $recordid = AddAuthority($marc_record, undef, GuessAuthTypeCode($marc_record));
628 $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
630 my $sth = $dbh->prepare_cached($query);
631 $sth->execute($recordid, $rowref->{'import_record_id'});
633 SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
634 } elsif ($record_result eq 'replace') {
636 $recordid = $record_match;
638 if ($record_type eq 'biblio') {
639 my $oldbiblio = Koha::Biblios->find( $recordid );
640 $oldxml = GetXmlBiblio($recordid);
642 # remove item fields so that they don't get
643 # added again if record is reverted
644 # FIXME: GetXmlBiblio output should not contain item info any more! So the next foreach should not be needed. Does not hurt either; may remove old 952s that should not have been there anymore.
645 my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'}, $marc_type);
646 foreach my $item_field ($old_marc->field($item_tag)) {
647 $old_marc->delete_field($item_field);
649 $oldxml = $old_marc->as_xml($marc_type);
651 my $context = { source => 'batchimport' };
652 if ($logged_in_patron) {
653 $context->{categorycode} = $logged_in_patron->categorycode;
654 $context->{userid} = $logged_in_patron->userid;
660 $overlay_framework // $oldbiblio->frameworkcode,
662 overlay_context => $context,
663 skip_record_index => 1,
664 skip_holds_queue => 1,
667 push @biblio_ids, $recordid;
668 push @updated_ids, $recordid;
669 $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; # FIXME call SetMatchedBiblionumber instead
671 if ($item_result eq 'create_new' || $item_result eq 'replace') {
672 my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = _batchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result);
673 $num_items_added += $bib_items_added;
674 $num_items_replaced += $bib_items_replaced;
675 $num_items_errored += $bib_items_errored;
678 $oldxml = GetAuthorityXML($recordid);
680 ModAuthority($recordid, $marc_record, GuessAuthTypeCode($marc_record));
681 $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
683 # Combine xml update, SetImportRecordOverlayStatus, and SetImportRecordStatus updates into a single update for efficiency, especially in a transaction
684 my $sth = $dbh->prepare_cached("UPDATE import_records SET marcxml_old = ?, status = ?, overlay_status = ? WHERE import_record_id = ?");
685 $sth->execute( $oldxml, 'imported', 'match_applied', $rowref->{'import_record_id'} );
687 my $sth2 = $dbh->prepare_cached($query);
688 $sth2->execute($recordid, $rowref->{'import_record_id'});
690 } elsif ($record_result eq 'ignore') {
691 $recordid = $record_match;
693 if ($record_type eq 'biblio' and defined $recordid and ( $item_result eq 'create_new' || $item_result eq 'replace' ) ) {
694 my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = _batchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result);
695 push @biblio_ids, $recordid if $bib_items_added || $bib_items_replaced;
696 $num_items_added += $bib_items_added;
697 $num_items_replaced += $bib_items_replaced;
698 $num_items_errored += $bib_items_errored;
699 # still need to record the matched biblionumber so that the
700 # items can be reverted
701 my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); # FIXME call SetMatchedBiblionumber instead
702 $sth2->execute($recordid, $rowref->{'import_record_id'});
703 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
705 SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored');
710 if ($progress_interval){
711 &$progress_callback($rec_num);
716 SetImportBatchStatus($batch_id, 'imported');
720 my $indexer = Koha::SearchEngine::Indexer->new( { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
721 $indexer->index_records( \@biblio_ids, "specialUpdate", "biblioserver" );
723 Koha::BackgroundJob::BatchUpdateBiblioHoldsQueue->new->enqueue( { biblio_ids => \@updated_ids } )
724 if ( @updated_ids && C4::Context->preference('RealTimeHoldsQueue') );
726 return ($num_added, $num_updated, $num_items_added, $num_items_replaced, $num_items_errored, $num_ignored);
729 =head2 _batchCommitItems
731 ($num_items_added, $num_items_errored) =
732 _batchCommitItems($import_record_id, $biblionumber, [$action, $biblioitemnumber]);
734 Private function for batch committing item changes. We do not trigger a re-index here, that is left to the caller.
738 sub _batchCommitItems {
739 my ( $import_record_id, $biblionumber, $action, $biblioitemnumber ) = @_;
741 my $dbh = C4::Context->dbh;
743 my $num_items_added = 0;
744 my $num_items_errored = 0;
745 my $num_items_replaced = 0;
747 my $sth = $dbh->prepare( "
748 SELECT import_items_id, import_items.marcxml, encoding
750 JOIN import_records USING (import_record_id)
751 WHERE import_record_id = ?
752 ORDER BY import_items_id
754 $sth->bind_param( 1, $import_record_id );
757 while ( my $row = $sth->fetchrow_hashref() ) {
758 my $item_marc = MARC::Record->new_from_xml( StripNonXmlChars( $row->{'marcxml'} ), 'UTF-8', $row->{'encoding'} );
760 # Delete date_due subfield as to not accidentally delete item checkout due dates
761 my ( $MARCfield, $MARCsubfield ) = GetMarcFromKohaField( 'items.onloan' );
762 $item_marc->field($MARCfield)->delete_subfield( code => $MARCsubfield );
764 my $item = TransformMarcToKoha({ record => $item_marc, kohafields => ['items.barcode','items.itemnumber'] });
767 my $duplicate_barcode = exists( $item->{'barcode'} );
768 my $duplicate_itemnumber = exists( $item->{'itemnumber'} );
770 # We assume that when replacing items we do not want to move them - the onus is on the importer to
771 # ensure the correct items/records are being updated
772 my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, itemnumber = ?, import_error = ? WHERE import_items_id = ?");
774 $action eq "replace" &&
775 $duplicate_itemnumber &&
776 ( $item_match = Koha::Items->find( $item->{itemnumber} ))
778 # Duplicate itemnumbers have precedence, that way we can update barcodes by overlaying
779 ModItemFromMarc( $item_marc, $item_match->biblionumber, $item->{itemnumber}, { skip_record_index => 1 } );
780 $updsth->bind_param( 1, 'imported' );
781 $updsth->bind_param( 2, $item->{itemnumber} );
782 $updsth->bind_param( 3, undef );
783 $updsth->bind_param( 4, $row->{'import_items_id'} );
786 $num_items_replaced++;
788 $action eq "replace" &&
789 $duplicate_barcode &&
790 ( $item_match = Koha::Items->find({ barcode => $item->{'barcode'} }) )
792 ModItemFromMarc( $item_marc, $item_match->biblionumber, $item_match->itemnumber, { skip_record_index => 1 } );
793 $updsth->bind_param( 1, 'imported' );
794 $updsth->bind_param( 2, $item->{itemnumber} );
795 $updsth->bind_param( 3, undef );
796 $updsth->bind_param( 4, $row->{'import_items_id'} );
799 $num_items_replaced++;
801 # We aren't replacing, but the incoming file has a barcode, we need to check if it exists
802 $duplicate_barcode &&
803 ( $item_match = Koha::Items->find({ barcode => $item->{'barcode'} }) )
805 $updsth->bind_param( 1, 'error' );
806 $updsth->bind_param( 2, undef );
807 $updsth->bind_param( 3, 'duplicate item barcode' );
808 $updsth->bind_param( 4, $row->{'import_items_id'} );
810 $num_items_errored++;
812 # Remove the itemnumber if it exists, we want to create a new item
813 my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField( "items.itemnumber" );
814 $item_marc->field($itemtag)->delete_subfield( code => $itemsubfield );
816 my ( $item_biblionumber, $biblioitemnumber, $itemnumber ) = AddItemFromMarc( $item_marc, $biblionumber, { biblioitemnumber => $biblioitemnumber, skip_record_index => 1 } );
818 $updsth->bind_param( 1, 'imported' );
819 $updsth->bind_param( 2, $itemnumber );
820 $updsth->bind_param( 3, undef );
821 $updsth->bind_param( 4, $row->{'import_items_id'} );
829 return ( $num_items_added, $num_items_replaced, $num_items_errored );
832 =head2 BatchRevertRecords
834 my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted,
835 $num_ignored) = BatchRevertRecords($batch_id);
839 sub BatchRevertRecords {
840 my $batch_id = shift;
842 my $logger = Koha::Logger->get( { category => 'C4.ImportBatch' } );
844 $logger->trace("C4::ImportBatch::BatchRevertRecords( $batch_id )");
849 my $num_reverted = 0;
851 my $num_items_deleted = 0;
852 # commit (i.e., save, all records in the batch)
853 SetImportBatchStatus($batch_id, 'reverting');
854 my $overlay_action = GetImportBatchOverlayAction($batch_id);
855 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
856 my $dbh = C4::Context->dbh;
857 my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marcxml_old, encoding, matched_biblionumber, matched_authid
859 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
860 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
861 WHERE import_batch_id = ?");
862 $sth->execute($batch_id);
864 my $marcflavour = C4::Context->preference('marcflavour');
865 while (my $rowref = $sth->fetchrow_hashref) {
866 $record_type = $rowref->{'record_type'};
867 if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') {
871 if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') {
872 $marc_type = 'UNIMARCAUTH';
873 } elsif ($marcflavour eq 'UNIMARC') {
874 $marc_type = 'UNIMARC';
876 $marc_type = 'USMARC';
879 my $record_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'});
881 if ($record_result eq 'delete') {
883 if ($record_type eq 'biblio') {
884 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
885 $error = DelBiblio($rowref->{'matched_biblionumber'});
887 DelAuthority({ authid => $rowref->{'matched_authid'} });
889 if (defined $error) {
893 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
895 } elsif ($record_result eq 'restore') {
897 my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}, $marc_type);
898 if ($record_type eq 'biblio') {
899 my $biblionumber = $rowref->{'matched_biblionumber'};
900 my $oldbiblio = Koha::Biblios->find( $biblionumber );
902 $logger->info("C4::ImportBatch::BatchRevertRecords: Biblio record $biblionumber does not exist, restoration of this record was skipped") unless $oldbiblio;
903 next unless $oldbiblio; # Record has since been deleted. Deleted records should stay deleted.
905 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
906 ModBiblio($old_record, $biblionumber, $oldbiblio->frameworkcode);
908 my $authid = $rowref->{'matched_authid'};
909 ModAuthority($authid, $old_record, GuessAuthTypeCode($old_record));
911 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
912 } elsif ($record_result eq 'ignore') {
913 if ($record_type eq 'biblio') {
914 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
916 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
919 if ($record_type eq 'biblio') {
920 # remove matched_biblionumber only if there is no 'imported' item left
921 $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?"; # FIXME Remove me
922 $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ? AND NOT EXISTS (SELECT * FROM import_items WHERE import_items.import_record_id=import_biblios.import_record_id and status='imported')";
924 $query = "UPDATE import_auths SET matched_authid = NULL WHERE import_record_id = ?";
926 my $sth2 = $dbh->prepare_cached($query);
927 $sth2->execute($rowref->{'import_record_id'});
931 SetImportBatchStatus($batch_id, 'reverted');
932 return ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored);
935 =head2 BatchRevertItems
937 my $num_items_deleted = BatchRevertItems($import_record_id, $biblionumber);
941 sub BatchRevertItems {
942 my ($import_record_id, $biblionumber) = @_;
944 my $dbh = C4::Context->dbh;
945 my $num_items_deleted = 0;
947 my $sth = $dbh->prepare_cached("SELECT import_items_id, itemnumber
949 JOIN items USING (itemnumber)
950 WHERE import_record_id = ?");
951 $sth->bind_param(1, $import_record_id);
953 while (my $row = $sth->fetchrow_hashref()) {
954 my $item = Koha::Items->find($row->{itemnumber});
955 if ($item->safe_delete){
956 my $updsth = $dbh->prepare("UPDATE import_items SET status = ? WHERE import_items_id = ?");
957 $updsth->bind_param(1, 'reverted');
958 $updsth->bind_param(2, $row->{'import_items_id'});
961 $num_items_deleted++;
968 return $num_items_deleted;
973 CleanBatch($batch_id)
975 Deletes all staged records from the import batch
976 and sets the status of the batch to 'cleaned'. Note
977 that deleting a stage record does *not* affect
978 any record that has been committed to the database.
983 my $batch_id = shift;
984 return unless defined $batch_id;
986 C4::Context->dbh->do('DELETE FROM import_records WHERE import_batch_id = ?', {}, $batch_id);
987 SetImportBatchStatus($batch_id, 'cleaned');
992 DeleteBatch($batch_id)
994 Deletes the record from the database. This can only be done
995 once the batch has been cleaned.
1000 my $batch_id = shift;
1001 return unless defined $batch_id;
1003 my $dbh = C4::Context->dbh;
1004 my $sth = $dbh->prepare('DELETE FROM import_batches WHERE import_batch_id = ?');
1005 $sth->execute( $batch_id );
1008 =head2 GetAllImportBatches
1010 my $results = GetAllImportBatches();
1012 Returns a references to an array of hash references corresponding
1013 to all import_batches rows (of batch_type 'batch'), sorted in
1014 ascending order by import_batch_id.
1018 sub GetAllImportBatches {
1019 my $dbh = C4::Context->dbh;
1020 my $sth = $dbh->prepare_cached("SELECT * FROM import_batches
1021 WHERE batch_type IN ('batch', 'webservice')
1022 ORDER BY import_batch_id ASC");
1026 while (my $row = $sth->fetchrow_hashref) {
1027 push @$results, $row;
1033 =head2 GetStagedWebserviceBatches
1035 my $batch_ids = GetStagedWebserviceBatches();
1037 Returns a references to an array of batch id's
1038 of batch_type 'webservice' that are not imported
1042 my $PENDING_WEBSERVICE_BATCHES_QRY = <<EOQ;
1043 SELECT import_batch_id FROM import_batches
1044 WHERE batch_type = 'webservice'
1045 AND import_status = 'staged'
1047 sub GetStagedWebserviceBatches {
1048 my $dbh = C4::Context->dbh;
1049 return $dbh->selectcol_arrayref($PENDING_WEBSERVICE_BATCHES_QRY);
1052 =head2 GetImportBatchRangeDesc
1054 my $results = GetImportBatchRangeDesc($offset, $results_per_group);
1056 Returns a reference to an array of hash references corresponding to
1057 import_batches rows (sorted in descending order by import_batch_id)
1058 start at the given offset.
1062 sub GetImportBatchRangeDesc {
1063 my ($offset, $results_per_group) = @_;
1065 my $dbh = C4::Context->dbh;
1066 my $query = "SELECT b.*, p.name as profile FROM import_batches b
1067 LEFT JOIN import_batch_profiles p
1068 ON b.profile_id = p.id
1069 WHERE b.batch_type IN ('batch', 'webservice')
1070 ORDER BY b.import_batch_id DESC";
1072 if ($results_per_group){
1073 $query .= " LIMIT ?";
1074 push(@params, $results_per_group);
1077 $query .= " OFFSET ?";
1078 push(@params, $offset);
1080 my $sth = $dbh->prepare_cached($query);
1081 $sth->execute(@params);
1082 my $results = $sth->fetchall_arrayref({});
1087 =head2 GetItemNumbersFromImportBatch
1089 my @itemsnos = GetItemNumbersFromImportBatch($batch_id);
1093 sub GetItemNumbersFromImportBatch {
1094 my ($batch_id) = @_;
1095 my $dbh = C4::Context->dbh;
1097 SELECT itemnumber FROM import_items
1098 INNER JOIN items USING (itemnumber)
1099 INNER JOIN import_records USING (import_record_id)
1100 WHERE import_batch_id = ?|;
1101 my $sth = $dbh->prepare( $sql );
1102 $sth->execute($batch_id);
1104 while ( my ($itm) = $sth->fetchrow_array ) {
1110 =head2 GetNumberOfImportBatches
1112 my $count = GetNumberOfImportBatches();
1116 sub GetNumberOfNonZ3950ImportBatches {
1117 my $dbh = C4::Context->dbh;
1118 my $sth = $dbh->prepare("SELECT COUNT(*) FROM import_batches WHERE batch_type != 'z3950'");
1120 my ($count) = $sth->fetchrow_array();
1125 =head2 GetImportBiblios
1127 my $results = GetImportBiblios($importid);
1131 sub GetImportBiblios {
1132 my ($import_record_id) = @_;
1134 my $dbh = C4::Context->dbh;
1135 my $query = "SELECT * FROM import_biblios WHERE import_record_id = ?";
1136 return $dbh->selectall_arrayref(
1144 =head2 GetImportRecordsRange
1146 my $results = GetImportRecordsRange($batch_id, $offset, $results_per_group);
1148 Returns a reference to an array of hash references corresponding to
1149 import_biblios/import_auths/import_records rows for a given batch
1150 starting at the given offset.
1154 sub GetImportRecordsRange {
1155 my ( $batch_id, $offset, $results_per_group, $status, $parameters ) = @_;
1157 my $dbh = C4::Context->dbh;
1159 my $order_by = $parameters->{order_by} || 'import_record_id';
1160 ( $order_by ) = grep( { $_ eq $order_by } qw( import_record_id title status overlay_status ) ) ? $order_by : 'import_record_id';
1162 my $order_by_direction =
1163 uc( $parameters->{order_by_direction} // 'ASC' ) eq 'DESC' ? 'DESC' : 'ASC';
1165 $order_by .= " $order_by_direction, authorized_heading" if $order_by eq 'title';
1167 my $query = "SELECT title, author, isbn, issn, authorized_heading, import_records.import_record_id,
1168 record_sequence, status, overlay_status,
1169 matched_biblionumber, matched_authid, record_type
1171 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
1172 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
1173 WHERE import_batch_id = ?";
1175 push(@params, $batch_id);
1177 $query .= " AND status=?";
1178 push(@params,$status);
1181 $query.=" ORDER BY $order_by $order_by_direction";
1183 if($results_per_group){
1184 $query .= " LIMIT ?";
1185 push(@params, $results_per_group);
1188 $query .= " OFFSET ?";
1189 push(@params, $offset);
1191 my $sth = $dbh->prepare_cached($query);
1192 $sth->execute(@params);
1193 my $results = $sth->fetchall_arrayref({});
1199 =head2 GetBestRecordMatch
1201 my $record_id = GetBestRecordMatch($import_record_id);
1205 sub GetBestRecordMatch {
1206 my ($import_record_id) = @_;
1208 my $dbh = C4::Context->dbh;
1209 my $sth = $dbh->prepare("SELECT candidate_match_id
1210 FROM import_record_matches
1211 JOIN import_records ON ( import_record_matches.import_record_id = import_records.import_record_id )
1212 LEFT JOIN biblio ON ( candidate_match_id = biblio.biblionumber )
1213 LEFT JOIN auth_header ON ( candidate_match_id = auth_header.authid )
1214 WHERE import_record_matches.import_record_id = ? AND
1215 ( (import_records.record_type = 'biblio' AND biblio.biblionumber IS NOT NULL) OR
1216 (import_records.record_type = 'auth' AND auth_header.authid IS NOT NULL) )
1218 ORDER BY score DESC, candidate_match_id DESC");
1219 $sth->execute($import_record_id);
1220 my ($record_id) = $sth->fetchrow_array();
1225 =head2 GetImportBatchStatus
1227 my $status = GetImportBatchStatus($batch_id);
1231 sub GetImportBatchStatus {
1232 my ($batch_id) = @_;
1234 my $dbh = C4::Context->dbh;
1235 my $sth = $dbh->prepare("SELECT import_status FROM import_batches WHERE import_batch_id = ?");
1236 $sth->execute($batch_id);
1237 my ($status) = $sth->fetchrow_array();
1243 =head2 SetImportBatchStatus
1245 SetImportBatchStatus($batch_id, $new_status);
1249 sub SetImportBatchStatus {
1250 my ($batch_id, $new_status) = @_;
1252 my $dbh = C4::Context->dbh;
1253 my $sth = $dbh->prepare("UPDATE import_batches SET import_status = ? WHERE import_batch_id = ?");
1254 $sth->execute($new_status, $batch_id);
1259 =head2 SetMatchedBiblionumber
1261 SetMatchedBiblionumber($import_record_id, $biblionumber);
1265 sub SetMatchedBiblionumber {
1266 my ($import_record_id, $biblionumber) = @_;
1268 my $dbh = C4::Context->dbh;
1270 q|UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?|,
1271 undef, $biblionumber, $import_record_id
1275 =head2 GetImportBatchOverlayAction
1277 my $overlay_action = GetImportBatchOverlayAction($batch_id);
1281 sub GetImportBatchOverlayAction {
1282 my ($batch_id) = @_;
1284 my $dbh = C4::Context->dbh;
1285 my $sth = $dbh->prepare("SELECT overlay_action FROM import_batches WHERE import_batch_id = ?");
1286 $sth->execute($batch_id);
1287 my ($overlay_action) = $sth->fetchrow_array();
1289 return $overlay_action;
1294 =head2 SetImportBatchOverlayAction
1296 SetImportBatchOverlayAction($batch_id, $new_overlay_action);
1300 sub SetImportBatchOverlayAction {
1301 my ($batch_id, $new_overlay_action) = @_;
1303 my $dbh = C4::Context->dbh;
1304 my $sth = $dbh->prepare("UPDATE import_batches SET overlay_action = ? WHERE import_batch_id = ?");
1305 $sth->execute($new_overlay_action, $batch_id);
1310 =head2 GetImportBatchNoMatchAction
1312 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
1316 sub GetImportBatchNoMatchAction {
1317 my ($batch_id) = @_;
1319 my $dbh = C4::Context->dbh;
1320 my $sth = $dbh->prepare("SELECT nomatch_action FROM import_batches WHERE import_batch_id = ?");
1321 $sth->execute($batch_id);
1322 my ($nomatch_action) = $sth->fetchrow_array();
1324 return $nomatch_action;
1329 =head2 SetImportBatchNoMatchAction
1331 SetImportBatchNoMatchAction($batch_id, $new_nomatch_action);
1335 sub SetImportBatchNoMatchAction {
1336 my ($batch_id, $new_nomatch_action) = @_;
1338 my $dbh = C4::Context->dbh;
1339 my $sth = $dbh->prepare("UPDATE import_batches SET nomatch_action = ? WHERE import_batch_id = ?");
1340 $sth->execute($new_nomatch_action, $batch_id);
1345 =head2 GetImportBatchItemAction
1347 my $item_action = GetImportBatchItemAction($batch_id);
1351 sub GetImportBatchItemAction {
1352 my ($batch_id) = @_;
1354 my $dbh = C4::Context->dbh;
1355 my $sth = $dbh->prepare("SELECT item_action FROM import_batches WHERE import_batch_id = ?");
1356 $sth->execute($batch_id);
1357 my ($item_action) = $sth->fetchrow_array();
1359 return $item_action;
1364 =head2 SetImportBatchItemAction
1366 SetImportBatchItemAction($batch_id, $new_item_action);
1370 sub SetImportBatchItemAction {
1371 my ($batch_id, $new_item_action) = @_;
1373 my $dbh = C4::Context->dbh;
1374 my $sth = $dbh->prepare("UPDATE import_batches SET item_action = ? WHERE import_batch_id = ?");
1375 $sth->execute($new_item_action, $batch_id);
1380 =head2 GetImportBatchMatcher
1382 my $matcher_id = GetImportBatchMatcher($batch_id);
1386 sub GetImportBatchMatcher {
1387 my ($batch_id) = @_;
1389 my $dbh = C4::Context->dbh;
1390 my $sth = $dbh->prepare("SELECT matcher_id FROM import_batches WHERE import_batch_id = ?");
1391 $sth->execute($batch_id);
1392 my ($matcher_id) = $sth->fetchrow_array();
1399 =head2 SetImportBatchMatcher
1401 SetImportBatchMatcher($batch_id, $new_matcher_id);
1405 sub SetImportBatchMatcher {
1406 my ($batch_id, $new_matcher_id) = @_;
1408 my $dbh = C4::Context->dbh;
1409 my $sth = $dbh->prepare("UPDATE import_batches SET matcher_id = ? WHERE import_batch_id = ?");
1410 $sth->execute($new_matcher_id, $batch_id);
1415 =head2 GetImportRecordOverlayStatus
1417 my $overlay_status = GetImportRecordOverlayStatus($import_record_id);
1421 sub GetImportRecordOverlayStatus {
1422 my ($import_record_id) = @_;
1424 my $dbh = C4::Context->dbh;
1425 my $sth = $dbh->prepare("SELECT overlay_status FROM import_records WHERE import_record_id = ?");
1426 $sth->execute($import_record_id);
1427 my ($overlay_status) = $sth->fetchrow_array();
1429 return $overlay_status;
1434 =head2 SetImportRecordOverlayStatus
1436 SetImportRecordOverlayStatus($import_record_id, $new_overlay_status);
1440 sub SetImportRecordOverlayStatus {
1441 my ($import_record_id, $new_overlay_status) = @_;
1443 my $dbh = C4::Context->dbh;
1444 my $sth = $dbh->prepare("UPDATE import_records SET overlay_status = ? WHERE import_record_id = ?");
1445 $sth->execute($new_overlay_status, $import_record_id);
1450 =head2 GetImportRecordStatus
1452 my $status = GetImportRecordStatus($import_record_id);
1456 sub GetImportRecordStatus {
1457 my ($import_record_id) = @_;
1459 my $dbh = C4::Context->dbh;
1460 my $sth = $dbh->prepare("SELECT status FROM import_records WHERE import_record_id = ?");
1461 $sth->execute($import_record_id);
1462 my ($status) = $sth->fetchrow_array();
1469 =head2 SetImportRecordStatus
1471 SetImportRecordStatus($import_record_id, $new_status);
1475 sub SetImportRecordStatus {
1476 my ($import_record_id, $new_status) = @_;
1478 my $dbh = C4::Context->dbh;
1479 my $sth = $dbh->prepare("UPDATE import_records SET status = ? WHERE import_record_id = ?");
1480 $sth->execute($new_status, $import_record_id);
1485 =head2 GetImportRecordMatches
1487 my $results = GetImportRecordMatches($import_record_id, $best_only);
1491 sub GetImportRecordMatches {
1492 my $import_record_id = shift;
1493 my $best_only = @_ ? shift : 0;
1495 my $dbh = C4::Context->dbh;
1496 # FIXME currently biblio only
1497 my $sth = $dbh->prepare_cached("SELECT title, author, biblionumber,
1498 candidate_match_id, score, record_type,
1501 JOIN import_record_matches USING (import_record_id)
1502 LEFT JOIN biblio ON (biblionumber = candidate_match_id)
1503 WHERE import_record_id = ?
1504 ORDER BY score DESC, biblionumber DESC");
1505 $sth->bind_param(1, $import_record_id);
1508 while (my $row = $sth->fetchrow_hashref) {
1509 if ($row->{'record_type'} eq 'auth') {
1510 $row->{'authorized_heading'} = GetAuthorizedHeading( { authid => $row->{'candidate_match_id'} } );
1512 next if ($row->{'record_type'} eq 'biblio' && not $row->{'biblionumber'});
1513 push @$results, $row;
1522 =head2 SetImportRecordMatches
1524 SetImportRecordMatches($import_record_id, @matches);
1528 sub SetImportRecordMatches {
1529 my $import_record_id = shift;
1532 my $dbh = C4::Context->dbh;
1533 my $delsth = $dbh->prepare("DELETE FROM import_record_matches WHERE import_record_id = ?");
1534 $delsth->execute($import_record_id);
1537 my $sth = $dbh->prepare("INSERT INTO import_record_matches (import_record_id, candidate_match_id, score, chosen)
1538 VALUES (?, ?, ?, ?)");
1539 my $chosen = 1; #The first match is defaulted to be chosen
1540 foreach my $match (@matches) {
1541 $sth->execute($import_record_id, $match->{'record_id'}, $match->{'score'}, $chosen);
1542 $chosen = 0; #After the first we do not default to other matches
1546 =head2 RecordsFromISO2709File
1548 my ($errors, $records) = C4::ImportBatch::RecordsFromISO2709File($input_file, $record_type, $encoding);
1550 Reads ISO2709 binary porridge from the given file and creates MARC::Record-objects out of it.
1552 @PARAM1, String, absolute path to the ISO2709 file.
1553 @PARAM2, String, see stage_file.pl
1554 @PARAM3, String, should be utf8
1556 Returns two array refs.
1560 sub RecordsFromISO2709File {
1561 my ($input_file, $record_type, $encoding) = @_;
1564 my $marc_type = C4::Context->preference('marcflavour');
1565 $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth');
1567 open my $fh, '<', $input_file or die "$0: cannot open input file $input_file: $!\n";
1573 next unless $_; # skip if record has only whitespace, as might occur
1574 # if file includes newlines between each MARC record
1575 my ($marc_record, $charset_guessed, $char_errors) = MarcToUTF8Record($_, $marc_type, $encoding);
1576 push @marc_records, $marc_record;
1577 if ($charset_guessed ne $encoding) {
1579 "Unexpected charset $charset_guessed, expecting $encoding";
1583 return ( \@errors, \@marc_records );
1586 =head2 RecordsFromMARCXMLFile
1588 my ($errors, $records) = C4::ImportBatch::RecordsFromMARCXMLFile($input_file, $encoding);
1590 Creates MARC::Record-objects out of the given MARCXML-file.
1592 @PARAM1, String, absolute path to the MARCXML file.
1593 @PARAM2, String, should be utf8
1595 Returns two array refs.
1599 sub RecordsFromMARCXMLFile {
1600 my ( $filename, $encoding ) = @_;
1601 my $batch = MARC::File::XML->in( $filename );
1602 my ( @marcRecords, @errors, $record );
1604 eval { $record = $batch->next( $encoding ); };
1608 push @marcRecords, $record if $record;
1610 return (\@errors, \@marcRecords);
1613 =head2 RecordsFromMarcPlugin
1615 Converts text of input_file into array of MARC records with to_marc plugin
1619 sub RecordsFromMarcPlugin {
1620 my ($input_file, $plugin_class, $encoding) = @_;
1621 my ( $text, @return );
1622 return \@return if !$input_file || !$plugin_class;
1625 open my $fh, '<', $input_file or die "$0: cannot open input file $input_file: $!\n";
1635 # Convert to large MARC blob with plugin
1636 $text = Koha::Plugins::Handler->run({
1637 class => $plugin_class,
1638 method => 'to_marc',
1639 params => { data => $text },
1642 # Convert to array of MARC records
1644 my $marc_type = C4::Context->preference('marcflavour');
1645 foreach my $blob ( split(/\x1D/, $text) ) {
1646 next if $blob =~ /^\s*$/;
1647 my ($marcrecord) = MarcToUTF8Record($blob, $marc_type, $encoding);
1648 push @return, $marcrecord;
1654 # internal functions
1656 sub _create_import_record {
1657 my ($batch_id, $record_sequence, $marc_record, $record_type, $encoding, $marc_type) = @_;
1659 my $dbh = C4::Context->dbh;
1660 my $sth = $dbh->prepare("INSERT INTO import_records (import_batch_id, record_sequence, marc, marcxml, marcxml_old,
1661 record_type, encoding)
1662 VALUES (?, ?, ?, ?, ?, ?, ?)");
1663 $sth->execute($batch_id, $record_sequence, $marc_record->as_usmarc(), $marc_record->as_xml($marc_type), '',
1664 $record_type, $encoding);
1665 my $import_record_id = $dbh->{'mysql_insertid'};
1667 return $import_record_id;
1670 sub _add_auth_fields {
1671 my ($import_record_id, $marc_record) = @_;
1674 if ($marc_record->field('001')) {
1675 $controlnumber = $marc_record->field('001')->data();
1677 my $authorized_heading = GetAuthorizedHeading({ record => $marc_record });
1678 my $dbh = C4::Context->dbh;
1679 my $sth = $dbh->prepare("INSERT INTO import_auths (import_record_id, control_number, authorized_heading) VALUES (?, ?, ?)");
1680 $sth->execute($import_record_id, $controlnumber, $authorized_heading);
1684 sub _add_biblio_fields {
1685 my ($import_record_id, $marc_record) = @_;
1687 my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record);
1688 my $dbh = C4::Context->dbh;
1689 # FIXME no controlnumber, originalsource
1690 $isbn = C4::Koha::GetNormalizedISBN($isbn);
1691 my $sth = $dbh->prepare("INSERT INTO import_biblios (import_record_id, title, author, isbn, issn) VALUES (?, ?, ?, ?, ?)");
1692 $sth->execute($import_record_id, $title, $author, $isbn, $issn) or die $sth->errstr;
1697 sub _update_biblio_fields {
1698 my ($import_record_id, $marc_record) = @_;
1700 my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record);
1701 my $dbh = C4::Context->dbh;
1702 # FIXME no controlnumber, originalsource
1703 # FIXME 2 - should regularize normalization of ISBN wherever it is done
1707 my $sth = $dbh->prepare("UPDATE import_biblios SET title = ?, author = ?, isbn = ?, issn = ?
1708 WHERE import_record_id = ?");
1709 $sth->execute($title, $author, $isbn, $issn, $import_record_id);
1713 sub _parse_biblio_fields {
1714 my ($marc_record) = @_;
1716 my $dbh = C4::Context->dbh;
1717 my $bibliofields = TransformMarcToKoha({ record => $marc_record, kohafields => ['biblio.title','biblio.author','biblioitems.isbn','biblioitems.issn'] });
1718 return ($bibliofields->{'title'}, $bibliofields->{'author'}, $bibliofields->{'isbn'}, $bibliofields->{'issn'});
1722 sub _update_batch_record_counts {
1723 my ($batch_id) = @_;
1725 my $dbh = C4::Context->dbh;
1726 my ( $num_records ) = $dbh->selectrow_array(q|
1729 WHERE import_batch_id = ?
1730 |, undef, $batch_id );
1731 my ( $num_items ) = $dbh->selectrow_array(q|
1734 JOIN import_items USING (import_record_id)
1735 WHERE import_batch_id = ? AND record_type = 'biblio'
1736 |, undef, $batch_id );
1738 "UPDATE import_batches SET num_records=?, num_items=? WHERE import_batch_id=?",
1746 sub _get_commit_action {
1747 my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id, $record_type) = @_;
1749 if ($record_type eq 'biblio') {
1750 my ($bib_result, $bib_match, $item_result);
1752 $bib_match = GetBestRecordMatch($import_record_id);
1753 if ($overlay_status ne 'no_match' && defined($bib_match)) {
1755 $bib_result = $overlay_action;
1757 if($item_action eq 'always_add' or $item_action eq 'add_only_for_matches'){
1758 $item_result = 'create_new';
1759 } elsif($item_action eq 'replace'){
1760 $item_result = 'replace';
1762 $item_result = 'ignore';
1766 $bib_result = $nomatch_action;
1767 $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore';
1769 return ($bib_result, $item_result, $bib_match);
1770 } else { # must be auths
1771 my ($auth_result, $auth_match);
1773 $auth_match = GetBestRecordMatch($import_record_id);
1774 if ($overlay_status ne 'no_match' && defined($auth_match)) {
1775 $auth_result = $overlay_action;
1777 $auth_result = $nomatch_action;
1780 return ($auth_result, undef, $auth_match);
1785 sub _get_revert_action {
1786 my ($overlay_action, $overlay_status, $status) = @_;
1790 if ($status eq 'ignored') {
1791 $bib_result = 'ignore';
1793 if ($overlay_action eq 'create_new') {
1794 $bib_result = 'delete';
1796 $bib_result = ($overlay_status eq 'match_applied') ? 'restore' : 'delete';
1807 Koha Development Team <http://koha-community.org/>
1809 Galen Charlton <galen.charlton@liblime.com>