1 package C4::ImportBatch;
3 # Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it
8 # under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 3 of the License, or
10 # (at your option) any later version.
12 # Koha is distributed in the hope that it will be useful, but
13 # WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with Koha; if not, see <http://www.gnu.org/licenses>.
24 use C4::Koha qw( GetNormalizedISBN );
33 use C4::Items qw( AddItemFromMarc ModItemFromMarc );
34 use C4::Charset qw( MarcToUTF8Record SetUTF8Flag StripNonXmlChars );
35 use C4::AuthoritiesMarc qw( AddAuthority GuessAuthTypeCode GetAuthorityXML ModAuthority DelAuthority GetAuthorizedHeading );
36 use C4::MarcModificationTemplates qw( ModifyRecordWithTemplate );
37 use Koha::BackgroundJob::BatchUpdateBiblioHoldsQueue;
39 use Koha::SearchEngine;
40 use Koha::SearchEngine::Indexer;
41 use Koha::Plugins::Handler;
44 our (@ISA, @EXPORT_OK);
56 AddItemsToImportBiblio
67 GetStagedWebserviceBatches
68 GetImportBatchRangeDesc
69 GetNumberOfNonZ3950ImportBatches
72 GetItemNumbersFromImportBatch
76 GetImportBatchOverlayAction
77 SetImportBatchOverlayAction
78 GetImportBatchNoMatchAction
79 SetImportBatchNoMatchAction
80 GetImportBatchItemAction
81 SetImportBatchItemAction
84 GetImportRecordOverlayStatus
85 SetImportRecordOverlayStatus
88 SetMatchedBiblionumber
89 GetImportRecordMatches
90 SetImportRecordMatches
92 RecordsFromMARCXMLFile
93 RecordsFromISO2709File
100 C4::ImportBatch - manage batches of imported MARC records
108 =head2 GetZ3950BatchId
110 my $batchid = GetZ3950BatchId($z3950server);
112 Retrieves the ID of the import batch for the Z39.50
113 reservoir for the given target. If necessary,
114 creates the import batch.
118 sub GetZ3950BatchId {
119 my ($z3950server) = @_;
121 my $dbh = C4::Context->dbh;
122 my $sth = $dbh->prepare("SELECT import_batch_id FROM import_batches
123 WHERE batch_type = 'z3950'
125 $sth->execute($z3950server);
126 my $rowref = $sth->fetchrow_arrayref();
128 if (defined $rowref) {
131 my $batch_id = AddImportBatch( {
132 overlay_action => 'create_new',
133 import_status => 'staged',
134 batch_type => 'z3950',
135 file_name => $z3950server,
142 =head2 GetWebserviceBatchId
144 my $batchid = GetWebserviceBatchId();
146 Retrieves the ID of the import batch for webservice.
147 If necessary, creates the import batch.
151 my $WEBSERVICE_BASE_QRY = <<EOQ;
152 SELECT import_batch_id FROM import_batches
153 WHERE batch_type = 'webservice'
154 AND import_status = 'staged'
156 sub GetWebserviceBatchId {
159 my $dbh = C4::Context->dbh;
160 my $sql = $WEBSERVICE_BASE_QRY;
162 foreach my $field (qw(matcher_id overlay_action nomatch_action item_action)) {
163 if (my $val = $params->{$field}) {
164 $sql .= " AND $field = ?";
168 my $id = $dbh->selectrow_array($sql, undef, @args);
171 $params->{batch_type} = 'webservice';
172 $params->{import_status} = 'staged';
173 return AddImportBatch($params);
176 =head2 GetImportRecordMarc
178 my ($marcblob, $encoding) = GetImportRecordMarc($import_record_id);
182 sub GetImportRecordMarc {
183 my ($import_record_id) = @_;
185 my $dbh = C4::Context->dbh;
186 my ( $marc, $encoding ) = $dbh->selectrow_array(q|
187 SELECT marc, encoding
189 WHERE import_record_id = ?
190 |, undef, $import_record_id );
192 return $marc, $encoding;
195 sub EmbedItemsInImportBiblio {
196 my ( $record, $import_record_id ) = @_;
197 my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField( "items.itemnumber" );
198 my $dbh = C4::Context->dbh;
199 my $import_items = $dbh->selectall_arrayref(q|
200 SELECT import_items.marcxml
202 WHERE import_record_id = ?
203 |, { Slice => {} }, $import_record_id );
205 for my $import_item ( @$import_items ) {
206 my $item_marc = MARC::Record::new_from_xml($import_item->{marcxml}, 'UTF-8');
207 push @item_fields, $item_marc->field($itemtag);
209 $record->append_fields(@item_fields);
213 =head2 AddImportBatch
215 my $batch_id = AddImportBatch($params_hash);
223 foreach (qw( matcher_id template_id branchcode
224 overlay_action nomatch_action item_action
225 import_status batch_type file_name comments record_type )) {
226 if (exists $params->{$_}) {
228 push @vals, $params->{$_};
231 my $dbh = C4::Context->dbh;
232 $dbh->do("INSERT INTO import_batches (".join( ',', @fields).")
233 VALUES (".join( ',', map '?', @fields).")",
236 return $dbh->{'mysql_insertid'};
239 =head2 GetImportBatch
241 my $row = GetImportBatch($batch_id);
243 Retrieve a hashref of an import_batches row.
250 my $dbh = C4::Context->dbh;
251 my $sth = $dbh->prepare_cached("SELECT b.*, p.name as profile FROM import_batches b LEFT JOIN import_batch_profiles p ON p.id = b.profile_id WHERE import_batch_id = ?");
252 $sth->bind_param(1, $batch_id);
254 my $result = $sth->fetchrow_hashref;
260 =head2 AddBiblioToBatch
262 my $import_record_id = AddBiblioToBatch($batch_id, $record_sequence,
263 $marc_record, $encoding, $update_counts);
267 sub AddBiblioToBatch {
268 my $batch_id = shift;
269 my $record_sequence = shift;
270 my $marc_record = shift;
271 my $encoding = shift;
272 my $update_counts = @_ ? shift : 1;
274 my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'biblio', $encoding, C4::Context->preference('marcflavour'));
275 _add_biblio_fields($import_record_id, $marc_record);
276 _update_batch_record_counts($batch_id) if $update_counts;
277 return $import_record_id;
280 =head2 AddAuthToBatch
282 my $import_record_id = AddAuthToBatch($batch_id, $record_sequence,
283 $marc_record, $encoding, $update_counts, [$marc_type]);
288 my $batch_id = shift;
289 my $record_sequence = shift;
290 my $marc_record = shift;
291 my $encoding = shift;
292 my $update_counts = @_ ? shift : 1;
293 my $marc_type = shift || C4::Context->preference('marcflavour');
295 $marc_type = 'UNIMARCAUTH' if $marc_type eq 'UNIMARC';
297 my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $marc_type);
298 _add_auth_fields($import_record_id, $marc_record);
299 _update_batch_record_counts($batch_id) if $update_counts;
300 return $import_record_id;
303 =head2 BatchStageMarcRecords
305 ( $batch_id, $num_records, $num_items, @invalid_records ) =
306 BatchStageMarcRecords(
307 $record_type, $encoding,
308 $marc_records, $file_name,
309 $marc_modification_template, $comments,
310 $branch_code, $parse_items,
311 $leave_as_staging, $progress_interval,
317 sub BatchStageMarcRecords {
318 my $record_type = shift;
319 my $encoding = shift;
320 my $marc_records = shift;
321 my $file_name = shift;
322 my $marc_modification_template = shift;
323 my $comments = shift;
324 my $branch_code = shift;
325 my $parse_items = shift;
326 my $leave_as_staging = shift;
328 # optional callback to monitor status
330 my $progress_interval = 0;
331 my $progress_callback = undef;
333 $progress_interval = shift;
334 $progress_callback = shift;
335 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
336 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
339 my $batch_id = AddImportBatch( {
340 overlay_action => 'create_new',
341 import_status => 'staging',
342 batch_type => 'batch',
343 file_name => $file_name,
344 comments => $comments,
345 record_type => $record_type,
348 SetImportBatchItemAction($batch_id, 'always_add');
350 SetImportBatchItemAction($batch_id, 'ignore');
354 my $marc_type = C4::Context->preference('marcflavour');
355 $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth');
356 my @invalid_records = ();
359 # FIXME - for now, we're dealing only with bibs
361 foreach my $marc_record (@$marc_records) {
363 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
364 &$progress_callback($rec_num);
367 ModifyRecordWithTemplate( $marc_modification_template, $marc_record ) if ( $marc_modification_template );
369 my $import_record_id;
370 if (scalar($marc_record->fields()) == 0) {
371 push @invalid_records, $marc_record;
374 # Normalize the record so it doesn't have separated diacritics
375 SetUTF8Flag($marc_record);
378 if ($record_type eq 'biblio') {
379 $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, 0);
381 my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0);
382 $num_items += scalar(@import_items_ids);
384 } elsif ($record_type eq 'auth') {
385 $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, 0, $marc_type);
389 unless ($leave_as_staging) {
390 SetImportBatchStatus($batch_id, 'staged');
392 # FIXME branch_code, number of bibs, number of items
393 _update_batch_record_counts($batch_id);
394 if ($progress_interval){
395 &$progress_callback($rec_num);
398 return ($batch_id, $num_valid, $num_items, @invalid_records);
401 =head2 AddItemsToImportBiblio
403 my @import_items_ids = AddItemsToImportBiblio($batch_id,
404 $import_record_id, $marc_record, $update_counts);
408 sub AddItemsToImportBiblio {
409 my $batch_id = shift;
410 my $import_record_id = shift;
411 my $marc_record = shift;
412 my $update_counts = @_ ? shift : 0;
414 my @import_items_ids = ();
416 my $dbh = C4::Context->dbh;
417 my ($item_tag,$item_subfield) = &GetMarcFromKohaField( "items.itemnumber" );
418 foreach my $item_field ($marc_record->field($item_tag)) {
419 my $item_marc = MARC::Record->new();
420 $item_marc->leader("00000 a "); # must set Leader/09 to 'a'
421 $item_marc->append_fields($item_field);
422 $marc_record->delete_field($item_field);
423 my $sth = $dbh->prepare_cached("INSERT INTO import_items (import_record_id, status, marcxml)
425 $sth->bind_param(1, $import_record_id);
426 $sth->bind_param(2, 'staged');
427 $sth->bind_param(3, $item_marc->as_xml("USMARC"));
429 push @import_items_ids, $dbh->{'mysql_insertid'};
433 if ($#import_items_ids > -1) {
434 _update_batch_record_counts($batch_id) if $update_counts;
436 return @import_items_ids;
439 =head2 BatchFindDuplicates
441 my $num_with_matches = BatchFindDuplicates($batch_id, $matcher,
442 $max_matches, $progress_interval, $progress_callback);
444 Goes through the records loaded in the batch and attempts to
445 find duplicates for each one. Sets the matching status
446 of each record to "no_match" or "auto_match" as appropriate.
448 The $max_matches parameter is optional; if it is not supplied,
451 The $progress_interval and $progress_callback parameters are
452 optional; if both are supplied, the sub referred to by
453 $progress_callback will be invoked every $progress_interval
454 records using the number of records processed as the
459 sub BatchFindDuplicates {
460 my $batch_id = shift;
462 my $max_matches = @_ ? shift : 10;
464 # optional callback to monitor status
466 my $progress_interval = 0;
467 my $progress_callback = undef;
469 $progress_interval = shift;
470 $progress_callback = shift;
471 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
472 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
475 my $dbh = C4::Context->dbh;
477 my $sth = $dbh->prepare("SELECT import_record_id, record_type, marc
479 WHERE import_batch_id = ?");
480 $sth->execute($batch_id);
481 my $num_with_matches = 0;
483 while (my $rowref = $sth->fetchrow_hashref) {
485 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
486 &$progress_callback($rec_num);
488 my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
490 if (defined $matcher) {
491 @matches = $matcher->get_matches($marc_record, $max_matches);
493 if (scalar(@matches) > 0) {
495 SetImportRecordMatches($rowref->{'import_record_id'}, @matches);
496 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'auto_match');
498 SetImportRecordMatches($rowref->{'import_record_id'}, ());
499 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'no_match');
503 if ($progress_interval){
504 &$progress_callback($rec_num);
508 return $num_with_matches;
511 =head2 BatchCommitRecords
513 Takes a hashref containing params for committing the batch - optional parameters 'progress_interval' and
514 'progress_callback' will define code called every X records.
516 my ($num_added, $num_updated, $num_items_added, $num_items_replaced, $num_items_errored, $num_ignored) =
518 batch_id => $batch_id,
519 framework => $framework,
520 overlay_framework => $overlay_framework,
521 progress_interval => $progress_interval,
522 progress_callback => $progress_callback,
526 sub BatchCommitRecords {
528 my $batch_id = $params->{batch_id};
529 my $framework = $params->{framework};
530 my $overlay_framework = $params->{overlay_framework};
531 my $progress_interval = $params->{progress_interval} // 0;
532 my $progress_callback = $params->{progress_callback};
533 $progress_interval = 0 unless $progress_interval && $progress_interval =~ /^\d+$/;
534 $progress_interval = 0 unless ref($progress_callback) eq 'CODE';
536 my $schema = Koha::Database->schema;
541 my $num_items_added = 0;
542 my $num_items_replaced = 0;
543 my $num_items_errored = 0;
545 # commit (i.e., save, all records in the batch)
546 my $overlay_action = GetImportBatchOverlayAction($batch_id);
547 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
548 my $item_action = GetImportBatchItemAction($batch_id);
551 my $dbh = C4::Context->dbh;
552 my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marc, encoding
554 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
555 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
556 WHERE import_batch_id = ?");
557 $sth->execute($batch_id);
558 my $marcflavour = C4::Context->preference('marcflavour');
560 my $userenv = C4::Context->userenv;
561 my $logged_in_patron = Koha::Patrons->find( $userenv->{number} );
566 while (my $rowref = $sth->fetchrow_hashref) {
568 $record_type = $rowref->{'record_type'};
572 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
574 &$progress_callback( $rec_num );
576 if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'imported') {
582 if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') {
583 $marc_type = 'UNIMARCAUTH';
584 } elsif ($marcflavour eq 'UNIMARC') {
585 $marc_type = 'UNIMARC';
587 $marc_type = 'USMARC';
589 my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
591 if ($record_type eq 'biblio') {
592 # remove any item tags - rely on _batchCommitItems
593 ($item_tag,$item_subfield) = &GetMarcFromKohaField( "items.itemnumber" );
594 foreach my $item_field ($marc_record->field($item_tag)) {
595 $marc_record->delete_field($item_field);
597 if(C4::Context->preference('autoControlNumber') eq 'biblionumber'){
598 my @control_num = $marc_record->field('001');
599 $marc_record->delete_fields(@control_num);
603 my ($record_result, $item_result, $record_match) =
604 _get_commit_action($overlay_action, $nomatch_action, $item_action,
605 $rowref->{'overlay_status'}, $rowref->{'import_record_id'}, $record_type);
609 if ($record_result eq 'create_new') {
611 if ($record_type eq 'biblio') {
612 my $biblioitemnumber;
613 ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework, { skip_record_index => 1 });
614 push @biblio_ids, $recordid if $recordid;
615 $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; # FIXME call SetMatchedBiblionumber instead
616 if ($item_result eq 'create_new' || $item_result eq 'replace') {
617 my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = _batchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result, $biblioitemnumber);
618 $num_items_added += $bib_items_added;
619 $num_items_replaced += $bib_items_replaced;
620 $num_items_errored += $bib_items_errored;
623 $recordid = AddAuthority($marc_record, undef, GuessAuthTypeCode($marc_record));
624 $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
626 my $sth = $dbh->prepare_cached($query);
627 $sth->execute($recordid, $rowref->{'import_record_id'});
629 SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
630 } elsif ($record_result eq 'replace') {
632 $recordid = $record_match;
634 if ($record_type eq 'biblio') {
635 my $oldbiblio = Koha::Biblios->find( $recordid );
636 $oldxml = GetXmlBiblio($recordid);
638 # remove item fields so that they don't get
639 # added again if record is reverted
640 # FIXME: GetXmlBiblio output should not contain item info any more! So the next foreach should not be needed. Does not hurt either; may remove old 952s that should not have been there anymore.
641 my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'}, $marc_type);
642 foreach my $item_field ($old_marc->field($item_tag)) {
643 $old_marc->delete_field($item_field);
645 $oldxml = $old_marc->as_xml($marc_type);
647 my $context = { source => 'batchimport' };
648 if ($logged_in_patron) {
649 $context->{categorycode} = $logged_in_patron->categorycode;
650 $context->{userid} = $logged_in_patron->userid;
656 $overlay_framework // $oldbiblio->frameworkcode,
658 overlay_context => $context,
659 skip_record_index => 1,
660 skip_holds_queue => 1,
663 push @biblio_ids, $recordid;
664 push @updated_ids, $recordid;
665 $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; # FIXME call SetMatchedBiblionumber instead
667 if ($item_result eq 'create_new' || $item_result eq 'replace') {
668 my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = _batchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result);
669 $num_items_added += $bib_items_added;
670 $num_items_replaced += $bib_items_replaced;
671 $num_items_errored += $bib_items_errored;
674 $oldxml = GetAuthorityXML($recordid);
676 ModAuthority($recordid, $marc_record, GuessAuthTypeCode($marc_record));
677 $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
679 # Combine xml update, SetImportRecordOverlayStatus, and SetImportRecordStatus updates into a single update for efficiency, especially in a transaction
680 my $sth = $dbh->prepare_cached("UPDATE import_records SET marcxml_old = ?, status = ?, overlay_status = ? WHERE import_record_id = ?");
681 $sth->execute( $oldxml, 'imported', 'match_applied', $rowref->{'import_record_id'} );
683 my $sth2 = $dbh->prepare_cached($query);
684 $sth2->execute($recordid, $rowref->{'import_record_id'});
686 } elsif ($record_result eq 'ignore') {
687 $recordid = $record_match;
689 if ($record_type eq 'biblio' and defined $recordid and ( $item_result eq 'create_new' || $item_result eq 'replace' ) ) {
690 my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = _batchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result);
691 push @biblio_ids, $recordid if $bib_items_added || $bib_items_replaced;
692 $num_items_added += $bib_items_added;
693 $num_items_replaced += $bib_items_replaced;
694 $num_items_errored += $bib_items_errored;
695 # still need to record the matched biblionumber so that the
696 # items can be reverted
697 my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); # FIXME call SetMatchedBiblionumber instead
698 $sth2->execute($recordid, $rowref->{'import_record_id'});
699 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
701 SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored');
706 if ($progress_interval){
707 &$progress_callback($rec_num);
712 SetImportBatchStatus($batch_id, 'imported');
716 my $indexer = Koha::SearchEngine::Indexer->new( { index => $Koha::SearchEngine::BIBLIOS_INDEX } );
717 $indexer->index_records( \@biblio_ids, "specialUpdate", "biblioserver" );
719 Koha::BackgroundJob::BatchUpdateBiblioHoldsQueue->new->enqueue( { biblio_ids => \@updated_ids } )
720 if ( @updated_ids && C4::Context->preference('RealTimeHoldsQueue') );
722 return ($num_added, $num_updated, $num_items_added, $num_items_replaced, $num_items_errored, $num_ignored);
725 =head2 _batchCommitItems
727 ($num_items_added, $num_items_errored) =
728 _batchCommitItems($import_record_id, $biblionumber, [$action, $biblioitemnumber]);
730 Private function for batch committing item changes. We do not trigger a re-index here, that is left to the caller.
734 sub _batchCommitItems {
735 my ( $import_record_id, $biblionumber, $action, $biblioitemnumber ) = @_;
737 my $dbh = C4::Context->dbh;
739 my $num_items_added = 0;
740 my $num_items_errored = 0;
741 my $num_items_replaced = 0;
743 my $sth = $dbh->prepare( "
744 SELECT import_items_id, import_items.marcxml, encoding
746 JOIN import_records USING (import_record_id)
747 WHERE import_record_id = ?
748 ORDER BY import_items_id
750 $sth->bind_param( 1, $import_record_id );
753 while ( my $row = $sth->fetchrow_hashref() ) {
754 my $item_marc = MARC::Record->new_from_xml( StripNonXmlChars( $row->{'marcxml'} ), 'UTF-8', $row->{'encoding'} );
756 # Delete date_due subfield as to not accidentally delete item checkout due dates
757 my ( $MARCfield, $MARCsubfield ) = GetMarcFromKohaField( 'items.onloan' );
758 $item_marc->field($MARCfield)->delete_subfield( code => $MARCsubfield );
760 my $item = TransformMarcToKoha({ record => $item_marc, kohafields => ['items.barcode','items.itemnumber'] });
763 my $duplicate_barcode = exists( $item->{'barcode'} );
764 my $duplicate_itemnumber = exists( $item->{'itemnumber'} );
766 # We assume that when replacing items we do not want to move them - the onus is on the importer to
767 # ensure the correct items/records are being updated
768 my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, itemnumber = ?, import_error = ? WHERE import_items_id = ?");
770 $action eq "replace" &&
771 $duplicate_itemnumber &&
772 ( $item_match = Koha::Items->find( $item->{itemnumber} ))
774 # Duplicate itemnumbers have precedence, that way we can update barcodes by overlaying
775 ModItemFromMarc( $item_marc, $item_match->biblionumber, $item->{itemnumber}, { skip_record_index => 1 } );
776 $updsth->bind_param( 1, 'imported' );
777 $updsth->bind_param( 2, $item->{itemnumber} );
778 $updsth->bind_param( 3, undef );
779 $updsth->bind_param( 4, $row->{'import_items_id'} );
782 $num_items_replaced++;
784 $action eq "replace" &&
785 $duplicate_barcode &&
786 ( $item_match = Koha::Items->find({ barcode => $item->{'barcode'} }) )
788 ModItemFromMarc( $item_marc, $item_match->biblionumber, $item_match->itemnumber, { skip_record_index => 1 } );
789 $updsth->bind_param( 1, 'imported' );
790 $updsth->bind_param( 2, $item->{itemnumber} );
791 $updsth->bind_param( 3, undef );
792 $updsth->bind_param( 4, $row->{'import_items_id'} );
795 $num_items_replaced++;
797 # We aren't replacing, but the incoming file has a barcode, we need to check if it exists
798 $duplicate_barcode &&
799 ( $item_match = Koha::Items->find({ barcode => $item->{'barcode'} }) )
801 $updsth->bind_param( 1, 'error' );
802 $updsth->bind_param( 2, undef );
803 $updsth->bind_param( 3, 'duplicate item barcode' );
804 $updsth->bind_param( 4, $row->{'import_items_id'} );
806 $num_items_errored++;
808 # Remove the itemnumber if it exists, we want to create a new item
809 my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField( "items.itemnumber" );
810 $item_marc->field($itemtag)->delete_subfield( code => $itemsubfield );
812 my ( $item_biblionumber, $biblioitemnumber, $itemnumber ) = AddItemFromMarc( $item_marc, $biblionumber, { biblioitemnumber => $biblioitemnumber, skip_record_index => 1 } );
814 $updsth->bind_param( 1, 'imported' );
815 $updsth->bind_param( 2, $itemnumber );
816 $updsth->bind_param( 3, undef );
817 $updsth->bind_param( 4, $row->{'import_items_id'} );
825 return ( $num_items_added, $num_items_replaced, $num_items_errored );
828 =head2 BatchRevertRecords
830 my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted,
831 $num_ignored) = BatchRevertRecords($batch_id);
835 sub BatchRevertRecords {
836 my $batch_id = shift;
838 my $logger = Koha::Logger->get( { category => 'C4.ImportBatch' } );
840 $logger->trace("C4::ImportBatch::BatchRevertRecords( $batch_id )");
845 my $num_reverted = 0;
847 my $num_items_deleted = 0;
848 # commit (i.e., save, all records in the batch)
849 SetImportBatchStatus($batch_id, 'reverting');
850 my $overlay_action = GetImportBatchOverlayAction($batch_id);
851 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
852 my $dbh = C4::Context->dbh;
853 my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marcxml_old, encoding, matched_biblionumber, matched_authid
855 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
856 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
857 WHERE import_batch_id = ?");
858 $sth->execute($batch_id);
860 my $marcflavour = C4::Context->preference('marcflavour');
861 while (my $rowref = $sth->fetchrow_hashref) {
862 $record_type = $rowref->{'record_type'};
863 if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') {
867 if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') {
868 $marc_type = 'UNIMARCAUTH';
869 } elsif ($marcflavour eq 'UNIMARC') {
870 $marc_type = 'UNIMARC';
872 $marc_type = 'USMARC';
875 my $record_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'});
877 if ($record_result eq 'delete') {
879 if ($record_type eq 'biblio') {
880 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
881 $error = DelBiblio($rowref->{'matched_biblionumber'});
883 DelAuthority({ authid => $rowref->{'matched_authid'} });
885 if (defined $error) {
889 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
891 } elsif ($record_result eq 'restore') {
893 my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}, $marc_type);
894 if ($record_type eq 'biblio') {
895 my $biblionumber = $rowref->{'matched_biblionumber'};
896 my $oldbiblio = Koha::Biblios->find( $biblionumber );
898 $logger->info("C4::ImportBatch::BatchRevertRecords: Biblio record $biblionumber does not exist, restoration of this record was skipped") unless $oldbiblio;
899 next unless $oldbiblio; # Record has since been deleted. Deleted records should stay deleted.
901 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
902 ModBiblio($old_record, $biblionumber, $oldbiblio->frameworkcode);
904 my $authid = $rowref->{'matched_authid'};
905 ModAuthority($authid, $old_record, GuessAuthTypeCode($old_record));
907 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
908 } elsif ($record_result eq 'ignore') {
909 if ($record_type eq 'biblio') {
910 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
912 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
915 if ($record_type eq 'biblio') {
916 # remove matched_biblionumber only if there is no 'imported' item left
917 $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?"; # FIXME Remove me
918 $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ? AND NOT EXISTS (SELECT * FROM import_items WHERE import_items.import_record_id=import_biblios.import_record_id and status='imported')";
920 $query = "UPDATE import_auths SET matched_authid = NULL WHERE import_record_id = ?";
922 my $sth2 = $dbh->prepare_cached($query);
923 $sth2->execute($rowref->{'import_record_id'});
927 SetImportBatchStatus($batch_id, 'reverted');
928 return ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored);
931 =head2 BatchRevertItems
933 my $num_items_deleted = BatchRevertItems($import_record_id, $biblionumber);
937 sub BatchRevertItems {
938 my ($import_record_id, $biblionumber) = @_;
940 my $dbh = C4::Context->dbh;
941 my $num_items_deleted = 0;
943 my $sth = $dbh->prepare_cached("SELECT import_items_id, itemnumber
945 JOIN items USING (itemnumber)
946 WHERE import_record_id = ?");
947 $sth->bind_param(1, $import_record_id);
949 while (my $row = $sth->fetchrow_hashref()) {
950 my $item = Koha::Items->find($row->{itemnumber});
951 if ($item->safe_delete){
952 my $updsth = $dbh->prepare("UPDATE import_items SET status = ? WHERE import_items_id = ?");
953 $updsth->bind_param(1, 'reverted');
954 $updsth->bind_param(2, $row->{'import_items_id'});
957 $num_items_deleted++;
964 return $num_items_deleted;
969 CleanBatch($batch_id)
971 Deletes all staged records from the import batch
972 and sets the status of the batch to 'cleaned'. Note
973 that deleting a stage record does *not* affect
974 any record that has been committed to the database.
979 my $batch_id = shift;
980 return unless defined $batch_id;
982 C4::Context->dbh->do('DELETE FROM import_records WHERE import_batch_id = ?', {}, $batch_id);
983 SetImportBatchStatus($batch_id, 'cleaned');
988 DeleteBatch($batch_id)
990 Deletes the record from the database. This can only be done
991 once the batch has been cleaned.
996 my $batch_id = shift;
997 return unless defined $batch_id;
999 my $dbh = C4::Context->dbh;
1000 my $sth = $dbh->prepare('DELETE FROM import_batches WHERE import_batch_id = ?');
1001 $sth->execute( $batch_id );
1004 =head2 GetAllImportBatches
1006 my $results = GetAllImportBatches();
1008 Returns a references to an array of hash references corresponding
1009 to all import_batches rows (of batch_type 'batch'), sorted in
1010 ascending order by import_batch_id.
1014 sub GetAllImportBatches {
1015 my $dbh = C4::Context->dbh;
1016 my $sth = $dbh->prepare_cached("SELECT * FROM import_batches
1017 WHERE batch_type IN ('batch', 'webservice')
1018 ORDER BY import_batch_id ASC");
1022 while (my $row = $sth->fetchrow_hashref) {
1023 push @$results, $row;
1029 =head2 GetStagedWebserviceBatches
1031 my $batch_ids = GetStagedWebserviceBatches();
1033 Returns a references to an array of batch id's
1034 of batch_type 'webservice' that are not imported
1038 my $PENDING_WEBSERVICE_BATCHES_QRY = <<EOQ;
1039 SELECT import_batch_id FROM import_batches
1040 WHERE batch_type = 'webservice'
1041 AND import_status = 'staged'
1043 sub GetStagedWebserviceBatches {
1044 my $dbh = C4::Context->dbh;
1045 return $dbh->selectcol_arrayref($PENDING_WEBSERVICE_BATCHES_QRY);
1048 =head2 GetImportBatchRangeDesc
1050 my $results = GetImportBatchRangeDesc($offset, $results_per_group);
1052 Returns a reference to an array of hash references corresponding to
1053 import_batches rows (sorted in descending order by import_batch_id)
1054 start at the given offset.
1058 sub GetImportBatchRangeDesc {
1059 my ($offset, $results_per_group) = @_;
1061 my $dbh = C4::Context->dbh;
1062 my $query = "SELECT b.*, p.name as profile FROM import_batches b
1063 LEFT JOIN import_batch_profiles p
1064 ON b.profile_id = p.id
1065 WHERE b.batch_type IN ('batch', 'webservice')
1066 ORDER BY b.import_batch_id DESC";
1068 if ($results_per_group){
1069 $query .= " LIMIT ?";
1070 push(@params, $results_per_group);
1073 $query .= " OFFSET ?";
1074 push(@params, $offset);
1076 my $sth = $dbh->prepare_cached($query);
1077 $sth->execute(@params);
1078 my $results = $sth->fetchall_arrayref({});
1083 =head2 GetItemNumbersFromImportBatch
1085 my @itemsnos = GetItemNumbersFromImportBatch($batch_id);
1089 sub GetItemNumbersFromImportBatch {
1090 my ($batch_id) = @_;
1091 my $dbh = C4::Context->dbh;
1093 SELECT itemnumber FROM import_items
1094 INNER JOIN items USING (itemnumber)
1095 INNER JOIN import_records USING (import_record_id)
1096 WHERE import_batch_id = ?|;
1097 my $sth = $dbh->prepare( $sql );
1098 $sth->execute($batch_id);
1100 while ( my ($itm) = $sth->fetchrow_array ) {
1106 =head2 GetNumberOfImportBatches
1108 my $count = GetNumberOfImportBatches();
1112 sub GetNumberOfNonZ3950ImportBatches {
1113 my $dbh = C4::Context->dbh;
1114 my $sth = $dbh->prepare("SELECT COUNT(*) FROM import_batches WHERE batch_type != 'z3950'");
1116 my ($count) = $sth->fetchrow_array();
1121 =head2 GetImportBiblios
1123 my $results = GetImportBiblios($importid);
1127 sub GetImportBiblios {
1128 my ($import_record_id) = @_;
1130 my $dbh = C4::Context->dbh;
1131 my $query = "SELECT * FROM import_biblios WHERE import_record_id = ?";
1132 return $dbh->selectall_arrayref(
1140 =head2 GetImportRecordsRange
1142 my $results = GetImportRecordsRange($batch_id, $offset, $results_per_group);
1144 Returns a reference to an array of hash references corresponding to
1145 import_biblios/import_auths/import_records rows for a given batch
1146 starting at the given offset.
1150 sub GetImportRecordsRange {
1151 my ( $batch_id, $offset, $results_per_group, $status, $parameters ) = @_;
1153 my $dbh = C4::Context->dbh;
1155 my $order_by = $parameters->{order_by} || 'import_record_id';
1156 ( $order_by ) = grep( { $_ eq $order_by } qw( import_record_id title status overlay_status ) ) ? $order_by : 'import_record_id';
1158 my $order_by_direction =
1159 uc( $parameters->{order_by_direction} // 'ASC' ) eq 'DESC' ? 'DESC' : 'ASC';
1161 $order_by .= " $order_by_direction, authorized_heading" if $order_by eq 'title';
1163 my $query = "SELECT title, author, isbn, issn, authorized_heading, import_records.import_record_id,
1164 record_sequence, status, overlay_status,
1165 matched_biblionumber, matched_authid, record_type
1167 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
1168 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
1169 WHERE import_batch_id = ?";
1171 push(@params, $batch_id);
1173 $query .= " AND status=?";
1174 push(@params,$status);
1177 $query.=" ORDER BY $order_by $order_by_direction";
1179 if($results_per_group){
1180 $query .= " LIMIT ?";
1181 push(@params, $results_per_group);
1184 $query .= " OFFSET ?";
1185 push(@params, $offset);
1187 my $sth = $dbh->prepare_cached($query);
1188 $sth->execute(@params);
1189 my $results = $sth->fetchall_arrayref({});
1195 =head2 GetBestRecordMatch
1197 my $record_id = GetBestRecordMatch($import_record_id);
1201 sub GetBestRecordMatch {
1202 my ($import_record_id) = @_;
1204 my $dbh = C4::Context->dbh;
1205 my $sth = $dbh->prepare("SELECT candidate_match_id
1206 FROM import_record_matches
1207 JOIN import_records ON ( import_record_matches.import_record_id = import_records.import_record_id )
1208 LEFT JOIN biblio ON ( candidate_match_id = biblio.biblionumber )
1209 LEFT JOIN auth_header ON ( candidate_match_id = auth_header.authid )
1210 WHERE import_record_matches.import_record_id = ? AND
1211 ( (import_records.record_type = 'biblio' AND biblio.biblionumber IS NOT NULL) OR
1212 (import_records.record_type = 'auth' AND auth_header.authid IS NOT NULL) )
1214 ORDER BY score DESC, candidate_match_id DESC");
1215 $sth->execute($import_record_id);
1216 my ($record_id) = $sth->fetchrow_array();
1221 =head2 GetImportBatchStatus
1223 my $status = GetImportBatchStatus($batch_id);
1227 sub GetImportBatchStatus {
1228 my ($batch_id) = @_;
1230 my $dbh = C4::Context->dbh;
1231 my $sth = $dbh->prepare("SELECT import_status FROM import_batches WHERE import_batch_id = ?");
1232 $sth->execute($batch_id);
1233 my ($status) = $sth->fetchrow_array();
1239 =head2 SetImportBatchStatus
1241 SetImportBatchStatus($batch_id, $new_status);
1245 sub SetImportBatchStatus {
1246 my ($batch_id, $new_status) = @_;
1248 my $dbh = C4::Context->dbh;
1249 my $sth = $dbh->prepare("UPDATE import_batches SET import_status = ? WHERE import_batch_id = ?");
1250 $sth->execute($new_status, $batch_id);
1255 =head2 SetMatchedBiblionumber
1257 SetMatchedBiblionumber($import_record_id, $biblionumber);
1261 sub SetMatchedBiblionumber {
1262 my ($import_record_id, $biblionumber) = @_;
1264 my $dbh = C4::Context->dbh;
1266 q|UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?|,
1267 undef, $biblionumber, $import_record_id
1271 =head2 GetImportBatchOverlayAction
1273 my $overlay_action = GetImportBatchOverlayAction($batch_id);
1277 sub GetImportBatchOverlayAction {
1278 my ($batch_id) = @_;
1280 my $dbh = C4::Context->dbh;
1281 my $sth = $dbh->prepare("SELECT overlay_action FROM import_batches WHERE import_batch_id = ?");
1282 $sth->execute($batch_id);
1283 my ($overlay_action) = $sth->fetchrow_array();
1285 return $overlay_action;
1290 =head2 SetImportBatchOverlayAction
1292 SetImportBatchOverlayAction($batch_id, $new_overlay_action);
1296 sub SetImportBatchOverlayAction {
1297 my ($batch_id, $new_overlay_action) = @_;
1299 my $dbh = C4::Context->dbh;
1300 my $sth = $dbh->prepare("UPDATE import_batches SET overlay_action = ? WHERE import_batch_id = ?");
1301 $sth->execute($new_overlay_action, $batch_id);
1306 =head2 GetImportBatchNoMatchAction
1308 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
1312 sub GetImportBatchNoMatchAction {
1313 my ($batch_id) = @_;
1315 my $dbh = C4::Context->dbh;
1316 my $sth = $dbh->prepare("SELECT nomatch_action FROM import_batches WHERE import_batch_id = ?");
1317 $sth->execute($batch_id);
1318 my ($nomatch_action) = $sth->fetchrow_array();
1320 return $nomatch_action;
1325 =head2 SetImportBatchNoMatchAction
1327 SetImportBatchNoMatchAction($batch_id, $new_nomatch_action);
1331 sub SetImportBatchNoMatchAction {
1332 my ($batch_id, $new_nomatch_action) = @_;
1334 my $dbh = C4::Context->dbh;
1335 my $sth = $dbh->prepare("UPDATE import_batches SET nomatch_action = ? WHERE import_batch_id = ?");
1336 $sth->execute($new_nomatch_action, $batch_id);
1341 =head2 GetImportBatchItemAction
1343 my $item_action = GetImportBatchItemAction($batch_id);
1347 sub GetImportBatchItemAction {
1348 my ($batch_id) = @_;
1350 my $dbh = C4::Context->dbh;
1351 my $sth = $dbh->prepare("SELECT item_action FROM import_batches WHERE import_batch_id = ?");
1352 $sth->execute($batch_id);
1353 my ($item_action) = $sth->fetchrow_array();
1355 return $item_action;
1360 =head2 SetImportBatchItemAction
1362 SetImportBatchItemAction($batch_id, $new_item_action);
1366 sub SetImportBatchItemAction {
1367 my ($batch_id, $new_item_action) = @_;
1369 my $dbh = C4::Context->dbh;
1370 my $sth = $dbh->prepare("UPDATE import_batches SET item_action = ? WHERE import_batch_id = ?");
1371 $sth->execute($new_item_action, $batch_id);
1376 =head2 GetImportBatchMatcher
1378 my $matcher_id = GetImportBatchMatcher($batch_id);
1382 sub GetImportBatchMatcher {
1383 my ($batch_id) = @_;
1385 my $dbh = C4::Context->dbh;
1386 my $sth = $dbh->prepare("SELECT matcher_id FROM import_batches WHERE import_batch_id = ?");
1387 $sth->execute($batch_id);
1388 my ($matcher_id) = $sth->fetchrow_array();
1395 =head2 SetImportBatchMatcher
1397 SetImportBatchMatcher($batch_id, $new_matcher_id);
1401 sub SetImportBatchMatcher {
1402 my ($batch_id, $new_matcher_id) = @_;
1404 my $dbh = C4::Context->dbh;
1405 my $sth = $dbh->prepare("UPDATE import_batches SET matcher_id = ? WHERE import_batch_id = ?");
1406 $sth->execute($new_matcher_id, $batch_id);
1411 =head2 GetImportRecordOverlayStatus
1413 my $overlay_status = GetImportRecordOverlayStatus($import_record_id);
1417 sub GetImportRecordOverlayStatus {
1418 my ($import_record_id) = @_;
1420 my $dbh = C4::Context->dbh;
1421 my $sth = $dbh->prepare("SELECT overlay_status FROM import_records WHERE import_record_id = ?");
1422 $sth->execute($import_record_id);
1423 my ($overlay_status) = $sth->fetchrow_array();
1425 return $overlay_status;
1430 =head2 SetImportRecordOverlayStatus
1432 SetImportRecordOverlayStatus($import_record_id, $new_overlay_status);
1436 sub SetImportRecordOverlayStatus {
1437 my ($import_record_id, $new_overlay_status) = @_;
1439 my $dbh = C4::Context->dbh;
1440 my $sth = $dbh->prepare("UPDATE import_records SET overlay_status = ? WHERE import_record_id = ?");
1441 $sth->execute($new_overlay_status, $import_record_id);
1446 =head2 GetImportRecordStatus
1448 my $status = GetImportRecordStatus($import_record_id);
1452 sub GetImportRecordStatus {
1453 my ($import_record_id) = @_;
1455 my $dbh = C4::Context->dbh;
1456 my $sth = $dbh->prepare("SELECT status FROM import_records WHERE import_record_id = ?");
1457 $sth->execute($import_record_id);
1458 my ($status) = $sth->fetchrow_array();
1465 =head2 SetImportRecordStatus
1467 SetImportRecordStatus($import_record_id, $new_status);
1471 sub SetImportRecordStatus {
1472 my ($import_record_id, $new_status) = @_;
1474 my $dbh = C4::Context->dbh;
1475 my $sth = $dbh->prepare("UPDATE import_records SET status = ? WHERE import_record_id = ?");
1476 $sth->execute($new_status, $import_record_id);
1481 =head2 GetImportRecordMatches
1483 my $results = GetImportRecordMatches($import_record_id, $best_only);
1487 sub GetImportRecordMatches {
1488 my $import_record_id = shift;
1489 my $best_only = @_ ? shift : 0;
1491 my $dbh = C4::Context->dbh;
1492 # FIXME currently biblio only
1493 my $sth = $dbh->prepare_cached("SELECT title, author, biblionumber,
1494 candidate_match_id, score, record_type,
1497 JOIN import_record_matches USING (import_record_id)
1498 LEFT JOIN biblio ON (biblionumber = candidate_match_id)
1499 WHERE import_record_id = ?
1500 ORDER BY score DESC, biblionumber DESC");
1501 $sth->bind_param(1, $import_record_id);
1504 while (my $row = $sth->fetchrow_hashref) {
1505 if ($row->{'record_type'} eq 'auth') {
1506 $row->{'authorized_heading'} = GetAuthorizedHeading( { authid => $row->{'candidate_match_id'} } );
1508 next if ($row->{'record_type'} eq 'biblio' && not $row->{'biblionumber'});
1509 push @$results, $row;
1518 =head2 SetImportRecordMatches
1520 SetImportRecordMatches($import_record_id, @matches);
1524 sub SetImportRecordMatches {
1525 my $import_record_id = shift;
1528 my $dbh = C4::Context->dbh;
1529 my $delsth = $dbh->prepare("DELETE FROM import_record_matches WHERE import_record_id = ?");
1530 $delsth->execute($import_record_id);
1533 my $sth = $dbh->prepare("INSERT INTO import_record_matches (import_record_id, candidate_match_id, score, chosen)
1534 VALUES (?, ?, ?, ?)");
1535 my $chosen = 1; #The first match is defaulted to be chosen
1536 foreach my $match (@matches) {
1537 $sth->execute($import_record_id, $match->{'record_id'}, $match->{'score'}, $chosen);
1538 $chosen = 0; #After the first we do not default to other matches
1542 =head2 RecordsFromISO2709File
1544 my ($errors, $records) = C4::ImportBatch::RecordsFromISO2709File($input_file, $record_type, $encoding);
1546 Reads ISO2709 binary porridge from the given file and creates MARC::Record-objects out of it.
1548 @PARAM1, String, absolute path to the ISO2709 file.
1549 @PARAM2, String, see stage_file.pl
1550 @PARAM3, String, should be utf8
1552 Returns two array refs.
1556 sub RecordsFromISO2709File {
1557 my ($input_file, $record_type, $encoding) = @_;
1560 my $marc_type = C4::Context->preference('marcflavour');
1561 $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth');
1563 open my $fh, '<', $input_file or die "$0: cannot open input file $input_file: $!\n";
1569 next unless $_; # skip if record has only whitespace, as might occur
1570 # if file includes newlines between each MARC record
1571 my ($marc_record, $charset_guessed, $char_errors) = MarcToUTF8Record($_, $marc_type, $encoding);
1572 push @marc_records, $marc_record;
1573 if ($charset_guessed ne $encoding) {
1575 "Unexpected charset $charset_guessed, expecting $encoding";
1579 return ( \@errors, \@marc_records );
1582 =head2 RecordsFromMARCXMLFile
1584 my ($errors, $records) = C4::ImportBatch::RecordsFromMARCXMLFile($input_file, $encoding);
1586 Creates MARC::Record-objects out of the given MARCXML-file.
1588 @PARAM1, String, absolute path to the MARCXML file.
1589 @PARAM2, String, should be utf8
1591 Returns two array refs.
1595 sub RecordsFromMARCXMLFile {
1596 my ( $filename, $encoding ) = @_;
1597 my $batch = MARC::File::XML->in( $filename );
1598 my ( @marcRecords, @errors, $record );
1600 eval { $record = $batch->next( $encoding ); };
1604 push @marcRecords, $record if $record;
1606 return (\@errors, \@marcRecords);
1609 =head2 RecordsFromMarcPlugin
1611 Converts text of input_file into array of MARC records with to_marc plugin
1615 sub RecordsFromMarcPlugin {
1616 my ($input_file, $plugin_class, $encoding) = @_;
1617 my ( $text, @return );
1618 return \@return if !$input_file || !$plugin_class;
1621 open my $fh, '<', $input_file or die "$0: cannot open input file $input_file: $!\n";
1631 # Convert to large MARC blob with plugin
1632 $text = Koha::Plugins::Handler->run({
1633 class => $plugin_class,
1634 method => 'to_marc',
1635 params => { data => $text },
1638 # Convert to array of MARC records
1640 my $marc_type = C4::Context->preference('marcflavour');
1641 foreach my $blob ( split(/\x1D/, $text) ) {
1642 next if $blob =~ /^\s*$/;
1643 my ($marcrecord) = MarcToUTF8Record($blob, $marc_type, $encoding);
1644 push @return, $marcrecord;
1650 # internal functions
1652 sub _create_import_record {
1653 my ($batch_id, $record_sequence, $marc_record, $record_type, $encoding, $marc_type) = @_;
1655 my $dbh = C4::Context->dbh;
1656 my $sth = $dbh->prepare("INSERT INTO import_records (import_batch_id, record_sequence, marc, marcxml, marcxml_old,
1657 record_type, encoding)
1658 VALUES (?, ?, ?, ?, ?, ?, ?)");
1659 $sth->execute($batch_id, $record_sequence, $marc_record->as_usmarc(), $marc_record->as_xml($marc_type), '',
1660 $record_type, $encoding);
1661 my $import_record_id = $dbh->{'mysql_insertid'};
1663 return $import_record_id;
1666 sub _add_auth_fields {
1667 my ($import_record_id, $marc_record) = @_;
1670 if ($marc_record->field('001')) {
1671 $controlnumber = $marc_record->field('001')->data();
1673 my $authorized_heading = GetAuthorizedHeading({ record => $marc_record });
1674 my $dbh = C4::Context->dbh;
1675 my $sth = $dbh->prepare("INSERT INTO import_auths (import_record_id, control_number, authorized_heading) VALUES (?, ?, ?)");
1676 $sth->execute($import_record_id, $controlnumber, $authorized_heading);
1680 sub _add_biblio_fields {
1681 my ($import_record_id, $marc_record) = @_;
1683 my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record);
1684 my $dbh = C4::Context->dbh;
1685 # FIXME no controlnumber, originalsource
1686 $isbn = C4::Koha::GetNormalizedISBN($isbn);
1687 my $sth = $dbh->prepare("INSERT INTO import_biblios (import_record_id, title, author, isbn, issn) VALUES (?, ?, ?, ?, ?)");
1688 $sth->execute($import_record_id, $title, $author, $isbn, $issn) or die $sth->errstr;
1693 sub _update_biblio_fields {
1694 my ($import_record_id, $marc_record) = @_;
1696 my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record);
1697 my $dbh = C4::Context->dbh;
1698 # FIXME no controlnumber, originalsource
1699 # FIXME 2 - should regularize normalization of ISBN wherever it is done
1703 my $sth = $dbh->prepare("UPDATE import_biblios SET title = ?, author = ?, isbn = ?, issn = ?
1704 WHERE import_record_id = ?");
1705 $sth->execute($title, $author, $isbn, $issn, $import_record_id);
1709 sub _parse_biblio_fields {
1710 my ($marc_record) = @_;
1712 my $dbh = C4::Context->dbh;
1713 my $bibliofields = TransformMarcToKoha({ record => $marc_record, kohafields => ['biblio.title','biblio.author','biblioitems.isbn','biblioitems.issn'] });
1714 return ($bibliofields->{'title'}, $bibliofields->{'author'}, $bibliofields->{'isbn'}, $bibliofields->{'issn'});
1718 sub _update_batch_record_counts {
1719 my ($batch_id) = @_;
1721 my $dbh = C4::Context->dbh;
1722 my ( $num_records ) = $dbh->selectrow_array(q|
1725 WHERE import_batch_id = ?
1726 |, undef, $batch_id );
1727 my ( $num_items ) = $dbh->selectrow_array(q|
1730 JOIN import_items USING (import_record_id)
1731 WHERE import_batch_id = ? AND record_type = 'biblio'
1732 |, undef, $batch_id );
1734 "UPDATE import_batches SET num_records=?, num_items=? WHERE import_batch_id=?",
1742 sub _get_commit_action {
1743 my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id, $record_type) = @_;
1745 if ($record_type eq 'biblio') {
1746 my ($bib_result, $bib_match, $item_result);
1748 $bib_match = GetBestRecordMatch($import_record_id);
1749 if ($overlay_status ne 'no_match' && defined($bib_match)) {
1751 $bib_result = $overlay_action;
1753 if($item_action eq 'always_add' or $item_action eq 'add_only_for_matches'){
1754 $item_result = 'create_new';
1755 } elsif($item_action eq 'replace'){
1756 $item_result = 'replace';
1758 $item_result = 'ignore';
1762 $bib_result = $nomatch_action;
1763 $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore';
1765 return ($bib_result, $item_result, $bib_match);
1766 } else { # must be auths
1767 my ($auth_result, $auth_match);
1769 $auth_match = GetBestRecordMatch($import_record_id);
1770 if ($overlay_status ne 'no_match' && defined($auth_match)) {
1771 $auth_result = $overlay_action;
1773 $auth_result = $nomatch_action;
1776 return ($auth_result, undef, $auth_match);
1781 sub _get_revert_action {
1782 my ($overlay_action, $overlay_status, $status) = @_;
1786 if ($status eq 'ignored') {
1787 $bib_result = 'ignore';
1789 if ($overlay_action eq 'create_new') {
1790 $bib_result = 'delete';
1792 $bib_result = ($overlay_status eq 'match_applied') ? 'restore' : 'delete';
1803 Koha Development Team <http://koha-community.org/>
1805 Galen Charlton <galen.charlton@liblime.com>