1 package C4::ImportBatch;
3 # Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 2 of the License, or (at your option) any later
12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License along
17 # with Koha; if not, write to the Free Software Foundation, Inc.,
18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
28 use C4::AuthoritiesMarc;
30 use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
33 # set the version for version checking
34 $VERSION = 3.07.00.049;
41 GetImportRecordMarcXML
46 AddItemsToImportBiblio
57 GetStagedWebserviceBatches
58 GetImportBatchRangeDesc
59 GetNumberOfNonZ3950ImportBatches
61 GetItemNumbersFromImportBatch
65 GetImportBatchOverlayAction
66 SetImportBatchOverlayAction
67 GetImportBatchNoMatchAction
68 SetImportBatchNoMatchAction
69 GetImportBatchItemAction
70 SetImportBatchItemAction
73 GetImportRecordOverlayStatus
74 SetImportRecordOverlayStatus
77 GetImportRecordMatches
78 SetImportRecordMatches
84 C4::ImportBatch - manage batches of imported MARC records
92 =head2 GetZ3950BatchId
94 my $batchid = GetZ3950BatchId($z3950server);
96 Retrieves the ID of the import batch for the Z39.50
97 reservoir for the given target. If necessary,
98 creates the import batch.
102 sub GetZ3950BatchId {
103 my ($z3950server) = @_;
105 my $dbh = C4::Context->dbh;
106 my $sth = $dbh->prepare("SELECT import_batch_id FROM import_batches
107 WHERE batch_type = 'z3950'
109 $sth->execute($z3950server);
110 my $rowref = $sth->fetchrow_arrayref();
112 if (defined $rowref) {
115 my $batch_id = AddImportBatch( {
116 overlay_action => 'create_new',
117 import_status => 'staged',
118 batch_type => 'z3950',
119 file_name => $z3950server,
126 =head2 GetWebserviceBatchId
128 my $batchid = GetWebserviceBatchId();
130 Retrieves the ID of the import batch for webservice.
131 If necessary, creates the import batch.
135 my $WEBSERVICE_BASE_QRY = <<EOQ;
136 SELECT import_batch_id FROM import_batches
137 WHERE batch_type = 'webservice'
138 AND import_status = 'staged'
140 sub GetWebserviceBatchId {
143 my $dbh = C4::Context->dbh;
144 my $sql = $WEBSERVICE_BASE_QRY;
146 foreach my $field (qw(matcher_id overlay_action nomatch_action item_action)) {
147 if (my $val = $params->{$field}) {
148 $sql .= " AND $field = ?";
152 my $id = $dbh->selectrow_array($sql, undef, @args);
155 $params->{batch_type} = 'webservice';
156 $params->{import_status} = 'staged';
157 return AddImportBatch($params);
160 =head2 GetImportRecordMarc
162 my ($marcblob, $encoding) = GetImportRecordMarc($import_record_id);
166 sub GetImportRecordMarc {
167 my ($import_record_id) = @_;
169 my $dbh = C4::Context->dbh;
170 my $sth = $dbh->prepare("SELECT marc, encoding FROM import_records WHERE import_record_id = ?");
171 $sth->execute($import_record_id);
172 my ($marc, $encoding) = $sth->fetchrow();
174 return $marc, $encoding;
178 =head2 GetImportRecordMarcXML
180 my $marcxml = GetImportRecordMarcXML($import_record_id);
184 sub GetImportRecordMarcXML {
185 my ($import_record_id) = @_;
187 my $dbh = C4::Context->dbh;
188 my $sth = $dbh->prepare("SELECT marcxml FROM import_records WHERE import_record_id = ?");
189 $sth->execute($import_record_id);
190 my ($marcxml) = $sth->fetchrow();
196 =head2 AddImportBatch
198 my $batch_id = AddImportBatch($params_hash);
206 foreach (qw( matcher_id template_id branchcode
207 overlay_action nomatch_action item_action
208 import_status batch_type file_name comments record_type )) {
209 if (exists $params->{$_}) {
211 push @vals, $params->{$_};
214 my $dbh = C4::Context->dbh;
215 $dbh->do("INSERT INTO import_batches (".join( ',', @fields).")
216 VALUES (".join( ',', map '?', @fields).")",
219 return $dbh->{'mysql_insertid'};
222 =head2 GetImportBatch
224 my $row = GetImportBatch($batch_id);
226 Retrieve a hashref of an import_batches row.
233 my $dbh = C4::Context->dbh;
234 my $sth = $dbh->prepare_cached("SELECT * FROM import_batches WHERE import_batch_id = ?");
235 $sth->bind_param(1, $batch_id);
237 my $result = $sth->fetchrow_hashref;
243 =head2 AddBiblioToBatch
245 my $import_record_id = AddBiblioToBatch($batch_id, $record_sequence,
246 $marc_record, $encoding, $z3950random, $update_counts);
250 sub AddBiblioToBatch {
251 my $batch_id = shift;
252 my $record_sequence = shift;
253 my $marc_record = shift;
254 my $encoding = shift;
255 my $z3950random = shift;
256 my $update_counts = @_ ? shift : 1;
258 my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'biblio', $encoding, $z3950random, C4::Context->preference('marcflavour'));
259 _add_biblio_fields($import_record_id, $marc_record);
260 _update_batch_record_counts($batch_id) if $update_counts;
261 return $import_record_id;
264 =head2 ModBiblioInBatch
266 ModBiblioInBatch($import_record_id, $marc_record);
270 sub ModBiblioInBatch {
271 my ($import_record_id, $marc_record) = @_;
273 _update_import_record_marc($import_record_id, $marc_record, C4::Context->preference('marcflavour'));
274 _update_biblio_fields($import_record_id, $marc_record);
278 =head2 AddAuthToBatch
280 my $import_record_id = AddAuthToBatch($batch_id, $record_sequence,
281 $marc_record, $encoding, $z3950random, $update_counts, [$marc_type]);
286 my $batch_id = shift;
287 my $record_sequence = shift;
288 my $marc_record = shift;
289 my $encoding = shift;
290 my $z3950random = shift;
291 my $update_counts = @_ ? shift : 1;
292 my $marc_type = shift || C4::Context->preference('marcflavour');
294 $marc_type = 'UNIMARCAUTH' if $marc_type eq 'UNIMARC';
296 my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $z3950random, $marc_type);
297 _add_auth_fields($import_record_id, $marc_record);
298 _update_batch_record_counts($batch_id) if $update_counts;
299 return $import_record_id;
302 =head2 ModAuthInBatch
304 ModAuthInBatch($import_record_id, $marc_record);
309 my ($import_record_id, $marc_record) = @_;
311 my $marcflavour = C4::Context->preference('marcflavour');
312 _update_import_record_marc($import_record_id, $marc_record, $marcflavour eq 'UNIMARC' ? 'UNIMARCAUTH' : 'USMARC');
316 =head2 BatchStageMarcRecords
318 ($batch_id, $num_records, $num_items, @invalid_records) =
319 BatchStageMarcRecords($record_type, $encoding, $marc_records, $file_name,
320 $comments, $branch_code, $parse_items,
322 $progress_interval, $progress_callback);
326 sub BatchStageMarcRecords {
327 my $record_type = shift;
328 my $encoding = shift;
329 my $marc_records = shift;
330 my $file_name = shift;
331 my $comments = shift;
332 my $branch_code = shift;
333 my $parse_items = shift;
334 my $leave_as_staging = shift;
336 # optional callback to monitor status
338 my $progress_interval = 0;
339 my $progress_callback = undef;
341 $progress_interval = shift;
342 $progress_callback = shift;
343 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
344 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
347 my $batch_id = AddImportBatch( {
348 overlay_action => 'create_new',
349 import_status => 'staging',
350 batch_type => 'batch',
351 file_name => $file_name,
352 comments => $comments,
353 record_type => $record_type,
356 SetImportBatchItemAction($batch_id, 'always_add');
358 SetImportBatchItemAction($batch_id, 'ignore');
361 my $marc_type = C4::Context->preference('marcflavour');
362 $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth');
363 my @invalid_records = ();
366 # FIXME - for now, we're dealing only with bibs
368 foreach my $marc_blob (split(/\x1D/, $marc_records)) {
369 $marc_blob =~ s/^\s+//g;
370 $marc_blob =~ s/\s+$//g;
371 next unless $marc_blob;
373 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
374 &$progress_callback($rec_num);
376 my ($marc_record, $charset_guessed, $char_errors) =
377 MarcToUTF8Record($marc_blob, $marc_type, $encoding);
379 $encoding = $charset_guessed unless $encoding;
381 my $import_record_id;
382 if (scalar($marc_record->fields()) == 0) {
383 push @invalid_records, $marc_blob;
386 # Normalize the record so it doesn't have separated diacritics
387 SetUTF8Flag($marc_record);
390 if ($record_type eq 'biblio') {
391 $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0);
393 my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0);
394 $num_items += scalar(@import_items_ids);
396 } elsif ($record_type eq 'auth') {
397 $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0, $marc_type);
401 unless ($leave_as_staging) {
402 SetImportBatchStatus($batch_id, 'staged');
404 # FIXME branch_code, number of bibs, number of items
405 _update_batch_record_counts($batch_id);
406 return ($batch_id, $num_valid, $num_items, @invalid_records);
409 =head2 AddItemsToImportBiblio
411 my @import_items_ids = AddItemsToImportBiblio($batch_id,
412 $import_record_id, $marc_record, $update_counts);
416 sub AddItemsToImportBiblio {
417 my $batch_id = shift;
418 my $import_record_id = shift;
419 my $marc_record = shift;
420 my $update_counts = @_ ? shift : 0;
422 my @import_items_ids = ();
424 my $dbh = C4::Context->dbh;
425 my ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",'');
426 foreach my $item_field ($marc_record->field($item_tag)) {
427 my $item_marc = MARC::Record->new();
428 $item_marc->leader("00000 a "); # must set Leader/09 to 'a'
429 $item_marc->append_fields($item_field);
430 $marc_record->delete_field($item_field);
431 my $sth = $dbh->prepare_cached("INSERT INTO import_items (import_record_id, status, marcxml)
433 $sth->bind_param(1, $import_record_id);
434 $sth->bind_param(2, 'staged');
435 $sth->bind_param(3, $item_marc->as_xml());
437 push @import_items_ids, $dbh->{'mysql_insertid'};
441 if ($#import_items_ids > -1) {
442 _update_batch_record_counts($batch_id) if $update_counts;
443 _update_import_record_marc($import_record_id, $marc_record, C4::Context->preference('marcflavour'));
445 return @import_items_ids;
448 =head2 BatchFindDuplicates
450 my $num_with_matches = BatchFindDuplicates($batch_id, $matcher,
451 $max_matches, $progress_interval, $progress_callback);
453 Goes through the records loaded in the batch and attempts to
454 find duplicates for each one. Sets the matching status
455 of each record to "no_match" or "auto_match" as appropriate.
457 The $max_matches parameter is optional; if it is not supplied,
460 The $progress_interval and $progress_callback parameters are
461 optional; if both are supplied, the sub referred to by
462 $progress_callback will be invoked every $progress_interval
463 records using the number of records processed as the
468 sub BatchFindDuplicates {
469 my $batch_id = shift;
471 my $max_matches = @_ ? shift : 10;
473 # optional callback to monitor status
475 my $progress_interval = 0;
476 my $progress_callback = undef;
478 $progress_interval = shift;
479 $progress_callback = shift;
480 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
481 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
484 my $dbh = C4::Context->dbh;
486 my $sth = $dbh->prepare("SELECT import_record_id, record_type, marc
488 WHERE import_batch_id = ?");
489 $sth->execute($batch_id);
490 my $num_with_matches = 0;
492 while (my $rowref = $sth->fetchrow_hashref) {
494 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
495 &$progress_callback($rec_num);
497 my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
499 if (defined $matcher) {
500 @matches = $matcher->get_matches($marc_record, $max_matches);
502 if (scalar(@matches) > 0) {
504 SetImportRecordMatches($rowref->{'import_record_id'}, @matches);
505 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'auto_match');
507 SetImportRecordMatches($rowref->{'import_record_id'}, ());
508 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'no_match');
512 return $num_with_matches;
515 =head2 BatchCommitRecords
517 my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =
518 BatchCommitRecords($batch_id, $framework,
519 $progress_interval, $progress_callback);
523 sub BatchCommitRecords {
524 my $batch_id = shift;
525 my $framework = shift;
527 # optional callback to monitor status
529 my $progress_interval = 0;
530 my $progress_callback = undef;
532 $progress_interval = shift;
533 $progress_callback = shift;
534 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
535 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
541 my $num_items_added = 0;
542 my $num_items_errored = 0;
544 # commit (i.e., save, all records in the batch)
545 SetImportBatchStatus('importing');
546 my $overlay_action = GetImportBatchOverlayAction($batch_id);
547 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
548 my $item_action = GetImportBatchItemAction($batch_id);
551 my $dbh = C4::Context->dbh;
552 my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marc, encoding
554 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
555 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
556 WHERE import_batch_id = ?");
557 $sth->execute($batch_id);
558 my $marcflavour = C4::Context->preference('marcflavour');
560 while (my $rowref = $sth->fetchrow_hashref) {
561 $record_type = $rowref->{'record_type'};
563 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
564 &$progress_callback($rec_num);
566 if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'imported') {
572 if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') {
573 $marc_type = 'UNIMARCAUTH';
574 } elsif ($marcflavour eq 'UNIMARC') {
575 $marc_type = 'UNIMARC';
577 $marc_type = 'USMARC';
579 my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
581 if ($record_type eq 'biblio') {
582 # remove any item tags - rely on BatchCommitItems
583 ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",'');
584 foreach my $item_field ($marc_record->field($item_tag)) {
585 $marc_record->delete_field($item_field);
589 my ($record_result, $item_result, $record_match) =
590 _get_commit_action($overlay_action, $nomatch_action, $item_action,
591 $rowref->{'overlay_status'}, $rowref->{'import_record_id'}, $record_type);
595 if ($record_result eq 'create_new') {
597 if ($record_type eq 'biblio') {
598 my $biblioitemnumber;
599 ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework);
600 $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?";
601 if ($item_result eq 'create_new') {
602 my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid);
603 $num_items_added += $bib_items_added;
604 $num_items_errored += $bib_items_errored;
607 $recordid = AddAuthority($marc_record, undef, GuessAuthTypeCode($marc_record));
608 $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
610 my $sth = $dbh->prepare_cached($query);
611 $sth->execute($recordid, $rowref->{'import_record_id'});
613 SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
614 } elsif ($record_result eq 'replace') {
616 $recordid = $record_match;
618 if ($record_type eq 'biblio') {
619 my ($count, $oldbiblio) = GetBiblio($recordid);
620 $oldxml = GetXmlBiblio($recordid);
622 # remove item fields so that they don't get
623 # added again if record is reverted
624 # FIXME: GetXmlBiblio output should not contain item info any more! So the next foreach should not be needed. Does not hurt either; may remove old 952s that should not have been there anymore.
625 my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'}, $marc_type);
626 foreach my $item_field ($old_marc->field($item_tag)) {
627 $old_marc->delete_field($item_field);
629 $oldxml = $old_marc->as_xml($marc_type);
631 ModBiblio($marc_record, $recordid, $oldbiblio->{'frameworkcode'});
632 $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?";
634 if ($item_result eq 'create_new') {
635 my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid);
636 $num_items_added += $bib_items_added;
637 $num_items_errored += $bib_items_errored;
640 $oldxml = GetAuthorityXML($recordid);
642 ModAuthority($recordid, $marc_record, GuessAuthTypeCode($marc_record));
643 $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
645 my $sth = $dbh->prepare_cached("UPDATE import_records SET marcxml_old = ? WHERE import_record_id = ?");
646 $sth->execute($oldxml, $rowref->{'import_record_id'});
648 my $sth2 = $dbh->prepare_cached($query);
649 $sth2->execute($recordid, $rowref->{'import_record_id'});
651 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
652 SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
653 } elsif ($record_result eq 'ignore') {
655 $recordid = $record_match;
656 if ($record_type eq 'biblio' and defined $recordid and $item_result eq 'create_new') {
657 my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid);
658 $num_items_added += $bib_items_added;
659 $num_items_errored += $bib_items_errored;
660 # still need to record the matched biblionumber so that the
661 # items can be reverted
662 my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
663 $sth2->execute($recordid, $rowref->{'import_record_id'});
664 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
666 SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored');
670 SetImportBatchStatus($batch_id, 'imported');
671 return ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored);
674 =head2 BatchCommitItems
676 ($num_items_added, $num_items_errored) =
677 BatchCommitItems($import_record_id, $biblionumber);
681 sub BatchCommitItems {
682 my ($import_record_id, $biblionumber) = @_;
684 my $dbh = C4::Context->dbh;
686 my $num_items_added = 0;
687 my $num_items_errored = 0;
688 my $sth = $dbh->prepare("SELECT import_items_id, import_items.marcxml, encoding
690 JOIN import_records USING (import_record_id)
691 WHERE import_record_id = ?
692 ORDER BY import_items_id");
693 $sth->bind_param(1, $import_record_id);
695 while (my $row = $sth->fetchrow_hashref()) {
696 my $item_marc = MARC::Record->new_from_xml(StripNonXmlChars($row->{'marcxml'}), 'UTF-8', $row->{'encoding'});
697 # FIXME - duplicate barcode check needs to become part of AddItemFromMarc()
698 my $item = TransformMarcToKoha($dbh, $item_marc);
699 my $duplicate_barcode = exists($item->{'barcode'}) && GetItemnumberFromBarcode($item->{'barcode'});
700 if ($duplicate_barcode) {
701 my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, import_error = ? WHERE import_items_id = ?");
702 $updsth->bind_param(1, 'error');
703 $updsth->bind_param(2, 'duplicate item barcode');
704 $updsth->bind_param(3, $row->{'import_items_id'});
706 $num_items_errored++;
708 my ($item_biblionumber, $biblioitemnumber, $itemnumber) = AddItemFromMarc($item_marc, $biblionumber);
709 my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, itemnumber = ? WHERE import_items_id = ?");
710 $updsth->bind_param(1, 'imported');
711 $updsth->bind_param(2, $itemnumber);
712 $updsth->bind_param(3, $row->{'import_items_id'});
719 return ($num_items_added, $num_items_errored);
722 =head2 BatchRevertRecords
724 my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted,
725 $num_ignored) = BatchRevertRecords($batch_id);
729 sub BatchRevertRecords {
730 my $batch_id = shift;
735 my $num_reverted = 0;
737 my $num_items_deleted = 0;
738 # commit (i.e., save, all records in the batch)
739 SetImportBatchStatus('reverting');
740 my $overlay_action = GetImportBatchOverlayAction($batch_id);
741 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
742 my $dbh = C4::Context->dbh;
743 my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marcxml_old, encoding, matched_biblionumber, matched_authid
745 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
746 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
747 WHERE import_batch_id = ?");
748 $sth->execute($batch_id);
750 my $marcflavour = C4::Context->preference('marcflavour');
751 while (my $rowref = $sth->fetchrow_hashref) {
752 $record_type = $rowref->{'record_type'};
753 if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') {
757 if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') {
758 $marc_type = 'UNIMARCAUTH';
759 } elsif ($marcflavour eq 'UNIMARC') {
760 $marc_type = 'UNIMARC';
762 $marc_type = 'USMARC';
765 my $record_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'});
767 if ($record_result eq 'delete') {
769 if ($record_type eq 'biblio') {
770 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
771 $error = DelBiblio($rowref->{'matched_biblionumber'});
773 my $deletedauthid = DelAuthority($rowref->{'matched_authid'});
775 if (defined $error) {
779 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
781 } elsif ($record_result eq 'restore') {
783 my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}, $marc_type);
784 if ($record_type eq 'biblio') {
785 my $biblionumber = $rowref->{'matched_biblionumber'};
786 my ($count, $oldbiblio) = GetBiblio($biblionumber);
787 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
788 ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'});
790 my $authid = $rowref->{'matched_authid'};
791 ModAuthority($authid, $old_record, GuessAuthTypeCode($old_record));
793 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
794 } elsif ($record_result eq 'ignore') {
795 if ($record_type eq 'biblio') {
796 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
798 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
801 if ($record_type eq 'biblio') {
802 # remove matched_biblionumber only if there is no 'imported' item left
803 $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?";
804 $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ? AND NOT EXISTS (SELECT * FROM import_items WHERE import_items.import_record_id=import_biblios.import_record_id and status='imported')";
806 $query = "UPDATE import_auths SET matched_authid = NULL WHERE import_record_id = ?";
808 my $sth2 = $dbh->prepare_cached($query);
809 $sth2->execute($rowref->{'import_record_id'});
813 SetImportBatchStatus($batch_id, 'reverted');
814 return ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored);
817 =head2 BatchRevertItems
819 my $num_items_deleted = BatchRevertItems($import_record_id, $biblionumber);
823 sub BatchRevertItems {
824 my ($import_record_id, $biblionumber) = @_;
826 my $dbh = C4::Context->dbh;
827 my $num_items_deleted = 0;
829 my $sth = $dbh->prepare_cached("SELECT import_items_id, itemnumber
831 JOIN items USING (itemnumber)
832 WHERE import_record_id = ?");
833 $sth->bind_param(1, $import_record_id);
835 while (my $row = $sth->fetchrow_hashref()) {
836 my $error = DelItemCheck($dbh, $biblionumber, $row->{'itemnumber'});
838 my $updsth = $dbh->prepare("UPDATE import_items SET status = ? WHERE import_items_id = ?");
839 $updsth->bind_param(1, 'reverted');
840 $updsth->bind_param(2, $row->{'import_items_id'});
843 $num_items_deleted++;
850 return $num_items_deleted;
855 CleanBatch($batch_id)
857 Deletes all staged records from the import batch
858 and sets the status of the batch to 'cleaned'. Note
859 that deleting a stage record does *not* affect
860 any record that has been committed to the database.
865 my $batch_id = shift;
866 return unless defined $batch_id;
868 C4::Context->dbh->do('DELETE FROM import_records WHERE import_batch_id = ?', {}, $batch_id);
869 SetImportBatchStatus($batch_id, 'cleaned');
872 =head2 GetAllImportBatches
874 my $results = GetAllImportBatches();
876 Returns a references to an array of hash references corresponding
877 to all import_batches rows (of batch_type 'batch'), sorted in
878 ascending order by import_batch_id.
882 sub GetAllImportBatches {
883 my $dbh = C4::Context->dbh;
884 my $sth = $dbh->prepare_cached("SELECT * FROM import_batches
885 WHERE batch_type IN ('batch', 'webservice')
886 ORDER BY import_batch_id ASC");
890 while (my $row = $sth->fetchrow_hashref) {
891 push @$results, $row;
897 =head2 GetStagedWebserviceBatches
899 my $batch_ids = GetStagedWebserviceBatches();
901 Returns a references to an array of batch id's
902 of batch_type 'webservice' that are not imported
906 my $PENDING_WEBSERVICE_BATCHES_QRY = <<EOQ;
907 SELECT import_batch_id FROM import_batches
908 WHERE batch_type = 'webservice'
909 AND import_status = 'staged'
911 sub GetStagedWebserviceBatches {
912 my $dbh = C4::Context->dbh;
913 return $dbh->selectcol_arrayref($PENDING_WEBSERVICE_BATCHES_QRY);
916 =head2 GetImportBatchRangeDesc
918 my $results = GetImportBatchRangeDesc($offset, $results_per_group);
920 Returns a reference to an array of hash references corresponding to
921 import_batches rows (sorted in descending order by import_batch_id)
922 start at the given offset.
926 sub GetImportBatchRangeDesc {
927 my ($offset, $results_per_group) = @_;
929 my $dbh = C4::Context->dbh;
930 my $query = "SELECT * FROM import_batches
931 WHERE batch_type IN ('batch', 'webservice')
932 ORDER BY import_batch_id DESC";
934 if ($results_per_group){
935 $query .= " LIMIT ?";
936 push(@params, $results_per_group);
939 $query .= " OFFSET ?";
940 push(@params, $offset);
942 my $sth = $dbh->prepare_cached($query);
943 $sth->execute(@params);
944 my $results = $sth->fetchall_arrayref({});
949 =head2 GetItemNumbersFromImportBatch
951 my @itemsnos = GetItemNumbersFromImportBatch($batch_id);
955 sub GetItemNumbersFromImportBatch {
957 my $dbh = C4::Context->dbh;
958 my $sth = $dbh->prepare("SELECT itemnumber FROM import_batches,import_records,import_items WHERE import_batches.import_batch_id=import_records.import_batch_id AND import_records.import_record_id=import_items.import_record_id AND import_batches.import_batch_id=?");
959 $sth->execute($batch_id);
961 while ( my ($itm) = $sth->fetchrow_array ) {
967 =head2 GetNumberOfImportBatches
969 my $count = GetNumberOfImportBatches();
973 sub GetNumberOfNonZ3950ImportBatches {
974 my $dbh = C4::Context->dbh;
975 my $sth = $dbh->prepare("SELECT COUNT(*) FROM import_batches WHERE batch_type != 'z3950'");
977 my ($count) = $sth->fetchrow_array();
982 =head2 GetImportRecordsRange
984 my $results = GetImportRecordsRange($batch_id, $offset, $results_per_group);
986 Returns a reference to an array of hash references corresponding to
987 import_biblios/import_auths/import_records rows for a given batch
988 starting at the given offset.
992 sub GetImportRecordsRange {
993 my ($batch_id, $offset, $results_per_group, $status) = @_;
995 my $dbh = C4::Context->dbh;
996 my $query = "SELECT title, author, isbn, issn, authorized_heading, import_records.import_record_id,
997 record_sequence, status, overlay_status,
998 matched_biblionumber, matched_authid, record_type
1000 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
1001 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
1002 WHERE import_batch_id = ?";
1004 push(@params, $batch_id);
1006 $query .= " AND status=?";
1007 push(@params,$status);
1009 $query.=" ORDER BY import_record_id";
1011 if($results_per_group){
1012 $query .= " LIMIT ?";
1013 push(@params, $results_per_group);
1016 $query .= " OFFSET ?";
1017 push(@params, $offset);
1019 my $sth = $dbh->prepare_cached($query);
1020 $sth->execute(@params);
1021 my $results = $sth->fetchall_arrayref({});
1027 =head2 GetBestRecordMatch
1029 my $record_id = GetBestRecordMatch($import_record_id);
1033 sub GetBestRecordMatch {
1034 my ($import_record_id) = @_;
1036 my $dbh = C4::Context->dbh;
1037 my $sth = $dbh->prepare("SELECT candidate_match_id
1038 FROM import_record_matches
1039 JOIN import_records ON ( import_record_matches.import_record_id = import_records.import_record_id )
1040 LEFT JOIN biblio ON ( candidate_match_id = biblio.biblionumber )
1041 LEFT JOIN auth_header ON ( candidate_match_id = auth_header.authid )
1042 WHERE import_record_matches.import_record_id = ? AND
1043 ( (import_records.record_type = 'biblio' AND biblio.biblionumber IS NOT NULL) OR
1044 (import_records.record_type = 'auth' AND auth_header.authid IS NOT NULL) )
1045 ORDER BY score DESC, candidate_match_id DESC");
1046 $sth->execute($import_record_id);
1047 my ($record_id) = $sth->fetchrow_array();
1052 =head2 GetImportBatchStatus
1054 my $status = GetImportBatchStatus($batch_id);
1058 sub GetImportBatchStatus {
1059 my ($batch_id) = @_;
1061 my $dbh = C4::Context->dbh;
1062 my $sth = $dbh->prepare("SELECT import_status FROM import_batches WHERE import_batch_id = ?");
1063 $sth->execute($batch_id);
1064 my ($status) = $sth->fetchrow_array();
1070 =head2 SetImportBatchStatus
1072 SetImportBatchStatus($batch_id, $new_status);
1076 sub SetImportBatchStatus {
1077 my ($batch_id, $new_status) = @_;
1079 my $dbh = C4::Context->dbh;
1080 my $sth = $dbh->prepare("UPDATE import_batches SET import_status = ? WHERE import_batch_id = ?");
1081 $sth->execute($new_status, $batch_id);
1086 =head2 GetImportBatchOverlayAction
1088 my $overlay_action = GetImportBatchOverlayAction($batch_id);
1092 sub GetImportBatchOverlayAction {
1093 my ($batch_id) = @_;
1095 my $dbh = C4::Context->dbh;
1096 my $sth = $dbh->prepare("SELECT overlay_action FROM import_batches WHERE import_batch_id = ?");
1097 $sth->execute($batch_id);
1098 my ($overlay_action) = $sth->fetchrow_array();
1100 return $overlay_action;
1105 =head2 SetImportBatchOverlayAction
1107 SetImportBatchOverlayAction($batch_id, $new_overlay_action);
1111 sub SetImportBatchOverlayAction {
1112 my ($batch_id, $new_overlay_action) = @_;
1114 my $dbh = C4::Context->dbh;
1115 my $sth = $dbh->prepare("UPDATE import_batches SET overlay_action = ? WHERE import_batch_id = ?");
1116 $sth->execute($new_overlay_action, $batch_id);
1121 =head2 GetImportBatchNoMatchAction
1123 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
1127 sub GetImportBatchNoMatchAction {
1128 my ($batch_id) = @_;
1130 my $dbh = C4::Context->dbh;
1131 my $sth = $dbh->prepare("SELECT nomatch_action FROM import_batches WHERE import_batch_id = ?");
1132 $sth->execute($batch_id);
1133 my ($nomatch_action) = $sth->fetchrow_array();
1135 return $nomatch_action;
1140 =head2 SetImportBatchNoMatchAction
1142 SetImportBatchNoMatchAction($batch_id, $new_nomatch_action);
1146 sub SetImportBatchNoMatchAction {
1147 my ($batch_id, $new_nomatch_action) = @_;
1149 my $dbh = C4::Context->dbh;
1150 my $sth = $dbh->prepare("UPDATE import_batches SET nomatch_action = ? WHERE import_batch_id = ?");
1151 $sth->execute($new_nomatch_action, $batch_id);
1156 =head2 GetImportBatchItemAction
1158 my $item_action = GetImportBatchItemAction($batch_id);
1162 sub GetImportBatchItemAction {
1163 my ($batch_id) = @_;
1165 my $dbh = C4::Context->dbh;
1166 my $sth = $dbh->prepare("SELECT item_action FROM import_batches WHERE import_batch_id = ?");
1167 $sth->execute($batch_id);
1168 my ($item_action) = $sth->fetchrow_array();
1170 return $item_action;
1175 =head2 SetImportBatchItemAction
1177 SetImportBatchItemAction($batch_id, $new_item_action);
1181 sub SetImportBatchItemAction {
1182 my ($batch_id, $new_item_action) = @_;
1184 my $dbh = C4::Context->dbh;
1185 my $sth = $dbh->prepare("UPDATE import_batches SET item_action = ? WHERE import_batch_id = ?");
1186 $sth->execute($new_item_action, $batch_id);
1191 =head2 GetImportBatchMatcher
1193 my $matcher_id = GetImportBatchMatcher($batch_id);
1197 sub GetImportBatchMatcher {
1198 my ($batch_id) = @_;
1200 my $dbh = C4::Context->dbh;
1201 my $sth = $dbh->prepare("SELECT matcher_id FROM import_batches WHERE import_batch_id = ?");
1202 $sth->execute($batch_id);
1203 my ($matcher_id) = $sth->fetchrow_array();
1210 =head2 SetImportBatchMatcher
1212 SetImportBatchMatcher($batch_id, $new_matcher_id);
1216 sub SetImportBatchMatcher {
1217 my ($batch_id, $new_matcher_id) = @_;
1219 my $dbh = C4::Context->dbh;
1220 my $sth = $dbh->prepare("UPDATE import_batches SET matcher_id = ? WHERE import_batch_id = ?");
1221 $sth->execute($new_matcher_id, $batch_id);
1226 =head2 GetImportRecordOverlayStatus
1228 my $overlay_status = GetImportRecordOverlayStatus($import_record_id);
1232 sub GetImportRecordOverlayStatus {
1233 my ($import_record_id) = @_;
1235 my $dbh = C4::Context->dbh;
1236 my $sth = $dbh->prepare("SELECT overlay_status FROM import_records WHERE import_record_id = ?");
1237 $sth->execute($import_record_id);
1238 my ($overlay_status) = $sth->fetchrow_array();
1240 return $overlay_status;
1245 =head2 SetImportRecordOverlayStatus
1247 SetImportRecordOverlayStatus($import_record_id, $new_overlay_status);
1251 sub SetImportRecordOverlayStatus {
1252 my ($import_record_id, $new_overlay_status) = @_;
1254 my $dbh = C4::Context->dbh;
1255 my $sth = $dbh->prepare("UPDATE import_records SET overlay_status = ? WHERE import_record_id = ?");
1256 $sth->execute($new_overlay_status, $import_record_id);
1261 =head2 GetImportRecordStatus
1263 my $overlay_status = GetImportRecordStatus($import_record_id);
1267 sub GetImportRecordStatus {
1268 my ($import_record_id) = @_;
1270 my $dbh = C4::Context->dbh;
1271 my $sth = $dbh->prepare("SELECT status FROM import_records WHERE import_record_id = ?");
1272 $sth->execute($import_record_id);
1273 my ($overlay_status) = $sth->fetchrow_array();
1275 return $overlay_status;
1280 =head2 SetImportRecordStatus
1282 SetImportRecordStatus($import_record_id, $new_overlay_status);
1286 sub SetImportRecordStatus {
1287 my ($import_record_id, $new_overlay_status) = @_;
1289 my $dbh = C4::Context->dbh;
1290 my $sth = $dbh->prepare("UPDATE import_records SET status = ? WHERE import_record_id = ?");
1291 $sth->execute($new_overlay_status, $import_record_id);
1296 =head2 GetImportRecordMatches
1298 my $results = GetImportRecordMatches($import_record_id, $best_only);
1302 sub GetImportRecordMatches {
1303 my $import_record_id = shift;
1304 my $best_only = @_ ? shift : 0;
1306 my $dbh = C4::Context->dbh;
1307 # FIXME currently biblio only
1308 my $sth = $dbh->prepare_cached("SELECT title, author, biblionumber,
1309 candidate_match_id, score, record_type
1311 JOIN import_record_matches USING (import_record_id)
1312 LEFT JOIN biblio ON (biblionumber = candidate_match_id)
1313 WHERE import_record_id = ?
1314 ORDER BY score DESC, biblionumber DESC");
1315 $sth->bind_param(1, $import_record_id);
1318 while (my $row = $sth->fetchrow_hashref) {
1319 if ($row->{'record_type'} eq 'auth') {
1320 $row->{'authorized_heading'} = C4::AuthoritiesMarc::GetAuthorizedHeading( { authid => $row->{'candidate_match_id'} } );
1322 next if ($row->{'record_type'} eq 'biblio' && not $row->{'biblionumber'});
1323 push @$results, $row;
1333 =head2 SetImportRecordMatches
1335 SetImportRecordMatches($import_record_id, @matches);
1339 sub SetImportRecordMatches {
1340 my $import_record_id = shift;
1343 my $dbh = C4::Context->dbh;
1344 my $delsth = $dbh->prepare("DELETE FROM import_record_matches WHERE import_record_id = ?");
1345 $delsth->execute($import_record_id);
1348 my $sth = $dbh->prepare("INSERT INTO import_record_matches (import_record_id, candidate_match_id, score)
1350 foreach my $match (@matches) {
1351 $sth->execute($import_record_id, $match->{'record_id'}, $match->{'score'});
1356 # internal functions
1358 sub _create_import_record {
1359 my ($batch_id, $record_sequence, $marc_record, $record_type, $encoding, $z3950random, $marc_type) = @_;
1361 my $dbh = C4::Context->dbh;
1362 my $sth = $dbh->prepare("INSERT INTO import_records (import_batch_id, record_sequence, marc, marcxml,
1363 record_type, encoding, z3950random)
1364 VALUES (?, ?, ?, ?, ?, ?, ?)");
1365 $sth->execute($batch_id, $record_sequence, $marc_record->as_usmarc(), $marc_record->as_xml($marc_type),
1366 $record_type, $encoding, $z3950random);
1367 my $import_record_id = $dbh->{'mysql_insertid'};
1369 return $import_record_id;
1372 sub _update_import_record_marc {
1373 my ($import_record_id, $marc_record, $marc_type) = @_;
1375 my $dbh = C4::Context->dbh;
1376 my $sth = $dbh->prepare("UPDATE import_records SET marc = ?, marcxml = ?
1377 WHERE import_record_id = ?");
1378 $sth->execute($marc_record->as_usmarc(), $marc_record->as_xml($marc_type), $import_record_id);
1382 sub _add_auth_fields {
1383 my ($import_record_id, $marc_record) = @_;
1386 if ($marc_record->field('001')) {
1387 $controlnumber = $marc_record->field('001')->data();
1389 my $authorized_heading = C4::AuthoritiesMarc::GetAuthorizedHeading({ record => $marc_record });
1390 my $dbh = C4::Context->dbh;
1391 my $sth = $dbh->prepare("INSERT INTO import_auths (import_record_id, control_number, authorized_heading) VALUES (?, ?, ?)");
1392 $sth->execute($import_record_id, $controlnumber, $authorized_heading);
1396 sub _add_biblio_fields {
1397 my ($import_record_id, $marc_record) = @_;
1399 my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record);
1400 my $dbh = C4::Context->dbh;
1401 # FIXME no controlnumber, originalsource
1402 $isbn = C4::Koha::_isbn_cleanup($isbn); # FIXME C4::Koha::_isbn_cleanup should be made public
1403 my $sth = $dbh->prepare("INSERT INTO import_biblios (import_record_id, title, author, isbn, issn) VALUES (?, ?, ?, ?, ?)");
1404 $sth->execute($import_record_id, $title, $author, $isbn, $issn);
1409 sub _update_biblio_fields {
1410 my ($import_record_id, $marc_record) = @_;
1412 my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record);
1413 my $dbh = C4::Context->dbh;
1414 # FIXME no controlnumber, originalsource
1415 # FIXME 2 - should regularize normalization of ISBN wherever it is done
1419 my $sth = $dbh->prepare("UPDATE import_biblios SET title = ?, author = ?, isbn = ?, issn = ?
1420 WHERE import_record_id = ?");
1421 $sth->execute($title, $author, $isbn, $issn, $import_record_id);
1425 sub _parse_biblio_fields {
1426 my ($marc_record) = @_;
1428 my $dbh = C4::Context->dbh;
1429 my $bibliofields = TransformMarcToKoha($dbh, $marc_record, '');
1430 return ($bibliofields->{'title'}, $bibliofields->{'author'}, $bibliofields->{'isbn'}, $bibliofields->{'issn'});
1434 sub _update_batch_record_counts {
1435 my ($batch_id) = @_;
1437 my $dbh = C4::Context->dbh;
1438 my $sth = $dbh->prepare_cached("UPDATE import_batches SET
1442 WHERE import_batch_id = import_batches.import_batch_id),
1446 JOIN import_items USING (import_record_id)
1447 WHERE import_batch_id = import_batches.import_batch_id
1448 AND record_type = 'biblio')
1449 WHERE import_batch_id = ?");
1450 $sth->bind_param(1, $batch_id);
1455 sub _get_commit_action {
1456 my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id, $record_type) = @_;
1458 if ($record_type eq 'biblio') {
1459 my ($bib_result, $bib_match, $item_result);
1461 if ($overlay_status ne 'no_match') {
1462 $bib_match = GetBestRecordMatch($import_record_id);
1463 if ($overlay_action eq 'replace') {
1464 $bib_result = defined($bib_match) ? 'replace' : 'create_new';
1465 } elsif ($overlay_action eq 'create_new') {
1466 $bib_result = 'create_new';
1467 } elsif ($overlay_action eq 'ignore') {
1468 $bib_result = 'ignore';
1470 $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore';
1472 $bib_result = $nomatch_action;
1473 $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore';
1475 return ($bib_result, $item_result, $bib_match);
1476 } else { # must be auths
1477 my ($auth_result, $auth_match);
1479 if ($overlay_status ne 'no_match') {
1480 $auth_match = GetBestRecordMatch($import_record_id);
1481 if ($overlay_action eq 'replace') {
1482 $auth_result = defined($auth_match) ? 'replace' : 'create_new';
1483 } elsif ($overlay_action eq 'create_new') {
1484 $auth_result = 'create_new';
1485 } elsif ($overlay_action eq 'ignore') {
1486 $auth_result = 'ignore';
1489 $auth_result = $nomatch_action;
1492 return ($auth_result, undef, $auth_match);
1497 sub _get_revert_action {
1498 my ($overlay_action, $overlay_status, $status) = @_;
1502 if ($status eq 'ignored') {
1503 $bib_result = 'ignore';
1505 if ($overlay_action eq 'create_new') {
1506 $bib_result = 'delete';
1508 $bib_result = ($overlay_status eq 'match_applied') ? 'restore' : 'delete';
1519 Koha Development Team <http://koha-community.org/>
1521 Galen Charlton <galen.charlton@liblime.com>