1 package C4::ImportBatch;
3 # Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 2 of the License, or (at your option) any later
12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License along
17 # with Koha; if not, write to the Free Software Foundation, Inc.,
18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
28 use C4::AuthoritiesMarc;
30 use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
33 # set the version for version checking
34 $VERSION = 3.07.00.049;
41 GetImportRecordMarcXML
46 AddItemsToImportBiblio
57 GetStagedWebserviceBatches
58 GetImportBatchRangeDesc
59 GetNumberOfNonZ3950ImportBatches
61 GetItemNumbersFromImportBatch
65 GetImportBatchOverlayAction
66 SetImportBatchOverlayAction
67 GetImportBatchNoMatchAction
68 SetImportBatchNoMatchAction
69 GetImportBatchItemAction
70 SetImportBatchItemAction
73 GetImportRecordOverlayStatus
74 SetImportRecordOverlayStatus
77 GetImportRecordMatches
78 SetImportRecordMatches
84 C4::ImportBatch - manage batches of imported MARC records
92 =head2 GetZ3950BatchId
94 my $batchid = GetZ3950BatchId($z3950server);
96 Retrieves the ID of the import batch for the Z39.50
97 reservoir for the given target. If necessary,
98 creates the import batch.
102 sub GetZ3950BatchId {
103 my ($z3950server) = @_;
105 my $dbh = C4::Context->dbh;
106 my $sth = $dbh->prepare("SELECT import_batch_id FROM import_batches
107 WHERE batch_type = 'z3950'
109 $sth->execute($z3950server);
110 my $rowref = $sth->fetchrow_arrayref();
112 if (defined $rowref) {
115 my $batch_id = AddImportBatch( {
116 overlay_action => 'create_new',
117 import_status => 'staged',
118 batch_type => 'z3950',
119 file_name => $z3950server,
126 =head2 GetWebserviceBatchId
128 my $batchid = GetWebserviceBatchId();
130 Retrieves the ID of the import batch for webservice.
131 If necessary, creates the import batch.
135 my $WEBSERVICE_BASE_QRY = <<EOQ;
136 SELECT import_batch_id FROM import_batches
137 WHERE batch_type = 'webservice'
138 AND import_status = 'staged'
140 sub GetWebserviceBatchId {
143 my $dbh = C4::Context->dbh;
144 my $sql = $WEBSERVICE_BASE_QRY;
146 foreach my $field (qw(matcher_id overlay_action nomatch_action item_action)) {
147 if (my $val = $params->{$field}) {
148 $sql .= " AND $field = ?";
152 my $id = $dbh->selectrow_array($sql, undef, @args);
155 $params->{batch_type} = 'webservice';
156 $params->{import_status} = 'staged';
157 return AddImportBatch($params);
160 =head2 GetImportRecordMarc
162 my ($marcblob, $encoding) = GetImportRecordMarc($import_record_id);
166 sub GetImportRecordMarc {
167 my ($import_record_id) = @_;
169 my $dbh = C4::Context->dbh;
170 my $sth = $dbh->prepare("SELECT marc, encoding FROM import_records WHERE import_record_id = ?");
171 $sth->execute($import_record_id);
172 my ($marc, $encoding) = $sth->fetchrow();
174 return $marc, $encoding;
178 =head2 GetImportRecordMarcXML
180 my $marcxml = GetImportRecordMarcXML($import_record_id);
184 sub GetImportRecordMarcXML {
185 my ($import_record_id) = @_;
187 my $dbh = C4::Context->dbh;
188 my $sth = $dbh->prepare("SELECT marcxml FROM import_records WHERE import_record_id = ?");
189 $sth->execute($import_record_id);
190 my ($marcxml) = $sth->fetchrow();
196 =head2 AddImportBatch
198 my $batch_id = AddImportBatch($params_hash);
206 foreach (qw( matcher_id template_id branchcode
207 overlay_action nomatch_action item_action
208 import_status batch_type file_name comments record_type )) {
209 if (exists $params->{$_}) {
211 push @vals, $params->{$_};
214 my $dbh = C4::Context->dbh;
215 $dbh->do("INSERT INTO import_batches (".join( ',', @fields).")
216 VALUES (".join( ',', map '?', @fields).")",
219 return $dbh->{'mysql_insertid'};
222 =head2 GetImportBatch
224 my $row = GetImportBatch($batch_id);
226 Retrieve a hashref of an import_batches row.
233 my $dbh = C4::Context->dbh;
234 my $sth = $dbh->prepare_cached("SELECT * FROM import_batches WHERE import_batch_id = ?");
235 $sth->bind_param(1, $batch_id);
237 my $result = $sth->fetchrow_hashref;
243 =head2 AddBiblioToBatch
245 my $import_record_id = AddBiblioToBatch($batch_id, $record_sequence,
246 $marc_record, $encoding, $z3950random, $update_counts);
250 sub AddBiblioToBatch {
251 my $batch_id = shift;
252 my $record_sequence = shift;
253 my $marc_record = shift;
254 my $encoding = shift;
255 my $z3950random = shift;
256 my $update_counts = @_ ? shift : 1;
258 my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'biblio', $encoding, $z3950random, C4::Context->preference('marcflavour'));
259 _add_biblio_fields($import_record_id, $marc_record);
260 _update_batch_record_counts($batch_id) if $update_counts;
261 return $import_record_id;
264 =head2 ModBiblioInBatch
266 ModBiblioInBatch($import_record_id, $marc_record);
270 sub ModBiblioInBatch {
271 my ($import_record_id, $marc_record) = @_;
273 _update_import_record_marc($import_record_id, $marc_record, C4::Context->preference('marcflavour'));
274 _update_biblio_fields($import_record_id, $marc_record);
278 =head2 AddAuthToBatch
280 my $import_record_id = AddAuthToBatch($batch_id, $record_sequence,
281 $marc_record, $encoding, $z3950random, $update_counts, [$marc_type]);
286 my $batch_id = shift;
287 my $record_sequence = shift;
288 my $marc_record = shift;
289 my $encoding = shift;
290 my $z3950random = shift;
291 my $update_counts = @_ ? shift : 1;
292 my $marc_type = shift || C4::Context->preference('marcflavour');
294 $marc_type = 'UNIMARCAUTH' if $marc_type eq 'UNIMARC';
296 my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $z3950random, $marc_type);
297 _add_auth_fields($import_record_id, $marc_record);
298 _update_batch_record_counts($batch_id) if $update_counts;
299 return $import_record_id;
302 =head2 ModAuthInBatch
304 ModAuthInBatch($import_record_id, $marc_record);
309 my ($import_record_id, $marc_record) = @_;
311 my $marcflavour = C4::Context->preference('marcflavour');
312 _update_import_record_marc($import_record_id, $marc_record, $marcflavour eq 'UNIMARC' ? 'UNIMARCAUTH' : 'USMARC');
316 =head2 BatchStageMarcRecords
318 ($batch_id, $num_records, $num_items, @invalid_records) =
319 BatchStageMarcRecords($record_type, $encoding, $marc_records, $file_name,
320 $comments, $branch_code, $parse_items,
322 $progress_interval, $progress_callback);
326 sub BatchStageMarcRecords {
327 my $record_type = shift;
328 my $encoding = shift;
329 my $marc_records = shift;
330 my $file_name = shift;
331 my $comments = shift;
332 my $branch_code = shift;
333 my $parse_items = shift;
334 my $leave_as_staging = shift;
336 # optional callback to monitor status
338 my $progress_interval = 0;
339 my $progress_callback = undef;
341 $progress_interval = shift;
342 $progress_callback = shift;
343 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
344 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
347 my $batch_id = AddImportBatch( {
348 overlay_action => 'create_new',
349 import_status => 'staging',
350 batch_type => 'batch',
351 file_name => $file_name,
352 comments => $comments,
353 record_type => $record_type,
356 SetImportBatchItemAction($batch_id, 'always_add');
358 SetImportBatchItemAction($batch_id, 'ignore');
361 my $marc_type = C4::Context->preference('marcflavour');
362 $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth');
363 my @invalid_records = ();
366 # FIXME - for now, we're dealing only with bibs
368 foreach my $marc_blob (split(/\x1D/, $marc_records)) {
369 $marc_blob =~ s/^\s+//g;
370 $marc_blob =~ s/\s+$//g;
371 next unless $marc_blob;
373 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
374 &$progress_callback($rec_num);
376 my ($marc_record, $charset_guessed, $char_errors) =
377 MarcToUTF8Record($marc_blob, $marc_type, $encoding);
379 $encoding = $charset_guessed unless $encoding;
381 my $import_record_id;
382 if (scalar($marc_record->fields()) == 0) {
383 push @invalid_records, $marc_blob;
386 # Normalize the record so it doesn't have separated diacritics
387 SetUTF8Flag($marc_record);
390 if ($record_type eq 'biblio') {
391 $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0);
393 my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0);
394 $num_items += scalar(@import_items_ids);
396 } elsif ($record_type eq 'auth') {
397 $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0, $marc_type);
401 unless ($leave_as_staging) {
402 SetImportBatchStatus($batch_id, 'staged');
404 # FIXME branch_code, number of bibs, number of items
405 _update_batch_record_counts($batch_id);
406 return ($batch_id, $num_valid, $num_items, @invalid_records);
409 =head2 AddItemsToImportBiblio
411 my @import_items_ids = AddItemsToImportBiblio($batch_id,
412 $import_record_id, $marc_record, $update_counts);
416 sub AddItemsToImportBiblio {
417 my $batch_id = shift;
418 my $import_record_id = shift;
419 my $marc_record = shift;
420 my $update_counts = @_ ? shift : 0;
422 my @import_items_ids = ();
424 my $dbh = C4::Context->dbh;
425 my ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",'');
426 foreach my $item_field ($marc_record->field($item_tag)) {
427 my $item_marc = MARC::Record->new();
428 $item_marc->leader("00000 a "); # must set Leader/09 to 'a'
429 $item_marc->append_fields($item_field);
430 $marc_record->delete_field($item_field);
431 my $sth = $dbh->prepare_cached("INSERT INTO import_items (import_record_id, status, marcxml)
433 $sth->bind_param(1, $import_record_id);
434 $sth->bind_param(2, 'staged');
435 $sth->bind_param(3, $item_marc->as_xml());
437 push @import_items_ids, $dbh->{'mysql_insertid'};
441 if ($#import_items_ids > -1) {
442 _update_batch_record_counts($batch_id) if $update_counts;
443 _update_import_record_marc($import_record_id, $marc_record, C4::Context->preference('marcflavour'));
445 return @import_items_ids;
448 =head2 BatchFindDuplicates
450 my $num_with_matches = BatchFindDuplicates($batch_id, $matcher,
451 $max_matches, $progress_interval, $progress_callback);
453 Goes through the records loaded in the batch and attempts to
454 find duplicates for each one. Sets the matching status
455 of each record to "no_match" or "auto_match" as appropriate.
457 The $max_matches parameter is optional; if it is not supplied,
460 The $progress_interval and $progress_callback parameters are
461 optional; if both are supplied, the sub referred to by
462 $progress_callback will be invoked every $progress_interval
463 records using the number of records processed as the
468 sub BatchFindDuplicates {
469 my $batch_id = shift;
471 my $max_matches = @_ ? shift : 10;
473 # optional callback to monitor status
475 my $progress_interval = 0;
476 my $progress_callback = undef;
478 $progress_interval = shift;
479 $progress_callback = shift;
480 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
481 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
484 my $dbh = C4::Context->dbh;
486 my $sth = $dbh->prepare("SELECT import_record_id, record_type, marc
488 WHERE import_batch_id = ?");
489 $sth->execute($batch_id);
490 my $num_with_matches = 0;
492 while (my $rowref = $sth->fetchrow_hashref) {
494 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
495 &$progress_callback($rec_num);
497 my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
499 if (defined $matcher) {
500 @matches = $matcher->get_matches($marc_record, $max_matches);
502 if (scalar(@matches) > 0) {
504 SetImportRecordMatches($rowref->{'import_record_id'}, @matches);
505 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'auto_match');
507 SetImportRecordMatches($rowref->{'import_record_id'}, ());
508 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'no_match');
512 return $num_with_matches;
515 =head2 BatchCommitRecords
517 my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =
518 BatchCommitRecords($batch_id, $framework,
519 $progress_interval, $progress_callback);
523 sub BatchCommitRecords {
524 my $batch_id = shift;
525 my $framework = shift;
527 # optional callback to monitor status
529 my $progress_interval = 0;
530 my $progress_callback = undef;
532 $progress_interval = shift;
533 $progress_callback = shift;
534 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
535 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
541 my $num_items_added = 0;
542 my $num_items_errored = 0;
544 # commit (i.e., save, all records in the batch)
545 SetImportBatchStatus('importing');
546 my $overlay_action = GetImportBatchOverlayAction($batch_id);
547 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
548 my $item_action = GetImportBatchItemAction($batch_id);
551 my $dbh = C4::Context->dbh;
552 my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marc, encoding
554 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
555 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
556 WHERE import_batch_id = ?");
557 $sth->execute($batch_id);
558 my $marcflavour = C4::Context->preference('marcflavour');
560 while (my $rowref = $sth->fetchrow_hashref) {
561 $record_type = $rowref->{'record_type'};
563 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
564 &$progress_callback($rec_num);
566 if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'imported') {
572 if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') {
573 $marc_type = 'UNIMARCAUTH';
574 } elsif ($marcflavour eq 'UNIMARC') {
575 $marc_type = 'UNIMARC';
577 $marc_type = 'USMARC';
579 my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
581 if ($record_type eq 'biblio') {
582 # remove any item tags - rely on BatchCommitItems
583 ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",'');
584 foreach my $item_field ($marc_record->field($item_tag)) {
585 $marc_record->delete_field($item_field);
589 my ($record_result, $item_result, $record_match) =
590 _get_commit_action($overlay_action, $nomatch_action, $item_action,
591 $rowref->{'overlay_status'}, $rowref->{'import_record_id'}, $record_type);
595 if ($record_result eq 'create_new') {
597 if ($record_type eq 'biblio') {
598 my $biblioitemnumber;
599 ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework);
600 $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?";
601 if ($item_result eq 'create_new') {
602 my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid);
603 $num_items_added += $bib_items_added;
604 $num_items_errored += $bib_items_errored;
607 $recordid = AddAuthority($marc_record, undef, GuessAuthTypeCode($marc_record));
608 $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
610 my $sth = $dbh->prepare_cached($query);
611 $sth->execute($recordid, $rowref->{'import_record_id'});
613 SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
614 } elsif ($record_result eq 'replace') {
616 $recordid = $record_match;
618 if ($record_type eq 'biblio') {
619 my ($count, $oldbiblio) = GetBiblio($recordid);
620 $oldxml = GetXmlBiblio($recordid);
622 # remove item fields so that they don't get
623 # added again if record is reverted
624 # FIXME: GetXmlBiblio output should not contain item info any more! So the next foreach should not be needed. Does not hurt either; may remove old 952s that should not have been there anymore.
625 my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'}, $marc_type);
626 foreach my $item_field ($old_marc->field($item_tag)) {
627 $old_marc->delete_field($item_field);
629 $oldxml = $old_marc->as_xml($marc_type);
631 ModBiblio($marc_record, $recordid, $oldbiblio->{'frameworkcode'});
632 $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?";
634 if ($item_result eq 'create_new') {
635 my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid);
636 $num_items_added += $bib_items_added;
637 $num_items_errored += $bib_items_errored;
640 $oldxml = GetAuthorityXML($recordid);
642 ModAuthority($recordid, $marc_record, GuessAuthTypeCode($marc_record));
643 $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
645 my $sth = $dbh->prepare_cached("UPDATE import_records SET marcxml_old = ? WHERE import_record_id = ?");
646 $sth->execute($oldxml, $rowref->{'import_record_id'});
648 my $sth2 = $dbh->prepare_cached($query);
649 $sth2->execute($recordid, $rowref->{'import_record_id'});
651 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
652 SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
653 } elsif ($record_result eq 'ignore') {
655 if ($record_type eq 'biblio' and defined $recordid and $item_result eq 'create_new') {
656 my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid);
657 $num_items_added += $bib_items_added;
658 $num_items_errored += $bib_items_errored;
659 # still need to record the matched biblionumber so that the
660 # items can be reverted
661 my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
662 $sth2->execute($recordid, $rowref->{'import_record_id'});
663 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
665 SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored');
669 SetImportBatchStatus($batch_id, 'imported');
670 return ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored);
673 =head2 BatchCommitItems
675 ($num_items_added, $num_items_errored) =
676 BatchCommitItems($import_record_id, $biblionumber);
680 sub BatchCommitItems {
681 my ($import_record_id, $biblionumber) = @_;
683 my $dbh = C4::Context->dbh;
685 my $num_items_added = 0;
686 my $num_items_errored = 0;
687 my $sth = $dbh->prepare("SELECT import_items_id, import_items.marcxml, encoding
689 JOIN import_records USING (import_record_id)
690 WHERE import_record_id = ?
691 ORDER BY import_items_id");
692 $sth->bind_param(1, $import_record_id);
694 while (my $row = $sth->fetchrow_hashref()) {
695 my $item_marc = MARC::Record->new_from_xml(StripNonXmlChars($row->{'marcxml'}), 'UTF-8', $row->{'encoding'});
696 # FIXME - duplicate barcode check needs to become part of AddItemFromMarc()
697 my $item = TransformMarcToKoha($dbh, $item_marc);
698 my $duplicate_barcode = exists($item->{'barcode'}) && GetItemnumberFromBarcode($item->{'barcode'});
699 if ($duplicate_barcode) {
700 my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, import_error = ? WHERE import_items_id = ?");
701 $updsth->bind_param(1, 'error');
702 $updsth->bind_param(2, 'duplicate item barcode');
703 $updsth->bind_param(3, $row->{'import_items_id'});
705 $num_items_errored++;
707 my ($item_biblionumber, $biblioitemnumber, $itemnumber) = AddItemFromMarc($item_marc, $biblionumber);
708 my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, itemnumber = ? WHERE import_items_id = ?");
709 $updsth->bind_param(1, 'imported');
710 $updsth->bind_param(2, $itemnumber);
711 $updsth->bind_param(3, $row->{'import_items_id'});
718 return ($num_items_added, $num_items_errored);
721 =head2 BatchRevertRecords
723 my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted,
724 $num_ignored) = BatchRevertRecords($batch_id);
728 sub BatchRevertRecords {
729 my $batch_id = shift;
734 my $num_reverted = 0;
736 my $num_items_deleted = 0;
737 # commit (i.e., save, all records in the batch)
738 SetImportBatchStatus('reverting');
739 my $overlay_action = GetImportBatchOverlayAction($batch_id);
740 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
741 my $dbh = C4::Context->dbh;
742 my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marcxml_old, encoding, matched_biblionumber, matched_authid
744 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
745 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
746 WHERE import_batch_id = ?");
747 $sth->execute($batch_id);
749 my $marcflavour = C4::Context->preference('marcflavour');
750 while (my $rowref = $sth->fetchrow_hashref) {
751 $record_type = $rowref->{'record_type'};
752 if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') {
756 if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') {
757 $marc_type = 'UNIMARCAUTH';
758 } elsif ($marcflavour eq 'UNIMARC') {
759 $marc_type = 'UNIMARC';
761 $marc_type = 'USMARC';
764 my $record_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'});
766 if ($record_result eq 'delete') {
768 if ($record_type eq 'biblio') {
769 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
770 $error = DelBiblio($rowref->{'matched_biblionumber'});
772 my $deletedauthid = DelAuthority($rowref->{'matched_authid'});
774 if (defined $error) {
778 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
780 } elsif ($record_result eq 'restore') {
782 my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}, $marc_type);
783 if ($record_type eq 'biblio') {
784 my $biblionumber = $rowref->{'matched_biblionumber'};
785 my ($count, $oldbiblio) = GetBiblio($biblionumber);
786 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
787 ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'});
789 my $authid = $rowref->{'matched_authid'};
790 ModAuthority($authid, $old_record, GuessAuthTypeCode($old_record));
792 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
793 } elsif ($record_result eq 'ignore') {
794 if ($record_type eq 'biblio') {
795 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
797 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
800 if ($record_type eq 'biblio') {
801 # remove matched_biblionumber only if there is no 'imported' item left
802 $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?";
803 $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ? AND NOT EXISTS (SELECT * FROM import_items WHERE import_items.import_record_id=import_biblios.import_record_id and status='imported')";
805 $query = "UPDATE import_auths SET matched_authid = NULL WHERE import_record_id = ?";
807 my $sth2 = $dbh->prepare_cached($query);
808 $sth2->execute($rowref->{'import_record_id'});
812 SetImportBatchStatus($batch_id, 'reverted');
813 return ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored);
816 =head2 BatchRevertItems
818 my $num_items_deleted = BatchRevertItems($import_record_id, $biblionumber);
822 sub BatchRevertItems {
823 my ($import_record_id, $biblionumber) = @_;
825 my $dbh = C4::Context->dbh;
826 my $num_items_deleted = 0;
828 my $sth = $dbh->prepare_cached("SELECT import_items_id, itemnumber
830 JOIN items USING (itemnumber)
831 WHERE import_record_id = ?");
832 $sth->bind_param(1, $import_record_id);
834 while (my $row = $sth->fetchrow_hashref()) {
835 my $error = DelItemCheck($dbh, $biblionumber, $row->{'itemnumber'});
837 my $updsth = $dbh->prepare("UPDATE import_items SET status = ? WHERE import_items_id = ?");
838 $updsth->bind_param(1, 'reverted');
839 $updsth->bind_param(2, $row->{'import_items_id'});
842 $num_items_deleted++;
849 return $num_items_deleted;
854 CleanBatch($batch_id)
856 Deletes all staged records from the import batch
857 and sets the status of the batch to 'cleaned'. Note
858 that deleting a stage record does *not* affect
859 any record that has been committed to the database.
864 my $batch_id = shift;
865 return unless defined $batch_id;
867 C4::Context->dbh->do('DELETE FROM import_records WHERE import_batch_id = ?', {}, $batch_id);
868 SetImportBatchStatus($batch_id, 'cleaned');
871 =head2 GetAllImportBatches
873 my $results = GetAllImportBatches();
875 Returns a references to an array of hash references corresponding
876 to all import_batches rows (of batch_type 'batch'), sorted in
877 ascending order by import_batch_id.
881 sub GetAllImportBatches {
882 my $dbh = C4::Context->dbh;
883 my $sth = $dbh->prepare_cached("SELECT * FROM import_batches
884 WHERE batch_type IN ('batch', 'webservice')
885 ORDER BY import_batch_id ASC");
889 while (my $row = $sth->fetchrow_hashref) {
890 push @$results, $row;
896 =head2 GetStagedWebserviceBatches
898 my $batch_ids = GetStagedWebserviceBatches();
900 Returns a references to an array of batch id's
901 of batch_type 'webservice' that are not imported
905 my $PENDING_WEBSERVICE_BATCHES_QRY = <<EOQ;
906 SELECT import_batch_id FROM import_batches
907 WHERE batch_type = 'webservice'
908 AND import_status = 'staged'
910 sub GetStagedWebserviceBatches {
911 my $dbh = C4::Context->dbh;
912 return $dbh->selectcol_arrayref($PENDING_WEBSERVICE_BATCHES_QRY);
915 =head2 GetImportBatchRangeDesc
917 my $results = GetImportBatchRangeDesc($offset, $results_per_group);
919 Returns a reference to an array of hash references corresponding to
920 import_batches rows (sorted in descending order by import_batch_id)
921 start at the given offset.
925 sub GetImportBatchRangeDesc {
926 my ($offset, $results_per_group) = @_;
928 my $dbh = C4::Context->dbh;
929 my $query = "SELECT * FROM import_batches
930 WHERE batch_type IN ('batch', 'webservice')
931 ORDER BY import_batch_id DESC";
933 if ($results_per_group){
934 $query .= " LIMIT ?";
935 push(@params, $results_per_group);
938 $query .= " OFFSET ?";
939 push(@params, $offset);
941 my $sth = $dbh->prepare_cached($query);
942 $sth->execute(@params);
943 my $results = $sth->fetchall_arrayref({});
948 =head2 GetItemNumbersFromImportBatch
950 my @itemsnos = GetItemNumbersFromImportBatch($batch_id);
954 sub GetItemNumbersFromImportBatch {
956 my $dbh = C4::Context->dbh;
957 my $sth = $dbh->prepare("SELECT itemnumber FROM import_batches,import_records,import_items WHERE import_batches.import_batch_id=import_records.import_batch_id AND import_records.import_record_id=import_items.import_record_id AND import_batches.import_batch_id=?");
958 $sth->execute($batch_id);
960 while ( my ($itm) = $sth->fetchrow_array ) {
966 =head2 GetNumberOfImportBatches
968 my $count = GetNumberOfImportBatches();
972 sub GetNumberOfNonZ3950ImportBatches {
973 my $dbh = C4::Context->dbh;
974 my $sth = $dbh->prepare("SELECT COUNT(*) FROM import_batches WHERE batch_type != 'z3950'");
976 my ($count) = $sth->fetchrow_array();
981 =head2 GetImportRecordsRange
983 my $results = GetImportRecordsRange($batch_id, $offset, $results_per_group);
985 Returns a reference to an array of hash references corresponding to
986 import_biblios/import_auths/import_records rows for a given batch
987 starting at the given offset.
991 sub GetImportRecordsRange {
992 my ($batch_id, $offset, $results_per_group, $status) = @_;
994 my $dbh = C4::Context->dbh;
995 my $query = "SELECT title, author, isbn, issn, authorized_heading, import_records.import_record_id,
996 record_sequence, status, overlay_status,
997 matched_biblionumber, matched_authid, record_type
999 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
1000 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
1001 WHERE import_batch_id = ?";
1003 push(@params, $batch_id);
1005 $query .= " AND status=?";
1006 push(@params,$status);
1008 $query.=" ORDER BY import_record_id";
1010 if($results_per_group){
1011 $query .= " LIMIT ?";
1012 push(@params, $results_per_group);
1015 $query .= " OFFSET ?";
1016 push(@params, $offset);
1018 my $sth = $dbh->prepare_cached($query);
1019 $sth->execute(@params);
1020 my $results = $sth->fetchall_arrayref({});
1026 =head2 GetBestRecordMatch
1028 my $record_id = GetBestRecordMatch($import_record_id);
1032 sub GetBestRecordMatch {
1033 my ($import_record_id) = @_;
1035 my $dbh = C4::Context->dbh;
1036 my $sth = $dbh->prepare("SELECT candidate_match_id
1037 FROM import_record_matches
1038 WHERE import_record_id = ?
1039 ORDER BY score DESC, candidate_match_id DESC");
1040 $sth->execute($import_record_id);
1041 my ($record_id) = $sth->fetchrow_array();
1046 =head2 GetImportBatchStatus
1048 my $status = GetImportBatchStatus($batch_id);
1052 sub GetImportBatchStatus {
1053 my ($batch_id) = @_;
1055 my $dbh = C4::Context->dbh;
1056 my $sth = $dbh->prepare("SELECT import_status FROM import_batches WHERE import_batch_id = ?");
1057 $sth->execute($batch_id);
1058 my ($status) = $sth->fetchrow_array();
1064 =head2 SetImportBatchStatus
1066 SetImportBatchStatus($batch_id, $new_status);
1070 sub SetImportBatchStatus {
1071 my ($batch_id, $new_status) = @_;
1073 my $dbh = C4::Context->dbh;
1074 my $sth = $dbh->prepare("UPDATE import_batches SET import_status = ? WHERE import_batch_id = ?");
1075 $sth->execute($new_status, $batch_id);
1080 =head2 GetImportBatchOverlayAction
1082 my $overlay_action = GetImportBatchOverlayAction($batch_id);
1086 sub GetImportBatchOverlayAction {
1087 my ($batch_id) = @_;
1089 my $dbh = C4::Context->dbh;
1090 my $sth = $dbh->prepare("SELECT overlay_action FROM import_batches WHERE import_batch_id = ?");
1091 $sth->execute($batch_id);
1092 my ($overlay_action) = $sth->fetchrow_array();
1094 return $overlay_action;
1099 =head2 SetImportBatchOverlayAction
1101 SetImportBatchOverlayAction($batch_id, $new_overlay_action);
1105 sub SetImportBatchOverlayAction {
1106 my ($batch_id, $new_overlay_action) = @_;
1108 my $dbh = C4::Context->dbh;
1109 my $sth = $dbh->prepare("UPDATE import_batches SET overlay_action = ? WHERE import_batch_id = ?");
1110 $sth->execute($new_overlay_action, $batch_id);
1115 =head2 GetImportBatchNoMatchAction
1117 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
1121 sub GetImportBatchNoMatchAction {
1122 my ($batch_id) = @_;
1124 my $dbh = C4::Context->dbh;
1125 my $sth = $dbh->prepare("SELECT nomatch_action FROM import_batches WHERE import_batch_id = ?");
1126 $sth->execute($batch_id);
1127 my ($nomatch_action) = $sth->fetchrow_array();
1129 return $nomatch_action;
1134 =head2 SetImportBatchNoMatchAction
1136 SetImportBatchNoMatchAction($batch_id, $new_nomatch_action);
1140 sub SetImportBatchNoMatchAction {
1141 my ($batch_id, $new_nomatch_action) = @_;
1143 my $dbh = C4::Context->dbh;
1144 my $sth = $dbh->prepare("UPDATE import_batches SET nomatch_action = ? WHERE import_batch_id = ?");
1145 $sth->execute($new_nomatch_action, $batch_id);
1150 =head2 GetImportBatchItemAction
1152 my $item_action = GetImportBatchItemAction($batch_id);
1156 sub GetImportBatchItemAction {
1157 my ($batch_id) = @_;
1159 my $dbh = C4::Context->dbh;
1160 my $sth = $dbh->prepare("SELECT item_action FROM import_batches WHERE import_batch_id = ?");
1161 $sth->execute($batch_id);
1162 my ($item_action) = $sth->fetchrow_array();
1164 return $item_action;
1169 =head2 SetImportBatchItemAction
1171 SetImportBatchItemAction($batch_id, $new_item_action);
1175 sub SetImportBatchItemAction {
1176 my ($batch_id, $new_item_action) = @_;
1178 my $dbh = C4::Context->dbh;
1179 my $sth = $dbh->prepare("UPDATE import_batches SET item_action = ? WHERE import_batch_id = ?");
1180 $sth->execute($new_item_action, $batch_id);
1185 =head2 GetImportBatchMatcher
1187 my $matcher_id = GetImportBatchMatcher($batch_id);
1191 sub GetImportBatchMatcher {
1192 my ($batch_id) = @_;
1194 my $dbh = C4::Context->dbh;
1195 my $sth = $dbh->prepare("SELECT matcher_id FROM import_batches WHERE import_batch_id = ?");
1196 $sth->execute($batch_id);
1197 my ($matcher_id) = $sth->fetchrow_array();
1204 =head2 SetImportBatchMatcher
1206 SetImportBatchMatcher($batch_id, $new_matcher_id);
1210 sub SetImportBatchMatcher {
1211 my ($batch_id, $new_matcher_id) = @_;
1213 my $dbh = C4::Context->dbh;
1214 my $sth = $dbh->prepare("UPDATE import_batches SET matcher_id = ? WHERE import_batch_id = ?");
1215 $sth->execute($new_matcher_id, $batch_id);
1220 =head2 GetImportRecordOverlayStatus
1222 my $overlay_status = GetImportRecordOverlayStatus($import_record_id);
1226 sub GetImportRecordOverlayStatus {
1227 my ($import_record_id) = @_;
1229 my $dbh = C4::Context->dbh;
1230 my $sth = $dbh->prepare("SELECT overlay_status FROM import_records WHERE import_record_id = ?");
1231 $sth->execute($import_record_id);
1232 my ($overlay_status) = $sth->fetchrow_array();
1234 return $overlay_status;
1239 =head2 SetImportRecordOverlayStatus
1241 SetImportRecordOverlayStatus($import_record_id, $new_overlay_status);
1245 sub SetImportRecordOverlayStatus {
1246 my ($import_record_id, $new_overlay_status) = @_;
1248 my $dbh = C4::Context->dbh;
1249 my $sth = $dbh->prepare("UPDATE import_records SET overlay_status = ? WHERE import_record_id = ?");
1250 $sth->execute($new_overlay_status, $import_record_id);
1255 =head2 GetImportRecordStatus
1257 my $overlay_status = GetImportRecordStatus($import_record_id);
1261 sub GetImportRecordStatus {
1262 my ($import_record_id) = @_;
1264 my $dbh = C4::Context->dbh;
1265 my $sth = $dbh->prepare("SELECT status FROM import_records WHERE import_record_id = ?");
1266 $sth->execute($import_record_id);
1267 my ($overlay_status) = $sth->fetchrow_array();
1269 return $overlay_status;
1274 =head2 SetImportRecordStatus
1276 SetImportRecordStatus($import_record_id, $new_overlay_status);
1280 sub SetImportRecordStatus {
1281 my ($import_record_id, $new_overlay_status) = @_;
1283 my $dbh = C4::Context->dbh;
1284 my $sth = $dbh->prepare("UPDATE import_records SET status = ? WHERE import_record_id = ?");
1285 $sth->execute($new_overlay_status, $import_record_id);
1290 =head2 GetImportRecordMatches
1292 my $results = GetImportRecordMatches($import_record_id, $best_only);
1296 sub GetImportRecordMatches {
1297 my $import_record_id = shift;
1298 my $best_only = @_ ? shift : 0;
1300 my $dbh = C4::Context->dbh;
1301 # FIXME currently biblio only
1302 my $sth = $dbh->prepare_cached("SELECT title, author, biblionumber,
1303 candidate_match_id, score, record_type
1305 JOIN import_record_matches USING (import_record_id)
1306 LEFT JOIN biblio ON (biblionumber = candidate_match_id)
1307 WHERE import_record_id = ?
1308 ORDER BY score DESC, biblionumber DESC");
1309 $sth->bind_param(1, $import_record_id);
1312 while (my $row = $sth->fetchrow_hashref) {
1313 if ($row->{'record_type'} eq 'auth') {
1314 $row->{'authorized_heading'} = C4::AuthoritiesMarc::GetAuthorizedHeading( { authid => $row->{'candidate_match_id'} } );
1316 next if ($row->{'record_type'} eq 'biblio' && not $row->{'biblionumber'});
1317 push @$results, $row;
1327 =head2 SetImportRecordMatches
1329 SetImportRecordMatches($import_record_id, @matches);
1333 sub SetImportRecordMatches {
1334 my $import_record_id = shift;
1337 my $dbh = C4::Context->dbh;
1338 my $delsth = $dbh->prepare("DELETE FROM import_record_matches WHERE import_record_id = ?");
1339 $delsth->execute($import_record_id);
1342 my $sth = $dbh->prepare("INSERT INTO import_record_matches (import_record_id, candidate_match_id, score)
1344 foreach my $match (@matches) {
1345 $sth->execute($import_record_id, $match->{'record_id'}, $match->{'score'});
1350 # internal functions
1352 sub _create_import_record {
1353 my ($batch_id, $record_sequence, $marc_record, $record_type, $encoding, $z3950random, $marc_type) = @_;
1355 my $dbh = C4::Context->dbh;
1356 my $sth = $dbh->prepare("INSERT INTO import_records (import_batch_id, record_sequence, marc, marcxml,
1357 record_type, encoding, z3950random)
1358 VALUES (?, ?, ?, ?, ?, ?, ?)");
1359 $sth->execute($batch_id, $record_sequence, $marc_record->as_usmarc(), $marc_record->as_xml($marc_type),
1360 $record_type, $encoding, $z3950random);
1361 my $import_record_id = $dbh->{'mysql_insertid'};
1363 return $import_record_id;
1366 sub _update_import_record_marc {
1367 my ($import_record_id, $marc_record, $marc_type) = @_;
1369 my $dbh = C4::Context->dbh;
1370 my $sth = $dbh->prepare("UPDATE import_records SET marc = ?, marcxml = ?
1371 WHERE import_record_id = ?");
1372 $sth->execute($marc_record->as_usmarc(), $marc_record->as_xml($marc_type), $import_record_id);
1376 sub _add_auth_fields {
1377 my ($import_record_id, $marc_record) = @_;
1380 if ($marc_record->field('001')) {
1381 $controlnumber = $marc_record->field('001')->data();
1383 my $authorized_heading = C4::AuthoritiesMarc::GetAuthorizedHeading({ record => $marc_record });
1384 my $dbh = C4::Context->dbh;
1385 my $sth = $dbh->prepare("INSERT INTO import_auths (import_record_id, control_number, authorized_heading) VALUES (?, ?, ?)");
1386 $sth->execute($import_record_id, $controlnumber, $authorized_heading);
1390 sub _add_biblio_fields {
1391 my ($import_record_id, $marc_record) = @_;
1393 my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record);
1394 my $dbh = C4::Context->dbh;
1395 # FIXME no controlnumber, originalsource
1396 $isbn = C4::Koha::_isbn_cleanup($isbn); # FIXME C4::Koha::_isbn_cleanup should be made public
1397 my $sth = $dbh->prepare("INSERT INTO import_biblios (import_record_id, title, author, isbn, issn) VALUES (?, ?, ?, ?, ?)");
1398 $sth->execute($import_record_id, $title, $author, $isbn, $issn);
1403 sub _update_biblio_fields {
1404 my ($import_record_id, $marc_record) = @_;
1406 my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record);
1407 my $dbh = C4::Context->dbh;
1408 # FIXME no controlnumber, originalsource
1409 # FIXME 2 - should regularize normalization of ISBN wherever it is done
1413 my $sth = $dbh->prepare("UPDATE import_biblios SET title = ?, author = ?, isbn = ?, issn = ?
1414 WHERE import_record_id = ?");
1415 $sth->execute($title, $author, $isbn, $issn, $import_record_id);
1419 sub _parse_biblio_fields {
1420 my ($marc_record) = @_;
1422 my $dbh = C4::Context->dbh;
1423 my $bibliofields = TransformMarcToKoha($dbh, $marc_record, '');
1424 return ($bibliofields->{'title'}, $bibliofields->{'author'}, $bibliofields->{'isbn'}, $bibliofields->{'issn'});
1428 sub _update_batch_record_counts {
1429 my ($batch_id) = @_;
1431 my $dbh = C4::Context->dbh;
1432 my $sth = $dbh->prepare_cached("UPDATE import_batches SET
1436 WHERE import_batch_id = import_batches.import_batch_id),
1440 JOIN import_items USING (import_record_id)
1441 WHERE import_batch_id = import_batches.import_batch_id
1442 AND record_type = 'biblio')
1443 WHERE import_batch_id = ?");
1444 $sth->bind_param(1, $batch_id);
1449 sub _get_commit_action {
1450 my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id, $record_type) = @_;
1452 if ($record_type eq 'biblio') {
1453 my ($bib_result, $bib_match, $item_result);
1455 if ($overlay_status ne 'no_match') {
1456 $bib_match = GetBestRecordMatch($import_record_id);
1457 if ($overlay_action eq 'replace') {
1458 $bib_result = defined($bib_match) ? 'replace' : 'create_new';
1459 } elsif ($overlay_action eq 'create_new') {
1460 $bib_result = 'create_new';
1461 } elsif ($overlay_action eq 'ignore') {
1462 $bib_result = 'ignore';
1464 $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore';
1466 $bib_result = $nomatch_action;
1467 $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore';
1469 return ($bib_result, $item_result, $bib_match);
1470 } else { # must be auths
1471 my ($auth_result, $auth_match);
1473 if ($overlay_status ne 'no_match') {
1474 $auth_match = GetBestRecordMatch($import_record_id);
1475 if ($overlay_action eq 'replace') {
1476 $auth_result = defined($auth_match) ? 'replace' : 'create_new';
1477 } elsif ($overlay_action eq 'create_new') {
1478 $auth_result = 'create_new';
1479 } elsif ($overlay_action eq 'ignore') {
1480 $auth_result = 'ignore';
1483 $auth_result = $nomatch_action;
1486 return ($auth_result, undef, $auth_match);
1491 sub _get_revert_action {
1492 my ($overlay_action, $overlay_status, $status) = @_;
1496 if ($status eq 'ignored') {
1497 $bib_result = 'ignore';
1499 if ($overlay_action eq 'create_new') {
1500 $bib_result = 'delete';
1502 $bib_result = ($overlay_status eq 'match_applied') ? 'restore' : 'delete';
1513 Koha Development Team <http://koha-community.org/>
1515 Galen Charlton <galen.charlton@liblime.com>