From 438ed2333787a37101ce5476dbd656005ab17537 Mon Sep 17 00:00:00 2001 From: Galen Charlton Date: Wed, 30 Apr 2008 15:45:44 -0500 Subject: [PATCH] staging import - enhance record overlay behavior Enhanced the ability of catalogers to specify how bib and item records should be added, replaced, or ignored during a staging import. When an import batch of bib records is staged and commit, the user can now explicitly specify what should occur when an incoming bib record has, or does not have, a match with a record already in the database. The options are: if match found (overlay_action): create_new (just add the incoming record) replace (replace the matched record with the incoming one) use_template (option not implemented) ignore (do nothing with the incoming bib; however, the items attached to it may still be processed based on the item action) if no match is found (nomatch_action): create_new (just add the incoming record) ignore (do nothing with the incoming bib; in this case, any items attached to it will be ignored since there will be nothing to attach them to) The following options for handling items embedded in the bib record are now available: always_add (add the items to the new or replaced bib) add_only_if_match (add the items only if the incoming bib matches an existing bib) add_only_if_add (add the items only if the incoming bib does *not* match an existing bib) ignore (ignore the items entirely) With these changes, it is now possible to support the following use cases: [1] A library joining an existing Koha database wishes to add their items to existing bib records if they match, but does not want to overlay the bib records themselves. [2] A library wants to load a file of records, but only handle the new ones, not ones that are already in the database. [3] A library wants to load a file of records, but only handle the ones that match existing records (e.g., if the records are coming back from an authority control vendor). Documentation changes: * See description above; also, screenshots of the 'stage MARC records for import' and 'manage staged MARC records' should be updated. Test cases: * Added test cases to exercise staging and committing import batches. UI changes: * The pages for staging and managing import batches now have controls for setting the overlay action, action if no match, and item action separately. * in the manage import batch tool, user is notified when they change overlay action, no-match action, and item action * HTML for manage import batch tool now uses fieldsets Database changes (DB rev 076): * added import_batches.item_action * added import_batches.nomatch_action * added 'ignore' as a valid value for import_batches.overlay_action * added 'ignored' as a valid value for import_records.status * added 'status' as a valid value for import_items.status API changes: * new accessor routines for C4::ImportBatch GetImportBatchNoMatchAction SetImportBatchNoMatchAction GetImportBatchItemAction SetImportBatchItemAction * new internal functions for C4::ImportBatch to determine how a given bib and item are to be processed, based on overlay_action, nomatch_action, and item_action: _get_commit_action _get_revert_action Signed-off-by: Joshua Ferraro --- C4/ImportBatch.pm | 200 ++++++++++++-- installer/data/mysql/kohastructure.sql | 8 +- installer/data/mysql/updatedatabase.pl | 20 ++ .../prog/en/includes/tools-item-action.inc | 26 ++ .../prog/en/includes/tools-nomatch-action.inc | 14 + .../prog/en/includes/tools-overlay-action.inc | 20 ++ .../en/modules/tools/manage-marc-import.tmpl | 65 +++-- .../en/modules/tools/stage-marc-import.tmpl | 14 +- kohaversion.pl | 2 +- misc/stage_biblios_file.pl | 26 +- t/lib/KohaTest/ImportBatch.pm | 106 ++++++++ .../ImportBatch/BatchStageCommitRevert.pm | 252 ++++++++++++++++++ tools/manage-marc-import.pl | 46 +++- tools/stage-marc-import.pl | 9 +- 14 files changed, 752 insertions(+), 56 deletions(-) create mode 100644 koha-tmpl/intranet-tmpl/prog/en/includes/tools-item-action.inc create mode 100644 koha-tmpl/intranet-tmpl/prog/en/includes/tools-nomatch-action.inc create mode 100644 koha-tmpl/intranet-tmpl/prog/en/includes/tools-overlay-action.inc create mode 100644 t/lib/KohaTest/ImportBatch.pm create mode 100644 t/lib/KohaTest/ImportBatch/BatchStageCommitRevert.pm diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index bf4e1862a1..9f9c7f645e 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -54,6 +54,10 @@ BEGIN { SetImportBatchStatus GetImportBatchOverlayAction SetImportBatchOverlayAction + GetImportBatchNoMatchAction + SetImportBatchNoMatchAction + GetImportBatchItemAction + SetImportBatchItemAction GetImportBatchMatcher SetImportBatchMatcher GetImportRecordOverlayStatus @@ -261,6 +265,12 @@ sub BatchStageMarcRecords { } my $batch_id = AddImportBatch('create_new', 'staging', 'batch', $file_name, $comments); + if ($parse_items) { + SetImportBatchItemAction($batch_id, 'always_add'); + } else { + SetImportBatchItemAction($batch_id, 'ignore'); + } + my @invalid_records = (); my $num_valid = 0; my $num_items = 0; @@ -347,8 +357,7 @@ my $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, $max_matches, =back Goes through the records loaded in the batch and attempts to -find duplicates for each one. Sets the overlay action to -"replace" if it was "create_new", and sets the overlay status +find duplicates for each one. Sets the matching status of each record to "no_match" or "auto_match" as appropriate. The $max_matches parameter is optional; if it is not supplied, @@ -379,10 +388,6 @@ sub BatchFindBibDuplicates { } my $dbh = C4::Context->dbh; - my $old_overlay_action = GetImportBatchOverlayAction($batch_id); - if ($old_overlay_action eq "create_new") { - SetImportBatchOverlayAction($batch_id, 'replace'); - } my $sth = $dbh->prepare("SELECT import_record_id, marc FROM import_records @@ -448,6 +453,8 @@ sub BatchCommitBibRecords { # FIXME biblio only at the moment SetImportBatchStatus('importing'); my $overlay_action = GetImportBatchOverlayAction($batch_id); + my $nomatch_action = GetImportBatchNoMatchAction($batch_id); + my $item_action = GetImportBatchItemAction($batch_id); my $dbh = C4::Context->dbh; my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marc, encoding FROM import_records @@ -472,20 +479,26 @@ sub BatchCommitBibRecords { $marc_record->delete_field($item_field); } - if ($overlay_action eq 'create_new' or - ($overlay_action eq 'replace' and $rowref->{'overlay_status'} eq 'no_match')) { + # decide what what to do with the bib and item records + my ($bib_result, $item_result, $bib_match) = + _get_commit_action($overlay_action, $nomatch_action, $item_action, + $rowref->{'overlay_status'}, $rowref->{'import_record_id'}); + + if ($bib_result eq 'create_new') { $num_added++; my ($biblionumber, $biblioitemnumber) = AddBiblio($marc_record, ''); my $sth = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); $sth->execute($biblionumber, $rowref->{'import_record_id'}); $sth->finish(); - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); - $num_items_added += $bib_items_added; - $num_items_errored += $bib_items_errored; + if ($item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); + $num_items_added += $bib_items_added; + $num_items_errored += $bib_items_errored; + } SetImportRecordStatus($rowref->{'import_record_id'}, 'imported'); - } else { + } elsif ($bib_result eq 'replace') { $num_updated++; - my $biblionumber = GetBestRecordMatch($rowref->{'import_record_id'}); + my $biblionumber = $bib_match; my ($count, $oldbiblio) = GetBiblio($biblionumber); my $oldxml = GetXmlBiblio($biblionumber); @@ -503,11 +516,27 @@ sub BatchCommitBibRecords { my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); $sth2->execute($biblionumber, $rowref->{'import_record_id'}); $sth2->finish(); - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); - $num_items_added += $bib_items_added; - $num_items_errored += $bib_items_errored; + if ($item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); + $num_items_added += $bib_items_added; + $num_items_errored += $bib_items_errored; + } SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied'); SetImportRecordStatus($rowref->{'import_record_id'}, 'imported'); + } elsif ($bib_result eq 'ignore') { + $num_ignored++; + my $biblionumber = $bib_match; + if (defined $biblionumber and $item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); + $num_items_added += $bib_items_added; + $num_items_errored += $bib_items_errored; + # still need to record the matched biblionumber so that the + # items can be reverted + my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); + $sth2->execute($biblionumber, $rowref->{'import_record_id'}); + SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied'); + } + SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored'); } } $sth->finish(); @@ -588,6 +617,7 @@ sub BatchRevertBibRecords { # FIXME biblio only at the moment SetImportBatchStatus('reverting'); my $overlay_action = GetImportBatchOverlayAction($batch_id); + my $nomatch_action = GetImportBatchNoMatchAction($batch_id); my $dbh = C4::Context->dbh; my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marcxml_old, encoding, matched_biblionumber FROM import_records @@ -598,8 +628,10 @@ sub BatchRevertBibRecords { if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') { $num_ignored++; } - if ($overlay_action eq 'create_new' or - ($overlay_action eq 'replace' and $rowref->{'overlay_status'} eq 'no_match')) { + + my $bib_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'}); + + if ($bib_result eq 'delete') { $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); my $error = DelBiblio($rowref->{'matched_biblionumber'}); if (defined $error) { @@ -608,7 +640,7 @@ sub BatchRevertBibRecords { $num_deleted++; SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); } - } else { + } elsif ($bib_result eq 'restore') { $num_reverted++; my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}); my $biblionumber = $rowref->{'matched_biblionumber'}; @@ -616,8 +648,12 @@ sub BatchRevertBibRecords { $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'}); SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); + } elsif ($bib_result eq 'ignore') { + $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); + SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); } } + $sth->finish(); SetImportBatchStatus($batch_id, 'reverted'); return ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored); @@ -905,6 +941,92 @@ sub SetImportBatchOverlayAction { } +=head2 GetImportBatchNoMatchAction + +=over 4 + +my $nomatch_action = GetImportBatchNoMatchAction($batch_id); + +=back + +=cut + +sub GetImportBatchNoMatchAction { + my ($batch_id) = @_; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("SELECT nomatch_action FROM import_batches WHERE import_batch_id = ?"); + $sth->execute($batch_id); + my ($nomatch_action) = $sth->fetchrow_array(); + $sth->finish(); + return $nomatch_action; + +} + + +=head2 SetImportBatchNoMatchAction + +=over 4 + +SetImportBatchNoMatchAction($batch_id, $new_nomatch_action); + +=back + +=cut + +sub SetImportBatchNoMatchAction { + my ($batch_id, $new_nomatch_action) = @_; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("UPDATE import_batches SET nomatch_action = ? WHERE import_batch_id = ?"); + $sth->execute($new_nomatch_action, $batch_id); + $sth->finish(); + +} + +=head2 GetImportBatchItemAction + +=over 4 + +my $item_action = GetImportBatchItemAction($batch_id); + +=back + +=cut + +sub GetImportBatchItemAction { + my ($batch_id) = @_; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("SELECT item_action FROM import_batches WHERE import_batch_id = ?"); + $sth->execute($batch_id); + my ($item_action) = $sth->fetchrow_array(); + $sth->finish(); + return $item_action; + +} + + +=head2 SetImportBatchItemAction + +=over 4 + +SetImportBatchItemAction($batch_id, $new_item_action); + +=back + +=cut + +sub SetImportBatchItemAction { + my ($batch_id, $new_item_action) = @_; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("UPDATE import_batches SET item_action = ? WHERE import_batch_id = ?"); + $sth->execute($new_item_action, $batch_id); + $sth->finish(); + +} + =head2 GetImportBatchMatcher =over 4 @@ -1190,6 +1312,46 @@ sub _update_batch_record_counts { } +sub _get_commit_action { + my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id) = @_; + + my ($bib_result, $bib_match, $item_result); + + if ($overlay_status ne 'no_match') { + $bib_match = GetBestRecordMatch($import_record_id); + if ($overlay_action eq 'replace') { + $bib_result = defined($bib_match) ? 'replace' : 'create_new'; + } elsif ($overlay_action eq 'create_new') { + $bib_result = 'create_new'; + } elsif ($overlay_action eq 'ignore') { + $bib_result = 'ignore'; + } + $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore'; + } else { + $bib_result = $nomatch_action; + $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore'; + } + + return ($bib_result, $item_result, $bib_match); +} + +sub _get_revert_action { + my ($overlay_action, $overlay_status, $status) = @_; + + my $bib_result; + + if ($status eq 'ignored') { + $bib_result = 'ignore'; + } else { + if ($overlay_action eq 'create_new') { + $bib_result = 'delete'; + } else { + $bib_result = ($overlay_status eq 'match_applied') ? 'restore' : 'delete'; + } + } + return $bib_result; +} + 1; __END__ diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index 964614ddbe..ff55985fdd 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -872,7 +872,9 @@ CREATE TABLE `import_batches` ( `num_biblios` int(11) NOT NULL default 0, `num_items` int(11) NOT NULL default 0, `upload_timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, - `overlay_action` enum('replace', 'create_new', 'use_template') NOT NULL default 'create_new', + `overlay_action` enum('replace', 'create_new', 'use_template', 'ignore') NOT NULL default 'create_new', + `nomatch_action` enum('create_new', 'ignore') NOT NULL default 'create_new', + `item_action` enum('always_add', 'add_only_for_matches', 'add_only_for_new', 'ignore') NOT NULL default 'always_add', `import_status` enum('staging', 'staged', 'importing', 'imported', 'reverting', 'reverted', 'cleaned') NOT NULL default 'staging', `batch_type` enum('batch', 'z3950') NOT NULL default 'batch', `file_name` varchar(100), @@ -898,7 +900,7 @@ CREATE TABLE `import_records` ( `marcxml_old` longtext NOT NULL, `record_type` enum('biblio', 'auth', 'holdings') NOT NULL default 'biblio', `overlay_status` enum('no_match', 'auto_match', 'manual_match', 'match_applied') NOT NULL default 'no_match', - `status` enum('error', 'staged', 'imported', 'reverted', 'items_reverted') NOT NULL default 'staged', + `status` enum('error', 'staged', 'imported', 'reverted', 'items_reverted', 'ignored') NOT NULL default 'staged', `import_error` mediumtext, `encoding` varchar(40) NOT NULL default '', `z3950random` varchar(40) default NULL, @@ -954,7 +956,7 @@ CREATE TABLE `import_items` ( `import_record_id` int(11) NOT NULL, `itemnumber` int(11) default NULL, `branchcode` varchar(10) default NULL, - `status` enum('error', 'staged', 'imported', 'reverted') NOT NULL default 'staged', + `status` enum('error', 'staged', 'imported', 'reverted', 'ignored') NOT NULL default 'staged', `marcxml` longtext NOT NULL, `import_error` mediumtext, PRIMARY KEY (`import_items_id`), diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl index 0399292e94..592170d022 100755 --- a/installer/data/mysql/updatedatabase.pl +++ b/installer/data/mysql/updatedatabase.pl @@ -1397,6 +1397,26 @@ if (C4::Context->preference("Version") < TransformToNum($DBversion)) { SetVersion ($DBversion); } +$DBversion = "3.00.00.076"; +if (C4::Context->preference("Version") < TransformToNum($DBversion)) { + $dbh->do("ALTER TABLE import_batches + ADD COLUMN nomatch_action enum('create_new', 'ignore') NOT NULL default 'create_new' AFTER overlay_action"); + $dbh->do("ALTER TABLE import_batches + ADD COLUMN item_action enum('always_add', 'add_only_for_matches', 'add_only_for_new', 'ignore') + NOT NULL default 'always_add' AFTER nomatch_action"); + $dbh->do("ALTER TABLE import_batches + MODIFY overlay_action enum('replace', 'create_new', 'use_template', 'ignore') + NOT NULL default 'create_new'"); + $dbh->do("ALTER TABLE import_records + MODIFY status enum('error', 'staged', 'imported', 'reverted', 'items_reverted', + 'ignored') NOT NULL default 'staged'"); + $dbh->do("ALTER TABLE import_items + MODIFY status enum('error', 'staged', 'imported', 'reverted', 'ignored') NOT NULL default 'staged'"); + + print "Upgrade to $DBversion done (changes to import_batches and import_records) "; + SetVersion ($DBversion); +} + =item DropAllForeignKeys($table) Drop all foreign keys of the table $table diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-item-action.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-item-action.inc new file mode 100644 index 0000000000..7540b5fec6 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-item-action.inc @@ -0,0 +1,26 @@ + diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-nomatch-action.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-nomatch-action.inc new file mode 100644 index 0000000000..87d2ac097c --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-nomatch-action.inc @@ -0,0 +1,14 @@ + diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-overlay-action.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-overlay-action.inc new file mode 100644 index 0000000000..d12f497dde --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-overlay-action.inc @@ -0,0 +1,20 @@ + diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-marc-import.tmpl b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-marc-import.tmpl index 190bcaf98b..ea0a5e5f4f 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-marc-import.tmpl +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-marc-import.tmpl @@ -39,23 +39,33 @@ -

-

-

-

-

+

+
    +
  • +
  • +
  • +
  • +
  • () No matching rule in effect -

    +
  • +
  • +
  • +
  • +
+
+
" method="post"> " /> " /> +
    +
  • +
  • +
  • + +
  • +
  • + +
  • +
  • + +
  • +
+ + +

Failed to apply different matching rule

+ +

Applied different matching rule. Number of records matched now + +

+ + + +

Changed action if matching record found

+ + +

Changed action if no match found

+ + +

Changed item processing option

+ +
+
" method="post"> @@ -97,15 +138,16 @@ +

Completed import of records

+ -
Number of records added
Number of records updated
Number of records ignored
Number of items added
Number of items ignored because of duplicate barcode
Number of records ignored
@@ -118,15 +160,6 @@ Number of records ignored - - -

Failed to apply different matching rule

- -

Applied different matching rule. Number of records matched now - -

- -
diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tmpl b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tmpl index 00c5a50d41..36906b1404 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tmpl +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/stage-marc-import.tmpl @@ -100,7 +100,14 @@ function CheckForm(f) { - + +
  • + +
  • +
  • + +
  • +
    Check for embedded item record data? @@ -118,6 +125,11 @@ function CheckForm(f) { +
      +
    1. + +
    2. +
    diff --git a/kohaversion.pl b/kohaversion.pl index 2c726b4086..5d26b41a49 100644 --- a/kohaversion.pl +++ b/kohaversion.pl @@ -10,7 +10,7 @@ use strict; sub kohaversion { - our $VERSION = "3.00.00.075"; + our $VERSION = "3.00.00.076"; # version needs to be set this way # so that it can be picked up by Makefile.PL # during install diff --git a/misc/stage_biblios_file.pl b/misc/stage_biblios_file.pl index cf140c26f8..c54915076a 100755 --- a/misc/stage_biblios_file.pl +++ b/misc/stage_biblios_file.pl @@ -74,16 +74,20 @@ sub process_batch { my $num_with_matches = 0; if ($match_bibs) { - my $matcher = C4::Matcher->fetch($match_bibs) ; - if( ! defined $matcher) { - $matcher = C4::Matcher->new('biblio'); - $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, ''); - $matcher->add_simple_required_check('245', 'a', -1, 0, '', + my $matcher = C4::Matcher->fetch($match_bibs) ; + if (! defined $matcher) { + $matcher = C4::Matcher->new('biblio'); + $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, ''); + $matcher->add_simple_required_check('245', 'a', -1, 0, '', '245', 'a', -1, 0, ''); - } else { - SetImportBatchMatcher($batch_id, $match_bibs); - } - print "... looking for matches with records already in database\n"; + } else { + SetImportBatchMatcher($batch_id, $match_bibs); + } + # set default record overlay behavior + SetImportBatchOverlayAction($batch_id, 'replace'); + SetImportBatchNoMatchAction($batch_id, 'create_new'); + SetImportBatchItemAction($batch_id, 'always_add'); + print "... looking for matches with records already in database\n"; $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 100, \&print_progress_and_commit); print "... finished looking for matches\n"; } @@ -137,8 +141,8 @@ Parameters: --match-bibs use this option to match bibs in the file with bibs already in the database for future overlay. - If isn't defined, a default - MARC21 ISBN & title match rule will be applied. + If isn't defined, a default + MARC21 ISBN & title match rule will be applied. --add-items use this option to specify that item data is embedded in the MARC bibs and should be parsed. diff --git a/t/lib/KohaTest/ImportBatch.pm b/t/lib/KohaTest/ImportBatch.pm new file mode 100644 index 0000000000..82c703f153 --- /dev/null +++ b/t/lib/KohaTest/ImportBatch.pm @@ -0,0 +1,106 @@ +package KohaTest::ImportBatch; +use base qw(KohaTest); + +use strict; +use warnings; + +use Test::More; + +use C4::ImportBatch; +use C4::Matcher; +sub testing_class { 'C4::ImportBatch' }; + + +sub routines : Test( 1 ) { + my $self = shift; + my @routines = qw( + GetZ3950BatchId + GetImportRecordMarc + AddImportBatch + GetImportBatch + AddBiblioToBatch + ModBiblioInBatch + BatchStageMarcRecords + AddItemsToImportBiblio + BatchFindBibDuplicates + BatchCommitBibRecords + BatchCommitItems + BatchRevertBibRecords + BatchRevertItems + GetAllImportBatches + GetImportBatchRangeDesc + GetItemNumbersFromImportBatch + GetNumberOfNonZ3950ImportBatches + GetImportBibliosRange + GetBestRecordMatch + GetImportBatchStatus + SetImportBatchStatus + GetImportBatchOverlayAction + SetImportBatchOverlayAction + GetImportBatchNoMatchAction + SetImportBatchNoMatchAction + GetImportBatchItemAction + SetImportBatchItemAction + GetImportBatchItemAction + SetImportBatchItemAction + GetImportBatchMatcher + SetImportBatchMatcher + GetImportRecordOverlayStatus + SetImportRecordOverlayStatus + GetImportRecordStatus + SetImportRecordStatus + GetImportRecordMatches + SetImportRecordMatches + _create_import_record + _update_import_record_marc + _add_biblio_fields + _update_biblio_fields + _parse_biblio_fields + _update_batch_record_counts + _get_commit_action + _get_revert_action + ); + + can_ok($self->testing_class, @routines); +} + +sub startup_50_add_matcher : Test( startup => 1 ) { + my $self = shift; + # create test MARC21 ISBN matcher + my $matcher = C4::Matcher->new('biblio'); + $matcher->threshold(1000); + $matcher->code('TESTISBN'); + $matcher->description('test MARC21 ISBN matcher'); + $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, ''); + my $matcher_id = $matcher->store(); + like($matcher_id, qr/^\d+$/, "store new matcher and get back ID"); + + $self->{'matcher_id'} = $matcher_id; +} + +sub shutdown_50_remove_matcher : Test( shutdown => 6) { + my $self = shift; + my @matchers = C4::Matcher::GetMatcherList(); + cmp_ok(scalar(@matchers), ">=", 1, "at least one matcher present"); + my $matcher_id; + my $testisbn_count = 0; + # look for TESTISBN + foreach my $matcher (@matchers) { + if ($matcher->{'code'} eq 'TESTISBN') { + $testisbn_count++; + $matcher_id = $matcher->{'matcher_id'}; + } + } + ok($testisbn_count == 1, "only one TESTISBN matcher"); + like($matcher_id, qr/^\d+$/, "matcher ID is valid"); + my $matcher = C4::Matcher->fetch($matcher_id); + ok(defined($matcher), "got back a matcher"); + ok($matcher_id == $matcher->{'id'}, "got back the correct matcher"); + C4::Matcher->delete($matcher_id); + my $matcher2 = C4::Matcher->fetch($matcher_id); + ok(not(defined($matcher2)), "matcher removed"); + + delete $self->{'matcher_id'}; +} + +1; diff --git a/t/lib/KohaTest/ImportBatch/BatchStageCommitRevert.pm b/t/lib/KohaTest/ImportBatch/BatchStageCommitRevert.pm new file mode 100644 index 0000000000..94f811528c --- /dev/null +++ b/t/lib/KohaTest/ImportBatch/BatchStageCommitRevert.pm @@ -0,0 +1,252 @@ +package KohaTest::ImportBatch::BatchStageCommitRevert; +use base qw( KohaTest::ImportBatch ); + +use strict; +use warnings; + +use Test::More; + +use C4::ImportBatch; +use C4::Matcher; +use C4::Biblio; + +# define test records for various batches +sub startup_60_make_test_records : Test( startup ) { + my $self = shift; + $self->{'batches'} = { + 'batch1' => { + marc => _make_marc_batch([ + ['isbn001', 'title 1', ['batch-item-1'] ], + ['isbn002', 'title 2', [] ], + ['isbn003', 'title 3', ['batch-item-2','batch-item-3'] ], + ['isbn004', 'title 4', [ 'batch-item-4' ] ], + ['isbn005', 'title 5', [ 'batch-item-5', 'batch-item-6', 'batch-item-7' ] ], + ]), + args => { + parse_items => 1, + overlay_action => 'create_new', + nomatch_action => 'create_new', + item_action => 'always_add', + }, + results => { + num_bibs => 5, + num_items => 7, + num_invalid => 0, + num_matches => 0, + num_added => 5, + num_updated => 0, + num_items_added => 7, + num_items_errored => 0, + num_ignored => 0, + }, + }, + 'batch2' => { + marc => _make_marc_batch([ + ['isbn001', 'overlay title 1', ['batch-item-8'] ], + ['isbn002', 'overlay title 2', ['batch-item-9'] ], + ['isbn006', 'title 6', ['batch-item-10'] ], + ]), + args => { + parse_items => 1, + overlay_action => 'replace', + nomatch_action => 'create_new', + item_action => 'always_add', + }, + results => { + num_bibs => 3, + num_items => 3, + num_invalid => 0, + num_matches => 2, + num_added => 1, + num_updated => 2, + num_items_added => 3, + num_items_errored => 0, + num_ignored => 0, + }, + }, + 'batch3' => { + marc => _make_marc_batch([ + ['isbn007', 'title 7', ['batch-item-11'] ], + ['isbn006', 'overlay title 6', ['batch-item-12'] ], + ]), + args => { + parse_items => 1, + overlay_action => 'ignore', + nomatch_action => 'ignore', + item_action => 'always_add', + }, + results => { + num_bibs => 2, + num_items => 2, + num_invalid => 0, + num_matches => 1, + num_added => 0, + num_updated => 0, + num_items_added => 1, + num_items_errored => 0, + num_ignored => 2, + }, + }, + 'batch4' => { + marc => _make_marc_batch([ + ['isbn008', 'title 8', ['batch-item-13'] ], # not loading this item + ]), + args => { + parse_items => 0, + overlay_action => undef, + nomatch_action => 'create_new', + item_action => 'ignore', + }, + results => { + num_bibs => 1, + num_items => 0, + num_invalid => 0, + num_matches => 0, + num_added => 1, + num_updated => 0, + num_items_added => 0, + num_items_errored => 0, + num_ignored => 0, + }, + }, + 'batch5' => { + marc => _make_marc_batch([ + ['isbn009', 'title 9', ['batch-item-1'] ], # trigger dup barcode error + 'junkjunkjunkjunk', # trigger invalid bib + ]), + args => { + parse_items => 1, + overlay_action => undef, + nomatch_action => undef, + item_action => undef, + }, + results => { + num_bibs => 1, + num_items => 1, + num_invalid => 1, + num_matches => 0, + num_added => 1, + num_updated => 0, + num_items_added => 0, + num_items_errored => 1, + num_ignored => 0, + }, + }, + 'batch6' => { + marc => _make_marc_batch([ + ['isbn001', 'match title 1', ['batch-item-14', 'batch-item-15'] ], + ['isbn010', 'title 10', ['batch-item-16', 'batch-item-17'] ], + ]), + args => { + parse_items => 1, + overlay_action => 'ignore', + nomatch_action => 'create_new', + item_action => 'always_add', + }, + results => { + num_bibs => 2, + num_items => 4, + num_invalid => 0, + num_matches => 1, + num_added => 1, + num_updated => 0, + num_items_added => 4, + num_items_errored => 0, + num_ignored => 1, + }, + }, + }; + +} + +sub _make_marc_batch { + my $defs = shift; + my @marc = (); + foreach my $rec (@$defs) { + if (ref($rec) eq 'ARRAY') { + my $isbn = $rec->[0]; + my $title = $rec->[1]; + my $items = $rec->[2]; + my $bib = MARC::Record->new(); + $bib->leader(' nam a22 7a 4500'); + $bib->append_fields(MARC::Field->new('020', ' ', ' ', a => $isbn), + MARC::Field->new('245', ' ', ' ', a => $title)); + foreach my $barcode (@$items) { + my ($itemtag, $toss, $barcodesf, $branchsf); + ($itemtag, $toss) = GetMarcFromKohaField('items.itemnumber', ''); + ($toss, $barcodesf) = GetMarcFromKohaField('items.barcode', ''); + ($toss, $branchsf) = GetMarcFromKohaField('items.homebranch', ''); + $bib->append_fields(MARC::Field->new($itemtag, ' ', ' ', $barcodesf => $barcode, $branchsf => 'CPL')); + # FIXME: define branch in KohaTest + } + push @marc, $bib->as_usmarc(); + } else { + push @marc, $rec; + } + } + return join('', @marc); +} + +sub stage_commit_batches : Test( 75 ) { + my $self = shift; + + my $matcher = C4::Matcher->fetch($self->{'matcher_id'}); + ok(ref($matcher) eq 'C4::Matcher', "retrieved matcher"); + + for my $batch_key (sort keys %{ $self->{'batches'} }) { + my $batch = $self->{'batches'}->{$batch_key}; + my $args = $batch->{'args'}; + my $results = $batch->{'results'}; + my ($batch_id, $num_bibs, $num_items, @invalid) = + BatchStageMarcRecords('MARC21', $batch->{marc}, "$batch_key.mrc", "$batch_key comments", + '', $args->{'parse_items'}, 0); + like($batch_id, qr/^\d+$/, "staged $batch_key"); + cmp_ok($num_bibs, "==", $results->{'num_bibs'}, "$batch_key: correct number of bibs"); + cmp_ok($num_items, "==", $results->{'num_items'}, "$batch_key: correct number of items"); + cmp_ok(scalar(@invalid), "==", $results->{'num_invalid'}, "$batch_key: correct number of invalid bibs"); + + my $num_matches = BatchFindBibDuplicates($batch_id, $matcher, 10); + cmp_ok($num_matches, "==", $results->{'num_matches'}, "$batch_key: correct number of bib matches"); + + if (defined $args->{'overlay_action'}) { + if ($args->{'overlay_action'} eq 'create_new') { + cmp_ok(GetImportBatchOverlayAction($batch_id), "eq", 'create_new', "$batch_key: verify default overlay action"); + } else { + SetImportBatchOverlayAction($batch_id, $args->{'overlay_action'}); + cmp_ok(GetImportBatchOverlayAction($batch_id), "eq", $args->{'overlay_action'}, + "$batch_key: changed overlay action"); + } + } + if (defined $args->{'nomatch_action'}) { + if ($args->{'nomatch_action'} eq 'create_new') { + cmp_ok(GetImportBatchNoMatchAction($batch_id), "eq", 'create_new', "$batch_key: verify default nomatch action"); + } else { + SetImportBatchNoMatchAction($batch_id, $args->{'nomatch_action'}); + cmp_ok(GetImportBatchNoMatchAction($batch_id), "eq", $args->{'nomatch_action'}, + "$batch_key: changed nomatch action"); + } + } + if (defined $args->{'item_action'}) { + if ($args->{'item_action'} eq 'create_new') { + cmp_ok(GetImportBatchItemAction($batch_id), "eq", 'always_add', "$batch_key: verify default item action"); + } else { + SetImportBatchItemAction($batch_id, $args->{'item_action'}); + cmp_ok(GetImportBatchItemAction($batch_id), "eq", $args->{'item_action'}, + "$batch_key: changed item action"); + } + } + + my ($num_added, $num_updated, $num_items_added, + $num_items_errored, $num_ignored) = BatchCommitBibRecords($batch_id); + cmp_ok($num_added, "==", $results->{'num_added'}, "$batch_key: added correct number of bibs"); + cmp_ok($num_updated, "==", $results->{'num_updated'}, "$batch_key: updated correct number of bibs"); + cmp_ok($num_items_added, "==", $results->{'num_items_added'}, "$batch_key: added correct number of items"); + cmp_ok($num_items_errored, "==", $results->{'num_items_errored'}, "$batch_key: correct number of item add errors"); + cmp_ok($num_ignored, "==", $results->{'num_ignored'}, "$batch_key: ignored correct number of bibs"); + + $self->reindex_marc(); + } + +} + +1; diff --git a/tools/manage-marc-import.pl b/tools/manage-marc-import.pl index 2501a6bc6c..eaf9815ada 100755 --- a/tools/manage-marc-import.pl +++ b/tools/manage-marc-import.pl @@ -99,7 +99,11 @@ if ($op eq "") { } elsif ($op eq "redo-matching") { my $new_matcher_id = $input->param('new_matcher_id'); my $current_matcher_id = $input->param('current_matcher_id'); - redo_matching($template, $import_batch_id, $new_matcher_id, $current_matcher_id); + my $overlay_action = $input->param('overlay_action'); + my $nomatch_action = $input->param('nomatch_action'); + my $item_action = $input->param('item_action'); + redo_matching($template, $import_batch_id, $new_matcher_id, $current_matcher_id, + $overlay_action, $nomatch_action, $item_action); import_biblios_list($template, $import_batch_id, $offset, $results_per_page); } @@ -108,10 +112,34 @@ output_html_with_http_headers $input, $cookie, $template->output; exit 0; sub redo_matching { - my ($template, $import_batch_id, $new_matcher_id, $current_matcher_id) = @_; + my ($template, $import_batch_id, $new_matcher_id, $current_matcher_id, $overlay_action, $nomatch_action, $item_action) = @_; my $rematch_failed = 0; return if not defined $new_matcher_id and not defined $current_matcher_id; - return if $new_matcher_id == $current_matcher_id; + my $old_overlay_action = GetImportBatchOverlayAction($import_batch_id); + my $old_nomatch_action = GetImportBatchNoMatchAction($import_batch_id); + my $old_item_action = GetImportBatchItemAction($import_batch_id); + return if $new_matcher_id == $current_matcher_id and + $old_overlay_action eq $overlay_action and + $old_nomatch_action eq $nomatch_action and + $old_item_action eq $item_action; + + if ($old_overlay_action ne $overlay_action) { + SetImportBatchOverlayAction($import_batch_id, $overlay_action); + $template->param('changed_overlay_action' => 1); + } + if ($old_nomatch_action ne $nomatch_action) { + SetImportBatchNoMatchAction($import_batch_id, $nomatch_action); + $template->param('changed_nomatch_action' => 1); + } + if ($old_item_action ne $item_action) { + SetImportBatchItemAction($import_batch_id, $item_action); + $template->param('changed_item_action' => 1); + } + + if ($new_matcher_id == $current_matcher_id) { + return; + } + my $num_with_matches = 0; if (defined $new_matcher_id and $new_matcher_id ne "") { my $matcher = C4::Matcher->fetch($new_matcher_id); @@ -123,7 +151,8 @@ sub redo_matching { } } else { $num_with_matches = BatchFindBibDuplicates($import_batch_id, undef); - SetImportBatchMatcher($import_batch_id, undef); + SetImportBatchMatcher($import_batch_id, undef); + SetImportBatchOverlayAction('create_new'); } $template->param(rematch_failed => $rematch_failed); $template->param(rematch_attempted => 1); @@ -314,6 +343,15 @@ sub import_biblios_list { $template->param(num_results => $num_biblios); $template->param(results_per_page => $results_per_page); $template->param(import_batch_id => $import_batch_id); + my $overlay_action = GetImportBatchOverlayAction($import_batch_id); + $template->param("overlay_action_${overlay_action}" => 1); + $template->param(overlay_action => $overlay_action); + my $nomatch_action = GetImportBatchNoMatchAction($import_batch_id); + $template->param("nomatch_action_${nomatch_action}" => 1); + $template->param(nomatch_action => $nomatch_action); + my $item_action = GetImportBatchItemAction($import_batch_id); + $template->param("item_action_${item_action}" => 1); + $template->param(item_action => $item_action); batch_info($template, $batch); } diff --git a/tools/stage-marc-import.pl b/tools/stage-marc-import.pl index b926da643d..9c09d8c06d 100755 --- a/tools/stage-marc-import.pl +++ b/tools/stage-marc-import.pl @@ -50,7 +50,10 @@ my $fileID=$input->param('uploadedfileid'); my $runinbackground = $input->param('runinbackground'); my $completedJobID = $input->param('completedJobID'); my $matcher_id = $input->param('matcher'); +my $overlay_action = $input->param('overlay_action'); +my $nomatch_action = $input->param('nomatch_action'); my $parse_items = $input->param('parse_items'); +my $item_action = $input->param('item_action'); my $comments = $input->param('comments'); my $syntax = $input->param('syntax'); my ($template, $loggedinuser, $cookie) @@ -140,8 +143,12 @@ if ($completedJobID) { if (defined $matcher) { $checked_matches = 1; $matcher_code = $matcher->code(); - $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 50, matching_progress_callback($job, $dbh)); + $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, + 10, 50, matching_progress_callback($job, $dbh)); SetImportBatchMatcher($batch_id, $matcher_id); + SetImportBatchOverlayAction($batch_id, $overlay_action); + SetImportBatchNoMatchAction($batch_id, $nomatch_action); + SetImportBatchItemAction($batch_id, $item_action); $dbh->commit(); } else { $matcher_failed = 1; -- 2.20.1