From 438ed2333787a37101ce5476dbd656005ab17537 Mon Sep 17 00:00:00 2001
From: Galen Charlton
Date: Wed, 30 Apr 2008 15:45:44 -0500
Subject: [PATCH] staging import - enhance record overlay behavior
Enhanced the ability of catalogers to specify how
bib and item records should be added, replaced, or
ignored during a staging import.
When an import batch of bib records is staged and commit,
the user can now explicitly specify what should occur
when an incoming bib record has, or does not have, a match
with a record already in the database. The options are:
if match found (overlay_action):
create_new (just add the incoming record)
replace (replace the matched record with the incoming one)
use_template (option not implemented)
ignore (do nothing with the incoming bib; however, the
items attached to it may still be processed
based on the item action)
if no match is found (nomatch_action):
create_new (just add the incoming record)
ignore (do nothing with the incoming bib; in this
case, any items attached to it will be
ignored since there will be nothing to
attach them to)
The following options for handling items embedded in the
bib record are now available:
always_add (add the items to the new or replaced bib)
add_only_if_match (add the items only if the incoming bib
matches an existing bib)
add_only_if_add (add the items only if the incoming bib
does *not* match an existing bib)
ignore (ignore the items entirely)
With these changes, it is now possible to support the following use cases:
[1] A library joining an existing Koha database wishes to add their
items to existing bib records if they match, but does not want
to overlay the bib records themselves.
[2] A library wants to load a file of records, but only handle
the new ones, not ones that are already in the database.
[3] A library wants to load a file of records, but only
handle the ones that match existing records (e.g., if
the records are coming back from an authority control vendor).
Documentation changes:
* See description above; also, screenshots of the 'stage MARC records
for import' and 'manage staged MARC records' should be updated.
Test cases:
* Added test cases to exercise staging and committing import batches.
UI changes:
* The pages for staging and managing import batches now have
controls for setting the overlay action, action if no match,
and item action separately.
* in the manage import batch tool, user is notified when they
change overlay action, no-match action, and item action
* HTML for manage import batch tool now uses fieldsets
Database changes (DB rev 076):
* added import_batches.item_action
* added import_batches.nomatch_action
* added 'ignore' as a valid value for import_batches.overlay_action
* added 'ignored' as a valid value for import_records.status
* added 'status' as a valid value for import_items.status
API changes:
* new accessor routines for C4::ImportBatch
GetImportBatchNoMatchAction
SetImportBatchNoMatchAction
GetImportBatchItemAction
SetImportBatchItemAction
* new internal functions for C4::ImportBatch to
determine how a given bib and item are to be
processed, based on overlay_action, nomatch_action,
and item_action:
_get_commit_action
_get_revert_action
Signed-off-by: Joshua Ferraro
---
C4/ImportBatch.pm | 200 ++++++++++++--
installer/data/mysql/kohastructure.sql | 8 +-
installer/data/mysql/updatedatabase.pl | 20 ++
.../prog/en/includes/tools-item-action.inc | 26 ++
.../prog/en/includes/tools-nomatch-action.inc | 14 +
.../prog/en/includes/tools-overlay-action.inc | 20 ++
.../en/modules/tools/manage-marc-import.tmpl | 65 +++--
.../en/modules/tools/stage-marc-import.tmpl | 14 +-
kohaversion.pl | 2 +-
misc/stage_biblios_file.pl | 26 +-
t/lib/KohaTest/ImportBatch.pm | 106 ++++++++
.../ImportBatch/BatchStageCommitRevert.pm | 252 ++++++++++++++++++
tools/manage-marc-import.pl | 46 +++-
tools/stage-marc-import.pl | 9 +-
14 files changed, 752 insertions(+), 56 deletions(-)
create mode 100644 koha-tmpl/intranet-tmpl/prog/en/includes/tools-item-action.inc
create mode 100644 koha-tmpl/intranet-tmpl/prog/en/includes/tools-nomatch-action.inc
create mode 100644 koha-tmpl/intranet-tmpl/prog/en/includes/tools-overlay-action.inc
create mode 100644 t/lib/KohaTest/ImportBatch.pm
create mode 100644 t/lib/KohaTest/ImportBatch/BatchStageCommitRevert.pm
diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm
index bf4e1862a1..9f9c7f645e 100644
--- a/C4/ImportBatch.pm
+++ b/C4/ImportBatch.pm
@@ -54,6 +54,10 @@ BEGIN {
SetImportBatchStatus
GetImportBatchOverlayAction
SetImportBatchOverlayAction
+ GetImportBatchNoMatchAction
+ SetImportBatchNoMatchAction
+ GetImportBatchItemAction
+ SetImportBatchItemAction
GetImportBatchMatcher
SetImportBatchMatcher
GetImportRecordOverlayStatus
@@ -261,6 +265,12 @@ sub BatchStageMarcRecords {
}
my $batch_id = AddImportBatch('create_new', 'staging', 'batch', $file_name, $comments);
+ if ($parse_items) {
+ SetImportBatchItemAction($batch_id, 'always_add');
+ } else {
+ SetImportBatchItemAction($batch_id, 'ignore');
+ }
+
my @invalid_records = ();
my $num_valid = 0;
my $num_items = 0;
@@ -347,8 +357,7 @@ my $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, $max_matches,
=back
Goes through the records loaded in the batch and attempts to
-find duplicates for each one. Sets the overlay action to
-"replace" if it was "create_new", and sets the overlay status
+find duplicates for each one. Sets the matching status
of each record to "no_match" or "auto_match" as appropriate.
The $max_matches parameter is optional; if it is not supplied,
@@ -379,10 +388,6 @@ sub BatchFindBibDuplicates {
}
my $dbh = C4::Context->dbh;
- my $old_overlay_action = GetImportBatchOverlayAction($batch_id);
- if ($old_overlay_action eq "create_new") {
- SetImportBatchOverlayAction($batch_id, 'replace');
- }
my $sth = $dbh->prepare("SELECT import_record_id, marc
FROM import_records
@@ -448,6 +453,8 @@ sub BatchCommitBibRecords {
# FIXME biblio only at the moment
SetImportBatchStatus('importing');
my $overlay_action = GetImportBatchOverlayAction($batch_id);
+ my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
+ my $item_action = GetImportBatchItemAction($batch_id);
my $dbh = C4::Context->dbh;
my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marc, encoding
FROM import_records
@@ -472,20 +479,26 @@ sub BatchCommitBibRecords {
$marc_record->delete_field($item_field);
}
- if ($overlay_action eq 'create_new' or
- ($overlay_action eq 'replace' and $rowref->{'overlay_status'} eq 'no_match')) {
+ # decide what what to do with the bib and item records
+ my ($bib_result, $item_result, $bib_match) =
+ _get_commit_action($overlay_action, $nomatch_action, $item_action,
+ $rowref->{'overlay_status'}, $rowref->{'import_record_id'});
+
+ if ($bib_result eq 'create_new') {
$num_added++;
my ($biblionumber, $biblioitemnumber) = AddBiblio($marc_record, '');
my $sth = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
$sth->execute($biblionumber, $rowref->{'import_record_id'});
$sth->finish();
- my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber);
- $num_items_added += $bib_items_added;
- $num_items_errored += $bib_items_errored;
+ if ($item_result eq 'create_new') {
+ my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber);
+ $num_items_added += $bib_items_added;
+ $num_items_errored += $bib_items_errored;
+ }
SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
- } else {
+ } elsif ($bib_result eq 'replace') {
$num_updated++;
- my $biblionumber = GetBestRecordMatch($rowref->{'import_record_id'});
+ my $biblionumber = $bib_match;
my ($count, $oldbiblio) = GetBiblio($biblionumber);
my $oldxml = GetXmlBiblio($biblionumber);
@@ -503,11 +516,27 @@ sub BatchCommitBibRecords {
my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
$sth2->execute($biblionumber, $rowref->{'import_record_id'});
$sth2->finish();
- my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber);
- $num_items_added += $bib_items_added;
- $num_items_errored += $bib_items_errored;
+ if ($item_result eq 'create_new') {
+ my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber);
+ $num_items_added += $bib_items_added;
+ $num_items_errored += $bib_items_errored;
+ }
SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
+ } elsif ($bib_result eq 'ignore') {
+ $num_ignored++;
+ my $biblionumber = $bib_match;
+ if (defined $biblionumber and $item_result eq 'create_new') {
+ my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber);
+ $num_items_added += $bib_items_added;
+ $num_items_errored += $bib_items_errored;
+ # still need to record the matched biblionumber so that the
+ # items can be reverted
+ my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
+ $sth2->execute($biblionumber, $rowref->{'import_record_id'});
+ SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
+ }
+ SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored');
}
}
$sth->finish();
@@ -588,6 +617,7 @@ sub BatchRevertBibRecords {
# FIXME biblio only at the moment
SetImportBatchStatus('reverting');
my $overlay_action = GetImportBatchOverlayAction($batch_id);
+ my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
my $dbh = C4::Context->dbh;
my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marcxml_old, encoding, matched_biblionumber
FROM import_records
@@ -598,8 +628,10 @@ sub BatchRevertBibRecords {
if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') {
$num_ignored++;
}
- if ($overlay_action eq 'create_new' or
- ($overlay_action eq 'replace' and $rowref->{'overlay_status'} eq 'no_match')) {
+
+ my $bib_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'});
+
+ if ($bib_result eq 'delete') {
$num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
my $error = DelBiblio($rowref->{'matched_biblionumber'});
if (defined $error) {
@@ -608,7 +640,7 @@ sub BatchRevertBibRecords {
$num_deleted++;
SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
}
- } else {
+ } elsif ($bib_result eq 'restore') {
$num_reverted++;
my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'});
my $biblionumber = $rowref->{'matched_biblionumber'};
@@ -616,8 +648,12 @@ sub BatchRevertBibRecords {
$num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'});
SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
+ } elsif ($bib_result eq 'ignore') {
+ $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
+ SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
}
}
+
$sth->finish();
SetImportBatchStatus($batch_id, 'reverted');
return ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored);
@@ -905,6 +941,92 @@ sub SetImportBatchOverlayAction {
}
+=head2 GetImportBatchNoMatchAction
+
+=over 4
+
+my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
+
+=back
+
+=cut
+
+sub GetImportBatchNoMatchAction {
+ my ($batch_id) = @_;
+
+ my $dbh = C4::Context->dbh;
+ my $sth = $dbh->prepare("SELECT nomatch_action FROM import_batches WHERE import_batch_id = ?");
+ $sth->execute($batch_id);
+ my ($nomatch_action) = $sth->fetchrow_array();
+ $sth->finish();
+ return $nomatch_action;
+
+}
+
+
+=head2 SetImportBatchNoMatchAction
+
+=over 4
+
+SetImportBatchNoMatchAction($batch_id, $new_nomatch_action);
+
+=back
+
+=cut
+
+sub SetImportBatchNoMatchAction {
+ my ($batch_id, $new_nomatch_action) = @_;
+
+ my $dbh = C4::Context->dbh;
+ my $sth = $dbh->prepare("UPDATE import_batches SET nomatch_action = ? WHERE import_batch_id = ?");
+ $sth->execute($new_nomatch_action, $batch_id);
+ $sth->finish();
+
+}
+
+=head2 GetImportBatchItemAction
+
+=over 4
+
+my $item_action = GetImportBatchItemAction($batch_id);
+
+=back
+
+=cut
+
+sub GetImportBatchItemAction {
+ my ($batch_id) = @_;
+
+ my $dbh = C4::Context->dbh;
+ my $sth = $dbh->prepare("SELECT item_action FROM import_batches WHERE import_batch_id = ?");
+ $sth->execute($batch_id);
+ my ($item_action) = $sth->fetchrow_array();
+ $sth->finish();
+ return $item_action;
+
+}
+
+
+=head2 SetImportBatchItemAction
+
+=over 4
+
+SetImportBatchItemAction($batch_id, $new_item_action);
+
+=back
+
+=cut
+
+sub SetImportBatchItemAction {
+ my ($batch_id, $new_item_action) = @_;
+
+ my $dbh = C4::Context->dbh;
+ my $sth = $dbh->prepare("UPDATE import_batches SET item_action = ? WHERE import_batch_id = ?");
+ $sth->execute($new_item_action, $batch_id);
+ $sth->finish();
+
+}
+
=head2 GetImportBatchMatcher
=over 4
@@ -1190,6 +1312,46 @@ sub _update_batch_record_counts {
}
+sub _get_commit_action {
+ my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id) = @_;
+
+ my ($bib_result, $bib_match, $item_result);
+
+ if ($overlay_status ne 'no_match') {
+ $bib_match = GetBestRecordMatch($import_record_id);
+ if ($overlay_action eq 'replace') {
+ $bib_result = defined($bib_match) ? 'replace' : 'create_new';
+ } elsif ($overlay_action eq 'create_new') {
+ $bib_result = 'create_new';
+ } elsif ($overlay_action eq 'ignore') {
+ $bib_result = 'ignore';
+ }
+ $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore';
+ } else {
+ $bib_result = $nomatch_action;
+ $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore';
+ }
+
+ return ($bib_result, $item_result, $bib_match);
+}
+
+sub _get_revert_action {
+ my ($overlay_action, $overlay_status, $status) = @_;
+
+ my $bib_result;
+
+ if ($status eq 'ignored') {
+ $bib_result = 'ignore';
+ } else {
+ if ($overlay_action eq 'create_new') {
+ $bib_result = 'delete';
+ } else {
+ $bib_result = ($overlay_status eq 'match_applied') ? 'restore' : 'delete';
+ }
+ }
+ return $bib_result;
+}
+
1;
__END__
diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql
index 964614ddbe..ff55985fdd 100644
--- a/installer/data/mysql/kohastructure.sql
+++ b/installer/data/mysql/kohastructure.sql
@@ -872,7 +872,9 @@ CREATE TABLE `import_batches` (
`num_biblios` int(11) NOT NULL default 0,
`num_items` int(11) NOT NULL default 0,
`upload_timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP,
- `overlay_action` enum('replace', 'create_new', 'use_template') NOT NULL default 'create_new',
+ `overlay_action` enum('replace', 'create_new', 'use_template', 'ignore') NOT NULL default 'create_new',
+ `nomatch_action` enum('create_new', 'ignore') NOT NULL default 'create_new',
+ `item_action` enum('always_add', 'add_only_for_matches', 'add_only_for_new', 'ignore') NOT NULL default 'always_add',
`import_status` enum('staging', 'staged', 'importing', 'imported', 'reverting', 'reverted', 'cleaned') NOT NULL default 'staging',
`batch_type` enum('batch', 'z3950') NOT NULL default 'batch',
`file_name` varchar(100),
@@ -898,7 +900,7 @@ CREATE TABLE `import_records` (
`marcxml_old` longtext NOT NULL,
`record_type` enum('biblio', 'auth', 'holdings') NOT NULL default 'biblio',
`overlay_status` enum('no_match', 'auto_match', 'manual_match', 'match_applied') NOT NULL default 'no_match',
- `status` enum('error', 'staged', 'imported', 'reverted', 'items_reverted') NOT NULL default 'staged',
+ `status` enum('error', 'staged', 'imported', 'reverted', 'items_reverted', 'ignored') NOT NULL default 'staged',
`import_error` mediumtext,
`encoding` varchar(40) NOT NULL default '',
`z3950random` varchar(40) default NULL,
@@ -954,7 +956,7 @@ CREATE TABLE `import_items` (
`import_record_id` int(11) NOT NULL,
`itemnumber` int(11) default NULL,
`branchcode` varchar(10) default NULL,
- `status` enum('error', 'staged', 'imported', 'reverted') NOT NULL default 'staged',
+ `status` enum('error', 'staged', 'imported', 'reverted', 'ignored') NOT NULL default 'staged',
`marcxml` longtext NOT NULL,
`import_error` mediumtext,
PRIMARY KEY (`import_items_id`),
diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl
index 0399292e94..592170d022 100755
--- a/installer/data/mysql/updatedatabase.pl
+++ b/installer/data/mysql/updatedatabase.pl
@@ -1397,6 +1397,26 @@ if (C4::Context->preference("Version") < TransformToNum($DBversion)) {
SetVersion ($DBversion);
}
+$DBversion = "3.00.00.076";
+if (C4::Context->preference("Version") < TransformToNum($DBversion)) {
+ $dbh->do("ALTER TABLE import_batches
+ ADD COLUMN nomatch_action enum('create_new', 'ignore') NOT NULL default 'create_new' AFTER overlay_action");
+ $dbh->do("ALTER TABLE import_batches
+ ADD COLUMN item_action enum('always_add', 'add_only_for_matches', 'add_only_for_new', 'ignore')
+ NOT NULL default 'always_add' AFTER nomatch_action");
+ $dbh->do("ALTER TABLE import_batches
+ MODIFY overlay_action enum('replace', 'create_new', 'use_template', 'ignore')
+ NOT NULL default 'create_new'");
+ $dbh->do("ALTER TABLE import_records
+ MODIFY status enum('error', 'staged', 'imported', 'reverted', 'items_reverted',
+ 'ignored') NOT NULL default 'staged'");
+ $dbh->do("ALTER TABLE import_items
+ MODIFY status enum('error', 'staged', 'imported', 'reverted', 'ignored') NOT NULL default 'staged'");
+
+ print "Upgrade to $DBversion done (changes to import_batches and import_records) ";
+ SetVersion ($DBversion);
+}
+
=item DropAllForeignKeys($table)
Drop all foreign keys of the table $table
diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-item-action.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-item-action.inc
new file mode 100644
index 0000000000..7540b5fec6
--- /dev/null
+++ b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-item-action.inc
@@ -0,0 +1,26 @@
+
diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-nomatch-action.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-nomatch-action.inc
new file mode 100644
index 0000000000..87d2ac097c
--- /dev/null
+++ b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-nomatch-action.inc
@@ -0,0 +1,14 @@
+
diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-overlay-action.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-overlay-action.inc
new file mode 100644
index 0000000000..d12f497dde
--- /dev/null
+++ b/koha-tmpl/intranet-tmpl/prog/en/includes/tools-overlay-action.inc
@@ -0,0 +1,20 @@
+
diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-marc-import.tmpl b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-marc-import.tmpl
index 190bcaf98b..ea0a5e5f4f 100644
--- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-marc-import.tmpl
+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-marc-import.tmpl
@@ -39,23 +39,33 @@
-
-
-
-
-
+
+
+
+
+
+
+
+
+
Completed import of records
Number of records added
Number of records updated
+
Number of records ignored
Number of items added
Number of items ignored because of duplicate barcode
-
Number of records ignored
@@ -118,15 +160,6 @@
Number of records ignored
-
-
-
Failed to apply different matching rule
-
-
Applied different matching rule. Number of records matched now
-
-