From 991cdc31bcae3c037236c513c74dd3dfdee725a0 Mon Sep 17 00:00:00 2001 From: Galen Charlton Date: Thu, 3 Jan 2008 12:36:41 -0600 Subject: [PATCH] speed boost: command-line stage and commit biblios Turned off autocommit and commit every 100 records. Signed-off-by: Chris Cormack Signed-off-by: Joshua Ferraro --- misc/commit_biblios_file.pl | 8 ++++++-- misc/stage_biblios_file.pl | 10 +++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/misc/commit_biblios_file.pl b/misc/commit_biblios_file.pl index 4c8d8ac3d1..2f9851fe82 100755 --- a/misc/commit_biblios_file.pl +++ b/misc/commit_biblios_file.pl @@ -39,12 +39,15 @@ if ($list_batches) { # in future, probably should tie to a real user account C4::Context->set_userenv(0, 'batch', 0, 'batch', 'batch', 'batch', 'batch', 'batch'); +my $dbh = C4::Context->dbh; +$dbh->{AutoCommit} = 0; if ($batch_number =~ /^\d+$/ and $batch_number > 0) { my $batch = GetImportBatch($batch_number); die "$0: import batch $batch_number does not exist in database\n" unless defined $batch; die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n" unless $batch->{'import_status'} eq "staged" or $batch->{'import_status'} eq "reverted"; process_batch($batch_number); + $dbh->commit(); } else { die "$0: please specify a numeric batch ID\n"; } @@ -71,7 +74,7 @@ sub process_batch { print "... importing MARC records -- please wait\n"; my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = - BatchCommitBibRecords($import_batch_id, 100, \&print_progress); + BatchCommitBibRecords($import_batch_id, 100, \&print_progress_and_commit); print "... finished importing MARC records\n"; print <<_SUMMARY_; @@ -90,9 +93,10 @@ duplicate of one already in the database. _SUMMARY_ } -sub print_progress { +sub print_progress_and_commit { my $recs = shift; print "... processed $recs records\n"; + $dbh->commit(); } sub print_usage { diff --git a/misc/stage_biblios_file.pl b/misc/stage_biblios_file.pl index bfe26ce972..a692748640 100755 --- a/misc/stage_biblios_file.pl +++ b/misc/stage_biblios_file.pl @@ -39,7 +39,10 @@ unless (-r $input_file) { die "$0: cannot open input file $input_file: $!\n"; } +my $dbh = C4::Context->dbh; +$dbh->{AutoCommit} = 0; process_batch($input_file, $match_bibs, $add_items, $batch_comment); +$dbh->commit(); exit 0; @@ -62,7 +65,7 @@ sub process_batch { print "... staging MARC records -- please wait\n"; my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($marc_flavor, $marc_records, $input_file, $batch_comment, '', $add_items, 0, - 100, \&print_progress); + 100, \&print_progress_and_commit); print "... finished staging MARC records\n"; my $num_with_matches = 0; @@ -72,7 +75,7 @@ sub process_batch { $matcher->add_simple_required_check('245', 'a', -1, 0, '', '245', 'a', -1, 0, ''); print "... looking for matches with records already in database\n"; - $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 100, \&print_progress); + $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 100, \&print_progress_and_commit); print "... finished looking for matches\n"; } @@ -102,8 +105,9 @@ _SUMMARY_ print "\n"; } -sub print_progress { +sub print_progress_and_commit { my $recs = shift; + $dbh->commit(); print "... processed $recs records\n"; } -- 2.39.5