Koha-community/Koha
13
7
Fork 0
Code Issues Releases Wiki Activity

speed boost: tools stage and commit bib records

Browse source 
Turned off autocommit; commit every 50 records.

Signed-off-by: Chris Cormack <crc@liblime.com>
Signed-off-by: Joshua Ferraro <jmf@liblime.com>
This commit is contained in:
Galen Charlton 2008-01-03 12:36:42 -06:00 committed by Joshua Ferraro
parent 991cdc31bc
commit 8f854c0867
2 changed files with 19 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
tools/manage-marc-import.pl
View file

@ -152,13 +152,15 @@ sub commit_batch {
my ($template, $import_batch_id) = @_; my ($template, $import_batch_id) = @_;
my $job = undef; my $job = undef;
$dbh->{AutoCommit} = 0;
my $callback = sub {}; my $callback = sub {};
if ($runinbackground) { if ($runinbackground) {
$job = put_in_background($import_batch_id); $job = put_in_background($import_batch_id);
$callback = progress_callback($job); $callback = progress_callback($job, $dbh);
} }
my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =
BatchCommitBibRecords($import_batch_id, 50, $callback); BatchCommitBibRecords($import_batch_id, 50, $callback);
$dbh->commit();
my $results = { my $results = {
did_commit => 1, did_commit => 1,
@ -178,14 +180,16 @@ sub commit_batch {
sub revert_batch { sub revert_batch {
my ($template, $import_batch_id) = @_; my ($template, $import_batch_id) = @_;
$dbh->{AutoCommit} = 0;
my $job = undef; my $job = undef;
my $callback = sub {}; my $callback = sub {};
if ($runinbackground) { if ($runinbackground) {
$job = put_in_background($import_batch_id); $job = put_in_background($import_batch_id);
$callback = progress_callback($job); $callback = progress_callback($job, $dbh);
} }
my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) = my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) =
BatchRevertBibRecords($import_batch_id, 50, $callback); BatchRevertBibRecords($import_batch_id, 50, $callback);
$dbh->commit();
my $results = { my $results = {
did_revert => 1, did_revert => 1,
@ -239,9 +243,11 @@ sub put_in_background {
sub progress_callback { sub progress_callback {
my $job = shift; my $job = shift;
my $dbh = shift;
return sub { return sub {
my $progress = shift; my $progress = shift;
$job->progress($progress); $job->progress($progress);
$dbh->commit();
} }
} }

15
tools/stage-marc-import.pl
View file

@ -44,6 +44,7 @@ use C4::BackgroundJob;
my $input = new CGI; my $input = new CGI;
my $dbh = C4::Context->dbh; my $dbh = C4::Context->dbh;
$dbh->{AutoCommit} = 0;
my $fileID=$input->param('uploadedfileid'); my $fileID=$input->param('uploadedfileid');
my $runinbackground = $input->param('runinbackground'); my $runinbackground = $input->param('runinbackground');
@ -117,15 +118,16 @@ if ($completedJobID) {
# if we get here, we're a child that has detached # if we get here, we're a child that has detached
# itself from Apache # itself from Apache
$staging_callback = staging_progress_callback($job); $staging_callback = staging_progress_callback($job, $dbh);
$matching_callback = matching_progress_callback($job); $matching_callback = matching_progress_callback($job, $dbh);
} }
# FIXME branch code # FIXME branch code
my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($syntax, $marcrecord, $filename, my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($syntax, $marcrecord, $filename,
$comments, '', $parse_items, 0, $comments, '', $parse_items, 0,
50, staging_progress_callback($job)); 50, staging_progress_callback($job, $dbh));
$dbh->commit();
my $num_with_matches = 0; my $num_with_matches = 0;
my $checked_matches = 0; my $checked_matches = 0;
my $matcher_failed = 0; my $matcher_failed = 0;
@ -135,8 +137,9 @@ if ($completedJobID) {
if (defined $matcher) { if (defined $matcher) {
$checked_matches = 1; $checked_matches = 1;
$matcher_code = $matcher->code(); $matcher_code = $matcher->code();
$num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 50, matching_progress_callback($job)); $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 50, matching_progress_callback($job, $dbh));
SetImportBatchMatcher($batch_id, $matcher_id); SetImportBatchMatcher($batch_id, $matcher_id);
$dbh->commit();
} else { } else {
$matcher_failed = 1; $matcher_failed = 1;
} }
@ -180,17 +183,21 @@ exit 0;
sub staging_progress_callback { sub staging_progress_callback {
my $job = shift; my $job = shift;
my $dbh = shift;
return sub { return sub {
my $progress = shift; my $progress = shift;
$job->progress($progress); $job->progress($progress);
$dbh->commit();
} }
} }
sub matching_progress_callback { sub matching_progress_callback {
my $job = shift; my $job = shift;
my $dbh = shift;
my $start_progress = $job->progress(); my $start_progress = $job->progress();
return sub { return sub {
my $progress = shift; my $progress = shift;
$job->progress($start_progress + $progress); $job->progress($start_progress + $progress);
$dbh->commit();
} }
} }