Koha-community/Koha
13
7
Fork 0
Code Issues Releases Wiki Activity

speed boost: tools stage and commit bib records

Browse source 
Turned off autocommit; commit every 50 records.

Signed-off-by: Chris Cormack <crc@liblime.com>
Signed-off-by: Joshua Ferraro <jmf@liblime.com>
This commit is contained in:
Galen Charlton 2008-01-03 12:36:42 -06:00 committed by Joshua Ferraro
parent 991cdc31bc
commit 8f854c0867
2 changed files with 19 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
tools/manage-marc-import.pl
View file

@ -152,13 +152,15 @@ sub commit_batch {
my ($template, $import_batch_id) = @_;
my $job = undef;
$dbh->{AutoCommit} = 0;
my $callback = sub {};
if ($runinbackground) {
$job = put_in_background($import_batch_id);
$callback = progress_callback($job);
$callback = progress_callback($job, $dbh);
}
my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =
BatchCommitBibRecords($import_batch_id, 50, $callback);
$dbh->commit();
my $results = {
did_commit => 1,
@ -178,14 +180,16 @@ sub commit_batch {
sub revert_batch {
my ($template, $import_batch_id) = @_;
$dbh->{AutoCommit} = 0;
my $job = undef;
my $callback = sub {};
if ($runinbackground) {
$job = put_in_background($import_batch_id);
$callback = progress_callback($job);
$callback = progress_callback($job, $dbh);
}
my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) =
BatchRevertBibRecords($import_batch_id, 50, $callback);
$dbh->commit();
my $results = {
did_revert => 1,
@ -239,9 +243,11 @@ sub put_in_background {
sub progress_callback {
my $job = shift;
my $dbh = shift;
return sub {
my $progress = shift;
$job->progress($progress);
$dbh->commit();
}
}

15
tools/stage-marc-import.pl
View file

@ -44,6 +44,7 @@ use C4::BackgroundJob;
my $input = new CGI;
my $dbh = C4::Context->dbh;
$dbh->{AutoCommit} = 0;
my $fileID=$input->param('uploadedfileid');
my $runinbackground = $input->param('runinbackground');
@ -117,15 +118,16 @@ if ($completedJobID) {
# if we get here, we're a child that has detached
# itself from Apache
$staging_callback = staging_progress_callback($job);
$matching_callback = matching_progress_callback($job);
$staging_callback = staging_progress_callback($job, $dbh);
$matching_callback = matching_progress_callback($job, $dbh);
}
# FIXME branch code
my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($syntax, $marcrecord, $filename,
$comments, '', $parse_items, 0,
50, staging_progress_callback($job));
50, staging_progress_callback($job, $dbh));
$dbh->commit();
my $num_with_matches = 0;
my $checked_matches = 0;
my $matcher_failed = 0;
@ -135,8 +137,9 @@ if ($completedJobID) {
if (defined $matcher) {
$checked_matches = 1;
$matcher_code = $matcher->code();
$num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 50, matching_progress_callback($job));
$num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 50, matching_progress_callback($job, $dbh));
SetImportBatchMatcher($batch_id, $matcher_id);
$dbh->commit();
} else {
$matcher_failed = 1;
}
@ -180,17 +183,21 @@ exit 0;
sub staging_progress_callback {
my $job = shift;
my $dbh = shift;
return sub {
my $progress = shift;
$job->progress($progress);
$dbh->commit();
}
}
sub matching_progress_callback {
my $job = shift;
my $dbh = shift;
my $start_progress = $job->progress();
return sub {
my $progress = shift;
$job->progress($start_progress + $progress);
$dbh->commit();
}
}