speed boost: tools stage and commit bib records
Turned off autocommit; commit every 50 records. Signed-off-by: Chris Cormack <crc@liblime.com> Signed-off-by: Joshua Ferraro <jmf@liblime.com>
This commit is contained in:
parent
991cdc31bc
commit
8f854c0867
2 changed files with 19 additions and 6 deletions
|
@ -152,13 +152,15 @@ sub commit_batch {
|
||||||
my ($template, $import_batch_id) = @_;
|
my ($template, $import_batch_id) = @_;
|
||||||
|
|
||||||
my $job = undef;
|
my $job = undef;
|
||||||
|
$dbh->{AutoCommit} = 0;
|
||||||
my $callback = sub {};
|
my $callback = sub {};
|
||||||
if ($runinbackground) {
|
if ($runinbackground) {
|
||||||
$job = put_in_background($import_batch_id);
|
$job = put_in_background($import_batch_id);
|
||||||
$callback = progress_callback($job);
|
$callback = progress_callback($job, $dbh);
|
||||||
}
|
}
|
||||||
my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =
|
my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =
|
||||||
BatchCommitBibRecords($import_batch_id, 50, $callback);
|
BatchCommitBibRecords($import_batch_id, 50, $callback);
|
||||||
|
$dbh->commit();
|
||||||
|
|
||||||
my $results = {
|
my $results = {
|
||||||
did_commit => 1,
|
did_commit => 1,
|
||||||
|
@ -178,14 +180,16 @@ sub commit_batch {
|
||||||
sub revert_batch {
|
sub revert_batch {
|
||||||
my ($template, $import_batch_id) = @_;
|
my ($template, $import_batch_id) = @_;
|
||||||
|
|
||||||
|
$dbh->{AutoCommit} = 0;
|
||||||
my $job = undef;
|
my $job = undef;
|
||||||
my $callback = sub {};
|
my $callback = sub {};
|
||||||
if ($runinbackground) {
|
if ($runinbackground) {
|
||||||
$job = put_in_background($import_batch_id);
|
$job = put_in_background($import_batch_id);
|
||||||
$callback = progress_callback($job);
|
$callback = progress_callback($job, $dbh);
|
||||||
}
|
}
|
||||||
my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) =
|
my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) =
|
||||||
BatchRevertBibRecords($import_batch_id, 50, $callback);
|
BatchRevertBibRecords($import_batch_id, 50, $callback);
|
||||||
|
$dbh->commit();
|
||||||
|
|
||||||
my $results = {
|
my $results = {
|
||||||
did_revert => 1,
|
did_revert => 1,
|
||||||
|
@ -239,9 +243,11 @@ sub put_in_background {
|
||||||
|
|
||||||
sub progress_callback {
|
sub progress_callback {
|
||||||
my $job = shift;
|
my $job = shift;
|
||||||
|
my $dbh = shift;
|
||||||
return sub {
|
return sub {
|
||||||
my $progress = shift;
|
my $progress = shift;
|
||||||
$job->progress($progress);
|
$job->progress($progress);
|
||||||
|
$dbh->commit();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -44,6 +44,7 @@ use C4::BackgroundJob;
|
||||||
|
|
||||||
my $input = new CGI;
|
my $input = new CGI;
|
||||||
my $dbh = C4::Context->dbh;
|
my $dbh = C4::Context->dbh;
|
||||||
|
$dbh->{AutoCommit} = 0;
|
||||||
|
|
||||||
my $fileID=$input->param('uploadedfileid');
|
my $fileID=$input->param('uploadedfileid');
|
||||||
my $runinbackground = $input->param('runinbackground');
|
my $runinbackground = $input->param('runinbackground');
|
||||||
|
@ -117,15 +118,16 @@ if ($completedJobID) {
|
||||||
|
|
||||||
# if we get here, we're a child that has detached
|
# if we get here, we're a child that has detached
|
||||||
# itself from Apache
|
# itself from Apache
|
||||||
$staging_callback = staging_progress_callback($job);
|
$staging_callback = staging_progress_callback($job, $dbh);
|
||||||
$matching_callback = matching_progress_callback($job);
|
$matching_callback = matching_progress_callback($job, $dbh);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# FIXME branch code
|
# FIXME branch code
|
||||||
my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($syntax, $marcrecord, $filename,
|
my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($syntax, $marcrecord, $filename,
|
||||||
$comments, '', $parse_items, 0,
|
$comments, '', $parse_items, 0,
|
||||||
50, staging_progress_callback($job));
|
50, staging_progress_callback($job, $dbh));
|
||||||
|
$dbh->commit();
|
||||||
my $num_with_matches = 0;
|
my $num_with_matches = 0;
|
||||||
my $checked_matches = 0;
|
my $checked_matches = 0;
|
||||||
my $matcher_failed = 0;
|
my $matcher_failed = 0;
|
||||||
|
@ -135,8 +137,9 @@ if ($completedJobID) {
|
||||||
if (defined $matcher) {
|
if (defined $matcher) {
|
||||||
$checked_matches = 1;
|
$checked_matches = 1;
|
||||||
$matcher_code = $matcher->code();
|
$matcher_code = $matcher->code();
|
||||||
$num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 50, matching_progress_callback($job));
|
$num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 50, matching_progress_callback($job, $dbh));
|
||||||
SetImportBatchMatcher($batch_id, $matcher_id);
|
SetImportBatchMatcher($batch_id, $matcher_id);
|
||||||
|
$dbh->commit();
|
||||||
} else {
|
} else {
|
||||||
$matcher_failed = 1;
|
$matcher_failed = 1;
|
||||||
}
|
}
|
||||||
|
@ -180,17 +183,21 @@ exit 0;
|
||||||
|
|
||||||
sub staging_progress_callback {
|
sub staging_progress_callback {
|
||||||
my $job = shift;
|
my $job = shift;
|
||||||
|
my $dbh = shift;
|
||||||
return sub {
|
return sub {
|
||||||
my $progress = shift;
|
my $progress = shift;
|
||||||
$job->progress($progress);
|
$job->progress($progress);
|
||||||
|
$dbh->commit();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
sub matching_progress_callback {
|
sub matching_progress_callback {
|
||||||
my $job = shift;
|
my $job = shift;
|
||||||
|
my $dbh = shift;
|
||||||
my $start_progress = $job->progress();
|
my $start_progress = $job->progress();
|
||||||
return sub {
|
return sub {
|
||||||
my $progress = shift;
|
my $progress = shift;
|
||||||
$job->progress($start_progress + $progress);
|
$job->progress($start_progress + $progress);
|
||||||
|
$dbh->commit();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue