speed boost: tools stage and commit bib records
Turned off autocommit; commit every 50 records. Signed-off-by: Chris Cormack <crc@liblime.com> Signed-off-by: Joshua Ferraro <jmf@liblime.com>
This commit is contained in:
parent
991cdc31bc
commit
8f854c0867
2 changed files with 19 additions and 6 deletions
|
@ -152,13 +152,15 @@ sub commit_batch {
|
|||
my ($template, $import_batch_id) = @_;
|
||||
|
||||
my $job = undef;
|
||||
$dbh->{AutoCommit} = 0;
|
||||
my $callback = sub {};
|
||||
if ($runinbackground) {
|
||||
$job = put_in_background($import_batch_id);
|
||||
$callback = progress_callback($job);
|
||||
$callback = progress_callback($job, $dbh);
|
||||
}
|
||||
my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) =
|
||||
BatchCommitBibRecords($import_batch_id, 50, $callback);
|
||||
$dbh->commit();
|
||||
|
||||
my $results = {
|
||||
did_commit => 1,
|
||||
|
@ -178,14 +180,16 @@ sub commit_batch {
|
|||
sub revert_batch {
|
||||
my ($template, $import_batch_id) = @_;
|
||||
|
||||
$dbh->{AutoCommit} = 0;
|
||||
my $job = undef;
|
||||
my $callback = sub {};
|
||||
if ($runinbackground) {
|
||||
$job = put_in_background($import_batch_id);
|
||||
$callback = progress_callback($job);
|
||||
$callback = progress_callback($job, $dbh);
|
||||
}
|
||||
my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) =
|
||||
BatchRevertBibRecords($import_batch_id, 50, $callback);
|
||||
$dbh->commit();
|
||||
|
||||
my $results = {
|
||||
did_revert => 1,
|
||||
|
@ -239,9 +243,11 @@ sub put_in_background {
|
|||
|
||||
sub progress_callback {
|
||||
my $job = shift;
|
||||
my $dbh = shift;
|
||||
return sub {
|
||||
my $progress = shift;
|
||||
$job->progress($progress);
|
||||
$dbh->commit();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -44,6 +44,7 @@ use C4::BackgroundJob;
|
|||
|
||||
my $input = new CGI;
|
||||
my $dbh = C4::Context->dbh;
|
||||
$dbh->{AutoCommit} = 0;
|
||||
|
||||
my $fileID=$input->param('uploadedfileid');
|
||||
my $runinbackground = $input->param('runinbackground');
|
||||
|
@ -117,15 +118,16 @@ if ($completedJobID) {
|
|||
|
||||
# if we get here, we're a child that has detached
|
||||
# itself from Apache
|
||||
$staging_callback = staging_progress_callback($job);
|
||||
$matching_callback = matching_progress_callback($job);
|
||||
$staging_callback = staging_progress_callback($job, $dbh);
|
||||
$matching_callback = matching_progress_callback($job, $dbh);
|
||||
|
||||
}
|
||||
|
||||
# FIXME branch code
|
||||
my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($syntax, $marcrecord, $filename,
|
||||
$comments, '', $parse_items, 0,
|
||||
50, staging_progress_callback($job));
|
||||
50, staging_progress_callback($job, $dbh));
|
||||
$dbh->commit();
|
||||
my $num_with_matches = 0;
|
||||
my $checked_matches = 0;
|
||||
my $matcher_failed = 0;
|
||||
|
@ -135,8 +137,9 @@ if ($completedJobID) {
|
|||
if (defined $matcher) {
|
||||
$checked_matches = 1;
|
||||
$matcher_code = $matcher->code();
|
||||
$num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 50, matching_progress_callback($job));
|
||||
$num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 50, matching_progress_callback($job, $dbh));
|
||||
SetImportBatchMatcher($batch_id, $matcher_id);
|
||||
$dbh->commit();
|
||||
} else {
|
||||
$matcher_failed = 1;
|
||||
}
|
||||
|
@ -180,17 +183,21 @@ exit 0;
|
|||
|
||||
sub staging_progress_callback {
|
||||
my $job = shift;
|
||||
my $dbh = shift;
|
||||
return sub {
|
||||
my $progress = shift;
|
||||
$job->progress($progress);
|
||||
$dbh->commit();
|
||||
}
|
||||
}
|
||||
|
||||
sub matching_progress_callback {
|
||||
my $job = shift;
|
||||
my $dbh = shift;
|
||||
my $start_progress = $job->progress();
|
||||
return sub {
|
||||
my $progress = shift;
|
||||
$job->progress($start_progress + $progress);
|
||||
$dbh->commit();
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue