Koha-community
/
Koha
13
5
Fork 0
Code Issues Releases Wiki Activity
Browse Source

kohabug 2048 - improve zebraqueue_daemon performance 

[1] Increase sleep interval between checks of zebraqueue
    from 0.01 seconds to 0.50.
[2] Batch up commits of changes to the zebraqueue table
[3] If the same record appears multiple times in the queue,
    handle only once.
[4] Properly postpone failures to process record deletes to
    avoid spinning.
[5] Correct how queue entries are marked done - avoid skipping
    an authority record update, e.g., if it has the same
    ID number as a bib that was updated.
[6] Added a FIXME about a possible later enhancement to
    batch up updates so that Zebra isn't told to commit
    after each record.

No documentation changes.

Signed-off-by: Joshua Ferraro <jmf@liblime.com>
3.0.x
Galen Charlton 17 years ago
committed by Joshua Ferraro
parent
709619013d
commit
e3f473187e
1 changed files with 37 additions and 13 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 50
      misc/bin/zebraqueue_daemon.pl

50
misc/bin/zebraqueue_daemon.pl
View File

@ -110,7 +110,7 @@ sub handler_sleep {
# can be used to slow down loop execution if needed
my ( $kernel, $heap, $session ) = @_[ KERNEL, HEAP, SESSION ];
use Time::HiRes qw (sleep);
Time::HiRes::sleep(0.01);
Time::HiRes::sleep(0.5);
#sleep 1;
$kernel->yield('status_check');
}
@ -125,10 +125,14 @@ sub handler_check {
if ($data->{'opcount'} > 0) {
Unix::Syslog::syslog LOG_INFO, "$data->{'opcount'} operations waiting to be run\n";
$sth->finish();
$dbh->commit(); # needed so that we get current state of zebraqueue next time
# we enter handler_check
$kernel->yield('do_ops');
}
else {
$sth->finish();
$dbh->commit(); # needed so that we get current state of zebraqueue next time
# we enter handler_check
$kernel->yield('sleep');
}
}
@ -137,10 +141,11 @@ sub zebraop {
# execute operations waiting in the zebraqueue
my ( $kernel, $heap, $session ) = @_[ KERNEL, HEAP, SESSION ];
my $dbh = get_db_connection();
my $readsth = $dbh->prepare("SELECT id, biblio_auth_number, operation, server FROM zebraqueue WHERE done = 0");
my $readsth = $dbh->prepare("SELECT id, biblio_auth_number, operation, server FROM zebraqueue WHERE done = 0 ORDER BY id DESC");
$readsth->execute();
Unix::Syslog::syslog LOG_INFO, "Executing zebra operations\n";
my $completed_updates = {};
ZEBRAQUEUE: while (my $data = $readsth->fetchrow_hashref()) {
warn "Inside while loop" if $debug;
@ -152,6 +157,7 @@ sub zebraop {
my $server = $data->{'server'};
next ZEBRAQUEUE if exists $postponed_updates->{$server}->{$record_number};
next ZEBRAQUEUE if exists $completed_updates->{$server}->{$record_number}->{$op};
my $ok = 0;
my $record;
@ -162,11 +168,16 @@ sub zebraop {
$ok = process_update($dbh, $server, $record_number, $id);
}
if ($ok == 1) {
mark_done($dbh, $record_number, $op);
mark_done($dbh, $record_number, $op, $server);
$completed_updates->{$server}->{$record_number}->{$op} = 1;
if ($op eq 'recordDelete') {
$completed_updates->{$server}->{$record_number}->{'specialUpdate'} = 1;
}
}
}
$readsth->finish();
$kernel->yield('status_check');
$dbh->commit();
$kernel->yield('sleep');
}
sub process_delete {
@ -190,8 +201,8 @@ sub process_delete {
$ok = 1;
} else {
# caught a ZOOM::Exception
my $error = _format_zoom_error_message($@);
warn "ERROR: $error";
my $message = _format_zoom_error_message($@);
postpone_update($server, $record_number, $message);
}
} else {
# then, delete the record
@ -227,7 +238,7 @@ sub process_update {
## it's Broken XML-- Should not reach here-- but if it does -lets protect ZEBRA
if ($@) {
Unix::Syslog::syslog LOG_ERR, "$server record $record_number is malformed: $@";
mark_done_by_id($dbh, $id);
mark_done_by_id($dbh, $id, $server);
$ok = 0;
} else {
# ok, we have everything, do the operation in zebra !
@ -239,14 +250,16 @@ sub process_update {
sub mark_done_by_id {
my $dbh = shift;
my $id = shift;
my $delsth = $dbh->prepare("UPDATE zebraqueue SET done = 1 WHERE id = ?");
$delsth->execute($id);
my $server = shift;
my $delsth = $dbh->prepare("UPDATE zebraqueue SET done = 1 WHERE id = ? AND server = ? AND done = 0");
$delsth->execute($id, $server);
}
sub mark_done {
my $dbh = shift;
my $record_number = shift;
my $op = shift;
my $server = shift;
my $delsth;
if ($op eq 'recordDelete') {
@ -254,15 +267,21 @@ sub mark_done {
# did a modif (or item deletion) just before biblio deletion, there are some specialUpdate
# that are pending and can't succeed, as we don't have the XML anymore
# so, delete everything for this biblionumber
$delsth = $dbh->prepare_cached("UPDATE zebraqueue SET done=1 WHERE biblio_auth_number = ? and operation = ?");
$delsth->execute($record_number, $op);
$delsth = $dbh->prepare_cached("UPDATE zebraqueue SET done = 1
WHERE biblio_auth_number = ?
AND server = ?
AND done = 0");
$delsth->execute($record_number, $server);
} else {
# if it's not a deletion, delete every pending specialUpdate for this biblionumber
# in case the user add biblio, then X items, before this script runs
# this avoid indexing X+1 times where just 1 is enough.
$delsth = $dbh->prepare("UPDATE zebraqueue SET done = 1
WHERE biblio_auth_number = ? and operation = 'specialUpdate'");
$delsth->execute($record_number);
WHERE biblio_auth_number = ?
AND operation = 'specialUpdate'
AND server = ?
AND done = 0");
$delsth->execute($record_number, $server);
}
}
@ -308,6 +327,9 @@ sub zebrado {
postpone_update($server, $record_number, $message);
}
}
# FIXME - would be more efficient to send a ES commit
# after a batch of records, rather than commiting after
# each one - Zebra handles updates relatively slowly.
eval { $Zpackage->send('commit'); };
if ($@) {
# operation succeeded, but commit
@ -370,6 +392,8 @@ sub get_db_connection {
# C4::Context->dbh dies if it cannot
# establish a connection
$db_connection_wait = $min_connection_wait;
$dbh->{AutoCommit} = 0; # do this to reduce number of
# commits to zebraqueue
return $dbh;
}

Write Preview
Loading…
Cancel
Save