From ae9721491db7983f7aaaa32f2093779e35eadd0f Mon Sep 17 00:00:00 2001 From: tonnesen Date: Fri, 28 Jun 2002 17:45:39 +0000 Subject: [PATCH] z3950queue now listens for a -HUP signal before processing the queue. Z3950.pm sends the -HUP signal when queries are added to the queue. --- C4/Z3950.pm | 7 + acqui.simple/processz3950queue | 363 +++++++++++++++-------------- acqui.simple/z3950-daemon-shell.sh | 6 +- 3 files changed, 196 insertions(+), 180 deletions(-) diff --git a/C4/Z3950.pm b/C4/Z3950.pm index 65c4f9ca3f..4b33e968e1 100755 --- a/C4/Z3950.pm +++ b/C4/Z3950.pm @@ -148,11 +148,18 @@ sub addz3950queue { (term,type,servers, identifier) values (?, ?, ?, ?)"); $sth->execute($query, $type, $serverlist, $requestid); + my $pid=`cat /var/log/koha/processz3950queue.pid`; + chomp $pid; + kill 1, $pid; } } # sub addz3950queue #-------------------------------------- # $Log$ +# Revision 1.1.2.3 2002/06/28 17:45:39 tonnesen +# z3950queue now listens for a -HUP signal before processing the queue. Z3950.pm +# sends the -HUP signal when queries are added to the queue. +# # Revision 1.1.2.2 2002/06/26 20:54:31 tonnesen # use warnings breaks on perl 5.005... # diff --git a/acqui.simple/processz3950queue b/acqui.simple/processz3950queue index f9398b1c5a..3077f1263a 100755 --- a/acqui.simple/processz3950queue +++ b/acqui.simple/processz3950queue @@ -12,216 +12,217 @@ my $sth=$dbh->prepare("update z3950results set active=0"); $sth->execute; $sth->finish; $SIG{CHLD}='reap'; +$SIG{HUP}='checkqueue'; + +my $logdir=$ARGV[0]; + +open PID, ">$logdir/processz3950queue.pid"; +print PID $$."\n"; +close PID; my $reapcounter=0; my $forkcounter=0; +my $checkqueue=1; my $pid=$$; my $lastrun=0; while (1) { if ((time-$lastrun)>5) { - my $sth=$dbh->prepare("select id,term,type,servers from z3950queue order by id"); - $sth->execute; - while (my ($id, $term, $type, $servers) = $sth->fetchrow) { - if ($forkcounter<12) { - my $now=time(); - $stk=$dbh->prepare("select id,server,startdate,enddate,numrecords,active from z3950results where queryid=$id"); - ($stk->execute) || (next); - my %serverdone; - unless ($stk->rows) { - my $sti=$dbh->prepare("update z3950queue set done=-1,startdate=$now where id=$id"); - $sti->execute; - } - while (my ($r_id, $r_server,$r_startdate,$r_enddate,$r_numrecords,$active) = $stk->fetchrow) { - if ($r_enddate >0) { - $serverdone{$r_server}=1; - } elsif ($active) { - $serverdone{$r_server}=1; - } else { - $serverdone{$r_server}=-1; + if ($checkqueue) { + $checkqueue=0; + my $sth=$dbh->prepare("select id,term,type,servers from z3950queue order by id"); + $sth->execute; + while (my ($id, $term, $type, $servers) = $sth->fetchrow) { + if ($forkcounter<12) { + my $now=time(); + $stk=$dbh->prepare("select id,server,startdate,enddate,numrecords,active from z3950results where queryid=$id"); + ($stk->execute) || (next); + my %serverdone; + unless ($stk->rows) { + my $sti=$dbh->prepare("update z3950queue set done=-1,startdate=$now where id=$id"); + $sti->execute; + } + while (my ($r_id, $r_server,$r_startdate,$r_enddate,$r_numrecords,$active) = $stk->fetchrow) { + if ($r_enddate >0) { + $serverdone{$r_server}=1; + } elsif ($active) { + $serverdone{$r_server}=1; + } else { + $serverdone{$r_server}=-1; + } } - } - $stk->finish; - my $attr=''; - if ($type eq 'isbn') { - $attr='1=7'; - } elsif ($type eq 'title') { - $attr='1=4'; - } elsif ($type eq 'author') { - $attr='1=1003'; - } elsif ($type eq 'lccn') { - $attr='1=9'; - } elsif ($type eq 'keyword') { - $attr='1=1016'; - } - $term='"'.$term.'"'; - $query="\@attr $attr $term"; - my $totalrecords=0; - my $serverinfo; - my $stillprocessing=0; - foreach $serverinfo (split(/\s+/, $servers)) { - (next) if ($serverdone{$serverinfo} == 1); - my $stillprocessing=1; - if (my $pid=fork()) { - $forkcounter++; - } else { - #$sth->finish; - #$sti->finish; - #$dbh->disconnect; - my $dbi=C4Connect; - my ($name, $server, $database, $user, $password) = split(/\//, $serverinfo, 5); - $server=~/(.*)\:(\d+)/; - my $servername=$1; - my $port=$2; - print "Processing $type=$term at $name $server $database (".($forkcounter+1)." forks)\n"; - $now=time(); - my $q_serverinfo=$dbi->quote($serverinfo); - my $resultsid; - if ($serverdone{$serverinfo}==-1) { - my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id"); - $stj->execute; - ($resultsid) = $stj->fetchrow; + $stk->finish; + my $attr=''; + if ($type eq 'isbn') { + $attr='1=7'; + } elsif ($type eq 'title') { + $attr='1=4'; + } elsif ($type eq 'author') { + $attr='1=1003'; + } elsif ($type eq 'lccn') { + $attr='1=9'; + } elsif ($type eq 'keyword') { + $attr='1=1016'; + } + $term='"'.$term.'"'; + $query="\@attr $attr $term"; + my $totalrecords=0; + my $serverinfo; + my $stillprocessing=0; + foreach $serverinfo (split(/\s+/, $servers)) { + (next) if ($serverdone{$serverinfo} == 1); + my $stillprocessing=1; + if (my $pid=fork()) { + $forkcounter++; } else { - my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id"); - $stj->execute; - ($resultsid) = $stj->fetchrow; - unless ($resultsid) { - my $stj=$dbi->prepare("insert into z3950results (server, queryid, startdate) values ($q_serverinfo, $id, $now)"); + #$sth->finish; + #$sti->finish; + #$dbh->disconnect; + my $dbi=C4Connect; + my ($name, $server, $database, $user, $password) = split(/\//, $serverinfo, 5); + $server=~/(.*)\:(\d+)/; + my $servername=$1; + my $port=$2; + print "Processing $type=$term at $name $server $database (".($forkcounter+1)." forks)\n"; + $now=time(); + my $q_serverinfo=$dbi->quote($serverinfo); + my $resultsid; + if ($serverdone{$serverinfo}==-1) { + my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id"); $stj->execute; - $resultsid=$dbi->{'mysql_insertid'}; - } - } - my $stj=$dbh->prepare("update z3950results set active=1 where id=$resultsid"); - $stj->execute; - my $conn; - my $noconnection=0; - my $error=0; - if ($user) { - eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database, user => $user, password => $password); }; - if ($@) { - $noconnection=1; + ($resultsid) = $stj->fetchrow; } else { - $error=pe(); + my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id"); + $stj->execute; + ($resultsid) = $stj->fetchrow; + unless ($resultsid) { + my $stj=$dbi->prepare("insert into z3950results (server, queryid, startdate) values ($q_serverinfo, $id, $now)"); + $stj->execute; + $resultsid=$dbi->{'mysql_insertid'}; + } } - } else { - eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database); }; - if ($@) { - $noconnection=1; + my $stj=$dbh->prepare("update z3950results set active=1 where id=$resultsid"); + $stj->execute; + my $conn; + my $noconnection=0; + my $error=0; + if ($user) { + eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database, user => $user, password => $password); }; + if ($@) { + $noconnection=1; + } else { + $error=pe(); + } } else { - $error=pe(); + eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database); }; + if ($@) { + $noconnection=1; + } else { + $error=pe(); + } } - } - if ($noconnection || $error) { - } else { - print "Q: $query\n"; - my $rs=$conn->search($query); - pe(); - eval { $rs->option(preferredRecordSyntax => Net::Z3950::RecordSyntax::USMARC);}; - if ($@) { - print "ERROR: $@\n"; + if ($noconnection || $error) { } else { + print "Q: $query\n"; + my $rs=$conn->search($query); pe(); - my $numresults=$rs->size(); - pe(); - my $i; - my $result=''; - my $scantimerstart=time(); - for ($i=1; $i<=(($numresults<80) ? ($numresults) : (80)); $i++) { - my $rec=$rs->record($i); - my $marcdata=$rec->rawdata(); - $result.=$marcdata; - } - my $scantimerend=time(); - my $numrecords; - ($numresults<80) ? ($numrecords=$numresults) : ($numrecords=80); - my $elapsed=$scantimerend-$scantimerstart; - if ($elapsed) { - my $speed=int($numresults/$elapsed*100)/100; - print " SPEED: $speed $server done $numrecords\n"; - } + eval { $rs->option(preferredRecordSyntax => Net::Z3950::RecordSyntax::USMARC);}; + if ($@) { + print "ERROR: $@\n"; + } else { + pe(); + my $numresults=$rs->size(); + pe(); + my $i; + my $result=''; + my $scantimerstart=time(); + for ($i=1; $i<=(($numresults<80) ? ($numresults) : (80)); $i++) { + my $rec=$rs->record($i); + my $marcdata=$rec->rawdata(); + $result.=$marcdata; + } + my $scantimerend=time(); + my $numrecords; + ($numresults<80) ? ($numrecords=$numresults) : ($numrecords=80); + my $elapsed=$scantimerend-$scantimerstart; + if ($elapsed) { + my $speed=int($numresults/$elapsed*100)/100; + print " SPEED: $speed $server done $numrecords\n"; + } - my $q_result=$dbi->quote($result); - ($q_result) || ($q_result='""'); - $now=time(); - my $task="update z3950results set numrecords=$numresults,numdownloaded=$numrecords,highestseen=0,results=$q_result,enddate=$now where id=$resultsid"; - my $stj=$dbi->prepare($task); - $stj->execute; - my $counter=0; - while ($counter<60 && $numrecords<$numresults) { - $counter++; - my $stj=$dbi->prepare("select highestseen from z3950results where id=$resultsid"); + my $q_result=$dbi->quote($result); + ($q_result) || ($q_result='""'); + $now=time(); + my $task="update z3950results set numrecords=$numresults,numdownloaded=$numrecords,highestseen=0,results=$q_result,enddate=$now where id=$resultsid"; + my $stj=$dbi->prepare($task); $stj->execute; - my ($highestseen) = $stj->fetchrow; - if ($highestseen>($numrecords-30)) { - $counter=0; - print " $server rescanning\n"; - my $scantimerstart=time(); - for ($i=$numrecords+1; $i<=(($numresults<($numrecords+40)) ? ($numresults) : ($numrecords+40)); $i++) { - my $rec=$rs->record($i); - my $marcdata=$rec->rawdata(); - $result.=$marcdata; - } - my $scantimerend=time(); - ($numresults<$numrecords+40) ? ($numrecords=$numresults) : ($numrecords=$numrecords+40); - my $elapsed=$scantimerend-$scantimerstart; - if ($elapsed) { - my $speed=int($numresults/$elapsed*100)/100; - print " SPEED: $speed $server done $numrecords\n"; - } - - my $q_result=$dbi->quote($result); - ($q_result) || ($q_result='""'); - $now=time(); - my $task="update z3950results set numdownloaded=$numrecords,results=$q_result where id=$resultsid"; - my $stj=$dbi->prepare($task); + my $counter=0; + while ($counter<60 && $numrecords<$numresults) { + $counter++; + my $stj=$dbi->prepare("select highestseen from z3950results where id=$resultsid"); $stj->execute; + my ($highestseen) = $stj->fetchrow; + if ($highestseen>($numrecords-30)) { + $counter=0; + print " $server rescanning\n"; + my $scantimerstart=time(); + for ($i=$numrecords+1; $i<=(($numresults<($numrecords+40)) ? ($numresults) : ($numrecords+40)); $i++) { + my $rec=$rs->record($i); + my $marcdata=$rec->rawdata(); + $result.=$marcdata; + } + my $scantimerend=time(); + ($numresults<$numrecords+40) ? ($numrecords=$numresults) : ($numrecords=$numrecords+40); + my $elapsed=$scantimerend-$scantimerstart; + if ($elapsed) { + my $speed=int($numresults/$elapsed*100)/100; + print " SPEED: $speed $server done $numrecords\n"; + } + + my $q_result=$dbi->quote($result); + ($q_result) || ($q_result='""'); + $now=time(); + my $task="update z3950results set numdownloaded=$numrecords,results=$q_result where id=$resultsid"; + my $stj=$dbi->prepare($task); + $stj->execute; + } + sleep 5; } - sleep 5; } } - } - my $stj=$dbi->prepare("update z3950results set active=0 where id=$resultsid"); - $stj->execute; - eval {$stj->finish}; - $dbi->disconnect; - print " $server done.\n"; - exit; - sub pe { - return 0; - my $code=$conn->errcode(); - my $msg=$conn->errmsg(); - my $ai=$conn->addinfo(); - print << "EOF"; - CODE: $code - MSG: $msg - ADDTL: $ai + my $stj=$dbi->prepare("update z3950results set active=0 where id=$resultsid"); + $stj->execute; + eval {$stj->finish}; + $dbi->disconnect; + print " $server done.\n"; + exit; + sub pe { + return 0; + my $code=$conn->errcode(); + my $msg=$conn->errmsg(); + my $ai=$conn->addinfo(); + print << "EOF"; +CODE: $code +MSG: $msg +ADDTL: $ai EOF - # if ($msg =~/not yet available/) { - # return 1; - # } - return 0; + return 0; + } } + } unless ($stillprocessing) { + #my $sti=$dbh->prepare("select enddate from z3950queue where id=$id"); + #$sti->execute; + #my ($enddate) = $sti->fetchrow; + #unless ($enddate) { } - } unless ($stillprocessing) { - #my $sti=$dbh->prepare("select enddate from z3950queue where id=$id"); - #$sti->execute; - #my ($enddate) = $sti->fetchrow; - #unless ($enddate) { - # my $now=time; -# $sti=$dbh->prepare("update z3950queue set done=1,numrecords=$totalrecords,enddate=$now where id=$id"); -# $sti->execute; -# } + } else { } - } else { -# my $q_serverinfo=$dbh->quote($serverinfo); -# my $stj=$dbh->prepare("insert into z3950results (server, queryid, startdate) values ($q_serverinfo, $id, 0)"); -# $stj->execute; } + $lastrun=time(); } - $lastrun=time(); + sleep 10; } - sleep 10; } sub getrecord { @@ -260,4 +261,8 @@ sub reap { } +sub checkqueue { + $checkqueue=1; +} + diff --git a/acqui.simple/z3950-daemon-shell.sh b/acqui.simple/z3950-daemon-shell.sh index 1e5fd6878c..831f61703a 100755 --- a/acqui.simple/z3950-daemon-shell.sh +++ b/acqui.simple/z3950-daemon-shell.sh @@ -37,10 +37,14 @@ fi PERL5LIB=$KohaModuleDir export PERL5LIB -exec $KohaZ3950Script >>$LOGFILE 2>&1 +exec $KohaZ3950Script $LogDir >>$LOGFILE 2>&1 #------------------- # $Log$ +# Revision 1.1.2.3 2002/06/28 17:45:39 tonnesen +# z3950queue now listens for a -HUP signal before processing the queue. Z3950.pm +# sends the -HUP signal when queries are added to the queue. +# # Revision 1.1.2.2 2002/06/26 16:25:51 amillar # Make directory variable name more explanatory # -- 2.39.5