From 219e006412be8ffbf6bd27773f6a49311ec689c8 Mon Sep 17 00:00:00 2001 From: tonnesen Date: Tue, 2 Jul 2002 22:08:50 +0000 Subject: [PATCH] merging changes from rel-1-2 --- acqui.simple/processz3950queue | 385 +++++++++++++++------------- acqui.simple/z3950-daemon-launch.sh | 29 ++- acqui.simple/z3950-daemon-shell.sh | 23 +- 3 files changed, 248 insertions(+), 189 deletions(-) diff --git a/acqui.simple/processz3950queue b/acqui.simple/processz3950queue index 2a6223f107..68cc91bd89 100755 --- a/acqui.simple/processz3950queue +++ b/acqui.simple/processz3950queue @@ -6,222 +6,244 @@ use C4::Acquisitions; use C4::Biblio; use C4::Output; use Net::Z3950; + + +if ($< == 0) { + # Running as root, switch privs + if (-d "/var/run") { + open PID, ">/var/run/processz3950queue.pid"; + print PID $$."\n"; + close PID; + } + # Get real apacheuser from koha.conf or reparsing httpd.conf + my $apacheuser='www-data'; + my $uid=0; + unless ($uid = (getpwnam($apacheuser))[2]) { + die "Attempt to run daemon as non-existent or superuser\n"; + } + $>=$uid; + $<=$uid; +} + + my $dbh=C4Connect; my $sth=$dbh->prepare("update z3950results set active=0"); $sth->execute; $sth->finish; $SIG{CHLD}='reap'; +$SIG{HUP}='checkqueue'; + + +my $logdir=$ARGV[0]; + +open PID, ">$logdir/processz3950queue.pid"; +print PID $$."\n"; +close PID; my $reapcounter=0; my $forkcounter=0; +my $checkqueue=1; my $pid=$$; my $lastrun=0; while (1) { if ((time-$lastrun)>5) { - my $sth=$dbh->prepare("select id,term,type,servers from z3950queue order by id"); - $sth->execute; - while (my ($id, $term, $type, $servers) = $sth->fetchrow) { - if ($forkcounter<12) { - my $now=time(); - $stk=$dbh->prepare("select id,server,startdate,enddate,numrecords,active from z3950results where queryid=$id"); - ($stk->execute) || (next); - my %serverdone; - unless ($stk->rows) { - my $sti=$dbh->prepare("update z3950queue set done=-1,startdate=$now where id=$id"); - $sti->execute; - } - while (my ($r_id, $r_server,$r_startdate,$r_enddate,$r_numrecords,$active) = $stk->fetchrow) { - if ($r_enddate >0) { - $serverdone{$r_server}=1; - } elsif ($active) { - $serverdone{$r_server}=1; - } else { - $serverdone{$r_server}=-1; + if ($checkqueue) { + $checkqueue=0; + my $sth=$dbh->prepare("select id,term,type,servers from z3950queue order by id"); + $sth->execute; + while (my ($id, $term, $type, $servers) = $sth->fetchrow) { + if ($forkcounter<12) { + my $now=time(); + $stk=$dbh->prepare("select id,server,startdate,enddate,numrecords,active from z3950results where queryid=$id"); + ($stk->execute) || (next); + my %serverdone; + unless ($stk->rows) { + my $sti=$dbh->prepare("update z3950queue set done=-1,startdate=$now where id=$id"); + $sti->execute; + } + while (my ($r_id, $r_server,$r_startdate,$r_enddate,$r_numrecords,$active) = $stk->fetchrow) { + if ($r_enddate >0) { + $serverdone{$r_server}=1; + } elsif ($active) { + $serverdone{$r_server}=1; + } else { + $serverdone{$r_server}=-1; + } } - } - $stk->finish; - my $attr=''; - if ($type eq 'isbn') { - $attr='1=7'; - } elsif ($type eq 'title') { - $attr='1=4'; - } elsif ($type eq 'author') { - $attr='1=1003'; - } elsif ($type eq 'lccn') { - $attr='1=9'; - } elsif ($type eq 'keyword') { - $attr='1=1016'; - } - $term='"'.$term.'"'; - $query="\@attr $attr $term"; - my $totalrecords=0; - my $serverinfo; - my $stillprocessing=0; - foreach $serverinfo (split(/\s+/, $servers)) { - (next) if ($serverdone{$serverinfo} == 1); - my $stillprocessing=1; - if (my $pid=fork()) { - $forkcounter++; - } else { - #$sth->finish; - #$sti->finish; - #$dbh->disconnect; - my $dbi=C4Connect; - my ($name, $server, $database, $user, $password) = split(/\//, $serverinfo, 5); - $server=~/(.*)\:(\d+)/; - my $servername=$1; - my $port=$2; - print "Processing $type=$term at $name $server $database (".($forkcounter+1)." forks)\n"; - $now=time(); - my $q_serverinfo=$dbi->quote($serverinfo); - my $resultsid; - if ($serverdone{$serverinfo}==-1) { - my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id"); - $stj->execute; - ($resultsid) = $stj->fetchrow; + $stk->finish; + my $attr=''; + if ($type eq 'isbn') { + $attr='1=7'; + } elsif ($type eq 'title') { + $attr='1=4'; + } elsif ($type eq 'author') { + $attr='1=1003'; + } elsif ($type eq 'lccn') { + $attr='1=9'; + } elsif ($type eq 'keyword') { + $attr='1=1016'; + } + $term='"'.$term.'"'; + $query="\@attr $attr $term"; + my $totalrecords=0; + my $serverinfo; + my $stillprocessing=0; + foreach $serverinfo (split(/\s+/, $servers)) { + (next) if ($serverdone{$serverinfo} == 1); + my $stillprocessing=1; + if (my $pid=fork()) { + $forkcounter++; } else { - my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id"); - $stj->execute; - ($resultsid) = $stj->fetchrow; - unless ($resultsid) { - my $stj=$dbi->prepare("insert into z3950results (server, queryid, startdate) values ($q_serverinfo, $id, $now)"); + #$sth->finish; + #$sti->finish; + #$dbh->disconnect; + my $dbi=C4Connect; + my ($name, $server, $database, $user, $password) = split(/\//, $serverinfo, 5); + $server=~/(.*)\:(\d+)/; + my $servername=$1; + my $port=$2; + print "Processing $type=$term at $name $server $database (".($forkcounter+1)." forks)\n"; + $now=time(); + my $q_serverinfo=$dbi->quote($serverinfo); + my $resultsid; + if ($serverdone{$serverinfo}==-1) { + my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id"); $stj->execute; - $resultsid=$dbi->{'mysql_insertid'}; - } - } - my $stj=$dbh->prepare("update z3950results set active=1 where id=$resultsid"); - $stj->execute; - my $conn; - my $noconnection=0; - my $error=0; - if ($user) { - eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database, user => $user, password => $password); }; - if ($@) { - $noconnection=1; + ($resultsid) = $stj->fetchrow; } else { - $error=pe(); + my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id"); + $stj->execute; + ($resultsid) = $stj->fetchrow; + unless ($resultsid) { + my $stj=$dbi->prepare("insert into z3950results (server, queryid, startdate) values ($q_serverinfo, $id, $now)"); + $stj->execute; + $resultsid=$dbi->{'mysql_insertid'}; + } } - } else { - eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database); }; - if ($@) { - $noconnection=1; + my $stj=$dbh->prepare("update z3950results set active=1 where id=$resultsid"); + $stj->execute; + my $conn; + my $noconnection=0; + my $error=0; + if ($user) { + eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database, user => $user, password => $password); }; + if ($@) { + $noconnection=1; + } else { + $error=pe(); + } } else { - $error=pe(); + eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database); }; + if ($@) { + $noconnection=1; + } else { + $error=pe(); + } } - } - if ($noconnection || $error) { - } else { - print "Q: $query\n"; - my $rs=$conn->search($query); - pe(); - eval { $rs->option(preferredRecordSyntax => Net::Z3950::RecordSyntax::USMARC);}; - if ($@) { - print "ERROR: $@\n"; + if ($noconnection || $error) { } else { + print "Q: $query\n"; + my $rs=$conn->search($query); pe(); - my $numresults=$rs->size(); - pe(); - my $i; - my $result=''; - my $scantimerstart=time(); - for ($i=1; $i<=(($numresults<80) ? ($numresults) : (80)); $i++) { - my $rec=$rs->record($i); - my $marcdata=$rec->rawdata(); - $result.=$marcdata; - } - my $scantimerend=time(); - my $numrecords; - ($numresults<80) ? ($numrecords=$numresults) : ($numrecords=80); - my $elapsed=$scantimerend-$scantimerstart; - if ($elapsed) { - my $speed=int($numresults/$elapsed*100)/100; - print " SPEED: $speed $server done $numrecords\n"; - } + eval { $rs->option(preferredRecordSyntax => Net::Z3950::RecordSyntax::USMARC);}; + if ($@) { + print "ERROR: $@\n"; + } else { + pe(); + my $numresults=$rs->size(); + pe(); + my $i; + my $result=''; + my $scantimerstart=time(); + for ($i=1; $i<=(($numresults<80) ? ($numresults) : (80)); $i++) { + my $rec=$rs->record($i); + my $marcdata=$rec->rawdata(); + $result.=$marcdata; + } + my $scantimerend=time(); + my $numrecords; + ($numresults<80) ? ($numrecords=$numresults) : ($numrecords=80); + my $elapsed=$scantimerend-$scantimerstart; + if ($elapsed) { + my $speed=int($numresults/$elapsed*100)/100; + print " SPEED: $speed $server done $numrecords\n"; + } - my $q_result=$dbi->quote($result); - ($q_result) || ($q_result='""'); - $now=time(); - my $task="update z3950results set numrecords=$numresults,numdownloaded=$numrecords,highestseen=0,results=$q_result,enddate=$now where id=$resultsid"; - my $stj=$dbi->prepare($task); - $stj->execute; - my $counter=0; - while ($counter<60 && $numrecords<$numresults) { - $counter++; - my $stj=$dbi->prepare("select highestseen from z3950results where id=$resultsid"); + my $q_result=$dbi->quote($result); + ($q_result) || ($q_result='""'); + $now=time(); + my $task="update z3950results set numrecords=$numresults,numdownloaded=$numrecords,highestseen=0,results=$q_result,enddate=$now where id=$resultsid"; + my $stj=$dbi->prepare($task); $stj->execute; - my ($highestseen) = $stj->fetchrow; - if ($highestseen>($numrecords-30)) { - $counter=0; - print " $server rescanning\n"; - my $scantimerstart=time(); - for ($i=$numrecords+1; $i<=(($numresults<($numrecords+40)) ? ($numresults) : ($numrecords+40)); $i++) { - my $rec=$rs->record($i); - my $marcdata=$rec->rawdata(); - $result.=$marcdata; - } - my $scantimerend=time(); - ($numresults<$numrecords+40) ? ($numrecords=$numresults) : ($numrecords=$numrecords+40); - my $elapsed=$scantimerend-$scantimerstart; - if ($elapsed) { - my $speed=int($numresults/$elapsed*100)/100; - print " SPEED: $speed $server done $numrecords\n"; - } - - my $q_result=$dbi->quote($result); - ($q_result) || ($q_result='""'); - $now=time(); - my $task="update z3950results set numdownloaded=$numrecords,results=$q_result where id=$resultsid"; - my $stj=$dbi->prepare($task); + my $counter=0; + while ($counter<60 && $numrecords<$numresults) { + $counter++; + my $stj=$dbi->prepare("select highestseen from z3950results where id=$resultsid"); $stj->execute; + my ($highestseen) = $stj->fetchrow; + if ($highestseen>($numrecords-30)) { + $counter=0; + print " $server rescanning\n"; + my $scantimerstart=time(); + for ($i=$numrecords+1; $i<=(($numresults<($numrecords+40)) ? ($numresults) : ($numrecords+40)); $i++) { + my $rec=$rs->record($i); + my $marcdata=$rec->rawdata(); + $result.=$marcdata; + } + my $scantimerend=time(); + ($numresults<$numrecords+40) ? ($numrecords=$numresults) : ($numrecords=$numrecords+40); + my $elapsed=$scantimerend-$scantimerstart; + if ($elapsed) { + my $speed=int($numresults/$elapsed*100)/100; + print " SPEED: $speed $server done $numrecords\n"; + } + + my $q_result=$dbi->quote($result); + ($q_result) || ($q_result='""'); + $now=time(); + my $task="update z3950results set numdownloaded=$numrecords,results=$q_result where id=$resultsid"; + my $stj=$dbi->prepare($task); + $stj->execute; + } + sleep 5; } - sleep 5; } } - } - my $stj=$dbi->prepare("update z3950results set active=0 where id=$resultsid"); - $stj->execute; - eval {$stj->finish}; - $dbi->disconnect; - print " $server done.\n"; - exit; - sub pe { - return 0; - my $code=$conn->errcode(); - my $msg=$conn->errmsg(); - my $ai=$conn->addinfo(); - print << "EOF"; - CODE: $code - MSG: $msg - ADDTL: $ai + my $stj=$dbi->prepare("update z3950results set active=0 where id=$resultsid"); + $stj->execute; + eval {$stj->finish}; + $dbi->disconnect; + print " $server done.\n"; + exit; + sub pe { + return 0; + my $code=$conn->errcode(); + my $msg=$conn->errmsg(); + my $ai=$conn->addinfo(); + print << "EOF"; +CODE: $code +MSG: $msg +ADDTL: $ai EOF - # if ($msg =~/not yet available/) { - # return 1; - # } - return 0; + return 0; + } } + } unless ($stillprocessing) { + #my $sti=$dbh->prepare("select enddate from z3950queue where id=$id"); + #$sti->execute; + #my ($enddate) = $sti->fetchrow; + #unless ($enddate) { } - } unless ($stillprocessing) { - #my $sti=$dbh->prepare("select enddate from z3950queue where id=$id"); - #$sti->execute; - #my ($enddate) = $sti->fetchrow; - #unless ($enddate) { - # my $now=time; -# $sti=$dbh->prepare("update z3950queue set done=1,numrecords=$totalrecords,enddate=$now where id=$id"); -# $sti->execute; -# } + } else { } - } else { -# my $q_serverinfo=$dbh->quote($serverinfo); -# my $stj=$dbh->prepare("insert into z3950results (server, queryid, startdate) values ($q_serverinfo, $id, 0)"); -# $stj->execute; } + $lastrun=time(); } - $lastrun=time(); + sleep 10; } - sleep 10; } sub getrecord { @@ -256,7 +278,12 @@ EOF } sub reap { $forkcounter--; + wait; } +sub checkqueue { + $checkqueue=1; +} + diff --git a/acqui.simple/z3950-daemon-launch.sh b/acqui.simple/z3950-daemon-launch.sh index 0e6fab8672..495c837826 100755 --- a/acqui.simple/z3950-daemon-launch.sh +++ b/acqui.simple/z3950-daemon-launch.sh @@ -1,5 +1,7 @@ #!/bin/sh +# $Id$ + # Script to start Koha background Z39.50 search daemon # Part of the Koha Library Mgmt System - www.koha.org @@ -15,17 +17,17 @@ # User ID to run the daemon as. Don't use "root" RunAsUser=apache -KohaDir=/usr/local/www/koha/htdocs/cgi-bin/koha/acqui.simple -export KohaDir +KohaZ3950Dir=/usr/local/www/koha/htdocs/cgi-bin/koha/acqui.simple +export KohaZ3950Dir #---------------------------- -if [ ! -d $KohaDir ] +if [ ! -d $KohaZ3950Dir ] then - echo ERROR: Cannot find Koha directory $KohaDir + echo ERROR: Cannot find Koha directory $KohaZ3950Dir exit 1 fi -KohaZ3950Shell=$KohaDir/z3950-daemon-shell.sh +KohaZ3950Shell=$KohaZ3950Dir/z3950-daemon-shell.sh if [ ! -x $KohaZ3950Shell ] then @@ -33,4 +35,19 @@ then exit 1 fi -su -s /bin/sh -c '$KohaZ3950Shell &' - $RunAsUser & +su -s /bin/sh -c $KohaZ3950Shell - $RunAsUser & + +exit + +#-------------- +# $Log$ +# Revision 1.3 2002/07/02 22:08:50 tonnesen +# merging changes from rel-1-2 +# +# Revision 1.1.2.3 2002/06/26 19:56:57 tonnesen +# Bug fix. Single quotes were causing $KohaZ3950Shell variable to not get +# expanded +# +# Revision 1.1.2.2 2002/06/26 16:25:51 amillar +# Make directory variable name more explanatory +# diff --git a/acqui.simple/z3950-daemon-shell.sh b/acqui.simple/z3950-daemon-shell.sh index b51710c1a3..f831e60ebc 100755 --- a/acqui.simple/z3950-daemon-shell.sh +++ b/acqui.simple/z3950-daemon-shell.sh @@ -1,17 +1,19 @@ #!/bin/sh +# $Id$ + # Script to start Koha background Z39.50 search daemon # Part of the Koha Library Mgmt System - www.koha.org # Licensed under the GPL #---------------------------- -# Do NOT run this script directly from system startup- +# Do NOT run this script directly from system startup-- this should not run as root # Call z3950-daemon-launch.sh instead #---------------------------- -KohaDir=/usr/local/www/koha/htdocs/cgi-bin/koha/acqui.simple +KohaZ3950Dir=/usr/local/www/koha/htdocs/cgi-bin/koha/acqui.simple KohaModuleDir=/usr/local/koha/modules LogDir=/var/log/koha @@ -25,7 +27,7 @@ then exit 1 fi -KohaZ3950Script=$KohaDir/processz3950queue +KohaZ3950Script=$KohaZ3950Dir/processz3950queue if [ ! -x $KohaZ3950Script ] then echo ERROR: Cannot find Koha Z39.50 daemon script $KohaZ3950Script @@ -35,4 +37,17 @@ fi PERL5LIB=$KohaModuleDir export PERL5LIB -exec $KohaDir/processz3950queue >>$LOGFILE 2>&1 +exec $KohaZ3950Script $LogDir >>$LOGFILE 2>&1 + +#------------------- +# $Log$ +# Revision 1.3 2002/07/02 22:08:50 tonnesen +# merging changes from rel-1-2 +# +# Revision 1.1.2.3 2002/06/28 17:45:39 tonnesen +# z3950queue now listens for a -HUP signal before processing the queue. Z3950.pm +# sends the -HUP signal when queries are added to the queue. +# +# Revision 1.1.2.2 2002/06/26 16:25:51 amillar +# Make directory variable name more explanatory +# -- 2.39.5