c1fcb1b4e9
to do Z39.50 searches. Daemon will also wait to see if the user looks beyond the first couple of pages of results, and will download more results if necessary.
244 lines
6.6 KiB
Perl
Executable file
244 lines
6.6 KiB
Perl
Executable file
#!/usr/bin/perl
|
|
use C4::Database;
|
|
use DBI;
|
|
#use strict;
|
|
use C4::Acquisitions;
|
|
use C4::Output;
|
|
use Net::Z3950;
|
|
my $dbh=C4Connect;
|
|
|
|
my $sth=$dbh->prepare("update z3950results set active=0");
|
|
$sth->execute;
|
|
$sth->finish;
|
|
$SIG{CHLD}='reap';
|
|
|
|
my $reapcounter=0;
|
|
my $forkcounter=0;
|
|
my $pid=$$;
|
|
my $lastrun=0;
|
|
while (1) {
|
|
if ((time-$lastrun)>5) {
|
|
my $sth=$dbh->prepare("select id,term,type,servers from z3950queue order by id");
|
|
$sth->execute;
|
|
while (my ($id, $term, $type, $servers) = $sth->fetchrow) {
|
|
if ($forkcounter<12) {
|
|
my $now=time();
|
|
$stk=$dbh->prepare("select id,server,startdate,enddate,numrecords,active from z3950results where queryid=$id");
|
|
$stk->execute;
|
|
my %serverdone;
|
|
unless ($stk->rows) {
|
|
my $sti=$dbh->prepare("update z3950queue set done=-1,startdate=$now where id=$id");
|
|
$sti->execute;
|
|
}
|
|
while (my ($r_id, $r_server,$r_startdate,$r_enddate,$r_numrecords,$active) = $stk->fetchrow) {
|
|
if ($r_enddate >0) {
|
|
$serverdone{$r_server}=1;
|
|
} elsif ($active) {
|
|
$serverdone{$r_server}=1;
|
|
} else {
|
|
$serverdone{$r_server}=-1;
|
|
}
|
|
}
|
|
|
|
$stk->finish;
|
|
my $attr='';
|
|
if ($type eq 'isbn') {
|
|
$attr='1=7';
|
|
} elsif ($type eq 'title') {
|
|
$attr='1=4';
|
|
} elsif ($type eq 'author') {
|
|
$attr='1=1003';
|
|
} elsif ($type eq 'lccn') {
|
|
$attr='1=9';
|
|
} elsif ($type eq 'keyword') {
|
|
$attr='1=1016';
|
|
}
|
|
$term='"'.$term.'"';
|
|
$query="\@attr $attr $term";
|
|
my $totalrecords=0;
|
|
my $serverinfo;
|
|
my $stillprocessing=0;
|
|
foreach $serverinfo (split(/\s+/, $servers)) {
|
|
(next) if ($serverdone{$serverinfo} == 1);
|
|
my $stillprocessing=1;
|
|
if (my $pid=fork()) {
|
|
$forkcounter++;
|
|
} else {
|
|
#$sth->finish;
|
|
#$sti->finish;
|
|
#$dbh->disconnect;
|
|
my $dbi=C4Connect;
|
|
my ($name, $server, $database, $user, $password) = split(/\//, $serverinfo, 5);
|
|
$server=~/(.*)\:(\d+)/;
|
|
my $servername=$1;
|
|
my $port=$2;
|
|
print "Processing $type=$term at $name $server $database (".($forkcounter+1)." forks)\n";
|
|
$now=time();
|
|
my $q_serverinfo=$dbi->quote($serverinfo);
|
|
my $resultsid;
|
|
if ($serverdone{$serverinfo}==-1) {
|
|
my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id");
|
|
$stj->execute;
|
|
($resultsid) = $stj->fetchrow;
|
|
} else {
|
|
my $stj=$dbi->prepare("insert into z3950results (server, queryid, startdate) values ($q_serverinfo, $id, $now)");
|
|
$stj->execute;
|
|
$resultsid=$dbi->{'mysql_insertid'};
|
|
}
|
|
my $stj=$dbh->prepare("update z3950results set active=1 where id=$resultsid");
|
|
$stj->execute;
|
|
my $conn;
|
|
my $noconnection=0;
|
|
if ($user) {
|
|
eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database, user => $user, password => $password); };
|
|
if ($@) {
|
|
$noconnection=1;
|
|
}
|
|
pe();
|
|
} else {
|
|
eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database); };
|
|
if ($@) {
|
|
$noconnection=1;
|
|
}
|
|
pe();
|
|
}
|
|
if ($noconnection) {
|
|
} else {
|
|
my $rs=$conn->search($query);
|
|
pe();
|
|
$rs->option(preferredRecordSyntax => Net::Z3950::RecordSyntax::USMARC);
|
|
pe();
|
|
my $numresults=$rs->size();
|
|
pe();
|
|
my $i;
|
|
my $result='';
|
|
my $scantimerstart=time();
|
|
for ($i=1; $i<=(($numresults<80) ? ($numresults) : (80)); $i++) {
|
|
my $rec=$rs->record($i);
|
|
my $marcdata=$rec->rawdata();
|
|
$result.=$marcdata;
|
|
}
|
|
my $scantimerend=time();
|
|
my $numrecords;
|
|
($numresults<80) ? ($numrecords=$numresults) : ($numrecords=80);
|
|
my $elapsed=$scantimerend-$scantimerstart;
|
|
if ($elapsed) {
|
|
my $speed=int($numresults/$elapsed*100)/100;
|
|
print " SPEED: $speed $server done $numrecords\n";
|
|
}
|
|
|
|
my $q_result=$dbi->quote($result);
|
|
($q_result) || ($q_result='""');
|
|
$now=time();
|
|
my $task="update z3950results set numrecords=$numresults,numdownloaded=$numrecords,highestseen=0,results=$q_result,enddate=$now where id=$resultsid";
|
|
my $stj=$dbi->prepare($task);
|
|
$stj->execute;
|
|
my $counter=0;
|
|
while ($counter<60 && $numrecords<$numresults) {
|
|
$counter++;
|
|
my $stj=$dbi->prepare("select highestseen from z3950results where id=$resultsid");
|
|
$stj->execute;
|
|
my ($highestseen) = $stj->fetchrow;
|
|
if ($highestseen>($numrecords-30)) {
|
|
$counter=0;
|
|
print " $server rescanning\n";
|
|
my $scantimerstart=time();
|
|
for ($i=$numrecords+1; $i<=(($numresults<($numrecords+40)) ? ($numresults) : ($numrecords+40)); $i++) {
|
|
my $rec=$rs->record($i);
|
|
my $marcdata=$rec->rawdata();
|
|
$result.=$marcdata;
|
|
}
|
|
my $scantimerend=time();
|
|
($numresults<$numrecords+40) ? ($numrecords=$numresults) : ($numrecords=$numrecords+40);
|
|
my $elapsed=$scantimerend-$scantimerstart;
|
|
if ($elapsed) {
|
|
my $speed=int($numresults/$elapsed*100)/100;
|
|
print " SPEED: $speed $server done $numrecords\n";
|
|
}
|
|
|
|
my $q_result=$dbi->quote($result);
|
|
($q_result) || ($q_result='""');
|
|
$now=time();
|
|
my $task="update z3950results set numdownloaded=$numrecords,results=$q_result where id=$resultsid";
|
|
my $stj=$dbi->prepare($task);
|
|
$stj->execute;
|
|
}
|
|
sleep 5;
|
|
}
|
|
}
|
|
my $stj=$dbi->prepare("update z3950results set active=0 where id=$resultsid");
|
|
$stj->execute;
|
|
eval {$stj->finish};
|
|
$dbi->disconnect;
|
|
print " $server done.\n";
|
|
exit;
|
|
sub pe {
|
|
(return) unless ($code);
|
|
my $code=$conn->errcode();
|
|
my $msg=$conn->errmsg();
|
|
my $ai=$conn->addinfo();
|
|
print << "EOF";
|
|
CODE: $code
|
|
MSG: $msg
|
|
ADDTL: $ai
|
|
|
|
EOF
|
|
}
|
|
}
|
|
} unless ($stillprocessing) {
|
|
#my $sti=$dbh->prepare("select enddate from z3950queue where id=$id");
|
|
#$sti->execute;
|
|
#my ($enddate) = $sti->fetchrow;
|
|
#unless ($enddate) {
|
|
# my $now=time;
|
|
# $sti=$dbh->prepare("update z3950queue set done=1,numrecords=$totalrecords,enddate=$now where id=$id");
|
|
# $sti->execute;
|
|
# }
|
|
}
|
|
} else {
|
|
# my $q_serverinfo=$dbh->quote($serverinfo);
|
|
# my $stj=$dbh->prepare("insert into z3950results (server, queryid, startdate) values ($q_serverinfo, $id, 0)");
|
|
# $stj->execute;
|
|
}
|
|
}
|
|
$lastrun=time();
|
|
}
|
|
sleep 1;
|
|
}
|
|
|
|
sub getrecord {
|
|
my $server=shift;
|
|
my $base=shift;
|
|
my $query=shift;
|
|
my $auth=shift;
|
|
my $id=shift;
|
|
open (M, "|yaz-client -m yaz-$id.mrc >>yaz.out 2>>yaz.err");
|
|
select M;
|
|
$|=1;
|
|
select STDOUT;
|
|
($auth) && ($auth="authentication $auth\n");
|
|
print M << "EOF";
|
|
$auth\open $server
|
|
base $base
|
|
setnames
|
|
$query
|
|
s
|
|
s
|
|
s
|
|
s
|
|
s
|
|
s
|
|
s
|
|
s
|
|
s
|
|
s
|
|
quit
|
|
EOF
|
|
close M;
|
|
}
|
|
sub reap {
|
|
$forkcounter--;
|
|
}
|
|
|
|
|
|
|