z3950/processz3950queue

   1 #!/usr/bin/perl
   2
   3 # $Id$
   4
   5 use C4::Context;
   6 use DBI;
   7 use strict;
   8 use C4::Biblio;
   9 use C4::Output;
  10 use C4::Breeding;
  11 use Net::Z3950;
  12
  13
  14 if ($< == 0) {
  15     # Running as root, switch privs
  16     if (-d "/var/run") {
  17         open PID, ">/var/run/processz3950queue.pid";
  18         print PID $$."\n";
  19         close PID;
  20     }
  21     # Get real apacheuser from koha.conf or reparsing httpd.conf
  22     my $apacheuser='paul';
  23     my $uid=0;
  24     unless ($uid = (getpwnam($apacheuser))[2]) {
  25         die "Attempt to run daemon as non-existent or superuser\n";
  26     }
  27     $>=$uid;
  28     $<=$uid;
  29 }
  30 my $dbh = C4::Context->dbh;
  31
  32 my $sth=$dbh->prepare("update z3950results set active=0");
  33 $sth->execute;
  34 $sth->finish;
  35 $SIG{CHLD}='reap';
  36 $SIG{HUP}='checkqueue';
  37
  38
  39 my $logdir=$ARGV[0];
  40
  41 open PID, ">$logdir/processz3950queue.pid";
  42 print PID $$."\n";
  43 close PID;
  44
  45 my $reapcounter=0;
  46 my $forkcounter=0;
  47 my $checkqueue=1;
  48 my $pid=$$;
  49 my $lastrun=0;
  50 while (1) {
  51         if ((time-$lastrun)>5) {
  52                 print "starting loop\n";
  53                 $checkqueue = 1; # FIXME during testing, this line forces the loop. REMOVE it to use SIG{HUP} when "daemonized" !
  54                 if ($checkqueue) { # everytime a SIG{HUP} is recieved
  55                         $checkqueue=0;
  56                         my $sth=$dbh->prepare("select id,term,type,servers,identifier from z3950queue order by id");
  57                         $sth->execute;
  58                         while (my ($id, $term, $type, $servers,$random) = $sth->fetchrow) {
  59                                 if ($forkcounter<12) {
  60                                         my $now=time();
  61                                         my $stk=$dbh->prepare("select id,server,startdate,enddate,numrecords,active from z3950results where queryid=$id");
  62                                         ($stk->execute) || (next);
  63                                         my %serverdone;
  64                                         unless ($stk->rows) {
  65                                                 my $sti=$dbh->prepare("update z3950queue set done=-1,startdate=$now where id=$id");
  66                                                 $sti->execute;
  67                                         }
  68                                         while (my ($r_id, $r_server,$r_startdate,$r_enddate,$r_numrecords,$active) = $stk->fetchrow) {
  69                                                 if ($r_enddate >0) {
  70                                                         $serverdone{$r_server}=1;
  71                                                 } elsif ($active) {
  72                                                         $serverdone{$r_server}=1;
  73                                                 } else {
  74                                                         $serverdone{$r_server}=-1;
  75                                                 }
  76                                         }
  77
  78                                         $stk->finish;
  79                                         my $attr='';
  80                                         if ($type eq 'isbn') {
  81                                                 $attr='1=7';
  82                                         } elsif ($type eq 'title') {
  83                                                 $attr='1=4';
  84                                         } elsif ($type eq 'author') {
  85                                                 $attr='1=1003';
  86                                         } elsif ($type eq 'lccn') {
  87                                                 $attr='1=9';
  88                                         } elsif ($type eq 'keyword') {
  89                                                 $attr='1=1016';
  90                                         }
  91                                         $term='"'.$term.'"';
  92                                         my $query="\@attr $attr $term";
  93                                         my $totalrecords=0;
  94                                         my $serverinfo;
  95                                         my $stillprocessing=0;
  96                                         my $globalname;
  97                                         my $globalsyntax;
  98                                         my $globalencoding;
  99                                         foreach $serverinfo (split(/\s+/, $servers)) {
 100                                                 (next) if ($serverdone{$serverinfo} == 1);
 101                                                 my $stillprocessing=1;
 102                                                 if (my $pid=fork()) {
 103                                                         $forkcounter++;
 104                                                 } else {
 105                                                         my $dbi = C4::Context->dbh;
 106                                                         my ($name, $server, $database, $user, $password,$syntax) = split(/\//, $serverinfo, 6);
 107                                                         $globalname=$name;
 108                                                         $globalsyntax = $syntax;
 109                                                         $server=~/(.*)\:(\d+)/;
 110                                                         my $servername=$1;
 111                                                         my $port=$2;
 112                                                         print "Processing $type=$term at $name $server $database $syntax (".($forkcounter+1)." forks)\n";
 113                                                         $now=time();
 114                                                         my $q_serverinfo=$dbi->quote($serverinfo);
 115                                                         my $resultsid;
 116                                                         if ($serverdone{$serverinfo}==-1) {
 117                                                                 my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id");
 118                                                                 $stj->execute;
 119                                                                 ($resultsid) = $stj->fetchrow;
 120                                                                 $stj->finish;
 121                                                         } else {
 122                                                                 my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id");
 123                                                                 $stj->execute;
 124                                                                 ($resultsid) = $stj->fetchrow;
 125                                                                 $stj->finish;
 126                                                                 unless ($resultsid) {
 127                                                                         $stj=$dbi->prepare("insert into z3950results (server, queryid, startdate) values ($q_serverinfo, $id, $now)");
 128                                                                         $stj->execute;
 129                                                                         $resultsid=$dbi->{'mysql_insertid'};
 130                                                                         $stj->finish;
 131                                                                 }
 132                                                         }
 133                                                         my $stj=$dbh->prepare("update z3950results set active=1 where id=$resultsid");
 134                                                         $stj->execute;
 135                                                         my $conn;
 136                                                         my $noconnection=0;
 137                                                         my $error=0;
 138                                                         if ($user) {
 139                                                                 eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database, user => $user, password => $password); };
 140                                                                 if ($@) {
 141                                                                         $noconnection=1;
 142                                                                 } else {
 143                                                                         $error=pe();
 144                                                                 }
 145                                                         } else {
 146                                                                 eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database); };
 147                                                                 if ($@) {
 148                                                                         $noconnection=1;
 149                                                                 } else {
 150                                                                         $error=pe();
 151                                                                 }
 152                                                         }
 153                                                         if ($noconnection || $error) {
 154                                                                 warn "no connection at $globalname ";
 155                                                         } else {
 156                                                                 warn "$globalname ==> $globalsyntax";
 157                                                                 eval {$conn->option(elementSetName => 'F')};
 158                                                                 eval { $conn->option(preferredRecordSyntax => Net::Z3950::RecordSyntax::USMARC);} if ($globalsyntax eq "USMARC");
 159                                                                 eval { $conn->option(preferredRecordSyntax => Net::Z3950::RecordSyntax::UNIMARC);} if ($globalsyntax eq "UNIMARC");
 160                                                                 if ($@) {
 161                                                                         print "$globalname ERROR: $@\n";
 162                                                                 } else {
 163 #                                                                       print "Q: $query\n";
 164                                                                         my $rs=$conn->search($query);
 165                                                                         pe();
 166                                                                         my $numresults=$rs->size();
 167                                                                         if ($numresults eq 0) {
 168                                                                                 warn "$globalname ==> answered : no records found";
 169                                                                         } else {
 170                                                                                 warn "$globalname ==> answered : $numresults found";
 171                                                                         }
 172                                                                         pe();
 173                                                                         my $i;
 174                                                                         my $result='';
 175                                                                         my $scantimerstart=time();
 176                                                                         for ($i=1; $i<=(($numresults<80) ? ($numresults) : (80)); $i++) {
 177                                                                                 my $rec=$rs->record($i);
 178                                                                                 my $marcdata;
 179                                                                                 # use render() or rawdata() depending on the type of the returned record
 180                                                                                 my $marcrecord;
 181                                                                                 if (ref($rec) eq "Net::Z3950::Record::USMARC") {
 182                                                                                         $marcdata = $rec->rawdata();
 183                                                                                         $marcrecord = MARC::File::USMARC::decode($rec->rawdata())
 184                                                                                 }
 185                                                                                 if (ref($rec) eq "Net::Z3950::Record::UNIMARC") {
 186                                                                                         $marcdata = $rec->render();
 187                                                                                         $marcrecord = MARC::File::USMARC::decode($rec->render())
 188                                                                                 }
 189                                                                                 $globalencoding = ref($rec);
 190                                                                                 $result.=$marcdata;
 191                                                                         }
 192                                                                         my @x=split /::/,$globalencoding;
 193                                                                         my ($notmarcrecord,$alreadyindb,$alreadyinfarm,$imported) = ImportBreeding($result,-1,"Z3950-$globalname",$x[3],$random);
 194                                                                         my $scantimerend=time();
 195                                                                         my $numrecords;
 196                                                                         ($numresults<80) ? ($numrecords=$numresults) : ($numrecords=80);
 197                                                                         my $elapsed=$scantimerend-$scantimerstart;
 198                                                                         if ($elapsed) {
 199                                                                                 my $speed=int($numresults/$elapsed*100)/100;
 200                                                                                 print "$globalname   SPEED: $speed  $server done $numrecords\n";
 201                                                                         }
 202                                                                         my $q_result=$dbi->quote($result);
 203                                                                         ($q_result) || ($q_result='""');
 204                                                                         $now=time();
 205                                                                         if ($numresults >0) {
 206                                                                                 my $task="update z3950results set numrecords=$numresults,numdownloaded=$numrecords,highestseen=0,results=$q_result,enddate=$now where id=$resultsid";
 207                                                                                 my $stj=$dbi->prepare($task);
 208                                                                                 $stj->execute;
 209                                                                         } else { # no results...
 210                                                                                 my $task="update z3950results set numrecords=$numresults,numdownloaded=$numrecords,highestseen=0,results='',enddate=$now where id=$resultsid";
 211                                                                                 my $stj=$dbi->prepare($task);
 212                                                                                 $stj->execute;
 213                                                                         }
 214                                                                         my $counter=0;
 215                                                                         while ($counter<60 && $numrecords<$numresults) {
 216                                                                                 $counter++;
 217                                                                                 my $stj=$dbi->prepare("select highestseen from z3950results where id=$resultsid");
 218                                                                                 $stj->execute;
 219                                                                                 my ($highestseen) = $stj->fetchrow;
 220                                                                                 if ($highestseen>($numrecords-30)) {
 221                                                                                         $counter=0;
 222                                                                                         print "   $server rescanning\n";
 223                                                                                         my $scantimerstart=time();
 224                                                                                         for ($i=$numrecords+1; $i<=(($numresults<($numrecords+40)) ? ($numresults) : ($numrecords+40)); $i++) {
 225                                                                                                 my $rec=$rs->record($i);
 226                                                                                                 my $marcdata=$rec->rawdata();
 227                                                                                                 $result.=$marcdata;
 228                                                                                         }
 229                                                                                         my $scantimerend=time();
 230                                                                                         ($numresults<$numrecords+40) ? ($numrecords=$numresults) : ($numrecords += 40);
 231                                                                                         my $elapsed=$scantimerend-$scantimerstart;
 232                                                                                         if ($elapsed) {
 233                                                                                                 my $speed=int($numresults/$elapsed*100)/100;
 234                                                                                                 print "  SPEED: $speed  $server done $numrecords\n";
 235                                                                                         }
 236
 237                                                                                         my $q_result=$dbi->quote($result);
 238                                                                                         ($q_result) || ($q_result='""');
 239                                                                                         $now=time();
 240                                                                                         my $task="update z3950results set numdownloaded=$numrecords,results=$q_result where id=$resultsid";
 241                                                                                         my $stj=$dbi->prepare($task);
 242                                                                                         $stj->execute;
 243                                                                                 }
 244                                                                                 sleep 5;
 245                                                                         }
 246                                                                 }
 247                                                         }
 248                                                         # FIXME - There's already a $stj in this scope
 249                                                         my $stj=$dbi->prepare("update z3950results set active=0 where id=$resultsid");
 250                                                         $stj->execute;
 251                                                         eval {$stj->finish};
 252                                                         print "    $server done.\n";
 253                                                         exit;
 254 sub pe {
 255         return 0;
 256         my $code=$conn->errcode();
 257         my $msg=$conn->errmsg();
 258         my $ai=$conn->addinfo();
 259         print << "EOF";
 260 CODE:  $code
 261 MSG:   $msg
 262 ADDTL: $ai
 263 EOF
 264         return 0;
 265 }
 266                                                 }
 267                                         }
 268                                         unless ($stillprocessing) {
 269                                                 #my $sti=$dbh->prepare("select enddate from z3950queue where id=$id");
 270                                                 #$sti->execute;
 271                                                 #my ($enddate) = $sti->fetchrow;
 272                                                 #unless ($enddate) {
 273                                         }
 274                                 } else {
 275                                 }
 276                         }
 277                         $lastrun=time();
 278                 }
 279                 sleep 10;
 280         }
 281 }
 282
 283 sub reap {
 284     $forkcounter--;
 285     wait;
 286 }
 287
 288
 289 sub checkqueue {
 290     $checkqueue=1;
 291 }
 292
 293