z3950/processz3950queue

   1 #!/usr/bin/perl
   2
   3 # $Id$
   4
   5 use C4::Context;
   6 use DBI;
   7 #use strict;
   8 use C4::Acquisitions;
   9 use C4::Biblio;
  10 use C4::Output;
  11 use Net::Z3950;
  12
  13
  14 if ($< == 0) {
  15     # Running as root, switch privs
  16     if (-d "/var/run") {
  17         open PID, ">/var/run/processz3950queue.pid";
  18         print PID $$."\n";
  19         close PID;
  20     }
  21     # Get real apacheuser from koha.conf or reparsing httpd.conf
  22     my $apacheuser='www-data';
  23     my $uid=0;
  24     unless ($uid = (getpwnam($apacheuser))[2]) {
  25         die "Attempt to run daemon as non-existent or superuser\n";
  26     }
  27     $>=$uid;
  28     $<=$uid;
  29 }
  30
  31
  32 my $dbh = C4::Context->dbh;
  33
  34 my $sth=$dbh->prepare("update z3950results set active=0");
  35 $sth->execute;
  36 $sth->finish;
  37 $SIG{CHLD}='reap';
  38 $SIG{HUP}='checkqueue';
  39
  40
  41 my $logdir=$ARGV[0];
  42
  43 open PID, ">$logdir/processz3950queue.pid";
  44 print PID $$."\n";
  45 close PID;
  46
  47 my $reapcounter=0;
  48 my $forkcounter=0;
  49 my $checkqueue=1;
  50 my $pid=$$;
  51 my $lastrun=0;
  52 while (1) {
  53     if ((time-$lastrun)>5) {
  54         if ($checkqueue) {
  55             $checkqueue=0;
  56             my $sth=$dbh->prepare("select id,term,type,servers from z3950queue order by id");
  57             $sth->execute;
  58             while (my ($id, $term, $type, $servers) = $sth->fetchrow) {
  59                 if ($forkcounter<12) {
  60                     my $now=time();
  61                     $stk=$dbh->prepare("select id,server,startdate,enddate,numrecords,active from z3950results where queryid=$id");
  62                     ($stk->execute) || (next);
  63                     my %serverdone;
  64                     unless ($stk->rows) {
  65                         my $sti=$dbh->prepare("update z3950queue set done=-1,startdate=$now where id=$id");
  66                         $sti->execute;
  67                     }
  68                     while (my ($r_id, $r_server,$r_startdate,$r_enddate,$r_numrecords,$active) = $stk->fetchrow) {
  69                         if ($r_enddate >0) {
  70                             $serverdone{$r_server}=1;
  71                         } elsif ($active) {
  72                             $serverdone{$r_server}=1;
  73                         } else {
  74                             $serverdone{$r_server}=-1;
  75                         }
  76                     }
  77
  78                     $stk->finish;
  79                     my $attr='';
  80                     if ($type eq 'isbn') {
  81                         $attr='1=7';
  82                     } elsif ($type eq 'title') {
  83                         $attr='1=4';
  84                     } elsif ($type eq 'author') {
  85                         $attr='1=1003';
  86                     } elsif ($type eq 'lccn') {
  87                         $attr='1=9';
  88                     } elsif ($type eq 'keyword') {
  89                         $attr='1=1016';
  90                     }
  91                     $term='"'.$term.'"';
  92                     $query="\@attr $attr $term";
  93                     my $totalrecords=0;
  94                     my $serverinfo;
  95                     my $stillprocessing=0;
  96                     foreach $serverinfo (split(/\s+/, $servers)) {
  97                         (next) if ($serverdone{$serverinfo} == 1);
  98                         my $stillprocessing=1;
  99                         if (my $pid=fork()) {
 100                             $forkcounter++;
 101                         } else {
 102                             #$sth->finish;
 103                             #$sti->finish;
 104                             #$dbh->disconnect;
 105                             my $dbi = C4::Context->dbh;
 106                             my ($name, $server, $database, $user, $password) = split(/\//, $serverinfo, 5);
 107                             $server=~/(.*)\:(\d+)/;
 108                             my $servername=$1;
 109                             my $port=$2;
 110                             print "Processing $type=$term at $name $server $database (".($forkcounter+1)." forks)\n";
 111                             $now=time();
 112                             my $q_serverinfo=$dbi->quote($serverinfo);
 113                             my $resultsid;
 114                             if ($serverdone{$serverinfo}==-1) {
 115                                 my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id");
 116                                 $stj->execute;
 117                                 ($resultsid) = $stj->fetchrow;
 118                             } else {
 119                                 my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id");
 120                                 $stj->execute;
 121                                 ($resultsid) = $stj->fetchrow;
 122                                 unless ($resultsid) {
 123                                     my $stj=$dbi->prepare("insert into z3950results (server, queryid, startdate) values ($q_serverinfo, $id, $now)");
 124                                     $stj->execute;
 125                                     $resultsid=$dbi->{'mysql_insertid'};
 126                                 }
 127                             }
 128                             my $stj=$dbh->prepare("update z3950results set active=1 where id=$resultsid");
 129                             $stj->execute;
 130                             my $conn;
 131                             my $noconnection=0;
 132                             my $error=0;
 133                             if ($user) {
 134                                 eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database, user => $user, password => $password); };
 135                                 if ($@) {
 136                                     $noconnection=1;
 137                                 } else {
 138                                     $error=pe();
 139                                 }
 140                             } else {
 141                                 eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database); };
 142                                 if ($@) {
 143                                     $noconnection=1;
 144                                 } else {
 145                                     $error=pe();
 146                                 }
 147                             }
 148                             if ($noconnection || $error) {
 149                             } else {
 150                                 print "Q: $query\n";
 151                                 my $rs=$conn->search($query);
 152                                 pe();
 153                                 eval { $rs->option(preferredRecordSyntax => Net::Z3950::RecordSyntax::USMARC);};
 154                                 if ($@) {
 155                                     print "ERROR: $@\n";
 156                                 } else {
 157                                     pe();
 158                                     my $numresults=$rs->size();
 159                                     pe();
 160                                     my $i;
 161                                     my $result='';
 162                                     my $scantimerstart=time();
 163                                     for ($i=1; $i<=(($numresults<80) ? ($numresults) : (80)); $i++) {
 164                                         my $rec=$rs->record($i);
 165                                         my $marcdata=$rec->rawdata();
 166                                         $result.=$marcdata;
 167                                     }
 168                                     my $scantimerend=time();
 169                                     my $numrecords;
 170                                     ($numresults<80) ? ($numrecords=$numresults) : ($numrecords=80);
 171                                     my $elapsed=$scantimerend-$scantimerstart;
 172                                     if ($elapsed) {
 173                                         my $speed=int($numresults/$elapsed*100)/100;
 174                                         print "  SPEED: $speed  $server done $numrecords\n";
 175                                     }
 176
 177                                     my $q_result=$dbi->quote($result);
 178                                     ($q_result) || ($q_result='""');
 179                                     $now=time();
 180                                     my $task="update z3950results set numrecords=$numresults,numdownloaded=$numrecords,highestseen=0,results=$q_result,enddate=$now where id=$resultsid";
 181                                     my $stj=$dbi->prepare($task);
 182                                     $stj->execute;
 183                                     my $counter=0;
 184                                     while ($counter<60 && $numrecords<$numresults) {
 185                                         $counter++;
 186                                         my $stj=$dbi->prepare("select highestseen from z3950results where id=$resultsid");
 187                                         $stj->execute;
 188                                         my ($highestseen) = $stj->fetchrow;
 189                                         if ($highestseen>($numrecords-30)) {
 190                                             $counter=0;
 191                                             print "   $server rescanning\n";
 192                                             my $scantimerstart=time();
 193                                             for ($i=$numrecords+1; $i<=(($numresults<($numrecords+40)) ? ($numresults) : ($numrecords+40)); $i++) {
 194                                                 my $rec=$rs->record($i);
 195                                                 my $marcdata=$rec->rawdata();
 196                                                 $result.=$marcdata;
 197                                             }
 198                                             my $scantimerend=time();
 199                                             ($numresults<$numrecords+40) ? ($numrecords=$numresults) : ($numrecords += 40);
 200                                             my $elapsed=$scantimerend-$scantimerstart;
 201                                             if ($elapsed) {
 202                                                 my $speed=int($numresults/$elapsed*100)/100;
 203                                                 print "  SPEED: $speed  $server done $numrecords\n";
 204                                             }
 205
 206                                             my $q_result=$dbi->quote($result);
 207                                             ($q_result) || ($q_result='""');
 208                                             $now=time();
 209                                             my $task="update z3950results set numdownloaded=$numrecords,results=$q_result where id=$resultsid";
 210                                             my $stj=$dbi->prepare($task);
 211                                             $stj->execute;
 212                                         }
 213                                         sleep 5;
 214                                     }
 215                                 }
 216                             }
 217                             # FIXME - There's already a $stj in this scope
 218                             my $stj=$dbi->prepare("update z3950results set active=0 where id=$resultsid");
 219                             $stj->execute;
 220                             eval {$stj->finish};
 221                             print "    $server done.\n";
 222                             exit;
 223                             sub pe {
 224                                 return 0;
 225                                 my $code=$conn->errcode();
 226                                 my $msg=$conn->errmsg();
 227                                 my $ai=$conn->addinfo();
 228                                 print << "EOF";
 229 CODE:  $code
 230 MSG:   $msg
 231 ADDTL: $ai
 232
 233 EOF
 234                                 return 0;
 235                             }
 236                         }
 237                     } unless ($stillprocessing) {
 238                         #my $sti=$dbh->prepare("select enddate from z3950queue where id=$id");
 239                         #$sti->execute;
 240                         #my ($enddate) = $sti->fetchrow;
 241                         #unless ($enddate) {
 242                     }
 243                 } else {
 244                 }
 245             }
 246             $lastrun=time();
 247         }
 248         sleep 10;
 249     }
 250 }
 251
 252 sub getrecord {
 253     my $server=shift;
 254     my $base=shift;
 255     my $query=shift;
 256     my $auth=shift;
 257     my $id=shift;
 258     open  (M, "|yaz-client -m yaz-$id.mrc >>yaz.out 2>>yaz.err");
 259     select M;
 260     $|=1;
 261     select STDOUT;
 262     ($auth) && ($auth="authentication $auth\n");
 263     print M << "EOF";
 264 $auth\open $server
 265 base $base
 266 setnames
 267 $query
 268 s
 269 s
 270 s
 271 s
 272 s
 273 s
 274 s
 275 s
 276 s
 277 s
 278 quit
 279 EOF
 280     close M;
 281 }
 282 sub reap {
 283     $forkcounter--;
 284     wait;
 285 }
 286
 287
 288 sub checkqueue {
 289     $checkqueue=1;
 290 }
 291
 292