acqui.simple/processz3950queue

   1 #!/usr/bin/perl
   2 use C4::Database;
   3 use DBI;
   4 #use strict;
   5 use C4::Acquisitions;
   6 use C4::Output;
   7 use Net::Z3950;
   8 my $dbh=C4Connect;
   9
  10 my $sth=$dbh->prepare("update z3950results set active=0");
  11 $sth->execute;
  12 $sth->finish;
  13 $SIG{CHLD}='reap';
  14
  15 my $reapcounter=0;
  16 my $forkcounter=0;
  17 my $pid=$$;
  18 my $lastrun=0;
  19 while (1) {
  20     if ((time-$lastrun)>5) {
  21         my $sth=$dbh->prepare("select id,term,type,servers from z3950queue order by id");
  22         $sth->execute;
  23         while (my ($id, $term, $type, $servers) = $sth->fetchrow) {
  24             if ($forkcounter<12) {
  25                 my $now=time();
  26                 $stk=$dbh->prepare("select id,server,startdate,enddate,numrecords,active from z3950results where queryid=$id");
  27                 ($stk->execute) || (next);
  28                 my %serverdone;
  29                 unless ($stk->rows) {
  30                     my $sti=$dbh->prepare("update z3950queue set done=-1,startdate=$now where id=$id");
  31                     $sti->execute;
  32                 }
  33                 while (my ($r_id, $r_server,$r_startdate,$r_enddate,$r_numrecords,$active) = $stk->fetchrow) {
  34                     if ($r_enddate >0) {
  35                         $serverdone{$r_server}=1;
  36                     } elsif ($active) {
  37                         $serverdone{$r_server}=1;
  38                     } else {
  39                         $serverdone{$r_server}=-1;
  40                     }
  41                 }
  42
  43                 $stk->finish;
  44                 my $attr='';
  45                 if ($type eq 'isbn') {
  46                     $attr='1=7';
  47                 } elsif ($type eq 'title') {
  48                     $attr='1=4';
  49                 } elsif ($type eq 'author') {
  50                     $attr='1=1003';
  51                 } elsif ($type eq 'lccn') {
  52                     $attr='1=9';
  53                 } elsif ($type eq 'keyword') {
  54                     $attr='1=1016';
  55                 }
  56                 $term='"'.$term.'"';
  57                 $query="\@attr $attr $term";
  58                 my $totalrecords=0;
  59                 my $serverinfo;
  60                 my $stillprocessing=0;
  61                 foreach $serverinfo (split(/\s+/, $servers)) {
  62                     (next) if ($serverdone{$serverinfo} == 1);
  63                     my $stillprocessing=1;
  64                     if (my $pid=fork()) {
  65                         $forkcounter++;
  66                     } else {
  67                         #$sth->finish;
  68                         #$sti->finish;
  69                         #$dbh->disconnect;
  70                         my $dbi=C4Connect;
  71                         my ($name, $server, $database, $user, $password) = split(/\//, $serverinfo, 5);
  72                         $server=~/(.*)\:(\d+)/;
  73                         my $servername=$1;
  74                         my $port=$2;
  75                         print "Processing $type=$term at $name $server $database (".($forkcounter+1)." forks)\n";
  76                         $now=time();
  77                         my $q_serverinfo=$dbi->quote($serverinfo);
  78                         my $resultsid;
  79                         if ($serverdone{$serverinfo}==-1) {
  80                             my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id");
  81                             $stj->execute;
  82                             ($resultsid) = $stj->fetchrow;
  83                         } else {
  84                             my $stj=$dbi->prepare("select id from z3950results where server=$q_serverinfo and queryid=$id");
  85                             $stj->execute;
  86                             ($resultsid) = $stj->fetchrow;
  87                             unless ($resultsid) {
  88                                 my $stj=$dbi->prepare("insert into z3950results (server, queryid, startdate) values ($q_serverinfo, $id, $now)");
  89                                 $stj->execute;
  90                                 $resultsid=$dbi->{'mysql_insertid'};
  91                             }
  92                         }
  93                         my $stj=$dbh->prepare("update z3950results set active=1 where id=$resultsid");
  94                         $stj->execute;
  95                         my $conn;
  96                         my $noconnection=0;
  97                         my $error=0;
  98                         if ($user) {
  99                             eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database, user => $user, password => $password); };
 100                             if ($@) {
 101                                 $noconnection=1;
 102                             } else {
 103                                 $error=pe();
 104                             }
 105                         } else {
 106                             eval { $conn= new Net::Z3950::Connection($servername, $port, databaseName => $database); };
 107                             if ($@) {
 108                                 $noconnection=1;
 109                             } else {
 110                                 $error=pe();
 111                             }
 112                         }
 113                         if ($noconnection || $error) {
 114                         } else {
 115                             print "Q: $query\n";
 116                             my $rs=$conn->search($query);
 117                             pe();
 118                             eval { $rs->option(preferredRecordSyntax => Net::Z3950::RecordSyntax::USMARC);};
 119                             if ($@) {
 120                                 print "ERROR: $@\n";
 121                             } else {
 122                                 pe();
 123                                 my $numresults=$rs->size();
 124                                 pe();
 125                                 my $i;
 126                                 my $result='';
 127                                 my $scantimerstart=time();
 128                                 for ($i=1; $i<=(($numresults<80) ? ($numresults) : (80)); $i++) {
 129                                     my $rec=$rs->record($i);
 130                                     my $marcdata=$rec->rawdata();
 131                                     $result.=$marcdata;
 132                                 }
 133                                 my $scantimerend=time();
 134                                 my $numrecords;
 135                                 ($numresults<80) ? ($numrecords=$numresults) : ($numrecords=80);
 136                                 my $elapsed=$scantimerend-$scantimerstart;
 137                                 if ($elapsed) {
 138                                     my $speed=int($numresults/$elapsed*100)/100;
 139                                     print "  SPEED: $speed  $server done $numrecords\n";
 140                                 }
 141
 142                                 my $q_result=$dbi->quote($result);
 143                                 ($q_result) || ($q_result='""');
 144                                 $now=time();
 145                                 my $task="update z3950results set numrecords=$numresults,numdownloaded=$numrecords,highestseen=0,results=$q_result,enddate=$now where id=$resultsid";
 146                                 my $stj=$dbi->prepare($task);
 147                                 $stj->execute;
 148                                 my $counter=0;
 149                                 while ($counter<60 && $numrecords<$numresults) {
 150                                     $counter++;
 151                                     my $stj=$dbi->prepare("select highestseen from z3950results where id=$resultsid");
 152                                     $stj->execute;
 153                                     my ($highestseen) = $stj->fetchrow;
 154                                     if ($highestseen>($numrecords-30)) {
 155                                         $counter=0;
 156                                         print "   $server rescanning\n";
 157                                         my $scantimerstart=time();
 158                                         for ($i=$numrecords+1; $i<=(($numresults<($numrecords+40)) ? ($numresults) : ($numrecords+40)); $i++) {
 159                                             my $rec=$rs->record($i);
 160                                             my $marcdata=$rec->rawdata();
 161                                             $result.=$marcdata;
 162                                         }
 163                                         my $scantimerend=time();
 164                                         ($numresults<$numrecords+40) ? ($numrecords=$numresults) : ($numrecords=$numrecords+40);
 165                                         my $elapsed=$scantimerend-$scantimerstart;
 166                                         if ($elapsed) {
 167                                             my $speed=int($numresults/$elapsed*100)/100;
 168                                             print "  SPEED: $speed  $server done $numrecords\n";
 169                                         }
 170
 171                                         my $q_result=$dbi->quote($result);
 172                                         ($q_result) || ($q_result='""');
 173                                         $now=time();
 174                                         my $task="update z3950results set numdownloaded=$numrecords,results=$q_result where id=$resultsid";
 175                                         my $stj=$dbi->prepare($task);
 176                                         $stj->execute;
 177                                     }
 178                                     sleep 5;
 179                                 }
 180                             }
 181                         }
 182                         my $stj=$dbi->prepare("update z3950results set active=0 where id=$resultsid");
 183                         $stj->execute;
 184                         eval {$stj->finish};
 185                         $dbi->disconnect;
 186                         print "    $server done.\n";
 187                         exit;
 188                         sub pe {
 189                             return 0;
 190                             my $code=$conn->errcode();
 191                             my $msg=$conn->errmsg();
 192                             my $ai=$conn->addinfo();
 193                             print << "EOF";
 194                         CODE:  $code
 195                         MSG:   $msg
 196                         ADDTL: $ai
 197
 198 EOF
 199                         #    if ($msg =~/not yet available/) {
 200                 #               return 1;
 201         #                   }
 202                             return 0;
 203                         }
 204                     }
 205                 } unless ($stillprocessing) {
 206                     #my $sti=$dbh->prepare("select enddate from z3950queue where id=$id");
 207                     #$sti->execute;
 208                     #my ($enddate) = $sti->fetchrow;
 209                     #unless ($enddate) {
 210         #               my $now=time;
 211 #                       $sti=$dbh->prepare("update z3950queue set done=1,numrecords=$totalrecords,enddate=$now where id=$id");
 212 #                       $sti->execute;
 213 #                   }
 214                 }
 215             } else {
 216 #           my $q_serverinfo=$dbh->quote($serverinfo);
 217 #           my $stj=$dbh->prepare("insert into z3950results (server, queryid, startdate) values ($q_serverinfo, $id, 0)");
 218 #           $stj->execute;
 219             }
 220         }
 221         $lastrun=time();
 222     }
 223     sleep 10;
 224 }
 225
 226 sub getrecord {
 227     my $server=shift;
 228     my $base=shift;
 229     my $query=shift;
 230     my $auth=shift;
 231     my $id=shift;
 232     open  (M, "|yaz-client -m yaz-$id.mrc >>yaz.out 2>>yaz.err");
 233     select M;
 234     $|=1;
 235     select STDOUT;
 236     ($auth) && ($auth="authentication $auth\n");
 237     print M << "EOF";
 238 $auth\open $server
 239 base $base
 240 setnames
 241 $query
 242 s
 243 s
 244 s
 245 s
 246 s
 247 s
 248 s
 249 s
 250 s
 251 s
 252 quit
 253 EOF
 254     close M;
 255 }
 256 sub reap {
 257     $forkcounter--;
 258 }
 259
 260
 261