From f1ea9624bd197b04f78750bcc4d8f07ef2f9675d Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fr=C3=A9d=C3=A9ric=20Demians?= Date: Mon, 12 Jan 2015 12:18:19 +0100 Subject: [PATCH] Bug 13568 OAI Server doesn't handle properly resumption token When responding to ListRecords and ListIdentifiers verbs, OAI server doesn't return proper resumption token. At the end of a result set, OAI server generates a resumption token even if there isn't anymore records. Consequently, OAI harverster will send a new request, based on this invalid resumption, token. OAI Server responds with an empty resultset, which is considered as an invalid response by most of the harvesters. TO TEST: - Find in your DB, a day where a few biblio records have been created. The number of created biblios must inferior to OAI-PMH:MaxCount. - Let say this day is 2014-01-09. Send an OAI-PMH request to Koha OAI Server: /cgi-bin/koha/oai.pl?verb=ListRecords&metadataPrefix=marcxml&from=2014-01-09&until=2014-01-09 - At the end of the result, you will see a resumption token which looks like that: marcxml/47/2014-01-09/2014-01-09/ This is wrong. No resumptiion token should be sent since there isn't anymore records to harvest. - Apply the patch. - Resend the OAI-PMH request. There is no resumption token at the end of the result. - You could test also with ListIdenfiers verb in place of ListRecord. Signed-off-by: Christophe Brocquet Signed-off-by: Jonathan Druart Signed-off-by: Tomas Cohen Arazi --- opac/oai.pl | 64 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/opac/oai.pl b/opac/oai.pl index d6ef2e9539..137a5f6599 100755 --- a/opac/oai.pl +++ b/opac/oai.pl @@ -71,7 +71,7 @@ my $repository = C4::OAI::Repository->new(); # - from # - until # - offset -# +# package C4::OAI::ResumptionToken; use strict; @@ -314,6 +314,7 @@ sub new { if(defined $token->{'set'}) { $set = GetOAISetBySpec($token->{'set'}); } + my $max = $repository->{koha_max_count}; my $sql = " SELECT biblioitems.biblionumber, biblioitems.timestamp FROM biblioitems @@ -322,32 +323,35 @@ sub new { $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? "; $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set; $sql .= " - LIMIT $repository->{'koha_max_count'} - OFFSET $token->{'offset'} + LIMIT " . ($max+1) . " + OFFSET $token->{offset} "; my $sth = $dbh->prepare( $sql ); my @bind_params = ($token->{'from'}, $token->{'until'}); push @bind_params, $set->{'id'} if defined $set; $sth->execute( @bind_params ); - my $pos = $token->{offset}; + my $count = 0; while ( my ($biblionumber, $timestamp) = $sth->fetchrow ) { + $count++; + if ( $count > $max ) { + $self->resumptionToken( + new C4::OAI::ResumptionToken( + metadataPrefix => $token->{metadata_prefix}, + from => $token->{from}, + until => $token->{until}, + offset => $token->{offset} + $max, + set => $token->{set} + ) + ); + last; + } $timestamp =~ s/ /T/, $timestamp .= 'Z'; $self->identifier( new HTTP::OAI::Header( identifier => $repository->{ koha_identifier} . ':' . $biblionumber, datestamp => $timestamp, ) ); - $pos++; } - $self->resumptionToken( - new C4::OAI::ResumptionToken( - metadataPrefix => $token->{metadata_prefix}, - from => $token->{from}, - until => $token->{until}, - offset => $pos, - set => $token->{set} - ) - ) if ($pos > $token->{offset}); return $self; } @@ -467,6 +471,7 @@ sub new { if(defined $token->{'set'}) { $set = GetOAISetBySpec($token->{'set'}); } + my $max = $repository->{koha_max_count}; my $sql = " SELECT biblioitems.biblionumber, biblioitems.marcxml, biblioitems.timestamp FROM biblioitems @@ -475,8 +480,8 @@ sub new { $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? "; $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set; $sql .= " - LIMIT $repository->{'koha_max_count'} - OFFSET $token->{'offset'} + LIMIT " . ($max + 1) . " + OFFSET $token->{offset} "; my $sth = $dbh->prepare( $sql ); @@ -484,8 +489,21 @@ sub new { push @bind_params, $set->{'id'} if defined $set; $sth->execute( @bind_params ); - my $pos = $token->{offset}; + my $count = 0; while ( my ($biblionumber, $marcxml, $timestamp) = $sth->fetchrow ) { + $count++; + if ( $count > $max ) { + $self->resumptionToken( + new C4::OAI::ResumptionToken( + metadataPrefix => $token->{metadata_prefix}, + from => $token->{from}, + until => $token->{until}, + offset => $token->{offset} + $max, + set => $token->{set} + ) + ); + last; + } my $oai_sets = GetOAISetsBiblio($biblionumber); my @setSpecs; foreach (@$oai_sets) { @@ -496,17 +514,7 @@ sub new { identifier => $repository->{ koha_identifier } . ':' . $biblionumber, metadataPrefix => $token->{metadata_prefix} ) ); - $pos++; } - $self->resumptionToken( - new C4::OAI::ResumptionToken( - metadataPrefix => $token->{metadata_prefix}, - from => $token->{from}, - until => $token->{until}, - offset => $pos, - set => $token->{set} - ) - ) if ($pos > $token->{offset}); return $self; } @@ -639,7 +647,7 @@ C4::OAI::Repository - Handles OAI-PMH requests for a Koha database. This object extend HTTP::OAI::Repository object. It accepts OAI-PMH HTTP requests and returns result. -This OAI-PMH server can operate in a simple mode and extended one. +This OAI-PMH server can operate in a simple mode and extended one. In simple mode, repository configuration comes entirely from Koha system preferences (OAI-PMH:archiveID and OAI-PMH:MaxCount) and the server returns -- 2.39.5