From fc95762725bd0af4d156c7643bd6f1a4aedab2c5 Mon Sep 17 00:00:00 2001 From: Matthias Meusburger Date: Thu, 28 May 2015 16:23:58 +0200 Subject: [PATCH] Bug 3206: OAI repository deleted records support. This patch allows Koha OAI repository to support deleted records. The OAI-PMH:DeletedRecord syspref is introduced and can be set to: - persistent (in case Koha's deletedbiblio table will never be emptied or truncated) - transient (in case Koha's deletedbiblio table might be emptied or truncated at some point) Test plan: - After applying the patch, test that: - Deleted records appear in ListRecords and ListIdentifiers requests. - Filter parameters (from, until, set and resumptionToken) still work and are applied to ListRecords and ListIdentifiers requests. - Identify request shows if the repository is considered persistent or transient, according to the OAI-PMH:DeletedRecord syspref. - Deleted records that used to belong to a set are still displayed in those sets and marked as deleted. - GetRecord requests work on deleted records, which are marked as deleted. Requests examples: /cgi-bin/koha/oai.pl?verb=ListRecords&metadataPrefix=oai_dc /cgi-bin/koha/oai.pl?verb=ListRecords&metadataPrefix=oai_dc&from=2015-02-20T11:08:33Z /cgi-bin/koha/oai.pl?verb=ListRecords&metadataPrefix=oai_dc&set=new_specSet1 /cgi-bin/koha/oai.pl?verb=GetRecord&identifier=KOHA-OAI-TEST:2&metadataPrefix=oai_dc /cgi-bin/koha/oai.pl?verb=Identify Signed-off-by: Frederic Demians It works in all situations described in the test plan. Great addition. Thanks. Signed-off-by: Jonathan Druart Signed-off-by: Jonathan Druart Signed-off-by: Tomas Cohen Arazi --- ...XXXX-add_OAI-PMH_DeletedRecord_syspref.sql | 3 + installer/data/mysql/kohastructure.sql | 1 - installer/data/mysql/sysprefs.sql | 3 +- .../admin/preferences/web_services.pref | 7 ++ opac/oai.pl | 90 ++++++++++++++++--- 5 files changed, 88 insertions(+), 16 deletions(-) create mode 100644 installer/data/mysql/atomicupdate/bug_XXXX-add_OAI-PMH_DeletedRecord_syspref.sql diff --git a/installer/data/mysql/atomicupdate/bug_XXXX-add_OAI-PMH_DeletedRecord_syspref.sql b/installer/data/mysql/atomicupdate/bug_XXXX-add_OAI-PMH_DeletedRecord_syspref.sql new file mode 100644 index 0000000000..233bcbeed6 --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_XXXX-add_OAI-PMH_DeletedRecord_syspref.sql @@ -0,0 +1,3 @@ +INSERT IGNORE INTO systempreferences (variable,value,explanation,options,type) +VALUES ('OAI-PMH:DeletedRecord','persistent','Koha\'s deletedbiblio table will never be deleted (persistent) or might be deleted (transient)','transient|persistent','Choice'); +ALTER TABLE oai_sets_biblios DROP FOREIGN KEY oai_sets_biblios_ibfk_1; diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index 66895d1628..df123ee6fd 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -1610,7 +1610,6 @@ CREATE TABLE `oai_sets_biblios` ( `biblionumber` int(11) NOT NULL, `set_id` int(11) NOT NULL, PRIMARY KEY (`biblionumber`, `set_id`), - CONSTRAINT `oai_sets_biblios_ibfk_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE CASCADE, CONSTRAINT `oai_sets_biblios_ibfk_2` FOREIGN KEY (`set_id`) REFERENCES `oai_sets` (`id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; diff --git a/installer/data/mysql/sysprefs.sql b/installer/data/mysql/sysprefs.sql index 84ba3631b2..f7ea475799 100644 --- a/installer/data/mysql/sysprefs.sql +++ b/installer/data/mysql/sysprefs.sql @@ -480,5 +480,6 @@ INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` ('XSLTDetailsDisplay','default','','Enable XSL stylesheet control over details page display on intranet','Free'), ('XSLTResultsDisplay','default','','Enable XSL stylesheet control over results page display on intranet','Free'), ('z3950AuthorAuthFields','701,702,700',NULL,'Define the MARC biblio fields for Personal Name Authorities to fill biblio.author','free'), -('z3950NormalizeAuthor','0','','If ON, Personal Name Authorities will replace authors in biblio.author','YesNo') +('z3950NormalizeAuthor','0','','If ON, Personal Name Authorities will replace authors in biblio.author','YesNo'), +('OAI-PMH:DeletedRecord','persistent','Koha\'s deletedbiblio table will never be deleted (persistent) or might be deleted (transient)','transient|persistent','Choice') ; diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref index 075ffb720a..518dfbd115 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref @@ -27,6 +27,13 @@ Web services: yes: Enable no: Disable - automatic update of OAI-PMH sets when a bibliographic record is created or updated + - + - Koha's deletedbiblio table + - pref: "OAI-PMH:DeletedRecord" + choices: + persistent: will never be emptied or truncated (persistent) + transient: might be emptied or truncated at some point (transient) + - "." ILS-DI: - - pref: ILS-DI diff --git a/opac/oai.pl b/opac/oai.pl index c80ebfa99c..82af58c394 100755 --- a/opac/oai.pl +++ b/opac/oai.pl @@ -152,7 +152,7 @@ sub new { MaxCount => C4::Context->preference("OAI-PMH:MaxCount"), granularity => 'YYYY-MM-DD', earliestDatestamp => '0001-01-01', - deletedRecord => 'no', + deletedRecord => C4::Context->preference("OAI-PMH:DeletedRecord") || 'no', ); # FIXME - alas, the description element is not so simple; to validate @@ -252,6 +252,36 @@ sub new { # __END__ C4::OAI::Record +package C4::OAI::DeletedRecord; + +use strict; +use warnings; +use HTTP::OAI; +use HTTP::OAI::Metadata::OAI_DC; + +use base ("HTTP::OAI::Record"); + +sub new { + my ($class, $timestamp, $setSpecs, %args) = @_; + + my $self = $class->SUPER::new(%args); + + $timestamp =~ s/ /T/, $timestamp .= 'Z'; + $self->header( new HTTP::OAI::Header( + status => 'deleted', + identifier => $args{identifier}, + datestamp => $timestamp, + ) ); + + foreach my $setSpec (@$setSpecs) { + $self->header->setSpec($setSpec); + } + + return $self; +} + +# __END__ C4::OAI::DeletedRecord + package C4::OAI::GetRecord; @@ -278,7 +308,16 @@ sub new { my ($biblionumber) = $args{identifier} =~ /^$prefix(.*)/; $sth->execute( $biblionumber ); my ($marcxml, $timestamp); + my $deleted = 0; unless ( ($marcxml, $timestamp) = $sth->fetchrow ) { + $sth = $dbh->prepare(" + SELECT biblionumber, timestamp + FROM deletedbiblio + WHERE biblionumber=? " ); + $sth->execute( $biblionumber ); + + unless ( ($marcxml, $timestamp) = $sth->fetchrow ) { + return HTTP::OAI::Response->new( requestURL => $repository->self_url(), errors => [ new HTTP::OAI::Error( @@ -286,8 +325,10 @@ sub new { message => "There is no biblio record with this identifier", ) ] , ); + } else { + $deleted = 1; + } } - my $oai_sets = GetOAISetsBiblio($biblionumber); my @setSpecs; foreach (@$oai_sets) { @@ -295,9 +336,10 @@ sub new { } #$self->header( HTTP::OAI::Header->new( identifier => $args{identifier} ) ); - $self->record( C4::OAI::Record->new( + ($deleted == 1) ? $self->record( C4::OAI::DeletedRecord->new( + $timestamp, \@setSpecs, %args ) ) + : $self->record( C4::OAI::Record->new( $repository, $marcxml, $timestamp, \@setSpecs, %args ) ); - return $self; } @@ -328,19 +370,27 @@ sub new { } my $max = $repository->{koha_max_count}; my $sql = " - SELECT biblioitems.biblionumber, biblioitems.timestamp + (SELECT biblioitems.biblionumber, timestamp FROM biblioitems "; $sql .= " JOIN oai_sets_biblios ON biblioitems.biblionumber = oai_sets_biblios.biblionumber " if defined $set; $sql .= " WHERE timestamp >= ? AND timestamp <= ? "; $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set; - $sql .= " + $sql .= ") UNION + (SELECT deletedbiblio.biblionumber, timestamp FROM deletedbiblio"; + $sql .= " JOIN oai_sets_biblios ON deletedbiblio.biblionumber = oai_sets_biblios.biblionumber " if defined $set; + $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? "; + $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set; + + $sql .= ") ORDER BY biblionumber LIMIT " . ($max+1) . " OFFSET $token->{offset} "; my $sth = $dbh->prepare( $sql ); my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'}); push @bind_params, $set->{'id'} if defined $set; + push @bind_params, ($token->{'from'}, $token->{'until'}); + push @bind_params, $set->{'id'} if defined $set; $sth->execute( @bind_params ); my $count = 0; @@ -485,20 +535,27 @@ sub new { } my $max = $repository->{koha_max_count}; my $sql = " - SELECT biblioitems.biblionumber, biblioitems.marcxml, biblioitems.timestamp + (SELECT biblioitems.biblionumber, marcxml, timestamp FROM biblioitems "; $sql .= " JOIN oai_sets_biblios ON biblioitems.biblionumber = oai_sets_biblios.biblionumber " if defined $set; $sql .= " WHERE timestamp >= ? AND timestamp <= ? "; $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set; - $sql .= " + $sql .= ") UNION + (SELECT deletedbiblio.biblionumber, null as marcxml, timestamp FROM deletedbiblio"; + $sql .= " JOIN oai_sets_biblios ON deletedbiblio.biblionumber = oai_sets_biblios.biblionumber " if defined $set; + $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? "; + $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set; + + $sql .= ") ORDER BY biblionumber LIMIT " . ($max + 1) . " OFFSET $token->{offset} "; - my $sth = $dbh->prepare( $sql ); my @bind_params = ($token->{'from_arg'}, $token->{'until_arg'}); push @bind_params, $set->{'id'} if defined $set; + push @bind_params, ($token->{'from'}, $token->{'until'}); + push @bind_params, $set->{'id'} if defined $set; $sth->execute( @bind_params ); my $count = 0; @@ -521,11 +578,16 @@ sub new { foreach (@$oai_sets) { push @setSpecs, $_->{spec}; } - $self->record( C4::OAI::Record->new( - $repository, $marcxml, $timestamp, \@setSpecs, - identifier => $repository->{ koha_identifier } . ':' . $biblionumber, - metadataPrefix => $token->{metadata_prefix} - ) ); + if ($marcxml) { + $self->record( C4::OAI::Record->new( + $repository, $marcxml, $timestamp, \@setSpecs, + identifier => $repository->{ koha_identifier } . ':' . $biblionumber, + metadataPrefix => $token->{metadata_prefix} + ) ); + } else { + $self->record( C4::OAI::DeletedRecord->new( + $timestamp, \@setSpecs, identifier => $repository->{ koha_identifier } . ':' . $biblionumber ) ); + } } return $self; -- 2.39.5