Koha/misc/maintenance/search_for_data_inconsistencies.pl
Fridolin Somers 3d0015fe69
Bug 31299: Fix duplicate output in search_for_data_inconsistencies.pl
In search_for_data_inconsistencies.pl when there are several inconsistencies on same framework, the second output of items contains the first one.

Looks like it is since Bug 21466

Test plan using koha-testing-docker :
1) Create 2 inconsistencies on the same item via SQL :
   update items set itemlost = 9 where itemnumber=900;
   update items set notforloan = 8 where itemnumber=900;
2) Without patch
3) Run ./misc/maintenance/search_for_data_inconsistencies.pl
=> You see duplicate output :
== Wrong values linked to authorised values ==
	* The Framework *BKS* is using the authorised value's category *NOT_LOAN*, but the following items.notforloan do not have a value defined ({itemnumber => value }):
 {900 => 8}
	* The Framework *BKS* is using the authorised value's category *LOST*, but the following items.itemlost do not have a value defined ({itemnumber => value }):
 {900 => 8} {900 => 9}

4) Apply patch
5) Run ./misc/maintenance/search_for_data_inconsistencies.pl
=> Fixed :D
== Wrong values linked to authorised values ==
	* The Framework *BKS* is using the authorised value's category *LOST*, but the following items.itemlost do not have a value defined ({itemnumber => value }):
 {900 => 9}
	* The Framework *BKS* is using the authorised value's category *NOT_LOAN*, but the following items.notforloan do not have a value defined ({itemnumber => value }):
 {900 => 8}

Signed-off-by: Marcel de Rooy <m.de.rooy@rijksmuseum.nl>
Signed-off-by: Tomas Cohen Arazi <tomascohen@theke.io>
2022-09-16 16:46:41 -03:00

353 lines
14 KiB
Perl
Executable file
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/perl
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.
use Modern::Perl;
use Koha::Script;
use Koha::AuthorisedValues;
use Koha::Authorities;
use Koha::Biblios;
use Koha::BiblioFrameworks;
use Koha::Biblioitems;
use Koha::Items;
use Koha::ItemTypes;
use Koha::Patrons;
use C4::Biblio qw( GetMarcFromKohaField );
{
my $items = Koha::Items->search({ -or => { homebranch => undef, holdingbranch => undef }});
if ( $items->count ) { new_section("Not defined items.homebranch and/or items.holdingbranch")}
while ( my $item = $items->next ) {
if ( not $item->homebranch and not $item->holdingbranch ) {
new_item(sprintf "Item with itemnumber=%s does not have homebranch and holdingbranch defined", $item->itemnumber);
} elsif ( not $item->homebranch ) {
new_item(sprintf "Item with itemnumber=%s does not have homebranch defined", $item->itemnumber);
} else {
new_item(sprintf "Item with itemnumber=%s does not have holdingbranch defined", $item->itemnumber);
}
}
if ( $items->count ) { new_hint("Edit these items and set valid homebranch and/or holdingbranch")}
}
{
# No join possible, FK is missing at DB level
my @auth_types = Koha::Authority::Types->search->get_column('authtypecode');
my $authorities = Koha::Authorities->search({authtypecode => { 'not in' => \@auth_types } });
if ( $authorities->count ) {new_section("Invalid auth_header.authtypecode")}
while ( my $authority = $authorities->next ) {
new_item(sprintf "Authority with authid=%s does not have a code defined (%s)", $authority->authid, $authority->authtypecode);
}
if ( $authorities->count ) {new_hint("Go to 'Home Administration Authority types' to define them")}
}
{
if ( C4::Context->preference('item-level_itypes') ) {
my $items_without_itype = Koha::Items->search( { -or => [itype => undef,itype => ''] } );
if ( $items_without_itype->count ) {
new_section("Items do not have itype defined");
while ( my $item = $items_without_itype->next ) {
if (defined $item->biblioitem->itemtype && $item->biblioitem->itemtype ne '' ) {
new_item(
sprintf "Item with itemnumber=%s does not have a itype value, biblio's item type will be used (%s)",
$item->itemnumber, $item->biblioitem->itemtype
);
} else {
new_item(
sprintf "Item with itemnumber=%s does not have a itype value, additionally no item type defined for biblionumber=%s",
$item->itemnumber, $item->biblioitem->biblionumber
);
}
}
new_hint("The system preference item-level_itypes expects item types to be defined at item level");
}
}
else {
my $biblioitems_without_itemtype = Koha::Biblioitems->search( { itemtype => undef } );
if ( $biblioitems_without_itemtype->count ) {
new_section("Biblioitems do not have itemtype defined");
while ( my $biblioitem = $biblioitems_without_itemtype->next ) {
new_item(
sprintf "Biblioitem with biblioitemnumber=%s does not have a itemtype value",
$biblioitem->biblioitemnumber
);
}
new_hint("The system preference item-level_itypes expects item types to be defined at biblio level");
}
}
my @itemtypes = Koha::ItemTypes->search->get_column('itemtype');
if ( C4::Context->preference('item-level_itypes') ) {
my $items_with_invalid_itype = Koha::Items->search( { -and => [itype => { not_in => \@itemtypes }, itype => { '!=' => '' }] } );
if ( $items_with_invalid_itype->count ) {
new_section("Items have invalid itype defined");
while ( my $item = $items_with_invalid_itype->next ) {
new_item(
sprintf "Item with itemnumber=%s, biblionumber=%s does not have a valid itype value (%s)",
$item->itemnumber, $item->biblionumber, $item->itype
);
}
new_hint("The items must have a itype value that is defined in the item types of Koha (Home Administration Item types administration)");
}
}
else {
my $biblioitems_with_invalid_itemtype = Koha::Biblioitems->search(
{ itemtype => { not_in => \@itemtypes } }
);
if ( $biblioitems_with_invalid_itemtype->count ) {
new_section("Biblioitems do not have itemtype defined");
while ( my $biblioitem = $biblioitems_with_invalid_itemtype->next ) {
new_item(
sprintf "Biblioitem with biblioitemnumber=%s does not have a valid itemtype value",
$biblioitem->biblioitemnumber
);
}
new_hint("The biblioitems must have a itemtype value that is defined in the item types of Koha (Home Administration Item types administration)");
}
}
my ( @decoding_errors, @ids_not_in_marc );
my $biblios = Koha::Biblios->search;
my ( $biblio_tag, $biblio_subfield ) = C4::Biblio::GetMarcFromKohaField( "biblio.biblionumber" );
my ( $biblioitem_tag, $biblioitem_subfield ) = C4::Biblio::GetMarcFromKohaField( "biblioitems.biblioitemnumber" );
while ( my $biblio = $biblios->next ) {
my $record = eval{$biblio->metadata->record;};
if ($@) {
push @decoding_errors, $@;
next;
}
my $biblionumber = $record->subfield($biblio_tag, $biblio_subfield);
my $biblioitemnumber = $record->subfield($biblioitem_tag, $biblioitem_subfield);
if ( $biblionumber != $biblio->biblionumber ) {
push @ids_not_in_marc,
{
biblionumber => $biblio->biblionumber,
biblionumber_in_marc => $biblionumber,
};
}
if ( $biblioitemnumber != $biblio->biblioitem->biblioitemnumber ) {
push @ids_not_in_marc,
{
biblionumber => $biblio->biblionumber,
biblioitemnumber => $biblio->biblioitem->biblioitemnumber,
biblioitemnumber_in_marc => $biblionumber,
};
}
}
if ( @decoding_errors ) {
new_section("Bibliographic records have invalid MARCXML");
new_item($_) for @decoding_errors;
new_hint("The bibliographic records must have a valid MARCXML or you will face encoding issues or wrong displays");
}
if (@ids_not_in_marc) {
new_section("Bibliographic records have MARCXML without biblionumber or biblioitemnumber");
for my $id (@ids_not_in_marc) {
if ( exists $id->{biblioitemnumber} ) {
new_item(
sprintf(q{Biblionumber %s has biblioitemnumber '%s' but should be '%s' in %s$%s},
$id->{biblionumber},
$id->{biblioitemnumber},
$id->{biblioitemnumber_in_marc},
$biblioitem_tag,
$biblioitem_subfield,
)
);
}
else {
new_item(
sprintf(q{Biblionumber %s has '%s' in %s$%s},
$id->{biblionumber},
$id->{biblionumber_in_marc},
$biblio_tag,
$biblio_subfield,
)
);
}
}
new_hint("The bibliographic records must have the biblionumber and biblioitemnumber in MARCXML");
}
}
{
my @framework_codes = Koha::BiblioFrameworks->search()->get_column('frameworkcode');
push @framework_codes,""; # The default is not stored in frameworks, we need to force it
my $invalid_av_per_framework = {};
foreach my $frameworkcode ( @framework_codes ) {
# We are only checking fields that are mapped to DB fields
my $msss = Koha::MarcSubfieldStructures->search({
frameworkcode => $frameworkcode,
authorised_value => {
'!=' => [ -and => ( undef, '' ) ]
},
kohafield => {
'!=' => [ -and => ( undef, '' ) ]
}
});
while ( my $mss = $msss->next ) {
my $kohafield = $mss->kohafield;
my $av = $mss->authorised_value;
next if grep {$_ eq $av} qw( branches itemtypes cn_source ); # internal categories
my $avs = Koha::AuthorisedValues->search_by_koha_field(
{
frameworkcode => $frameworkcode,
kohafield => $kohafield,
}
);
my $tmp_kohafield = $kohafield;
if ( $tmp_kohafield =~ /^biblioitems/ ) {
$tmp_kohafield =~ s|biblioitems|biblioitem|;
} else {
$tmp_kohafield =~ s|items|me|;
}
# replace items.attr with me.attr
# We are only checking biblios with items
my $items = Koha::Items->search(
{
$tmp_kohafield =>
{
-not_in => [ $avs->get_column('authorised_value'), '' ],
'!=' => undef,
},
'biblio.frameworkcode' => $frameworkcode
},
{ join => [ 'biblioitem', 'biblio' ] }
);
if ( $items->count ) {
$invalid_av_per_framework->{ $frameworkcode }->{$av} =
{ items => $items, kohafield => $kohafield };
}
}
}
if (%$invalid_av_per_framework) {
new_section('Wrong values linked to authorised values');
for my $frameworkcode ( keys %$invalid_av_per_framework ) {
while ( my ( $av_category, $v ) = each %{$invalid_av_per_framework->{$frameworkcode}} ) {
my $items = $v->{items};
my $kohafield = $v->{kohafield};
my ( $table, $column ) = split '\.', $kohafield;
my $output;
while ( my $i = $items->next ) {
my $value = $table eq 'items'
? $i->$column
: $table eq 'biblio'
? $i->biblio->$column
: $i->biblioitem->$column;
$output .= " {" . $i->itemnumber . " => " . $value . "}";
}
new_item(
sprintf(
"The Framework *%s* is using the authorised value's category *%s*, "
. "but the following %s do not have a value defined ({itemnumber => value }):\n%s",
$frameworkcode, $av_category, $kohafield, $output
)
);
}
}
}
}
{
my $biblios = Koha::Biblios->search({
-or => [
title => '',
title => undef,
]
});
if ( $biblios->count ) {
my ( $title_tag, $title_subtag ) = C4::Biblio::GetMarcFromKohaField( 'biblio.title' );
my $title_field = $title_tag // '';
$title_field .= '$'.$title_subtag if $title_subtag;
new_section("Biblio without title $title_field");
while ( my $biblio = $biblios->next ) {
new_item(sprintf "Biblio with biblionumber=%s does not have title defined", $biblio->biblionumber);
}
new_hint("Edit these biblio records to defined a title");
}
}
{
my $aging_patrons = Koha::Patrons->search(
{
-not => {
-or => {
'me.dateofbirth' => undef,
-and => {
'categorycode.dateofbirthrequired' => undef,
'categorycode.upperagelimit' => undef,
}
}
}
},
{ prefetch => ['categorycode'] },
{ order_by => [ 'me.categorycode', 'me.borrowernumber' ] },
);
my @invalid_patrons;
while ( my $aging_patron = $aging_patrons->next ) {
push @invalid_patrons, $aging_patron unless $aging_patron->is_valid_age;
}
if (@invalid_patrons) {
new_section("Patrons with invalid age for category");
foreach my $invalid_patron (@invalid_patrons) {
my $category = $invalid_patron->category;
new_item(
sprintf "Patron borrowernumber=%s has an invalid age of %s for their category '%s' (%s to %s)",
$invalid_patron->borrowernumber, $invalid_patron->get_age, $category->categorycode,
$category->dateofbirthrequired // '0', $category->upperagelimit // 'unlimited'
);
}
new_hint("You may change the patron's category automatically with misc/cronjobs/update_patrons_category.pl");
}
}
sub new_section {
my ( $name ) = @_;
say "\n== $name ==";
}
sub new_item {
my ( $name ) = @_;
say "\t* $name";
}
sub new_hint {
my ( $name ) = @_;
say "=> $name";
}
=head1 NAME
search_for_data_inconsistencies.pl
=head1 SYNOPSIS
perl search_for_data_inconsistencies.pl
=head1 DESCRIPTION
Catch data inconsistencies in Koha database
* Items with undefined homebranch and/or holdingbranch
* Authorities with undefined authtypecodes/authority types
* Item types:
* if item types are defined at item level (item-level_itypes=specific item),
then items.itype must be set else biblioitems.itemtype must be set
* Item types defined in items or biblioitems must be defined in the itemtypes table
* Invalid MARCXML in bibliographic records
* Patrons with invalid category types due to lower and upper age limits
=cut