From d55ea648ac6fbc87cf9d4afa7217d01915de3b55 Mon Sep 17 00:00:00 2001 From: Pedro Amorim Date: Mon, 23 Oct 2023 12:36:15 +0000 Subject: [PATCH] Bug 34587: Improve SUSHI COUNTER error handling This is doing a few things, checking for sushi errors: 1) Abort background job and provide error message if SUSHI returned {Severity} in response, this means SUSHI error, docs here: https://app.swaggerhub.com/apis/COUNTER/counter-sushi_5_0_api/5.0.2#/SUSHI_error_model 2) Abort background job and provide error message if SUSHI provided proper response, but contains Exceptions in Report_Header 3) Abort background job and provide error message if SUSHI returned 0 report items (e.g. everything is fine but there are no items for the harvest period provided 4) Abort background job and provide error message if COUNTER file created is larger than allowed by the database. 5) Provide error message if manual upload file size exceeds the max allowed by the database Signed-off-by: Jessica Zairo Signed-off-by: Michaela Sieber Signed-off-by: Nick Clemens Signed-off-by: Tomas Cohen Arazi --- Koha/BackgroundJob/ErmSushiHarvester.pm | 1 + Koha/ERM/EUsage/UsageDataProvider.pm | 79 ++++++++++++++++++- Koha/REST/V1/ERM/EUsage/UsageDataProviders.pm | 6 +- .../paths/erm_usage_data_providers.yaml | 4 + .../background_jobs/erm_sushi_harvester.inc | 20 +++-- 5 files changed, 102 insertions(+), 8 deletions(-) diff --git a/Koha/BackgroundJob/ErmSushiHarvester.pm b/Koha/BackgroundJob/ErmSushiHarvester.pm index 8d3e8c11c8..5dadc0b60f 100644 --- a/Koha/BackgroundJob/ErmSushiHarvester.pm +++ b/Koha/BackgroundJob/ErmSushiHarvester.pm @@ -155,6 +155,7 @@ sub set_job_size { type => 'success', # success, warning or error code => 'object_added', # object_added or object_already_exists title => $row->{Title}, + message => 'message', } ); diff --git a/Koha/ERM/EUsage/UsageDataProvider.pm b/Koha/ERM/EUsage/UsageDataProvider.pm index 0f23a9367f..b956b09382 100644 --- a/Koha/ERM/EUsage/UsageDataProvider.pm +++ b/Koha/ERM/EUsage/UsageDataProvider.pm @@ -215,11 +215,17 @@ sub harvest_sushi { return; } + my $decoded_response = decode_json( $response->decoded_content ); + + return if $self->_sushi_errors($decoded_response); + # Parse the SUSHI response my $sushi_counter = - Koha::ERM::EUsage::SushiCounter->new( { response => decode_json( $response->decoded_content ) } ); + Koha::ERM::EUsage::SushiCounter->new( { response => $decoded_response } ); my $counter_file = $sushi_counter->get_COUNTER_from_SUSHI; + return if $self->_counter_file_size_too_large($counter_file); + $self->counter_files( [ { @@ -440,6 +446,77 @@ sub _check_trailing_character { return $url; } +=head3 _sushi_errors + +Checks and handles possible errors in the SUSHI response +Additionally, adds background job report message(s) if that is the case + +=cut + +sub _sushi_errors { + my ( $self, $decoded_response ) = @_; + + if ( $decoded_response->{Severity} ) { + $self->{job_callbacks}->{add_message_callback}->( + { + type => 'error', + code => $decoded_response->{Code}, + message => $decoded_response->{Severity} . ' - ' . $decoded_response->{Message}, + } + ) if $self->{job_callbacks}; + return 1; + } + + if ( $decoded_response->{Report_Header}->{Exceptions} ) { + foreach my $exception ( @{ $decoded_response->{Report_Header}->{Exceptions} } ) { + $self->{job_callbacks}->{add_message_callback}->( + { + type => 'error', + code => $exception->{Code}, + message => $exception->{Message} . ' - ' . $exception->{Data}, + } + ) if $self->{job_callbacks}; + } + return 1; + } + + if ( scalar @{ $decoded_response->{Report_Items} } == 0 ) { + $self->{job_callbacks}->{add_message_callback}->( + { + type => 'error', + code => 'no_items', + } + ) if $self->{job_callbacks}; + return 1; + } + + return 0; +} + +=head3 _counter_file_size_too_large + +Checks whether a counter file size exceeds the size allowed by the database or not +Additionally, adds a background job report message if that is the case + +=cut + +sub _counter_file_size_too_large { + my ( $self, $counter_file ) = @_; + + my $max_allowed_packet = C4::Context->dbh->selectrow_array(q{SELECT @@max_allowed_packet}); + if ( length($counter_file) > $max_allowed_packet ) { + $self->{job_callbacks}->{add_message_callback}->( + { + type => 'error', + code => 'payload_too_large', + message => $max_allowed_packet / 1024 / 1024, + } + ) if $self->{job_callbacks}; + return 1; + } + return 0; +} + =head3 _type =cut diff --git a/Koha/REST/V1/ERM/EUsage/UsageDataProviders.pm b/Koha/REST/V1/ERM/EUsage/UsageDataProviders.pm index 8f93299334..e05b529d4e 100644 --- a/Koha/REST/V1/ERM/EUsage/UsageDataProviders.pm +++ b/Koha/REST/V1/ERM/EUsage/UsageDataProviders.pm @@ -313,6 +313,10 @@ sub process_COUNTER_file { ? decode_base64( $body->{file_content} ) : ""; + my $max_allowed_packet = C4::Context->dbh->selectrow_array(q{SELECT @@max_allowed_packet}); + Koha::Exceptions::PayloadTooLarge->throw("File size exceeds limit defined by server") + if length($file_content) > $max_allowed_packet; + # Validate the file_content without storing, it'll throw an exception if fail my $counter_file_validation = Koha::ERM::EUsage::CounterFile->new( { file_content => $file_content } ); $counter_file_validation->validate; @@ -356,7 +360,7 @@ sub process_COUNTER_file { } elsif ( $_->isa('Koha::Exceptions::PayloadTooLarge') ) { return $c->render( status => 413, - openapi => { error => $_->error } + openapi => { error => $_->description } ); } elsif ( $_->isa('Koha::Exceptions::ERM::EUsage::CounterFile::UnsupportedRelease') ) { return $c->render( diff --git a/api/v1/swagger/paths/erm_usage_data_providers.yaml b/api/v1/swagger/paths/erm_usage_data_providers.yaml index 4f034ef0d2..f244d8b138 100644 --- a/api/v1/swagger/paths/erm_usage_data_providers.yaml +++ b/api/v1/swagger/paths/erm_usage_data_providers.yaml @@ -437,6 +437,10 @@ description: Access forbidden schema: $ref: "../swagger.yaml#/definitions/error" + 413: + description: Payload too large + schema: + $ref: "../swagger.yaml#/definitions/error" 500: description: |- Internal server error. Possible `error_code` attribute values: diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc index 9670e8f321..e52fa732b4 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc @@ -84,12 +84,20 @@ [% END %] [% SWITCH m.code %] - [% CASE 'object_already_exists' %] - [% m.title %] already exists in this data provider and was not created. New usage statistics were verified anyway - [% CASE 'object_added' %] - [% m.title %] successfully added - [% CASE 'object_could_not_be_added' %] - [% m.title %] could not be processed - please check the logs + [% CASE 'object_already_exists' %] + [% m.title %] already exists in this data provider and was not created. New usage statistics were verified anyway + [% CASE 'object_added' %] + [% m.title %] successfully added + [% CASE 'object_could_not_be_added' %] + [% m.title %] could not be processed - please check the logs + [% CASE 'payload_too_large' %] + COUNTER report file size exceeds limit defined by the database. Limit is [% m.message %] MB.
+ Please increase this limit or harvest a shorter period. + [% CASE 'no_items' %] + SUSHI response returned no items.
+ Please try a different harvest period. + [% CASE %] + SUSHI returned: [% m.message %] [% m.code %] [% END %] [% END %] -- 2.20.1