From d4afc38c7c64851fe8f5982dce46c056ca355c45 Mon Sep 17 00:00:00 2001 From: Matt Blenkinsop Date: Wed, 20 Sep 2023 13:13:12 +0000 Subject: [PATCH] Bug 34587: Add try catch blocks to object creation Currently a harvest fails completely if a single title fails to harvest. Rather than failing the entire job, this patch introduces try catch blocks to stop this and allow the job to continue. Failed imports are recorded and displayed in the background job UI on job completion Signed-off-by: Jessica Zairo Signed-off-by: Michaela Sieber Signed-off-by: Nick Clemens Signed-off-by: Tomas Cohen Arazi --- Koha/ERM/EUsage/CounterFile.pm | 122 ++++++++++-------- .../background_jobs/erm_sushi_harvester.inc | 2 + 2 files changed, 68 insertions(+), 56 deletions(-) diff --git a/Koha/ERM/EUsage/CounterFile.pm b/Koha/ERM/EUsage/CounterFile.pm index e9ed5b52dc..6c0ebfa415 100644 --- a/Koha/ERM/EUsage/CounterFile.pm +++ b/Koha/ERM/EUsage/CounterFile.pm @@ -20,6 +20,7 @@ package Koha::ERM::EUsage::CounterFile; use Modern::Perl; use Text::CSV_XS qw( csv ); +use Try::Tiny; use Koha::ERM::EUsage::CounterLog; use Koha::ERM::EUsage::UsagePlatform; @@ -130,77 +131,86 @@ sub _add_usage_objects { # Set job size to the amount of rows we're processing $self->{job_callbacks}->{set_size_callback}->( scalar( @{$rows} ) ) if $self->{job_callbacks}; - foreach my $row ( @{$rows} ) { - - # INFO: A single row may have multiple instances in the COUNTER report, one for each metric_type or access_type - # If we're on a row that we've already gone through, use the same usage object - # and add usage statistics for the different metric_type or access_type - if ( $self->_is_same_usage_object( $previous_object, $row ) ) { - $usage_object = $previous_object; - } else { - - # Check if usage object already exists in this data provider, e.g. from a previous harvest - $usage_object = $self->_search_for_usage_object($row); - - if ($usage_object) { - - # Usage object already exists, add job warning message and do nothing else - $self->_add_job_message( - 'warning', 'object_already_exists', - $row - ); + try { + # INFO: A single row may have multiple instances in the COUNTER report, one for each metric_type or access_type + # If we're on a row that we've already gone through, use the same usage object + # and add usage statistics for the different metric_type or access_type + if ( $self->_is_same_usage_object( $previous_object, $row ) ) { + $usage_object = $previous_object; } else { - # Fresh usage object, create it - $usage_object = $self->_add_usage_object_entry($row); - - # Usage object created, add job success message - $self->_add_job_message( 'success', 'object_added', $row ); + # Check if usage object already exists in this data provider, e.g. from a previous harvest + $usage_object = $self->_search_for_usage_object($row); + + if ($usage_object) { + + # Usage object already exists, add job warning message and do nothing else + $self->_add_job_message( + 'warning', 'object_already_exists', + $row + ); + } else { + try { + # Fresh usage object, create it + $usage_object = $self->_add_usage_object_entry($row); + + # Usage object created, add job success message + $self->_add_job_message( 'success', 'object_added', $row ); + } catch { + $self->_add_job_message( + 'error', 'object_could_not_be_added', + $row + ); + $self->{job_callbacks}->{step_callback}->() if $self->{job_callbacks}; + }; + } } - } - # Regex match for Mmm-yyyy expected format, e.g. "Jan 2022" - my @date_fields = - map( $_ =~ /\b[A-Z][a-z][a-z]\b [0-9]{4}\b/ ? $_ : (), keys %{$row} ); + # Regex match for Mmm-yyyy expected format, e.g. "Jan 2022" + my @date_fields = + map( $_ =~ /\b[A-Z][a-z][a-z]\b [0-9]{4}\b/ ? $_ : (), keys %{$row} ); - unless (@date_fields) { - warn "No monthly usage fields retrieved"; - } + unless (@date_fields) { + warn "No monthly usage fields retrieved"; + } - # Add monthly usage statistics for this usage object - my %yearly_usages = (); - foreach my $year_month (@date_fields) { - my $usage = %{$row}{$year_month}; + # Add monthly usage statistics for this usage object + my %yearly_usages = (); + foreach my $year_month (@date_fields) { + my $usage = %{$row}{$year_month}; - # Skip this monthly usage entry if it's 0 - next if $usage eq "0"; + # Skip this monthly usage entry if it's 0 + next if $usage eq "0"; - my $month = substr( $year_month, 0, 3 ); - my $year = substr( $year_month, 4, 4 ); + my $month = substr( $year_month, 0, 3 ); + my $year = substr( $year_month, 4, 4 ); - if ( !exists $yearly_usages{$year} ) { - $yearly_usages{$year} = $usage; - } else { - $yearly_usages{$year} += $usage; + if ( !exists $yearly_usages{$year} ) { + $yearly_usages{$year} = $usage; + } else { + $yearly_usages{$year} += $usage; + } + + $self->_add_monthly_usage_entries( + $usage_object, + $row->{Metric_Type}, $row, $year, $month, $usage + ); } - $self->_add_monthly_usage_entries( - $usage_object, - $row->{Metric_Type}, $row, $year, $month, $usage + # Add yearly usage statistics for this usage object + $self->_add_yearly_usage_entries( + $usage_object, $row->{Metric_Type}, + $row, \%yearly_usages ); - } - - # Add yearly usage statistics for this usage object - $self->_add_yearly_usage_entries( - $usage_object, $row->{Metric_Type}, - $row, \%yearly_usages - ); - $previous_object = $usage_object; + $previous_object = $usage_object; - # Update background job step - $self->{job_callbacks}->{step_callback}->() if $self->{job_callbacks}; + # Update background job step + $self->{job_callbacks}->{step_callback}->() if $self->{job_callbacks}; + } catch { + warn $_; + } } } diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc index e89d623788..fa28b9be3a 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc @@ -79,6 +79,8 @@ [% m.title %] already exists in this data provider and was not created. New usage statistics were verified anyway [% CASE 'object_added' %] [% m.title %] successfully added + [% CASE 'object_could_not_be_added' %] + [% m.title %] could not be processed - please check the logs [% END %] [% END %] -- 2.20.1