From cf87082682343503a6fdff66fcf2cae35fb232d2 Mon Sep 17 00:00:00 2001 From: Pedro Amorim Date: Mon, 21 Aug 2023 14:50:45 +0000 Subject: [PATCH] Bug 34587: File upload - Background job ErmSushiHarvester background job will now be initiated by either enqueue_counter_file_processing_job or by enqueue_sushi_harvest_jobs from Koha/ERM/UsageDataProvider.pm, the former if triggered by a manual file upload, the latter if by the 'run now' button or by the cron script. This commit also includes some rewording/refactoring, namely: - COUNTER file validation now happens in the API, before enqueuing the job. - Removal of no longer used POST /erm/counter_files endpoint - Koha/ERM/UsageDataProvider.pm: -- run method is now enqueue_sushi_harvest_jobs -- new enqueue_counter_file_processing_job method -- harvest method is now harvest_sushi -- new set_background_job_callbacks method to set the background job callbacks - REST/V1/ERM/UsageDataProviders -- run method is now process_SUSHI_response -- new process_COUNTER_file endpoint Signed-off-by: Jessica Zairo Signed-off-by: Michaela Sieber Signed-off-by: Nick Clemens Signed-off-by: Tomas Cohen Arazi --- Koha/BackgroundJob/ErmSushiHarvester.pm | 32 ++++- Koha/ERM/CounterFile.pm | 5 +- Koha/ERM/UsageDataProvider.pm | 61 ++++++--- Koha/REST/V1/ERM/CounterFiles.pm | 1 - Koha/REST/V1/ERM/UsageDataProviders.pm | 120 +++++++++++++++++- api/v1/swagger/paths/erm_counter_files.yaml | 60 --------- .../paths/erm_usage_data_providers.yaml | 54 +++++++- api/v1/swagger/swagger.yaml | 6 +- .../background_jobs/erm_sushi_harvester.inc | 24 ++-- ...UsageStatisticsDataProvidersFileImport.vue | 24 +++- .../ERM/UsageStatisticsDataProvidersList.vue | 2 +- .../prog/js/vue/fetch/erm-api-client.js | 13 +- 12 files changed, 287 insertions(+), 115 deletions(-) diff --git a/Koha/BackgroundJob/ErmSushiHarvester.pm b/Koha/BackgroundJob/ErmSushiHarvester.pm index 0f8e2916b9..f1ecf09c7f 100644 --- a/Koha/BackgroundJob/ErmSushiHarvester.pm +++ b/Koha/BackgroundJob/ErmSushiHarvester.pm @@ -75,10 +75,7 @@ sub process { my $ud_provider = Koha::ERM::UsageDataProviders->find( $args->{ud_provider_id} ); - $ud_provider->harvest( - $args->{begin_date}, - $args->{end_date}, - $args->{report_type}, + $ud_provider->set_background_job_callbacks( { report_info_callback => sub { $self->report_info(@_); }, step_callback => sub { $self->step; }, @@ -87,9 +84,34 @@ sub process { } ); + my $counter_file; + if ( $args->{file_content} ) { + my $counter_files_rs = $ud_provider->counter_files( + [ + { + usage_data_provider_id => $ud_provider->erm_usage_data_provider_id, + file_content => $args->{file_content}, + date_uploaded => POSIX::strftime( "%Y%m%d%H%M%S", localtime ), + filename => $ud_provider->name . "_" . $ud_provider->{report_type}, + } + ] + ); + $counter_file = $counter_files_rs->last; + } else { + $ud_provider->harvest_sushi( + { + begin_date => $args->{begin_date}, + end_date => $args->{end_date}, + report_type => $args->{report_type} + } + ); + } + + my $job_report_report_type = $ud_provider->{report_type} || $counter_file->type; + # Prepare job report my $report = { - report_type => $ud_provider->{report_type}, + report_type => $job_report_report_type, total_records => $ud_provider->{total_records}, us_report_info => $self->{us_report_info}, ud_provider_id => $ud_provider->erm_usage_data_provider_id, diff --git a/Koha/ERM/CounterFile.pm b/Koha/ERM/CounterFile.pm index c0580c937f..82437fa73a 100644 --- a/Koha/ERM/CounterFile.pm +++ b/Koha/ERM/CounterFile.pm @@ -81,7 +81,6 @@ Receive background_job_callbacks to be able to update job progress sub store { my ( $self, $background_job_callbacks ) = @_; - $self->_validate; $self->_set_report_type_from_file; my $result = $self->SUPER::store; @@ -275,7 +274,7 @@ sub _add_yearly_usage_entries { } } -=head3 _validate +=head3 validate Verifies if the given file_content is a valid COUNTER file or not @@ -284,7 +283,7 @@ A I exception is thrown =cut -sub _validate { +sub validate { my ($self) = @_; open my $fh, "<", \$self->file_content or die; diff --git a/Koha/ERM/UsageDataProvider.pm b/Koha/ERM/UsageDataProvider.pm index 4e9230d221..9e20793c42 100644 --- a/Koha/ERM/UsageDataProvider.pm +++ b/Koha/ERM/UsageDataProvider.pm @@ -126,17 +126,17 @@ sub enqueue_sushi_harvest_jobs { return \@jobs; } -=head3 harvest +=head3 harvest_sushi - $ud_provider->harvest( + $ud_provider->harvest_sushi( { - step_callback => sub { $self->step; }, - set_size_callback => sub { $self->set_job_size(@_); }, - add_message_callback => sub { $self->add_message(@_); }, + begin_date => $args->{begin_date}, + end_date => $args->{end_date}, + report_type => $args->{report_type} } ); -Run the SUSHI harvester of this usage data provider +Runs this usage data provider's SUSHI harvester Builds the URL query and requests the COUNTER 5 SUSHI service COUNTER SUSHI api spec: @@ -144,30 +144,37 @@ https://app.swaggerhub.com/apis/COUNTER/counter-sushi_5_0_api/5.0.2 =over -=item report_type +=item begin_date -Report type to run this harvest on +Begin date of the SUSHI harvest =back =over -=item background_job_callbacks +=item end_date -Receive background_job_callbacks to be able to update job +End date of the SUSHI harvest + +=back + +=over + +=item report_type + +Report type to run this harvest on =back =cut -sub harvest { - my ( $self, $begin_date, $end_date, $report_type, $background_job_callbacks ) = @_; +sub harvest_sushi { + my ( $self, $args ) = @_; # Set class wide vars - $self->{job_callbacks} = $background_job_callbacks; - $self->{report_type} = $report_type; - $self->{begin_date} = $begin_date; - $self->{end_date} = $end_date; + $self->{report_type} = $args->{report_type}; + $self->{begin_date} = $args->{begin_date}; + $self->{end_date} = $args->{end_date}; my $url = $self->_build_url_query; my $request = HTTP::Request->new( 'GET' => $url ); @@ -217,6 +224,28 @@ sub harvest { $self->parse_SUSHI_response( decode_json( $response->decoded_content ) ); } +=head3 set_background_job_callbacks + + $self->set_background_job_callbacks($background_job_callbacks); + +Sets the background job callbacks + +=over + +=item background_job_callbacks + +Background job callbacks + +=back + +=cut + +sub set_background_job_callbacks { + my ( $self, $background_job_callbacks ) = @_; + + $self->{job_callbacks} = $background_job_callbacks; +} + =head3 parse_SUSHI_response $self->parse_SUSHI_response( decode_json( $response->decoded_content ) ); diff --git a/Koha/REST/V1/ERM/CounterFiles.pm b/Koha/REST/V1/ERM/CounterFiles.pm index 263be308ee..a224a4e306 100644 --- a/Koha/REST/V1/ERM/CounterFiles.pm +++ b/Koha/REST/V1/ERM/CounterFiles.pm @@ -19,7 +19,6 @@ package Koha::REST::V1::ERM::CounterFiles; use Modern::Perl; -use MIME::Base64 qw( decode_base64 ); use Mojo::Base 'Mojolicious::Controller'; use Koha::ERM::CounterFiles; diff --git a/Koha/REST/V1/ERM/UsageDataProviders.pm b/Koha/REST/V1/ERM/UsageDataProviders.pm index 6d8ee7a11b..800db03f80 100644 --- a/Koha/REST/V1/ERM/UsageDataProviders.pm +++ b/Koha/REST/V1/ERM/UsageDataProviders.pm @@ -19,6 +19,7 @@ package Koha::REST::V1::ERM::UsageDataProviders; use Modern::Perl; +use MIME::Base64 qw( decode_base64 ); use Mojo::Base 'Mojolicious::Controller'; use Koha::ERM::UsageDataProviders; @@ -307,11 +308,113 @@ sub delete { }; } -=head3 run +=head3 process_COUNTER_file + +Controller function that handles processing of the COUNTER file +It will->enqueue_counter_file_processing_job for its respective usage data provider =cut -sub run { +sub process_COUNTER_file { + my $c = shift->openapi->valid_input or return; + + return try { + Koha::Database->new->schema->txn_do( + sub { + + my $body = $c->validation->param('body'); + + my $file_content = + defined( $body->{file_content} ) + ? decode_base64( $body->{file_content} ) + : ""; + + # Validate the file_content without storing, it'll throw an exception if fail + my $counter_file_validation = + Koha::ERM::CounterFile->new( + { file_content => $file_content } ); + $counter_file_validation->validate; + + # Validation was successful, enqueue the job + my $udprovider = + Koha::ERM::UsageDataProviders->find( + $c->validation->param('erm_usage_data_provider_id') ); + + my $jobs = $udprovider->enqueue_counter_file_processing_job( + { + file_content => $file_content, + } + ); + + return $c->render( + status => 200, + openapi => { jobs => [ @{$jobs} ] } + ); + } + ); + } + catch { + + my $to_api_mapping = Koha::ERM::CounterFile->new->to_api_mapping; + + if ( blessed $_ ) { + if ( $_->isa('Koha::Exceptions::Object::DuplicateID') ) { + return $c->render( + status => 409, + openapi => + { error => $_->error, conflict => $_->duplicate_id } + ); + } + elsif ( $_->isa('Koha::Exceptions::Object::FKConstraint') ) { + return $c->render( + status => 400, + openapi => { + error => "Given " + . $to_api_mapping->{ $_->broken_fk } + . " does not exist" + } + ); + } + elsif ( $_->isa('Koha::Exceptions::BadParameter') ) { + return $c->render( + status => 400, + openapi => { + error => "Given " + . $to_api_mapping->{ $_->parameter } + . " does not exist" + } + ); + } + elsif ( $_->isa('Koha::Exceptions::PayloadTooLarge') ) { + return $c->render( + status => 413, + openapi => { error => $_->error } + ); + } + elsif ( + $_->isa( + 'Koha::Exceptions::ERM::CounterFile::UnsupportedRelease') + ) + { + return $c->render( + status => 400, + openapi => { error => $_->description } + ); + } + } + + $c->unhandled_exception($_); + }; +} + +=head3 process_SUSHI_response + +Controller function that handles processing of the SUSHI response +It will ->enqueue_sushi_harvest_jobs for this usage data provider + +=cut + +sub process_SUSHI_response { my $c = shift->openapi->valid_input or return; my $body = $c->validation->param('body'); @@ -325,7 +428,8 @@ sub run { ); } - my $udprovider = Koha::ERM::UsageDataProviders->find( $c->validation->param('erm_usage_data_provider_id') ); + my $udprovider = Koha::ERM::UsageDataProviders->find( + $c->validation->param('erm_usage_data_provider_id') ); unless ($udprovider) { return $c->render( @@ -335,7 +439,12 @@ sub run { } return try { - my $jobs = $udprovider->run( { begin_date => $begin_date, end_date => $end_date } ); + my $jobs = $udprovider->enqueue_sushi_harvest_jobs( + { + begin_date => $begin_date, + end_date => $end_date + } + ); return $c->render( status => 200, @@ -354,7 +463,8 @@ sub run { sub test_connection { my $c = shift->openapi->valid_input or return; - my $udprovider = Koha::ERM::UsageDataProviders->find( $c->validation->param('erm_usage_data_provider_id') ); + my $udprovider = Koha::ERM::UsageDataProviders->find( + $c->validation->param('erm_usage_data_provider_id') ); unless ($udprovider) { return $c->render( diff --git a/api/v1/swagger/paths/erm_counter_files.yaml b/api/v1/swagger/paths/erm_counter_files.yaml index ec7ec071b1..3e5d3bfb87 100644 --- a/api/v1/swagger/paths/erm_counter_files.yaml +++ b/api/v1/swagger/paths/erm_counter_files.yaml @@ -129,66 +129,6 @@ x-koha-authorization: permissions: erm: 1 - post: - x-mojo-to: ERM::CounterFiles#add - operationId: addErmCounterFiles - tags: - - counter_file - summary: Add counter_file - consumes: - - application/json - produces: - - application/json - parameters: - - description: A JSON object containing information about the new counter_file - in: body - name: body - required: true - schema: - $ref: "../swagger.yaml#/definitions/erm_counter_file" - responses: - 201: - description: A successfully created counter_file - schema: - items: - $ref: "../swagger.yaml#/definitions/erm_counter_file" - 400: - description: Bad parameter - schema: - $ref: "../swagger.yaml#/definitions/error" - 401: - description: Authentication required - schema: - $ref: "../swagger.yaml#/definitions/error" - 403: - description: Access forbidden - schema: - $ref: "../swagger.yaml#/definitions/error" - 404: - description: Ressource not found - schema: - $ref: "../swagger.yaml#/definitions/error" - 409: - description: Conflict in creating resource - schema: - $ref: "../swagger.yaml#/definitions/error" - 413: - description: Payload too large - schema: - $ref: "../swagger.yaml#/definitions/error" - 500: - description: |- - Internal server error. Possible `error_code` attribute values: - * `internal_server_error` - schema: - $ref: "../swagger.yaml#/definitions/error" - 503: - description: Under maintenance - schema: - $ref: "../swagger.yaml#/definitions/error" - x-koha-authorization: - permissions: - erm: 1 "/erm/counter_files/{erm_counter_files_id}": delete: x-mojo-to: ERM::CounterFiles#delete diff --git a/api/v1/swagger/paths/erm_usage_data_providers.yaml b/api/v1/swagger/paths/erm_usage_data_providers.yaml index 7c5aa88e1d..c62d989a90 100644 --- a/api/v1/swagger/paths/erm_usage_data_providers.yaml +++ b/api/v1/swagger/paths/erm_usage_data_providers.yaml @@ -353,13 +353,13 @@ x-koha-authorization: permissions: erm: 1 -"/erm/usage_data_providers/{erm_usage_data_provider_id}/run": +"/erm/usage_data_providers/{erm_usage_data_provider_id}/process_SUSHI_response": post: - x-mojo-to: ERM::UsageDataProviders#run - operationId: runUsageDataProviderHarvester + x-mojo-to: ERM::UsageDataProviders#process_SUSHI_response + operationId: processSUSHICounterUsageDataProviderHarvester tags: - usage data provider harvester - summary: Run this data provider's harvester + summary: Process SUSHI COUNTER for this data provider's harvester produces: - application/json parameters: @@ -382,7 +382,51 @@ additionalProperties: false responses: 200: - description: Successful harvest run + description: Successful SUSHI COUNTER processing + schema: + items: + $ref: "../swagger.yaml#/definitions/erm_usage_data_provider" + 400: + description: Bad request + schema: + $ref: "../swagger.yaml#/definitions/error" + 403: + description: Access forbidden + schema: + $ref: "../swagger.yaml#/definitions/error" + 500: + description: |- + Internal server error. Possible `error_code` attribute values: + * `internal_server_error` + schema: + $ref: "../swagger.yaml#/definitions/error" + 503: + description: Under maintenance + schema: + $ref: "../swagger.yaml#/definitions/error" + x-koha-authorization: + permissions: + erm: 1 +"/erm/usage_data_providers/{erm_usage_data_provider_id}/process_COUNTER_file": + post: + x-mojo-to: ERM::UsageDataProviders#process_COUNTER_file + operationId: processCOUNTERFileUsageDataProviderHarvester + tags: + - usage data provider harvester + summary: Process COUNTER file upload for this data provider's harvester + produces: + - application/json + parameters: + - $ref: "../swagger.yaml#/parameters/erm_usage_data_provider_id_pp" + - description: A JSON object containing information about the new counter_file + in: body + name: body + required: true + schema: + $ref: "../swagger.yaml#/definitions/erm_counter_file" + responses: + 200: + description: Successful COUNTER file processing schema: items: $ref: "../swagger.yaml#/definitions/erm_usage_data_provider" diff --git a/api/v1/swagger/swagger.yaml b/api/v1/swagger/swagger.yaml index 2d9003dfd0..b8445cc9be 100644 --- a/api/v1/swagger/swagger.yaml +++ b/api/v1/swagger/swagger.yaml @@ -309,8 +309,10 @@ paths: $ref: ./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers "/erm/usage_data_providers/{erm_usage_data_provider_id}": $ref: "./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers~1{erm_usage_data_provider_id}" - "/erm/usage_data_providers/{erm_usage_data_provider_id}/run": - $ref: "./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers~1{erm_usage_data_provider_id}~1run" + "/erm/usage_data_providers/{erm_usage_data_provider_id}/process_SUSHI_response": + $ref: "./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers~1{erm_usage_data_provider_id}~1process_SUSHI_response" + "/erm/usage_data_providers/{erm_usage_data_provider_id}/process_COUNTER_file": + $ref: "./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers~1{erm_usage_data_provider_id}~1process_COUNTER_file" "/erm/usage_data_providers/{erm_usage_data_provider_id}/test_connection": $ref: "./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers~1{erm_usage_data_provider_id}~1test_connection" /erm/usage_databases: diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc index 12a84ed00a..e89d623788 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc @@ -18,18 +18,20 @@ Report rows processed [% job.progress %] - - [% IF report.report_type.match( '^PR' ) %] - Platforms processed - [% ELSIF report.report_type.match( '^DR' ) %] - Databases processed - [% ELSIF report.report_type.match( '^IR' ) %] - Items processed - [% ELSIF report.report_type.match( '^TR' ) %] - Titles processed + [% IF report.total_records %] + + [% IF report.report_type.match( '^PR' ) %] + SUSHI Platforms processed + [% ELSIF report.report_type.match( '^DR' ) %] + SUSHI Databases processed + [% ELSIF report.report_type.match( '^IR' ) %] + SUSHI Items processed + [% ELSIF report.report_type.match( '^TR' ) %] + SUSHI Titles processed + [% END %] + [% report.total_records | html %] + [% END %] - [% report.total_records | html %] - Monthly usages skipped [% report.us_report_info.skipped_mus | html %] diff --git a/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/UsageStatisticsDataProvidersFileImport.vue b/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/UsageStatisticsDataProvidersFileImport.vue index a3decf053c..abcaf414b6 100644 --- a/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/UsageStatisticsDataProvidersFileImport.vue +++ b/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/UsageStatisticsDataProvidersFileImport.vue @@ -35,11 +35,21 @@