1 package Koha::ERM::EUsage::UsageDataProvider;
3 # Copyright 2023 PTFS Europe
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it
8 # under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 3 of the License, or
10 # (at your option) any later version.
12 # Koha is distributed in the hope that it will be useful, but
13 # WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU General Public License for more details.
17 # You should have received a copy of the GNU General Public License
18 # along with Koha; if not, see <http://www.gnu.org/licenses>.
23 use JSON qw( decode_json );
28 use base qw(Koha::Object);
30 use Koha::ERM::EUsage::CounterFile;
31 use Koha::ERM::EUsage::CounterFiles;
32 use Koha::ERM::EUsage::UsageTitles;
33 use Koha::ERM::EUsage::UsageItems;
34 use Koha::ERM::EUsage::UsagePlatforms;
35 use Koha::ERM::EUsage::UsageDatabases;
36 use Koha::ERM::EUsage::MonthlyUsages;
37 use Koha::ERM::EUsage::SushiCounter;
38 use Koha::BackgroundJob::ErmSushiHarvester;
42 Koha::ERM::EUsage::UsageDataProvider - Koha ErmUsageDataProvider Object class
50 Getter/setter for counter_files for this usage data provider
55 my ( $self, $counter_files ) = @_;
58 for my $counter_file (@$counter_files) {
59 Koha::ERM::EUsage::CounterFile->new($counter_file)->store( $self->{job_callbacks} );
62 my $counter_files_rs = $self->_result->erm_counter_files;
63 return Koha::ERM::EUsage::CounterFiles->_new_from_dbic($counter_files_rs);
66 =head3 enqueue_counter_file_processing_job
68 Enqueues a background job to process a COUNTER file that has been uploaded
72 sub enqueue_counter_file_processing_job {
73 my ( $self, $args ) = @_;
76 my $job_id = Koha::BackgroundJob::ErmSushiHarvester->new->enqueue(
78 ud_provider_id => $self->erm_usage_data_provider_id,
79 file_content => $args->{file_content},
91 =head3 enqueue_sushi_harvest_jobs
93 Enqueues one harvest background job for each report type in this usage data provider
97 sub enqueue_sushi_harvest_jobs {
98 my ( $self, $args ) = @_;
100 my @report_types = split( /;/, $self->report_types );
103 foreach my $report_type (@report_types) {
105 my $job_id = Koha::BackgroundJob::ErmSushiHarvester->new->enqueue(
107 ud_provider_id => $self->erm_usage_data_provider_id,
108 report_type => $report_type,
109 begin_date => $args->{begin_date},
110 end_date => $args->{end_date},
111 ud_provider_name => $self->name,
118 report_type => $report_type,
129 $ud_provider->harvest_sushi(
131 begin_date => $args->{begin_date},
132 end_date => $args->{end_date},
133 report_type => $args->{report_type}
137 Runs this usage data provider's SUSHI harvester
138 Builds the URL query and requests the COUNTER 5 SUSHI service
140 COUNTER SUSHI api spec:
141 https://app.swaggerhub.com/apis/COUNTER/counter-sushi_5_0_api/5.0.2
147 Begin date of the SUSHI harvest
155 End date of the SUSHI harvest
163 Report type to run this harvest on
170 my ( $self, $args ) = @_;
172 # Set class wide vars
173 $self->{report_type} = $args->{report_type};
174 $self->{begin_date} = $args->{begin_date};
175 $self->{end_date} = $args->{end_date};
176 my $url = $self->_build_url_query;
177 my $request = HTTP::Request->new( 'GET' => $url );
178 my $ua = LWP::UserAgent->new;
179 my $response = $ua->simple_request($request);
181 if ( $response->code >= 400 ) {
182 my $result = decode_json( $response->decoded_content );
185 if ( ref($result) eq 'ARRAY' ) {
186 for my $r (@$result) {
187 $message .= $r->{message};
191 #TODO: May want to check $result->{Report_Header}->{Exceptions} here
192 $message = $result->{message} || $result->{Message} || q{};
193 if ( $result->{errors} ) {
194 for my $e ( @{ $result->{errors} } ) {
195 $message .= $e->{message};
200 #TODO: May want to add a job error message here?
201 warn sprintf "ERROR - SUSHI service %s returned %s - %s\n", $url,
202 $response->code, $message;
203 if ( $response->code == 404 ) {
204 Koha::Exceptions::ObjectNotFound->throw($message);
205 } elsif ( $response->code == 401 ) {
206 Koha::Exceptions::Authorization::Unauthorized->throw($message);
209 #TODO: May want to add a job error message here?
210 die sprintf "ERROR requesting SUSHI service\n%s\ncode %s: %s\n",
211 $url, $response->code,
214 } elsif ( $response->code == 204 ) { # No content
218 my $decoded_response = decode_json( $response->decoded_content );
220 return if $self->_sushi_errors($decoded_response);
222 # Parse the SUSHI response
224 Koha::ERM::EUsage::SushiCounter->new( { response => $decoded_response } );
225 my $counter_file = $sushi_counter->get_COUNTER_from_SUSHI;
227 return if $self->_counter_file_size_too_large($counter_file);
229 $self->counter_files(
232 usage_data_provider_id => $self->erm_usage_data_provider_id,
233 file_content => $counter_file,
234 date_uploaded => POSIX::strftime( "%Y%m%d%H%M%S", localtime ),
236 #TODO: add ".csv" to end of filename here
237 filename => $self->name . "_" . $self->{report_type},
244 =head3 set_background_job_callbacks
246 $self->set_background_job_callbacks($background_job_callbacks);
248 Sets the background job callbacks
252 =item background_job_callbacks
254 Background job callbacks
260 sub set_background_job_callbacks {
261 my ( $self, $background_job_callbacks ) = @_;
263 $self->{job_callbacks} = $background_job_callbacks;
266 =head3 test_connection
268 Tests the connection of the harvester to the SUSHI service and returns any alerts of planned SUSHI outages
272 sub test_connection {
275 my $url = _validate_url($self->service_url, 'status');
277 $url .= '?customer_id=' . $self->customer_id;
278 $url .= '&requestor_id=' . $self->requestor_id if $self->requestor_id;
279 $url .= '&api_key=' . $self->api_key if $self->api_key;
281 my $request = HTTP::Request->new( 'GET' => $url );
282 my $ua = LWP::UserAgent->new;
283 my $response = $ua->simple_request($request);
285 if ( $response->{_rc} >= 400 ) {
286 my $message = $response->{_msg};
287 if ( $response->{_rc} == 404 ) {
288 Koha::Exceptions::ObjectNotFound->throw($message);
289 } elsif ( $response->{_rc} == 401 ) {
290 Koha::Exceptions::Authorization::Unauthorized->throw($message);
292 die sprintf "ERROR testing SUSHI service\n%s\ncode %s: %s\n",
293 $url, $response->{_rc},
298 my $result = decode_json( $response->decoded_content );
300 if ( ref($result) eq 'ARRAY' ) {
301 for my $r (@$result) {
302 $status = $r->{Service_Active};
305 $status = $result->{Service_Active};
315 =head3 erm_usage_titles
317 Method to embed erm_usage_titles to titles for report formatting
321 sub erm_usage_titles {
323 my $usage_title_rs = $self->_result->erm_usage_titles;
324 return Koha::ERM::EUsage::UsageTitles->_new_from_dbic($usage_title_rs);
327 =head3 erm_usage_muses
329 Method to embed erm_usage_muses to titles for report formatting
333 sub erm_usage_muses {
335 my $usage_mus_rs = $self->_result->erm_usage_muses;
336 return Koha::ERM::EUsage::MonthlyUsages->_new_from_dbic($usage_mus_rs);
339 =head3 erm_usage_platforms
341 Method to embed erm_usage_platforms to platforms for report formatting
345 sub erm_usage_platforms {
347 my $usage_platform_rs = $self->_result->erm_usage_platforms;
348 return Koha::ERM::EUsage::UsagePlatforms->_new_from_dbic($usage_platform_rs);
351 =head3 erm_usage_items
353 Method to embed erm_usage_items to items for report formatting
357 sub erm_usage_items {
359 my $usage_item_rs = $self->_result->erm_usage_items;
360 return Koha::ERM::EUsage::UsageItems->_new_from_dbic($usage_item_rs);
363 =head3 erm_usage_databases
365 Method to embed erm_usage_databases to databases for report formatting
369 sub erm_usage_databases {
371 my $usage_database_rs = $self->_result->erm_usage_databases;
372 return Koha::ERM::EUsage::UsageDatabases->_new_from_dbic($usage_database_rs);
375 =head2 Internal methods
377 =head3 _build_url_query
379 Build the URL query params for COUNTER 5 SUSHI request
383 sub _build_url_query {
386 unless ( $self->service_url && $self->customer_id ) {
388 "SUSHI Harvesting config for usage data provider %d is missing service_url or customer_id\n",
389 $self->erm_usage_data_provider_id;
392 my $url = _validate_url($self->service_url, 'harvest');
394 $url .= lc $self->{report_type};
395 $url .= '?customer_id=' . $self->customer_id;
396 $url .= '&requestor_id=' . $self->requestor_id if $self->requestor_id;
397 $url .= '&api_key=' . $self->api_key if $self->api_key;
398 $url .= '&begin_date=' . substr $self->{begin_date}, 0, 7 if $self->{begin_date};
399 $url .= '&end_date=' . substr $self->{end_date}, 0, 7 if $self->{end_date};
407 Checks whether the url ends in a trailing "/" and adds one if not
409 my $url = _validate_url($url, 'harvest')
411 $caller is either the harvest_sushi function ("harvest") or the test_connection function ("status")
416 my ( $url, $caller ) = @_;
418 if($caller eq 'harvest') {
419 # Not all urls will end in "/" - add one so they are standardised
420 $url = _check_trailing_character($url);
421 # All SUSHI report requests should be to the "/reports" endpoint
422 # Not all providers in the counter registry include this in their data so we need to check and add it
423 my $reports_param = substr $url, -8;
424 $url .= 'reports/' if $reports_param ne 'reports/';
426 $url = _check_trailing_character($url);
432 =head3 _check_trailing_character
434 Checks whether a url string ends in a "/" before we concatenate further params to the end of the url
438 sub _check_trailing_character {
441 my $trailing_char = substr $url, -1;
442 if ( $trailing_char ne '/' ) {
451 Checks and handles possible errors in the SUSHI response
452 Additionally, adds background job report message(s) if that is the case
457 my ( $self, $decoded_response ) = @_;
459 if ( $decoded_response->{Severity} ) {
460 $self->{job_callbacks}->{add_message_callback}->(
463 code => $decoded_response->{Code},
464 message => $decoded_response->{Severity} . ' - ' . $decoded_response->{Message},
466 ) if $self->{job_callbacks};
470 if ( $decoded_response->{Report_Header}->{Exceptions} ) {
471 foreach my $exception ( @{ $decoded_response->{Report_Header}->{Exceptions} } ) {
472 $self->{job_callbacks}->{add_message_callback}->(
475 code => $exception->{Code},
476 message => $exception->{Message} . ' - ' . $exception->{Data},
478 ) if $self->{job_callbacks};
483 if ( scalar @{ $decoded_response->{Report_Items} } == 0 ) {
484 $self->{job_callbacks}->{add_message_callback}->(
489 ) if $self->{job_callbacks};
496 =head3 _counter_file_size_too_large
498 Checks whether a counter file size exceeds the size allowed by the database or not
499 Additionally, adds a background job report message if that is the case
503 sub _counter_file_size_too_large {
504 my ( $self, $counter_file ) = @_;
506 my $max_allowed_packet = C4::Context->dbh->selectrow_array(q{SELECT @@max_allowed_packet});
507 if ( length($counter_file) > $max_allowed_packet ) {
508 $self->{job_callbacks}->{add_message_callback}->(
511 code => 'payload_too_large',
512 message => $max_allowed_packet / 1024 / 1024,
514 ) if $self->{job_callbacks};
525 return 'ErmUsageDataProvider';