1 package Koha::ERM::UsageDataProvider;
3 # This file is part of Koha.
5 # Koha is free software; you can redistribute it and/or modify it
6 # under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # Koha is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with Koha; if not, see <http://www.gnu.org/licenses>.
21 use JSON qw( decode_json );
23 use Text::CSV_XS qw( csv );
27 use base qw(Koha::Object);
29 use Koha::ERM::CounterFile;
30 use Koha::ERM::CounterFiles;
31 use Koha::ERM::UsageTitles;
32 use Koha::ERM::UsageItems;
33 use Koha::ERM::UsagePlatforms;
34 use Koha::ERM::UsageDatabases;
35 use Koha::ERM::MonthlyUsages;
36 use Koha::BackgroundJob::ErmSushiHarvester;
40 Koha::ERM::UsageDataProvider - Koha ErmUsageDataProvider Object class
48 Getter/setter for counter_files for this usage data provider
53 my ( $self, $counter_files ) = @_;
56 for my $counter_file (@$counter_files) {
57 Koha::ERM::CounterFile->new($counter_file)
58 ->store( $self->{job_callbacks} );
61 my $counter_files_rs = $self->_result->erm_counter_files;
62 return Koha::ERM::CounterFiles->_new_from_dbic($counter_files_rs);
65 =head3 enqueue_counter_file_processing_job
67 Enqueues a background job to process a COUNTER file that has been uploaded
71 sub enqueue_counter_file_processing_job {
72 my ( $self, $args ) = @_;
75 my $job_id = Koha::BackgroundJob::ErmSushiHarvester->new->enqueue(
77 ud_provider_id => $self->erm_usage_data_provider_id,
78 file_content => $args->{file_content},
92 =head3 enqueue_sushi_harvest_jobs
94 Enqueues one harvest background job for each report type in this usage data provider
98 sub enqueue_sushi_harvest_jobs {
99 my ( $self, $args ) = @_;
101 my @report_types = split( /;/, $self->report_types );
104 foreach my $report_type (@report_types) {
106 my $job_id = Koha::BackgroundJob::ErmSushiHarvester->new->enqueue(
108 ud_provider_id => $self->erm_usage_data_provider_id,
109 report_type => $report_type
116 report_type => $report_type,
127 $ud_provider->harvest(
129 step_callback => sub { $self->step; },
130 set_size_callback => sub { $self->set_job_size(@_); },
131 add_message_callback => sub { $self->add_message(@_); },
135 Run the SUSHI harvester of this usage data provider
136 Builds the URL query and requests the COUNTER 5 SUSHI service
138 COUNTER SUSHI api spec:
139 https://app.swaggerhub.com/apis/COUNTER/counter-sushi_5_0_api/5.0.2
145 Report type to run this harvest on
151 =item background_job_callbacks
153 Receive background_job_callbacks to be able to update job
160 my ( $self, $report_type, $background_job_callbacks ) = @_;
162 # Set class wide vars
163 $self->{job_callbacks} = $background_job_callbacks;
164 $self->{report_type} = $report_type;
166 my $url = $self->_build_url_query;
167 my $request = HTTP::Request->new( 'GET' => $url );
168 my $ua = LWP::UserAgent->new;
169 my $response = $ua->simple_request($request);
171 if ( $response->code >= 400 ) {
172 my $result = decode_json( $response->decoded_content );
175 if ( ref($result) eq 'ARRAY' ) {
176 for my $r (@$result) {
177 $message .= $r->{message};
181 #TODO: May want to check $result->{Report_Header}->{Exceptions} here
182 $message = $result->{message} || $result->{Message} || q{};
183 if ( $result->{errors} ) {
184 for my $e ( @{ $result->{errors} } ) {
185 $message .= $e->{message};
190 #TODO: May want to add a job error message here?
191 warn sprintf "ERROR - SUSHI service %s returned %s - %s\n", $url,
192 $response->code, $message;
193 if ( $response->code == 404 ) {
194 Koha::Exceptions::ObjectNotFound->throw($message);
196 elsif ( $response->code == 401 ) {
197 Koha::Exceptions::Authorization::Unauthorized->throw($message);
200 #TODO: May want to add a job error message here?
201 die sprintf "ERROR requesting SUSHI service\n%s\ncode %s: %s\n",
202 $url, $response->code,
206 elsif ( $response->code == 204 ) { # No content
210 # Parse the SUSHI response
211 $self->parse_SUSHI_response( decode_json( $response->decoded_content ) );
214 =head3 parse_SUSHI_response
216 $self->parse_SUSHI_response( decode_json( $response->decoded_content ) );
218 Parse the SUSHI response, prepare the COUNTER report file header,
219 column headings and body
225 The result of the SUSHI response after json decoded
231 sub parse_SUSHI_response {
232 my ( $self, $result ) = @_;
234 # Set class wide sushi response content
236 header => $result->{Report_Header},
237 body => $result->{Report_Items}
240 #TODO: Handle empty $self->{sushi}->{body} here!
242 # Get ready to build COUNTER file
243 my @report_header = $self->_COUNTER_report_header;
244 my @report_column_headings = $self->_COUNTER_report_column_headings;
245 my @report_body = $self->_COUNTER_report_body;
247 $self->_build_COUNTER_report_file( \@report_header,
248 \@report_column_headings, \@report_body );
251 =head2 Internal methods
253 =head3 _build_url_query
255 Build the URL query params for COUNTER 5 SUSHI request
259 sub _build_url_query {
262 unless ( $self->service_url && $self->customer_id ) {
264 "SUSHI Harvesting config for usage data provider %d is missing service_url or customer_id\n",
265 $self->erm_usage_data_provider_id;
268 # FIXME: service_url needs to end in 'reports/'
269 # below concat will result in a badly formed URL otherwise
270 # Either validate this on UI form, here, or both
271 my $url = $self->service_url;
273 $url .= $self->{report_type};
274 $url .= '?customer_id=' . $self->customer_id;
275 $url .= '&requestor_id=' . $self->requestor_id if $self->requestor_id;
276 $url .= '&api_key=' . $self->api_key if $self->api_key;
277 $url .= '&begin_date=' . $self->begin_date if $self->begin_date;
278 $url .= '&end_date=' . $self->end_date if $self->end_date;
283 =head3 _build_COUNTER_report_file
285 Build the COUNTER file
286 https://cop5.projectcounter.org/en/5.0.2/03-specifications/02-formats-for-counter-reports.html#report-header
290 sub _build_COUNTER_report_file {
291 my ( $self, $header, $column_headings, $body ) = @_;
293 my @report = ( @{$header}, @{$column_headings}, @{$body} );
295 #TODO: change this to tab instead of comma
296 csv( in => \@report, out => \my $counter_file, encoding => "utf-8" );
298 $self->counter_files(
301 usage_data_provider_id => $self->erm_usage_data_provider_id,
302 file_content => $counter_file,
303 date_uploaded => POSIX::strftime( "%Y%m%d%H%M%S", localtime ),
305 #TODO: add ".csv" to end of filename here
306 filename => $self->name . "_" . $self->{report_type},
312 =head3 _COUNTER_report_header
314 Return a COUNTER report header
315 https://cop5.projectcounter.org/en/5.0.2/04-reports/03-title-reports.html
319 sub _COUNTER_report_header {
322 my $header = $self->{sushi}->{header};
324 my @metric_types_string =
325 $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Metric_Type" );
328 $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Begin_Date" );
330 $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "End_Date" );
333 [ Report_Name => $header->{Report_Name} || "" ],
334 [ Report_ID => $header->{Report_ID} || "" ],
335 [ Release => $header->{Release} || "" ],
336 [ Institution_Name => $header->{Institution_Name} || "" ],
338 Institution_ID => join(
340 map( $_->{Type} . ":" . $_->{Value},
341 @{ $header->{Institution_ID} } )
346 Metric_Types => join( "; ", split( /\|/, $metric_types_string[0] ) )
350 Report_Filters => join(
352 map( $_->{Name} . ":" . $_->{Value},
353 @{ $header->{Report_Filters} } )
358 #TODO: Report_Attributes may need parsing, test this with a SUSHI response that provides it
359 [ Report_Attributes => $header->{Report_Attributes} || "" ],
363 map( $_->{Code} . ": "
364 . $_->{Message} . " ("
366 @{ $header->{Exceptions} } )
371 Reporting_Period => "Begin_Date="
376 [ Created => $header->{Created} || "" ],
377 [ Created_By => $header->{Created_By} || "" ],
378 [""] #empty 13th line
382 =head3 _COUNTER_item_report_row
384 Return a COUNTER item for the COUNTER items report body
385 https://cop5.projectcounter.org/en/5.0.2/04-reports/04-item-reports.html#column-headings-elements
389 sub _COUNTER_item_report_row {
390 my ( $self, $item_row, $metric_type, $total_usage, $monthly_usages ) = @_;
394 $item_row->{Item} || "",
395 $item_row->{Publisher} || "",
396 $self->_get_SUSHI_Type_Value( $item_row->{Publisher_ID}, "ISNI" )
398 $item_row->{Platform} || "",
399 $self->_get_SUSHI_Type_Value( $item_row->{Item_ID}, "DOI" ) || "",
400 $item_row->{Proprietary_ID} || "",
401 "", #FIXME: What goes in URI?
409 =head3 _COUNTER_database_report_row
411 Return a COUNTER database for the COUNTER databases report body
412 https://cop5.projectcounter.org/en/5.0.2/04-reports/02-database-reports.html#column-headings-elements
416 sub _COUNTER_database_report_row {
417 my ( $self, $database_row, $metric_type, $total_usage, $monthly_usages ) =
422 $database_row->{Database} || "",
423 $database_row->{Publisher} || "",
424 $self->_get_SUSHI_Type_Value( $database_row->{Publisher_ID},
427 $database_row->{Platform} || "",
428 $database_row->{Proprietary_ID} || "",
436 =head3 _COUNTER_platform_report_row
438 Return a COUNTER platform for the COUNTER platforms report body
439 https://cop5.projectcounter.org/en/5.0.2/04-reports/01-platform-reports.html#column-headings-elements
443 sub _COUNTER_platform_report_row {
444 my ( $self, $platform_row, $metric_type, $total_usage, $monthly_usages ) =
449 $platform_row->{Platform} || "", $metric_type,
450 $total_usage, @{$monthly_usages}
455 =head3 _COUNTER_title_report_row
457 Return a COUNTER title for the COUNTER titles report body
458 https://cop5.projectcounter.org/en/5.0.2/04-reports/03-title-reports.html#column-headings-elements
462 sub _COUNTER_title_report_row {
463 my ( $self, $title_row, $metric_type, $total_usage, $monthly_usages ) = @_;
465 my $header = $self->{sushi}->{header};
466 my $specific_fields =
467 $self->get_report_type_specific_fields( $header->{Report_ID} );
472 $title_row->{Title} || "",
475 $title_row->{Publisher} || "",
478 $self->_get_SUSHI_Type_Value( $title_row->{Publisher_ID}, "ISNI" )
482 $title_row->{Platform} || "",
485 $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "DOI" ) || "",
488 $self->_get_SUSHI_Type_Value(
489 $title_row->{Item_ID}, "Proprietary"
494 grep ( /ISBN/, @{$specific_fields} )
495 ? ( $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "ISBN" )
500 $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "Print_ISSN" )
504 $self->_get_SUSHI_Type_Value(
505 $title_row->{Item_ID}, "Online_ISSN"
509 # URI - FIXME: What goes in URI?
513 grep ( /YOP/, @{$specific_fields} )
514 ? ( $title_row->{YOP} || "" )
518 grep ( /Access_Type/, @{$specific_fields} )
519 ? ( $title_row->{Access_Type} || "" )
525 # Report_Period_Total
528 # Monthly usage entries
534 =head3 _COUNTER_report_row
536 Return a COUNTER row for the COUNTER report body
540 sub _COUNTER_report_row {
541 my ( $self, $report_row, $metric_type ) = @_;
543 my $header = $self->{sushi}->{header};
545 my ( $total_usage, @monthly_usages ) =
546 $self->_get_row_usages( $report_row, $metric_type );
548 if ( $header->{Report_ID} =~ /PR/i ) {
549 return $self->_COUNTER_platform_report_row( $report_row, $metric_type,
550 $total_usage, \@monthly_usages );
552 elsif ( $header->{Report_ID} =~ /DR/i ) {
553 return $self->_COUNTER_database_report_row( $report_row, $metric_type,
554 $total_usage, \@monthly_usages );
556 elsif ( $header->{Report_ID} =~ /IR/i ) {
557 return $self->_COUNTER_item_report_row( $report_row, $metric_type,
558 $total_usage, \@monthly_usages );
560 elsif ( $header->{Report_ID} =~ /TR/i ) {
561 return $self->_COUNTER_title_report_row( $report_row, $metric_type,
562 $total_usage, \@monthly_usages );
566 =head3 _get_row_usages
568 Returns the total and monthly usages for a row
572 sub _get_row_usages {
573 my ( $self, $row, $metric_type ) = @_;
575 my @usage_months = $self->_get_usage_months( $self->{sushi}->{header} );
577 my @usage_months_fields = ();
580 foreach my $usage_month (@usage_months) {
581 my $month_is_empty = 1;
583 foreach my $performance ( @{ $row->{Performance} } ) {
584 my $period = $performance->{Period};
585 my $period_usage_month = substr( $period->{Begin_Date}, 0, 7 );
587 my $instances = $performance->{Instance};
588 my @metric_type_count =
589 map( $_->{Metric_Type} eq $metric_type ? $_->{Count} : (),
592 if ( $period_usage_month eq $usage_month && $metric_type_count[0] )
594 push( @usage_months_fields, $metric_type_count[0] );
595 $count_total += $metric_type_count[0];
600 if ($month_is_empty) {
601 push( @usage_months_fields, 0 );
604 return ( $count_total, @usage_months_fields );
607 =head3 _COUNTER_report_body
609 Return the COUNTER report body as an array
613 sub _COUNTER_report_body {
616 my $header = $self->{sushi}->{header};
617 my $body = $self->{sushi}->{body};
619 my @metric_types_string = $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Metric_Type" );
620 my @metric_types = split( /\|/, $metric_types_string[0] );
622 my @report_body = ();
624 my $total_records = 0;
625 foreach my $report_row ( @{$body} ) {
627 my @metric_types = ();
629 # Grab all metric_types this SUSHI result has statistics for
630 foreach my $performance ( @{ $report_row->{Performance} } ) {
631 my @SUSHI_metric_types =
632 map( $_->{Metric_Type}, @{ $performance->{Instance} } );
634 foreach my $sushi_metric_type (@SUSHI_metric_types) {
635 push( @metric_types, $sushi_metric_type )
636 unless grep { $_ eq $sushi_metric_type } @metric_types;
640 # Add one report row for each metric_type we're working with
641 foreach my $metric_type (@metric_types) {
643 $self->_COUNTER_report_row( $report_row, $metric_type ) );
645 $self->{total_records} = ++$total_records;
651 =head3 _get_SUSHI_Name_Value
653 Returns "Value" of a given "Name"
657 sub _get_SUSHI_Name_Value {
658 my ( $self, $item, $name ) = @_;
660 my @value = map( $_->{Name} eq $name ? $_->{Value} : (), @{$item} );
665 =head3 _get_SUSHI_Type_Value
667 Returns "Value" of a given "Type"
671 sub _get_SUSHI_Type_Value {
672 my ( $self, $item, $type ) = @_;
674 my @value = map( $_->{Type} eq $type ? $_->{Value} : (), @{$item} );
679 =head3 _COUNTER_report_column_headings
681 Returns column headings by report type
682 Check the report type from the COUNTER header
683 and return column headings accordingly
687 sub _COUNTER_report_column_headings {
690 my $header = $self->{sushi}->{header};
692 if ( $header->{Report_ID} =~ /PR/i ) {
693 return $self->_COUNTER_platforms_report_column_headings;
695 elsif ( $header->{Report_ID} =~ /DR/i ) {
696 return $self->_COUNTER_databases_report_column_headings;
698 elsif ( $header->{Report_ID} =~ /IR/i ) {
699 return $self->_COUNTER_items_report_column_headings;
701 elsif ( $header->{Report_ID} =~ /TR/i ) {
702 return $self->_COUNTER_titles_report_column_headings;
708 =head3 _COUNTER_items_report_column_headings
710 Return items report column headings
714 sub _COUNTER_items_report_column_headings {
717 my $header = $self->{sushi}->{header};
718 my @month_headings = $self->_get_usage_months( $header, 1 );
727 # "Authors", #IR_A1 only
728 # "Publication_Date", #IR_A1 only
729 # "Article_Version", #IR_A1 only
734 # "Print_ISSN", #IR_A1 only
735 # "Online_ISSN", #IR_A1 only
738 # "Parent_Title", #IR_A1 only
739 # "Parent_Authors", #IR_A1 only
740 # "Parent_Publication_Date", #IR only
741 # "Parent_Article_Version", #IR_A1 only
742 # "Parent_Data_Type", #IR only
743 # "Parent_DOI", #IR_A1 only
744 # "Parent_Proprietary_ID", #IR_A1 only
745 # "Parent_ISBN", #IR only
746 # "Parent_Print_ISSN", #IR_A1 only
747 # "Parent_Online_ISSN", #IR_A1 only
748 # "Parent_URI", #IR_A1 only
749 # "Component_Title", #IR only
750 # "Component_Authors", #IR only
751 # "Component_Publication_Date", #IR only
752 # "Component_Data_Type", #IR only
753 # "Component_DOI", #IR only
754 # "Component_Proprietary_ID", #IR only
755 # "Component_ISBN", #IR only
756 # "Component_Print_ISSN", #IR only
757 # "Component_Online_ISSN", #IR only
758 # "Component_URI", #IR only
759 # "Data_Type", #IR only
761 # "Access_Type", #IR_A1 only
762 # "Access_Method", #IR only
764 "Reporting_Period_Total",
766 # @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true
772 =head3 _COUNTER_databases_report_column_headings
774 Return databases report column headings
778 sub _COUNTER_databases_report_column_headings {
781 my $header = $self->{sushi}->{header};
782 my @month_headings = $self->_get_usage_months( $header, 1 );
792 "Reporting_Period_Total",
794 # @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true
800 =head3 _COUNTER_platforms_report_column_headings
802 Return platforms report column headings
806 sub _COUNTER_platforms_report_column_headings {
809 my $header = $self->{sushi}->{header};
810 my @month_headings = $self->_get_usage_months( $header, 1 );
816 "Reporting_Period_Total",
818 # @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true
824 =head3 _COUNTER_titles_report_column_headings
826 Return titles report column headings
830 sub _COUNTER_titles_report_column_headings {
833 my $header = $self->{sushi}->{header};
834 my @month_headings = $self->_get_usage_months( $header, 1 );
835 my $specific_fields =
836 $self->get_report_type_specific_fields( $header->{Report_ID} );
846 grep ( /ISBN/, @{$specific_fields} ) ? ("ISBN") : (),
851 #"Data_Type", #TODO: Only if requested (?)
852 #"Section_Type", #TODO: Only if requested (?)
853 grep ( /YOP/, @{$specific_fields} ) ? ("YOP") : (),
854 grep ( /Access_Type/, @{$specific_fields} ) ? ("Access_Type") : (),
856 #"Access_Method", #TODO: Only if requested (?)
858 "Reporting_Period_Total",
860 # @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true
866 =head3 _get_usage_months
868 Return report usage months. Formatted for column headings if $column_headings_formatting
872 sub _get_usage_months {
873 my ( $self, $header, $column_headings_formatting ) = @_;
876 "Jan", "Feb", "Mar", "Apr", "May", "Jun",
877 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
880 my @begin_date = map( $_->{Name} eq "Begin_Date" ? $_->{Value} : (),
881 @{ $header->{Report_Filters} } );
882 my $begin_month = substr( $begin_date[0], 5, 2 );
883 my $begin_year = substr( $begin_date[0], 0, 4 );
885 my @end_date = map( $_->{Name} eq "End_Date" ? $_->{Value} : (),
886 @{ $header->{Report_Filters} } );
887 my $end_month = substr( $end_date[0], 5, 2 );
888 my $end_year = substr( $end_date[0], 0, 4 );
890 my @month_headings = ();
891 while ( $begin_month <= $end_month || $begin_year < $end_year ) {
892 push( @month_headings,
893 $column_headings_formatting
894 ? $months[ $begin_month - 1 ] . " " . $begin_year
895 : $begin_year . "-" . $begin_month );
897 if ( $begin_month > 12 ) {
901 $begin_month = "0" . $begin_month if length($begin_month) == 1;
904 return @month_headings;
907 =head3 get_report_type_specific_fields
909 Returns the specific fields for a given report_type
913 sub get_report_type_specific_fields {
914 my ( $self, $report_type ) = @_;
916 my %report_type_map = (
917 "TR_B1" => [ 'YOP', 'ISBN' ],
918 "TR_B2" => [ 'YOP', 'ISBN' ],
919 "TR_B3" => [ 'YOP', 'Access_Type', 'ISBN' ],
920 "TR_J3" => ['Access_Type'],
924 return $report_type_map{$report_type};
928 =head3 test_connection
930 Tests the connection of the harvester to the SUSHI service and returns any alerts of planned SUSHI outages
934 sub test_connection {
937 my $url = $self->service_url;
939 $url .= '?customer_id=' . $self->customer_id;
940 $url .= '&requestor_id=' . $self->requestor_id if $self->requestor_id;
941 $url .= '&api_key=' . $self->api_key if $self->api_key;
943 my $request = HTTP::Request->new( 'GET' => $url );
944 my $ua = LWP::UserAgent->new;
945 my $response = $ua->simple_request($request);
947 my @result = decode_json( $response->decoded_content );
948 if ( $result[0][0]->{Service_Active} ) {
957 =head3 erm_usage_titles
959 Method to embed erm_usage_titles to titles for report formatting
963 sub erm_usage_titles {
965 my $usage_title_rs = $self->_result->erm_usage_titles;
966 return Koha::ERM::UsageTitles->_new_from_dbic($usage_title_rs);
969 =head3 erm_usage_muses
971 Method to embed erm_usage_muses to titles for report formatting
975 sub erm_usage_muses {
977 my $usage_mus_rs = $self->_result->erm_usage_muses;
978 return Koha::ERM::MonthlyUsages->_new_from_dbic($usage_mus_rs);
983 =head3 erm_usage_platforms
985 Method to embed erm_usage_platforms to platforms for report formatting
989 sub erm_usage_platforms {
991 my $usage_platform_rs = $self->_result->erm_usage_platforms;
992 return Koha::ERM::UsagePlatforms->_new_from_dbic($usage_platform_rs);
995 =head3 erm_usage_items
997 Method to embed erm_usage_items to items for report formatting
1001 sub erm_usage_items {
1003 my $usage_item_rs = $self->_result->erm_usage_items;
1004 return Koha::ERM::UsageItems->_new_from_dbic($usage_item_rs);
1007 =head3 erm_usage_databases
1009 Method to embed erm_usage_databases to databases for report formatting
1013 sub erm_usage_databases {
1015 my $usage_database_rs = $self->_result->erm_usage_databases;
1016 return Koha::ERM::UsageDatabases->_new_from_dbic($usage_database_rs);
1024 return 'ErmUsageDataProvider';