1 package Koha::ERM::UsageDataProvider;
3 # This file is part of Koha.
5 # Koha is free software; you can redistribute it and/or modify it
6 # under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # Koha is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with Koha; if not, see <http://www.gnu.org/licenses>.
21 use JSON qw( decode_json );
23 use Text::CSV_XS qw( csv );
27 use base qw(Koha::Object);
29 use Koha::ERM::CounterFile;
30 use Koha::ERM::CounterFiles;
31 use Koha::BackgroundJob::ErmSushiHarvester;
35 Koha::ERM::UsageDataProvider - Koha ErmUsageDataProvider Object class
43 Getter/setter for counter_files for this usage data provider
48 my ( $self, $counter_files ) = @_;
51 for my $counter_file (@$counter_files) {
52 Koha::ERM::CounterFile->new($counter_file)
53 ->store( $self->{job_callbacks} );
56 my $counter_files_rs = $self->_result->erm_counter_files;
57 return Koha::ERM::CounterFiles->_new_from_dbic($counter_files_rs);
60 =head3 enqueue_counter_file_processing_job
62 Enqueues a background job to process a COUNTER file that has been uploaded
66 sub enqueue_counter_file_processing_job {
67 my ( $self, $args ) = @_;
70 my $job_id = Koha::BackgroundJob::ErmSushiHarvester->new->enqueue(
72 ud_provider_id => $self->erm_usage_data_provider_id,
73 file_content => $args->{file_content},
87 =head3 enqueue_sushi_harvest_jobs
89 Enqueues one harvest background job for each report type in this usage data provider
93 sub enqueue_sushi_harvest_jobs {
94 my ( $self, $args ) = @_;
96 my @report_types = split( /;/, $self->report_types );
99 foreach my $report_type (@report_types) {
101 my $job_id = Koha::BackgroundJob::ErmSushiHarvester->new->enqueue(
103 ud_provider_id => $self->erm_usage_data_provider_id,
104 report_type => $report_type
111 report_type => $report_type,
122 $ud_provider->harvest(
124 step_callback => sub { $self->step; },
125 set_size_callback => sub { $self->set_job_size(@_); },
126 add_message_callback => sub { $self->add_message(@_); },
130 Run the SUSHI harvester of this usage data provider
131 Builds the URL query and requests the COUNTER 5 SUSHI service
133 COUNTER SUSHI api spec:
134 https://app.swaggerhub.com/apis/COUNTER/counter-sushi_5_0_api/5.0.2
140 Report type to run this harvest on
146 =item background_job_callbacks
148 Receive background_job_callbacks to be able to update job
155 my ( $self, $report_type, $background_job_callbacks ) = @_;
157 # Set class wide vars
158 $self->{job_callbacks} = $background_job_callbacks;
159 $self->{report_type} = $report_type;
161 my $url = $self->_build_url_query;
162 my $request = HTTP::Request->new( 'GET' => $url );
163 my $ua = LWP::UserAgent->new;
164 my $response = $ua->simple_request($request);
166 if ( $response->code >= 400 ) {
167 my $result = decode_json( $response->decoded_content );
170 if ( ref($result) eq 'ARRAY' ) {
171 for my $r (@$result) {
172 $message .= $r->{message};
176 #TODO: May want to check $result->{Report_Header}->{Exceptions} here
177 $message = $result->{message} || $result->{Message} || q{};
178 if ( $result->{errors} ) {
179 for my $e ( @{ $result->{errors} } ) {
180 $message .= $e->{message};
185 #TODO: May want to add a job error message here?
186 warn sprintf "ERROR - SUSHI service %s returned %s - %s\n", $url,
187 $response->code, $message;
188 if ( $response->code == 404 ) {
189 Koha::Exceptions::ObjectNotFound->throw($message);
191 elsif ( $response->code == 401 ) {
192 Koha::Exceptions::Authorization::Unauthorized->throw($message);
195 #TODO: May want to add a job error message here?
196 die sprintf "ERROR requesting SUSHI service\n%s\ncode %s: %s\n",
197 $url, $response->code,
201 elsif ( $response->code == 204 ) { # No content
205 # Parse the SUSHI response
206 $self->parse_SUSHI_response( decode_json( $response->decoded_content ) );
209 =head3 parse_SUSHI_response
211 $self->parse_SUSHI_response( decode_json( $response->decoded_content ) );
213 Parse the SUSHI response, prepare the COUNTER report file header,
214 column headings and body
220 The result of the SUSHI response after json decoded
226 sub parse_SUSHI_response {
227 my ( $self, $result ) = @_;
229 # Set class wide sushi response content
231 header => $result->{Report_Header},
232 body => $result->{Report_Items}
235 #TODO: Handle empty $self->{sushi}->{body} here!
237 # Get ready to build COUNTER file
238 my @report_header = $self->_COUNTER_report_header;
239 my @report_column_headings = $self->_COUNTER_report_column_headings;
240 my @report_body = $self->_COUNTER_report_body;
242 $self->_build_COUNTER_report_file( \@report_header,
243 \@report_column_headings, \@report_body );
246 =head2 Internal methods
248 =head3 _build_url_query
250 Build the URL query params for COUNTER 5 SUSHI request
254 sub _build_url_query {
257 unless ( $self->service_url && $self->customer_id ) {
259 "SUSHI Harvesting config for usage data provider %d is missing service_url or customer_id\n",
260 $self->erm_usage_data_provider_id;
263 # FIXME: service_url needs to end in 'reports/'
264 # below concat will result in a badly formed URL otherwise
265 # Either validate this on UI form, here, or both
266 my $url = $self->service_url;
268 $url .= $self->{report_type};
269 $url .= '?customer_id=' . $self->customer_id;
270 $url .= '&requestor_id=' . $self->requestor_id if $self->requestor_id;
271 $url .= '&api_key=' . $self->api_key if $self->api_key;
272 $url .= '&begin_date=' . $self->begin_date if $self->begin_date;
273 $url .= '&end_date=' . $self->end_date if $self->end_date;
278 =head3 _build_COUNTER_report_file
280 Build the COUNTER file
281 https://cop5.projectcounter.org/en/5.0.2/03-specifications/02-formats-for-counter-reports.html#report-header
285 sub _build_COUNTER_report_file {
286 my ( $self, $header, $column_headings, $body ) = @_;
288 my @report = ( @{$header}, @{$column_headings}, @{$body} );
290 #TODO: change this to tab instead of comma
291 csv( in => \@report, out => \my $counter_file, encoding => "utf-8" );
293 $self->counter_files(
296 usage_data_provider_id => $self->erm_usage_data_provider_id,
297 file_content => $counter_file,
298 date_uploaded => POSIX::strftime( "%Y%m%d%H%M%S", localtime ),
300 #TODO: add ".csv" to end of filename here
301 filename => $self->name . "_" . $self->{report_type},
302 type => $self->{report_type}
308 =head3 _COUNTER_report_header
310 Return a COUNTER report header
311 https://cop5.projectcounter.org/en/5.0.2/04-reports/03-title-reports.html
315 sub _COUNTER_report_header {
318 my $header = $self->{sushi}->{header};
320 my @metric_types_string =
321 $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Metric_Type" );
324 $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Begin_Date" );
326 $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "End_Date" );
329 [ Report_Name => $header->{Report_Name} || "" ],
330 [ Report_ID => $header->{Report_ID} || "" ],
331 [ Release => $header->{Release} || "" ],
332 [ Institution_Name => $header->{Institution_Name} || "" ],
334 Institution_ID => join(
336 map( $_->{Type} . ":" . $_->{Value},
337 @{ $header->{Institution_ID} } )
342 Metric_Types => join( "; ", split( /\|/, $metric_types_string[0] ) )
346 Report_Filters => join(
348 map( $_->{Name} . ":" . $_->{Value},
349 @{ $header->{Report_Filters} } )
354 #TODO: Report_Attributes may need parsing, test this with a SUSHI response that provides it
355 [ Report_Attributes => $header->{Report_Attributes} || "" ],
359 map( $_->{Code} . ": "
360 . $_->{Message} . " ("
362 @{ $header->{Exceptions} } )
367 Reporting_Period => "Begin_Date="
372 [ Created => $header->{Created} || "" ],
373 [ Created_By => $header->{Created_By} || "" ],
374 [""] #empty 13th line
378 =head3 _COUNTER_item_report_row
380 Return a COUNTER item for the COUNTER items report body
381 https://cop5.projectcounter.org/en/5.0.2/04-reports/04-item-reports.html#column-headings-elements
385 sub _COUNTER_item_report_row {
386 my ( $self, $item_row, $metric_type, $total_usage, $monthly_usages ) = @_;
390 $item_row->{Item} || "",
391 $item_row->{Publisher} || "",
392 $self->_get_SUSHI_Type_Value( $item_row->{Publisher_ID}, "ISNI" )
394 $item_row->{Platform} || "",
395 $self->_get_SUSHI_Type_Value( $item_row->{Item_ID}, "DOI" ) || "",
396 $item_row->{Proprietary_ID} || "",
397 "", #FIXME: What goes in URI?
405 =head3 _COUNTER_database_report_row
407 Return a COUNTER database for the COUNTER databases report body
408 https://cop5.projectcounter.org/en/5.0.2/04-reports/02-database-reports.html#column-headings-elements
412 sub _COUNTER_database_report_row {
413 my ( $self, $database_row, $metric_type, $total_usage, $monthly_usages ) =
418 $database_row->{Database} || "",
419 $database_row->{Publisher} || "",
420 $self->_get_SUSHI_Type_Value( $database_row->{Publisher_ID},
423 $database_row->{Platform} || "",
424 $database_row->{Proprietary_ID} || "",
432 =head3 _COUNTER_platform_report_row
434 Return a COUNTER platform for the COUNTER platforms report body
435 https://cop5.projectcounter.org/en/5.0.2/04-reports/01-platform-reports.html#column-headings-elements
439 sub _COUNTER_platform_report_row {
440 my ( $self, $platform_row, $metric_type, $total_usage, $monthly_usages ) =
445 $platform_row->{Platform} || "", $metric_type,
446 $total_usage, @{$monthly_usages}
451 =head3 _COUNTER_title_report_row
453 Return a COUNTER title for the COUNTER titles report body
454 https://cop5.projectcounter.org/en/5.0.2/04-reports/03-title-reports.html#column-headings-elements
458 sub _COUNTER_title_report_row {
459 my ( $self, $title_row, $metric_type, $total_usage, $monthly_usages ) = @_;
461 my $header = $self->{sushi}->{header};
462 my $specific_fields =
463 $self->get_report_type_specific_fields( $header->{Report_ID} );
468 $title_row->{Title} || "",
471 $title_row->{Publisher} || "",
474 $self->_get_SUSHI_Type_Value( $title_row->{Publisher_ID}, "ISNI" )
478 $title_row->{Platform} || "",
481 $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "DOI" ) || "",
484 $self->_get_SUSHI_Type_Value(
485 $title_row->{Item_ID}, "Proprietary"
490 grep ( /ISBN/, @{$specific_fields} )
491 ? ( $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "ISBN" )
496 $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "Print_ISSN" )
500 $self->_get_SUSHI_Type_Value(
501 $title_row->{Item_ID}, "Online_ISSN"
505 # URI - FIXME: What goes in URI?
509 grep ( /YOP/, @{$specific_fields} )
510 ? ( $title_row->{YOP} || "" )
514 grep ( /Access_Type/, @{$specific_fields} )
515 ? ( $title_row->{Access_Type} || "" )
521 # Report_Period_Total
524 # Monthly usage entries
530 =head3 _COUNTER_report_row
532 Return a COUNTER row for the COUNTER report body
536 sub _COUNTER_report_row {
537 my ( $self, $report_row, $metric_type ) = @_;
539 my $header = $self->{sushi}->{header};
541 my ( $total_usage, @monthly_usages ) =
542 $self->_get_row_usages( $report_row, $metric_type );
544 if ( $header->{Report_ID} =~ /PR/i ) {
545 return $self->_COUNTER_platform_report_row( $report_row, $metric_type,
546 $total_usage, \@monthly_usages );
548 elsif ( $header->{Report_ID} =~ /DR/i ) {
549 return $self->_COUNTER_database_report_row( $report_row, $metric_type,
550 $total_usage, \@monthly_usages );
552 elsif ( $header->{Report_ID} =~ /IR/i ) {
553 return $self->_COUNTER_item_report_row( $report_row, $metric_type,
554 $total_usage, \@monthly_usages );
556 elsif ( $header->{Report_ID} =~ /TR/i ) {
557 return $self->_COUNTER_title_report_row( $report_row, $metric_type,
558 $total_usage, \@monthly_usages );
562 =head3 _get_row_usages
564 Returns the total and monthly usages for a row
568 sub _get_row_usages {
569 my ( $self, $row, $metric_type ) = @_;
571 my @usage_months = $self->_get_usage_months( $self->{sushi}->{header} );
573 my @usage_months_fields = ();
576 foreach my $usage_month (@usage_months) {
577 my $month_is_empty = 1;
579 foreach my $performance ( @{ $row->{Performance} } ) {
580 my $period = $performance->{Period};
581 my $period_usage_month = substr( $period->{Begin_Date}, 0, 7 );
583 my $instances = $performance->{Instance};
584 my @metric_type_count =
585 map( $_->{Metric_Type} eq $metric_type ? $_->{Count} : (),
588 if ( $period_usage_month eq $usage_month && $metric_type_count[0] )
590 push( @usage_months_fields, $metric_type_count[0] );
591 $count_total += $metric_type_count[0];
596 if ($month_is_empty) {
597 push( @usage_months_fields, 0 );
600 return ( $count_total, @usage_months_fields );
603 =head3 _COUNTER_report_body
605 Return the COUNTER report body as an array
609 sub _COUNTER_report_body {
612 my $header = $self->{sushi}->{header};
613 my $body = $self->{sushi}->{body};
615 my @metric_types_string = $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Metric_Type" );
616 my @metric_types = split( /\|/, $metric_types_string[0] );
618 my @report_body = ();
620 my $total_records = 0;
621 foreach my $report_row ( @{$body} ) {
623 my @metric_types = ();
625 # Grab all metric_types this SUSHI result has statistics for
626 foreach my $performance ( @{ $report_row->{Performance} } ) {
627 my @SUSHI_metric_types =
628 map( $_->{Metric_Type}, @{ $performance->{Instance} } );
630 foreach my $sushi_metric_type (@SUSHI_metric_types) {
631 push( @metric_types, $sushi_metric_type )
632 unless grep { $_ eq $sushi_metric_type } @metric_types;
636 # Add one report row for each metric_type we're working with
637 foreach my $metric_type (@metric_types) {
639 $self->_COUNTER_report_row( $report_row, $metric_type ) );
641 $self->{total_records} = ++$total_records;
647 =head3 _get_SUSHI_Name_Value
649 Returns "Value" of a given "Name"
653 sub _get_SUSHI_Name_Value {
654 my ( $self, $item, $name ) = @_;
656 my @value = map( $_->{Name} eq $name ? $_->{Value} : (), @{$item} );
661 =head3 _get_SUSHI_Type_Value
663 Returns "Value" of a given "Type"
667 sub _get_SUSHI_Type_Value {
668 my ( $self, $item, $type ) = @_;
670 my @value = map( $_->{Type} eq $type ? $_->{Value} : (), @{$item} );
675 =head3 _COUNTER_report_column_headings
677 Returns column headings by report type
678 Check the report type from the COUNTER header
679 and return column headings accordingly
683 sub _COUNTER_report_column_headings {
686 my $header = $self->{sushi}->{header};
688 if ( $header->{Report_ID} =~ /PR/i ) {
689 return $self->_COUNTER_platforms_report_column_headings;
691 elsif ( $header->{Report_ID} =~ /DR/i ) {
692 return $self->_COUNTER_databases_report_column_headings;
694 elsif ( $header->{Report_ID} =~ /IR/i ) {
695 return $self->_COUNTER_items_report_column_headings;
697 elsif ( $header->{Report_ID} =~ /TR/i ) {
698 return $self->_COUNTER_titles_report_column_headings;
704 =head3 _COUNTER_items_report_column_headings
706 Return items report column headings
710 sub _COUNTER_items_report_column_headings {
713 my $header = $self->{sushi}->{header};
714 my @month_headings = $self->_get_usage_months( $header, 1 );
723 # "Authors", #IR_A1 only
724 # "Publication_Date", #IR_A1 only
725 # "Article_Version", #IR_A1 only
730 # "Print_ISSN", #IR_A1 only
731 # "Online_ISSN", #IR_A1 only
734 # "Parent_Title", #IR_A1 only
735 # "Parent_Authors", #IR_A1 only
736 # "Parent_Publication_Date", #IR only
737 # "Parent_Article_Version", #IR_A1 only
738 # "Parent_Data_Type", #IR only
739 # "Parent_DOI", #IR_A1 only
740 # "Parent_Proprietary_ID", #IR_A1 only
741 # "Parent_ISBN", #IR only
742 # "Parent_Print_ISSN", #IR_A1 only
743 # "Parent_Online_ISSN", #IR_A1 only
744 # "Parent_URI", #IR_A1 only
745 # "Component_Title", #IR only
746 # "Component_Authors", #IR only
747 # "Component_Publication_Date", #IR only
748 # "Component_Data_Type", #IR only
749 # "Component_DOI", #IR only
750 # "Component_Proprietary_ID", #IR only
751 # "Component_ISBN", #IR only
752 # "Component_Print_ISSN", #IR only
753 # "Component_Online_ISSN", #IR only
754 # "Component_URI", #IR only
755 # "Data_Type", #IR only
757 # "Access_Type", #IR_A1 only
758 # "Access_Method", #IR only
760 "Reporting_Period_Total",
762 # @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true
768 =head3 _COUNTER_databases_report_column_headings
770 Return databases report column headings
774 sub _COUNTER_databases_report_column_headings {
777 my $header = $self->{sushi}->{header};
778 my @month_headings = $self->_get_usage_months( $header, 1 );
788 "Reporting_Period_Total",
790 # @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true
796 =head3 _COUNTER_platforms_report_column_headings
798 Return platforms report column headings
802 sub _COUNTER_platforms_report_column_headings {
805 my $header = $self->{sushi}->{header};
806 my @month_headings = $self->_get_usage_months( $header, 1 );
812 "Reporting_Period_Total",
814 # @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true
820 =head3 _COUNTER_titles_report_column_headings
822 Return titles report column headings
826 sub _COUNTER_titles_report_column_headings {
829 my $header = $self->{sushi}->{header};
830 my @month_headings = $self->_get_usage_months( $header, 1 );
831 my $specific_fields =
832 $self->get_report_type_specific_fields( $header->{Report_ID} );
842 grep ( /ISBN/, @{$specific_fields} ) ? ("ISBN") : (),
847 #"Data_Type", #TODO: Only if requested (?)
848 #"Section_Type", #TODO: Only if requested (?)
849 grep ( /YOP/, @{$specific_fields} ) ? ("YOP") : (),
850 grep ( /Access_Type/, @{$specific_fields} ) ? ("Access_Type") : (),
852 #"Access_Method", #TODO: Only if requested (?)
854 "Reporting_Period_Total",
856 # @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true
862 =head3 _get_usage_months
864 Return report usage months. Formatted for column headings if $column_headings_formatting
868 sub _get_usage_months {
869 my ( $self, $header, $column_headings_formatting ) = @_;
872 "Jan", "Feb", "Mar", "Apr", "May", "Jun",
873 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
876 my @begin_date = map( $_->{Name} eq "Begin_Date" ? $_->{Value} : (),
877 @{ $header->{Report_Filters} } );
878 my $begin_month = substr( $begin_date[0], 5, 2 );
879 my $begin_year = substr( $begin_date[0], 0, 4 );
881 my @end_date = map( $_->{Name} eq "End_Date" ? $_->{Value} : (),
882 @{ $header->{Report_Filters} } );
883 my $end_month = substr( $end_date[0], 5, 2 );
884 my $end_year = substr( $end_date[0], 0, 4 );
886 my @month_headings = ();
887 while ( $begin_month <= $end_month || $begin_year < $end_year ) {
888 push( @month_headings,
889 $column_headings_formatting
890 ? $months[ $begin_month - 1 ] . " " . $begin_year
891 : $begin_year . "-" . $begin_month );
893 if ( $begin_month > 12 ) {
897 $begin_month = "0" . $begin_month if length($begin_month) == 1;
900 return @month_headings;
903 =head3 get_report_type_specific_fields
905 Returns the specific fields for a given report_type
909 sub get_report_type_specific_fields {
910 my ( $self, $report_type ) = @_;
912 my %report_type_map = (
913 "TR_B1" => [ 'YOP', 'ISBN' ],
914 "TR_B2" => [ 'YOP', 'ISBN' ],
915 "TR_B3" => [ 'YOP', 'Access_Type', 'ISBN' ],
916 "TR_J3" => ['Access_Type'],
920 return $report_type_map{$report_type};
924 =head3 test_connection
926 Tests the connection of the harvester to the SUSHI service and returns any alerts of planned SUSHI outages
930 sub test_connection {
933 my $url = $self->service_url;
935 $url .= '?customer_id=' . $self->customer_id;
936 $url .= '&requestor_id=' . $self->requestor_id if $self->requestor_id;
937 $url .= '&api_key=' . $self->api_key if $self->api_key;
939 my $request = HTTP::Request->new( 'GET' => $url );
940 my $ua = LWP::UserAgent->new;
941 my $response = $ua->simple_request($request);
943 my @result = decode_json( $response->decoded_content );
944 if ( $result[0][0]->{Service_Active} ) {
953 =head3 erm_usage_titles
955 Method to embed erm_usage_titles to titles for report formatting
959 sub erm_usage_titles {
961 my $usage_title_rs = $self->_result->erm_usage_titles;
962 return Koha::ERM::UsageTitles->_new_from_dbic($usage_title_rs);
965 =head3 erm_usage_muses
967 Method to embed erm_usage_muses to titles for report formatting
971 sub erm_usage_muses {
973 my $usage_mus_rs = $self->_result->erm_usage_muses;
974 return Koha::ERM::MonthlyUsages->_new_from_dbic($usage_mus_rs);
984 return 'ErmUsageDataProvider';