From 2951edc697ea68fcd8c65d608db7a3a743e17563 Mon Sep 17 00:00:00 2001 From: Pedro Amorim Date: Wed, 1 Mar 2023 17:38:20 +0000 Subject: [PATCH] Bug 34587: SUSHI COUNTER Harvester This commit is a squash of the following: SUSHI harvesting process in the data providers class: * Builds the URL query and requests the SUSHI service endpoint * Parses the JSON response and builds the csv COUNTER file and adds it to counter_files table Usage statistics data processing: * When a counter_files entry is stored, CounterFile.pm will: * Parse the csv COUNTER file and * Add a usage_titles entry for each unique title in the COUNTER file * Add the title's respective erm_usage_mus (monthly usage) entries, repeating for each metric_type * Add the title's respective erm_usage_yus (yearly usage) entries, repeating for each metric_type Harvesting cronjob; 'Run now': * API endpoint to start the harvesting process of a data provider * Button in the data providers list to run the harvesting process for each data provider upon clicked ERM SUSHI: Background job Job progress is updated to total amount of usage titles after retrieving the response from SUSHI; Job warning and success messages are added accordingly Redundant duplicate titles will not be added Redundant duplicate monthly and yearly usage statistics will not be added Data provider harvest background job harvests once per report_type Enqueue one background job for each report_type in the usage data provider Update the way we measure progress in the background job. It now uses the COUNTER report body rows instead of SUSHI response results. We're now incrementing and showing the number of skipped mus, skipped yus, added mus and added yus There's a bug in the way we calculate yus Updates to background job progress bar - Depends on 34468 Signed-off-by: Jessica Zairo Signed-off-by: Michaela Sieber Signed-off-by: Nick Clemens Signed-off-by: Tomas Cohen Arazi --- Koha/BackgroundJob.pm | 1 + Koha/BackgroundJob/ErmSushiHarvester.pm | 167 ++++ Koha/ERM/CounterFile.pm | 557 ++++++++++- Koha/ERM/UsageDataProvider.pm | 890 ++++++++++++++++++ Koha/ERM/UsageDatabase.pm | 120 +++ Koha/ERM/UsageDatabases.pm | 52 + Koha/ERM/UsageItem.pm | 120 +++ Koha/ERM/UsageItems.pm | 52 + Koha/ERM/UsagePlatform.pm | 120 +++ Koha/ERM/UsagePlatforms.pm | 52 + Koha/ERM/UsageTitle.pm | 56 +- Koha/REST/V1/ERM/CounterFiles.pm | 6 +- Koha/REST/V1/ERM/UsageDataProviders.pm | 28 + Koha/Schema/Result/ErmUsageDataProvider.pm | 55 +- Koha/Schema/Result/ErmUsageDatabase.pm | 156 +++ Koha/Schema/Result/ErmUsageItem.pm | 146 +++ Koha/Schema/Result/ErmUsageMus.pm | 104 +- Koha/Schema/Result/ErmUsagePlatform.pm | 133 +++ Koha/Schema/Result/ErmUsageTitle.pm | 44 +- Koha/Schema/Result/ErmUsageYus.pm | 104 +- api/v1/swagger/definitions/erm_usage_mus.yaml | 20 + .../swagger/definitions/erm_usage_title.yaml | 20 + api/v1/swagger/definitions/erm_usage_yus.yaml | 20 + api/v1/swagger/paths/erm_counter_files.yaml | 45 + .../paths/erm_usage_data_providers.yaml | 50 + api/v1/swagger/paths/erm_usage_titles.yaml | 20 + api/v1/swagger/swagger.yaml | 2 + .../erm_usage_statistics_tables.pl | 81 +- installer/data/mysql/kohastructure.sql | 75 +- .../background_jobs/erm_sushi_harvester.inc | 113 +++ .../en/includes/str/erm_sushi_harvester.inc | 5 + .../prog/en/modules/admin/background_jobs.tt | 4 + .../ERM/UsageStatisticsDataProvidersList.vue | 16 +- .../prog/js/vue/fetch/erm-api-client.js | 4 + misc/cronjobs/erm_run_harvester.pl | 129 +++ 35 files changed, 3542 insertions(+), 25 deletions(-) create mode 100644 Koha/BackgroundJob/ErmSushiHarvester.pm create mode 100644 Koha/ERM/UsageDatabase.pm create mode 100644 Koha/ERM/UsageDatabases.pm create mode 100644 Koha/ERM/UsageItem.pm create mode 100644 Koha/ERM/UsageItems.pm create mode 100644 Koha/ERM/UsagePlatform.pm create mode 100644 Koha/ERM/UsagePlatforms.pm create mode 100644 Koha/Schema/Result/ErmUsageDatabase.pm create mode 100644 Koha/Schema/Result/ErmUsageItem.pm create mode 100644 Koha/Schema/Result/ErmUsagePlatform.pm create mode 100644 koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc create mode 100644 koha-tmpl/intranet-tmpl/prog/en/includes/str/erm_sushi_harvester.inc create mode 100755 misc/cronjobs/erm_run_harvester.pl diff --git a/Koha/BackgroundJob.pm b/Koha/BackgroundJob.pm index ca3f471be8..bcfa68baca 100644 --- a/Koha/BackgroundJob.pm +++ b/Koha/BackgroundJob.pm @@ -420,6 +420,7 @@ sub core_types_to_classes { batch_biblio_record_modification => 'Koha::BackgroundJob::BatchUpdateBiblio', batch_item_record_deletion => 'Koha::BackgroundJob::BatchDeleteItem', batch_item_record_modification => 'Koha::BackgroundJob::BatchUpdateItem', + erm_sushi_harvester => 'Koha::BackgroundJob::ErmSushiHarvester', batch_hold_cancel => 'Koha::BackgroundJob::BatchCancelHold', create_eholdings_from_biblios => 'Koha::BackgroundJob::CreateEHoldingsFromBiblios', update_elastic_index => 'Koha::BackgroundJob::UpdateElasticIndex', diff --git a/Koha/BackgroundJob/ErmSushiHarvester.pm b/Koha/BackgroundJob/ErmSushiHarvester.pm new file mode 100644 index 0000000000..2501bfecd0 --- /dev/null +++ b/Koha/BackgroundJob/ErmSushiHarvester.pm @@ -0,0 +1,167 @@ +package Koha::BackgroundJob::ErmSushiHarvester; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + +use Try::Tiny; + +use Koha::DateUtils qw( dt_from_string ); +use Koha::ERM::UsageDataProviders; + +use base 'Koha::BackgroundJob'; + +=head1 NAME + +Koha::BackgroundJob::ErmSushiHarvester - Background job derived class to process the ERM Usage Statistics SUSHI Harvester +=head1 API + +=head2 Class methods + +=head3 job_type + +Define the job type of this job: erm_sushi_harvester + +=cut + +sub job_type { + return 'erm_sushi_harvester'; +} + +=head3 process + +Koha::BackgroundJob->find($id)->process( + { + ud_provider_id => $self->erm_usage_data_provider_id + } +); + +Process the harvesting. + +=cut + +sub process { + my ( $self, $args ) = @_; + + if ( $self->status eq 'cancelled' ) { + return; + } + + $self->{us_report_info} = { + skipped_mus => 0, + skipped_yus => 0, + added_mus => 0, + added_yus => 0 + }; + +# FIXME If the job has already been started, but started again (worker has been restart for instance) +# Then we will start from scratch and so double process the same records + + $self->start; + + my $ud_provider = + Koha::ERM::UsageDataProviders->find( $args->{ud_provider_id} ); + + $ud_provider->harvest( + $args->{report_type}, + { + report_info_callback => sub { $self->report_info(@_); }, + step_callback => sub { $self->step; }, + set_size_callback => sub { $self->set_job_size(@_); }, + add_message_callback => sub { $self->add_message(@_); }, + } + ); + + # Prepare job report + my $report = { + report_type => $ud_provider->{report_type}, + total_records => $ud_provider->{total_records}, + us_report_info => $self->{us_report_info}, + ud_provider_id => $ud_provider->erm_usage_data_provider_id, + ud_provider_name => $ud_provider->name, + }; + + my $data = $self->decoded_data; + $data->{report} = $report; + $data->{messages} = \@{ $self->{messages} }; + + $self->finish($data); +} + +=head3 report_info + +Setter for report_info + +=cut + +sub report_info { + my ( $self, $info ) = @_; + + $self->{us_report_info}{$info}++; +} + +=head3 set_job_size + +Setter for job_size + +=cut + +sub set_job_size { + my ( $self, $size ) = @_; + + $self->size($size)->store(); +} + +=head3 add_message + + $job->add_message( + { + type => 'success', # success, warning or error + code => 'object_added', # object_added or object_already_exists + title => $row->{Title}, + } + ); + +Add a new job message + +=cut + +sub add_message { + my ( $self, $message ) = @_; + + push @{ $self->{messages} }, $message; + +} + +=head3 enqueue + +Enqueue the new job + +=cut + +sub enqueue { + my ( $self, $args ) = @_; + + $self->SUPER::enqueue( + { + job_size => 1, + job_args => {%$args}, + job_queue => 'long_tasks', + } + ); +} + +1; diff --git a/Koha/ERM/CounterFile.pm b/Koha/ERM/CounterFile.pm index b6efb017c8..c6775fe227 100644 --- a/Koha/ERM/CounterFile.pm +++ b/Koha/ERM/CounterFile.pm @@ -17,6 +17,19 @@ package Koha::ERM::CounterFile; use Modern::Perl; +use Text::CSV_XS qw( csv ); + +use Koha::ERM::CounterLog; +use Koha::ERM::UsagePlatform; +use Koha::ERM::UsagePlatforms; +use Koha::ERM::UsageDatabase; +use Koha::ERM::UsageDatabases; +use Koha::ERM::UsageItem; +use Koha::ERM::UsageItems; +use Koha::ERM::UsageTitle; +use Koha::ERM::UsageTitles; +use Koha::ERM::UsageDataProvider; + use base qw(Koha::Object); use Koha::ERM::CounterLogs; @@ -33,18 +46,558 @@ Koha::ERM::CounterFile - Koha ErmCounterFile Object class =head3 counter_logs -Return the counter logs for this data provider +Return the counter_logs for this counter_file =cut sub counter_logs { - my ( $self ) = @_; + my ($self) = @_; + my $counter_logs_rs = $self->_result->erm_counter_logs; return Koha::ERM::CounterLogs->_new_from_dbic($counter_logs_rs); } +=head3 store + + Koha::ERM::CounterFile->new($counter_file)->store( $self->{job_callbacks} ); + +Stores the csv COUNTER file. +Adds usage titles from the file. +Adds the respective counter_log entry. + +=over + +=item background_job_callbacks + +Receive background_job_callbacks to be able to update job progress + +=back + +=cut + +sub store { + my ( $self, $background_job_callbacks ) = @_; + + my $result = $self->SUPER::store; + + # Set class wide background_job callbacks + $self->{job_callbacks} = $background_job_callbacks; + + $self->_add_usage_objects; + $self->_add_counter_log_entry; + + return $result; +} + +=head3 get_usage_data_provider + +Getter for the usage data provider of this counter_file + +=cut + +sub get_usage_data_provider { + my ($self) = @_; + + my $usage_data_provider = $self->_result->usage_data_provider; + return Koha::ERM::UsageDataProvider->_new_from_dbic($usage_data_provider); +} + =head2 Internal methods +=head3 _add_usage_objects + +Goes through COUNTER file and adds usage objects for each row +A usage object may be a erm_usage_title, erm_usage_platform, erm_usage_item or erm_usage_database + +#FIXME?: "Yearly" usage may be incorrect, it'll only add up the months in the current report, not necessarily the whole year + +=cut + +sub _add_usage_objects { + my ($self) = @_; + + my $rows = $self->_get_rows_from_COUNTER_file; + my $usage_data_provider = $self->get_usage_data_provider; + my $previous_object = undef; + my $usage_object = undef; + + # Set job size to the amount of rows we're processing + $self->{job_callbacks}->{set_size_callback}->( scalar( @{$rows} ) ) + if $self->{job_callbacks}; + + foreach my $row ( @{$rows} ) { + +# INFO: A single row may have multiple instances in the COUNTER report, one for each metric_type or access_type +# If we're on a row that we've already gone through, use the same usage object +# and add usage statistics for the different metric_type or access_type + if ( $self->_is_same_usage_object( $previous_object, $row ) ) { + $usage_object = $previous_object; + } + else { + # Check if usage object already exists in this data provider, e.g. from a previous harvest + $usage_object = $self->_search_for_usage_object($row); + + if ($usage_object) { + + # Usage object already exists, add job warning message and do nothing else + $self->_add_job_message( 'warning', 'object_already_exists', + $row ); + } + else { + # Fresh usage object, create it + $usage_object = $self->_add_usage_object_entry($row); + + # Usage object created, add job success message + $self->_add_job_message( 'success', 'object_added', $row ); + } + } + + # Regex match for Mmm-yyyy expected format, e.g. "Jan 2022" + my @date_fields = + map( $_ =~ /\b[A-Z][a-z][a-z]\b [0-9]{4}\b/ ? $_ : (), keys %{$row} ); + + unless (@date_fields) { + warn "No monthly usage fields retrieved"; + } + + # Add monthly usage statistics for this usage object + my %yearly_usages = (); + foreach my $year_month (@date_fields) { + my $usage = %{$row}{$year_month}; + + # Skip this monthly usage entry if it's 0 + next if $usage eq "0"; + + my $month = substr( $year_month, 0, 3 ); + my $year = substr( $year_month, 4, 4 ); + + if ( !exists $yearly_usages{$year} ) { + $yearly_usages{$year} = $usage; + } + else { + $yearly_usages{$year} += $usage; + } + + $self->_add_monthly_usage_entries( $usage_object, + $row->{Metric_Type}, $row, $year, $month, $usage ); + } + + # Add yearly usage statistics for this usage object + $self->_add_yearly_usage_entries( $usage_object, $row->{Metric_Type}, + $row, \%yearly_usages ); + + $previous_object = $usage_object; + + # Update background job step + $self->{job_callbacks}->{step_callback}->() if $self->{job_callbacks}; + } +} + +=head3 _add_monthly_usage_entries + +Adds erm_usage_mus database entries + +=cut + +sub _add_monthly_usage_entries { + my ( $self, $usage_object, $metric_type, $row, $year, $month, $usage ) = @_; + + my $usage_data_provider = $self->get_usage_data_provider; + my $usage_object_info = $self->_get_usage_object_id_hash($usage_object); + my $specific_fields = + $usage_data_provider->get_report_type_specific_fields( $self->type ); + + $usage_object->monthly_usages( + [ + { + %{$usage_object_info}, + usage_data_provider_id => + $usage_data_provider->erm_usage_data_provider_id, + year => $year, + month => $self->_get_month_number($month), + usage_count => $usage, + metric_type => $row->{Metric_Type}, + grep ( /Access_Type/, @{$specific_fields} ) + ? ( access_type => $row->{Access_Type} ) + : (), + report_type => $self->type + } + ], + $self->{job_callbacks} + ); +} + +=head3 _add_yearly_usage_entries + +Adds erm_usage_yus database entries + +=cut + +sub _add_yearly_usage_entries { + my ( $self, $usage_object, $metric_type, $row, $yearly_usages ) = @_; + + my $usage_data_provider = $self->get_usage_data_provider; + my $usage_object_info = $self->_get_usage_object_id_hash($usage_object); + my $specific_fields = + $usage_data_provider->get_report_type_specific_fields( $self->type ); + + while ( my ( $year, $usage ) = each( %{$yearly_usages} ) ) { + + # Skip this yearly usage entry if it's 0 + next if $usage eq "0"; + + $usage_object->yearly_usages( + [ + { + %{$usage_object_info}, + usage_data_provider_id => + $usage_data_provider->erm_usage_data_provider_id, + year => $year, + totalcount => $usage, + metric_type => $metric_type, + grep ( /Access_Type/, @{$specific_fields} ) + ? ( access_type => $row->{Access_Type} ) + : (), + report_type => $self->type + } + ], + $self->{job_callbacks} + ); + } +} + +=head3 validate + +Verifies if the given file_content is a valid COUNTER file or not + +A I exception is thrown + if the file is invalid . + +=cut + +sub validate { + my ($self) = @_; + + open my $fh, "<", \$self->file_content or die; + my $csv = Text::CSV_XS->new( + { binary => 1, always_quote => 1, eol => $/, decode_utf8 => 1 } ); + + $csv->column_names(qw( header_key header_value )); + my @header_rows = $csv->getline_hr_all( $fh, 0, 12 ); + my @header = $header_rows[0]; + + my @release_row = + map( $_->{header_key} eq 'Release' ? $_ : (), @{ $header[0] } ); + my $release = $release_row[0]; + + # TODO: Validate that there is an empty row between header and body + + Koha::Exceptions::ERM::CounterFile::UnsupportedRelease->throw + if $release && $release->{header_value} != 5; + +} + +=head3 _set_report_type_from_file + +Extracts Report_ID from file and sets report_type for this counter_file + +=cut + +sub _set_report_type_from_file { + my ($self) = @_; + + open my $fh, "<", \$self->file_content or die; + my $csv = Text::CSV_XS->new( + { binary => 1, always_quote => 1, eol => $/, decode_utf8 => 1 } ); + + $csv->column_names(qw( header_key header_value )); + my @header_rows = $csv->getline_hr_all( $fh, 0, 12 ); + my @header = $header_rows[0]; + + my @report_id_row = + map( $_->{header_key} eq 'Report_ID' ? $_ : (), @{ $header[0] } ); + my $report = $report_id_row[0]; + + $self->type( $report->{header_value} ); +} + +=head3 _get_rows_from_COUNTER_file + +Returns array of rows from COUNTER file + +=cut + +sub _get_rows_from_COUNTER_file { + my ($self) = @_; + + open my $fh, "<", \$self->file_content or die; + my $csv = Text::CSV_XS->new( + { binary => 1, always_quote => 1, eol => $/, decode_utf8 => 1 } ); + + my $header_columns = $csv->getline_all( $fh, 13, 1 ); + $csv->column_names( @{$header_columns}[0] ); + + # Get all rows from 14th onward + return $csv->getline_hr_all($fh); +} + +=head3 _add_job_message + +Add a message to be displayed in the background job + +=cut + +sub _add_job_message { + my ( $self, $type, $code, $row ) = @_; + + my $usage_data_provider = $self->get_usage_data_provider; + + my $object_title; + + if ( $self->type =~ /PR/i ) { + $object_title = $row->{Platform}; + } + elsif ( $self->type =~ /DR/i ) { + $object_title = $row->{Database}; + } + elsif ( $self->type =~ /IR/i ) { + $object_title = $row->{Item}; + } + elsif ( $self->type =~ /TR/i ) { + $object_title = $row->{Title}; + } + + $self->{job_callbacks}->{add_message_callback}->( + { + type => $type, + code => $code, + title => $object_title, + } + ) if $self->{job_callbacks}; +} + +=head3 _get_usage_object_id_hash + +Return a usage_object id hash to be used when adding new yus/mus + +=cut + +sub _get_usage_object_id_hash { + my ( $self, $usage_object ) = @_; + + if ( $self->type =~ /PR/i ) { + return { platform_id => $usage_object->platform_id }; + } + elsif ( $self->type =~ /DR/i ) { + return { database_id => $usage_object->database_id }; + } + elsif ( $self->type =~ /IR/i ) { + return { item_id => $usage_object->item_id }; + } + elsif ( $self->type =~ /TR/i ) { + return { title_id => $usage_object->title_id }; + } + return 0; +} + +=head3 _search_for_usage_object + +Returns usage object if found + +=cut + +sub _search_for_usage_object { + my ( $self, $row ) = @_; + + my $usage_data_provider = $self->get_usage_data_provider; + + if ( $self->type =~ /PR/i ) { + return Koha::ERM::UsagePlatforms->search( + { + platform => $row->{Platform}, + usage_data_provider_id => + $usage_data_provider->erm_usage_data_provider_id + } + )->last; + } + elsif ( $self->type =~ /DR/i ) { + return Koha::ERM::UsageDatabases->search( + { + database => $row->{Database}, + usage_data_provider_id => + $usage_data_provider->erm_usage_data_provider_id + } + )->last; + } + elsif ( $self->type =~ /IR/i ) { + return Koha::ERM::UsageItems->search( + { + item => $row->{Item}, + publisher => $row->{Publisher}, + usage_data_provider_id => + $usage_data_provider->erm_usage_data_provider_id + } + )->last; + } + elsif ( $self->type =~ /TR/i ) { + return Koha::ERM::UsageTitles->search( + { + title_doi => $row->{DOI}, + usage_data_provider_id => + $usage_data_provider->erm_usage_data_provider_id + } + )->last; + } + + return 0; +} + +=head3 _is_same_usage_object + +Returns true if is the same usage object + +=cut + +sub _is_same_usage_object { + my ( $self, $previous_object, $row ) = @_; + + if ( $self->type =~ /PR/i ) { + return $previous_object + && $previous_object->platform eq $row->{Platform}; + } + elsif ( $self->type =~ /DR/i ) { + return $previous_object + && $previous_object->database eq $row->{Database}; + } + elsif ( $self->type =~ /IR/i ) { + return + $previous_object + && $previous_object->item eq $row->{Item} + && $previous_object->publisher eq $row->{Publisher}; + } + elsif ( $self->type =~ /TR/i ) { + return $previous_object && $previous_object->title_doi eq $row->{DOI}; + } + + return 0; +} + +=head3 _get_month_number + +Returns month number for a given Mmm month + +=cut + +sub _get_month_number { + my ( $self, $month ) = @_; + + my %months = ( + "Jan" => 1, + "Feb" => 2, + "Mar" => 3, + "Apr" => 4, + "May" => 5, + "Jun" => 6, + "Jul" => 7, + "Aug" => 8, + "Sep" => 9, + "Oct" => 10, + "Nov" => 11, + "Dec" => 12 + ); + + return $months{$month}; +} + +=head3 _add_counter_log_entry + +Adds a erm_counter_logs database entry + +=cut + +sub _add_counter_log_entry { + my ($self) = @_; + + Koha::ERM::CounterLog->new( + { +#TODO: borrowernumber only required for manual uploads or "harvest now" button clicks + borrowernumber => undef, + counter_files_id => $self->erm_counter_files_id, + importdate => $self->date_uploaded, + filename => $self->filename, + + #TODO: add eventual exceptions coming from the COUNTER report to logdetails? + logdetails => undef + } + )->store; +} + +=head3 _add_usage_object_entry + +Adds a usage object database entry + +=cut + +sub _add_usage_object_entry { + my ( $self, $row ) = @_; + + my $usage_data_provider = $self->get_usage_data_provider; + my $specific_fields = + $usage_data_provider->get_report_type_specific_fields( $self->type ); + + if ( $self->type =~ /PR/i ) { + return Koha::ERM::UsagePlatform->new( + { + usage_data_provider_id => + $usage_data_provider->erm_usage_data_provider_id, + platform => $row->{Platform}, + } + )->store; + } + elsif ( $self->type =~ /DR/i ) { + return Koha::ERM::UsageDatabase->new( + { + database => $row->{Database}, + usage_data_provider_id => + $usage_data_provider->erm_usage_data_provider_id, + platform => $row->{Platform}, + publisher => $row->{Publisher}, + publisher_id => $row->{Publisher_ID}, + } + )->store; + } + elsif ( $self->type =~ /IR/i ) { + return Koha::ERM::UsageItem->new( + { + item => $row->{Item}, + usage_data_provider_id => + $usage_data_provider->erm_usage_data_provider_id, + platform => $row->{Platform}, + publisher => $row->{Publisher}, + } + )->store; + } + elsif ( $self->type =~ /TR/i ) { + return Koha::ERM::UsageTitle->new( + { + title => $row->{Title}, + usage_data_provider_id => + $usage_data_provider->erm_usage_data_provider_id, + title_doi => $row->{DOI}, + print_issn => $row->{Print_ISSN}, + online_issn => $row->{Online_ISSN}, + title_uri => $row->{URI}, + publisher => $row->{Publisher}, + publisher_id => $row->{Publisher_ID}, + grep ( /YOP/, @{$specific_fields} ) ? ( yop => $row->{YOP} ) + : (), + grep ( /ISBN/, @{$specific_fields} ) ? ( isbn => $row->{ISBN} ) + : (), + } + )->store; + } +} + =head3 _type =cut diff --git a/Koha/ERM/UsageDataProvider.pm b/Koha/ERM/UsageDataProvider.pm index f9a1588d71..90d3ff0e57 100644 --- a/Koha/ERM/UsageDataProvider.pm +++ b/Koha/ERM/UsageDataProvider.pm @@ -17,8 +17,19 @@ package Koha::ERM::UsageDataProvider; use Modern::Perl; +use HTTP::Request; +use JSON qw( decode_json ); +use LWP::UserAgent; +use Text::CSV_XS qw( csv ); + +use Koha::Exceptions; + use base qw(Koha::Object); +use Koha::ERM::CounterFile; +use Koha::ERM::CounterFiles; +use Koha::BackgroundJob::ErmSushiHarvester; + =head1 NAME Koha::ERM::UsageDataProvider - Koha ErmUsageDataProvider Object class @@ -27,10 +38,889 @@ Koha::ERM::UsageDataProvider - Koha ErmUsageDataProvider Object class =head2 Class Methods +=head3 counter_files + +Getter/setter for counter_files for this usage data provider + +=cut + +sub counter_files { + my ( $self, $counter_files ) = @_; + + if ($counter_files) { + for my $counter_file (@$counter_files) { + Koha::ERM::CounterFile->new($counter_file) + ->store( $self->{job_callbacks} ); + } + } + my $counter_files_rs = $self->_result->erm_counter_files; + return Koha::ERM::CounterFiles->_new_from_dbic($counter_files_rs); +} + +=head3 enqueue_counter_file_processing_job + +Enqueues a background job to process a COUNTER file that has been uploaded + +=cut + +sub enqueue_counter_file_processing_job { + my ( $self, $args ) = @_; + + my @jobs; + my $job_id = Koha::BackgroundJob::ErmSushiHarvester->new->enqueue( + { + ud_provider_id => $self->erm_usage_data_provider_id, + file_content => $args->{file_content}, + } + ); + + push( + @jobs, + { + job_id => $job_id + } + ); + + return \@jobs; +} + +=head3 enqueue_sushi_harvest_jobs + +Enqueues one harvest background job for each report type in this usage data provider + +=cut + +sub enqueue_sushi_harvest_jobs { + my ( $self, $args ) = @_; + + my @report_types = split( /;/, $self->report_types ); + + my @jobs; + foreach my $report_type (@report_types) { + + my $job_id = Koha::BackgroundJob::ErmSushiHarvester->new->enqueue( + { + ud_provider_id => $self->erm_usage_data_provider_id, + report_type => $report_type + } + ); + + push( + @jobs, + { + report_type => $report_type, + job_id => $job_id + } + ); + } + + return \@jobs; +} + +=head3 harvest + + $ud_provider->harvest( + { + step_callback => sub { $self->step; }, + set_size_callback => sub { $self->set_job_size(@_); }, + add_message_callback => sub { $self->add_message(@_); }, + } + ); + +Run the SUSHI harvester of this usage data provider +Builds the URL query and requests the COUNTER 5 SUSHI service + +COUNTER SUSHI api spec: +https://app.swaggerhub.com/apis/COUNTER/counter-sushi_5_0_api/5.0.2 + +=over + +=item report_type + +Report type to run this harvest on + +=back + +=over + +=item background_job_callbacks + +Receive background_job_callbacks to be able to update job + +=back + =cut +sub harvest { + my ( $self, $report_type, $background_job_callbacks ) = @_; + + # Set class wide vars + $self->{job_callbacks} = $background_job_callbacks; + $self->{report_type} = $report_type; + + my $url = $self->_build_url_query; + my $request = HTTP::Request->new( 'GET' => $url ); + my $ua = LWP::UserAgent->new; + my $response = $ua->simple_request($request); + + if ( $response->code >= 400 ) { + my $result = decode_json( $response->decoded_content ); + + my $message; + if ( ref($result) eq 'ARRAY' ) { + for my $r (@$result) { + $message .= $r->{message}; + } + } + else { + #TODO: May want to check $result->{Report_Header}->{Exceptions} here + $message = $result->{message} || $result->{Message} || q{}; + if ( $result->{errors} ) { + for my $e ( @{ $result->{errors} } ) { + $message .= $e->{message}; + } + } + } + + #TODO: May want to add a job error message here? + warn sprintf "ERROR - SUSHI service %s returned %s - %s\n", $url, + $response->code, $message; + if ( $response->code == 404 ) { + Koha::Exceptions::ObjectNotFound->throw($message); + } + elsif ( $response->code == 401 ) { + Koha::Exceptions::Authorization::Unauthorized->throw($message); + } + else { + #TODO: May want to add a job error message here? + die sprintf "ERROR requesting SUSHI service\n%s\ncode %s: %s\n", + $url, $response->code, + $message; + } + } + elsif ( $response->code == 204 ) { # No content + return; + } + + # Parse the SUSHI response + $self->parse_SUSHI_response( decode_json( $response->decoded_content ) ); +} + +=head3 parse_SUSHI_response + + $self->parse_SUSHI_response( decode_json( $response->decoded_content ) ); + +Parse the SUSHI response, prepare the COUNTER report file header, +column headings and body + +=over + +=item result + +The result of the SUSHI response after json decoded + +=back + +=cut + +sub parse_SUSHI_response { + my ( $self, $result ) = @_; + + # Set class wide sushi response content + $self->{sushi} = { + header => $result->{Report_Header}, + body => $result->{Report_Items} + }; + + #TODO: Handle empty $self->{sushi}->{body} here! + + # Get ready to build COUNTER file + my @report_header = $self->_COUNTER_report_header; + my @report_column_headings = $self->_COUNTER_report_column_headings; + my @report_body = $self->_COUNTER_report_body; + + $self->_build_COUNTER_report_file( \@report_header, + \@report_column_headings, \@report_body ); +} + =head2 Internal methods +=head3 _build_url_query + +Build the URL query params for COUNTER 5 SUSHI request + +=cut + +sub _build_url_query { + my ($self) = @_; + + unless ( $self->service_url && $self->customer_id ) { + die sprintf +"SUSHI Harvesting config for usage data provider %d is missing service_url or customer_id\n", + $self->erm_usage_data_provider_id; + } + + # FIXME: service_url needs to end in 'reports/' + # below concat will result in a badly formed URL otherwise + # Either validate this on UI form, here, or both + my $url = $self->service_url; + + $url .= $self->{report_type}; + $url .= '?customer_id=' . $self->customer_id; + $url .= '&requestor_id=' . $self->requestor_id if $self->requestor_id; + $url .= '&api_key=' . $self->api_key if $self->api_key; + $url .= '&begin_date=' . $self->begin_date if $self->begin_date; + $url .= '&end_date=' . $self->end_date if $self->end_date; + + return $url; +} + +=head3 _build_COUNTER_report_file + +Build the COUNTER file +https://cop5.projectcounter.org/en/5.0.2/03-specifications/02-formats-for-counter-reports.html#report-header + +=cut + +sub _build_COUNTER_report_file { + my ( $self, $header, $column_headings, $body ) = @_; + + my @report = ( @{$header}, @{$column_headings}, @{$body} ); + + #TODO: change this to tab instead of comma + csv( in => \@report, out => \my $counter_file, encoding => "utf-8" ); + + $self->counter_files( + [ + { + usage_data_provider_id => $self->erm_usage_data_provider_id, + file_content => $counter_file, + date_uploaded => POSIX::strftime( "%Y%m%d%H%M%S", localtime ), + + #TODO: add ".csv" to end of filename here + filename => $self->name . "_" . $self->{report_type}, + type => $self->{report_type} + } + ] + ); +} + +=head3 _COUNTER_report_header + +Return a COUNTER report header +https://cop5.projectcounter.org/en/5.0.2/04-reports/03-title-reports.html + +=cut + +sub _COUNTER_report_header { + my ($self) = @_; + + my $header = $self->{sushi}->{header}; + + my @metric_types_string = + $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Metric_Type" ); + + my $begin_date = + $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Begin_Date" ); + my $end_date = + $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "End_Date" ); + + return ( + [ Report_Name => $header->{Report_Name} || "" ], + [ Report_ID => $header->{Report_ID} || "" ], + [ Release => $header->{Release} || "" ], + [ Institution_Name => $header->{Institution_Name} || "" ], + [ + Institution_ID => join( + "; ", + map( $_->{Type} . ":" . $_->{Value}, + @{ $header->{Institution_ID} } ) + ) + || "" + ], + [ + Metric_Types => join( "; ", split( /\|/, $metric_types_string[0] ) ) + || "" + ], + [ + Report_Filters => join( + "; ", + map( $_->{Name} . ":" . $_->{Value}, + @{ $header->{Report_Filters} } ) + ) + || "" + ], + +#TODO: Report_Attributes may need parsing, test this with a SUSHI response that provides it + [ Report_Attributes => $header->{Report_Attributes} || "" ], + [ + Exceptions => join( + "; ", + map( $_->{Code} . ": " + . $_->{Message} . " (" + . $_->{Data} . ")", + @{ $header->{Exceptions} } ) + ) + || "" + ], + [ + Reporting_Period => "Begin_Date=" + . $begin_date + . "; End_Date=" + . $end_date + ], + [ Created => $header->{Created} || "" ], + [ Created_By => $header->{Created_By} || "" ], + [""] #empty 13th line + ); +} + +=head3 _COUNTER_item_report_row + +Return a COUNTER item for the COUNTER items report body +https://cop5.projectcounter.org/en/5.0.2/04-reports/04-item-reports.html#column-headings-elements + +=cut + +sub _COUNTER_item_report_row { + my ( $self, $item_row, $metric_type, $total_usage, $monthly_usages ) = @_; + + return ( + [ + $item_row->{Item} || "", + $item_row->{Publisher} || "", + $self->_get_SUSHI_Type_Value( $item_row->{Publisher_ID}, "ISNI" ) + || "", + $item_row->{Platform} || "", + $self->_get_SUSHI_Type_Value( $item_row->{Item_ID}, "DOI" ) || "", + $item_row->{Proprietary_ID} || "", + "", #FIXME: What goes in URI? + $metric_type, + $total_usage, + @{$monthly_usages} + ] + ); +} + +=head3 _COUNTER_database_report_row + +Return a COUNTER database for the COUNTER databases report body +https://cop5.projectcounter.org/en/5.0.2/04-reports/02-database-reports.html#column-headings-elements + +=cut + +sub _COUNTER_database_report_row { + my ( $self, $database_row, $metric_type, $total_usage, $monthly_usages ) = + @_; + + return ( + [ + $database_row->{Database} || "", + $database_row->{Publisher} || "", + $self->_get_SUSHI_Type_Value( $database_row->{Publisher_ID}, + "ISNI" ) + || "", + $database_row->{Platform} || "", + $database_row->{Proprietary_ID} || "", + $metric_type, + $total_usage, + @{$monthly_usages} + ] + ); +} + +=head3 _COUNTER_platform_report_row + +Return a COUNTER platform for the COUNTER platforms report body +https://cop5.projectcounter.org/en/5.0.2/04-reports/01-platform-reports.html#column-headings-elements + +=cut + +sub _COUNTER_platform_report_row { + my ( $self, $platform_row, $metric_type, $total_usage, $monthly_usages ) = + @_; + + return ( + [ + $platform_row->{Platform} || "", $metric_type, + $total_usage, @{$monthly_usages} + ] + ); +} + +=head3 _COUNTER_title_report_row + +Return a COUNTER title for the COUNTER titles report body +https://cop5.projectcounter.org/en/5.0.2/04-reports/03-title-reports.html#column-headings-elements + +=cut + +sub _COUNTER_title_report_row { + my ( $self, $title_row, $metric_type, $total_usage, $monthly_usages ) = @_; + + my $header = $self->{sushi}->{header}; + my $specific_fields = + $self->get_report_type_specific_fields( $header->{Report_ID} ); + + return ( + [ + # Title + $title_row->{Title} || "", + + # Publisher + $title_row->{Publisher} || "", + + # Publisher_ID + $self->_get_SUSHI_Type_Value( $title_row->{Publisher_ID}, "ISNI" ) + || "", + + # Platform + $title_row->{Platform} || "", + + # DOI + $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "DOI" ) || "", + + # Proprietary_ID + $self->_get_SUSHI_Type_Value( + $title_row->{Item_ID}, "Proprietary" + ) + || "", + + # ISBN + grep ( /ISBN/, @{$specific_fields} ) + ? ( $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "ISBN" ) + || "" ) + : (), + + # Print_ISSN + $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "Print_ISSN" ) + || "", + + # Online_ISSN + $self->_get_SUSHI_Type_Value( + $title_row->{Item_ID}, "Online_ISSN" + ) + || "", + + # URI - FIXME: What goes in URI? + "", + + # YOP + grep ( /YOP/, @{$specific_fields} ) + ? ( $title_row->{YOP} || "" ) + : (), + + # Access_Type + grep ( /Access_Type/, @{$specific_fields} ) + ? ( $title_row->{Access_Type} || "" ) + : (), + + # Metric_Type + $metric_type, + + # Report_Period_Total + $total_usage, + + # Monthly usage entries + @{$monthly_usages} + ] + ); +} + +=head3 _COUNTER_report_row + +Return a COUNTER row for the COUNTER report body + +=cut + +sub _COUNTER_report_row { + my ( $self, $report_row, $metric_type ) = @_; + + my $header = $self->{sushi}->{header}; + + my ( $total_usage, @monthly_usages ) = + $self->_get_row_usages( $report_row, $metric_type ); + + if ( $header->{Report_ID} =~ /PR/i ) { + return $self->_COUNTER_platform_report_row( $report_row, $metric_type, + $total_usage, \@monthly_usages ); + } + elsif ( $header->{Report_ID} =~ /DR/i ) { + return $self->_COUNTER_database_report_row( $report_row, $metric_type, + $total_usage, \@monthly_usages ); + } + elsif ( $header->{Report_ID} =~ /IR/i ) { + return $self->_COUNTER_item_report_row( $report_row, $metric_type, + $total_usage, \@monthly_usages ); + } + elsif ( $header->{Report_ID} =~ /TR/i ) { + return $self->_COUNTER_title_report_row( $report_row, $metric_type, + $total_usage, \@monthly_usages ); + } +} + +=head3 _get_row_usages + +Returns the total and monthly usages for a row + +=cut + +sub _get_row_usages { + my ( $self, $row, $metric_type ) = @_; + + my @usage_months = $self->_get_usage_months( $self->{sushi}->{header} ); + + my @usage_months_fields = (); + my $count_total = 0; + + foreach my $usage_month (@usage_months) { + my $month_is_empty = 1; + + foreach my $performance ( @{ $row->{Performance} } ) { + my $period = $performance->{Period}; + my $period_usage_month = substr( $period->{Begin_Date}, 0, 7 ); + + my $instances = $performance->{Instance}; + my @metric_type_count = + map( $_->{Metric_Type} eq $metric_type ? $_->{Count} : (), + @{$instances} ); + + if ( $period_usage_month eq $usage_month && $metric_type_count[0] ) + { + push( @usage_months_fields, $metric_type_count[0] ); + $count_total += $metric_type_count[0]; + $month_is_empty = 0; + } + } + + if ($month_is_empty) { + push( @usage_months_fields, 0 ); + } + } + return ( $count_total, @usage_months_fields ); +} + +=head3 _COUNTER_report_body + +Return the COUNTER report body as an array + +=cut + +sub _COUNTER_report_body { + my ($self) = @_; + + my $header = $self->{sushi}->{header}; + my $body = $self->{sushi}->{body}; + + my @metric_types_string = $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Metric_Type" ); + my @metric_types = split( /\|/, $metric_types_string[0] ); + + my @report_body = (); + + my $total_records = 0; + foreach my $report_row ( @{$body} ) { + + my @metric_types = (); + + # Grab all metric_types this SUSHI result has statistics for + foreach my $performance ( @{ $report_row->{Performance} } ) { + my @SUSHI_metric_types = + map( $_->{Metric_Type}, @{ $performance->{Instance} } ); + + foreach my $sushi_metric_type (@SUSHI_metric_types) { + push( @metric_types, $sushi_metric_type ) + unless grep { $_ eq $sushi_metric_type } @metric_types; + } + } + + # Add one report row for each metric_type we're working with + foreach my $metric_type (@metric_types) { + push( @report_body, + $self->_COUNTER_report_row( $report_row, $metric_type ) ); + } + $self->{total_records} = ++$total_records; + } + + return @report_body; +} + +=head3 _get_SUSHI_Name_Value + +Returns "Value" of a given "Name" + +=cut + +sub _get_SUSHI_Name_Value { + my ( $self, $item, $name ) = @_; + + my @value = map( $_->{Name} eq $name ? $_->{Value} : (), @{$item} ); + + return $value[0]; +} + +=head3 _get_SUSHI_Type_Value + +Returns "Value" of a given "Type" + +=cut + +sub _get_SUSHI_Type_Value { + my ( $self, $item, $type ) = @_; + + my @value = map( $_->{Type} eq $type ? $_->{Value} : (), @{$item} ); + + return $value[0]; +} + +=head3 _COUNTER_report_column_headings + +Returns column headings by report type + Check the report type from the COUNTER header + and return column headings accordingly + +=cut + +sub _COUNTER_report_column_headings { + my ($self) = @_; + + my $header = $self->{sushi}->{header}; + + if ( $header->{Report_ID} =~ /PR/i ) { + return $self->_COUNTER_platforms_report_column_headings; + } + elsif ( $header->{Report_ID} =~ /DR/i ) { + return $self->_COUNTER_databases_report_column_headings; + } + elsif ( $header->{Report_ID} =~ /IR/i ) { + return $self->_COUNTER_items_report_column_headings; + } + elsif ( $header->{Report_ID} =~ /TR/i ) { + return $self->_COUNTER_titles_report_column_headings; + } + + return; +} + +=head3 _COUNTER_items_report_column_headings + +Return items report column headings + +=cut + +sub _COUNTER_items_report_column_headings { + my ($self) = @_; + + my $header = $self->{sushi}->{header}; + my @month_headings = $self->_get_usage_months( $header, 1 ); + + return ( + [ + "Item", + "Publisher", + "Publisher_ID", + "Platform", + + # "Authors", #IR_A1 only + # "Publication_Date", #IR_A1 only + # "Article_Version", #IR_A1 only + "DOI", + "Proprietary_ID", + + # "ISBN", #IR only + # "Print_ISSN", #IR_A1 only + # "Online_ISSN", #IR_A1 only + "URI", + + # "Parent_Title", #IR_A1 only + # "Parent_Authors", #IR_A1 only + # "Parent_Publication_Date", #IR only + # "Parent_Article_Version", #IR_A1 only + # "Parent_Data_Type", #IR only + # "Parent_DOI", #IR_A1 only + # "Parent_Proprietary_ID", #IR_A1 only + # "Parent_ISBN", #IR only + # "Parent_Print_ISSN", #IR_A1 only + # "Parent_Online_ISSN", #IR_A1 only + # "Parent_URI", #IR_A1 only + # "Component_Title", #IR only + # "Component_Authors", #IR only + # "Component_Publication_Date", #IR only + # "Component_Data_Type", #IR only + # "Component_DOI", #IR only + # "Component_Proprietary_ID", #IR only + # "Component_ISBN", #IR only + # "Component_Print_ISSN", #IR only + # "Component_Online_ISSN", #IR only + # "Component_URI", #IR only + # "Data_Type", #IR only + # "YOP", #IR only + # "Access_Type", #IR_A1 only + # "Access_Method", #IR only + "Metric_Type", + "Reporting_Period_Total", + +# @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true + @month_headings + ] + ); +} + +=head3 _COUNTER_databases_report_column_headings + +Return databases report column headings + +=cut + +sub _COUNTER_databases_report_column_headings { + my ($self) = @_; + + my $header = $self->{sushi}->{header}; + my @month_headings = $self->_get_usage_months( $header, 1 ); + + return ( + [ + "Database", + "Publisher", + "Publisher_ID", + "Platform", + "Proprietary_ID", + "Metric_Type", + "Reporting_Period_Total", + +# @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true + @month_headings + ] + ); +} + +=head3 _COUNTER_platforms_report_column_headings + +Return platforms report column headings + +=cut + +sub _COUNTER_platforms_report_column_headings { + my ($self) = @_; + + my $header = $self->{sushi}->{header}; + my @month_headings = $self->_get_usage_months( $header, 1 ); + + return ( + [ + "Platform", + "Metric_Type", + "Reporting_Period_Total", + +# @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true + @month_headings + ] + ); +} + +=head3 _COUNTER_titles_report_column_headings + +Return titles report column headings + +=cut + +sub _COUNTER_titles_report_column_headings { + my ($self) = @_; + + my $header = $self->{sushi}->{header}; + my @month_headings = $self->_get_usage_months( $header, 1 ); + my $specific_fields = + $self->get_report_type_specific_fields( $header->{Report_ID} ); + + return ( + [ + "Title", + "Publisher", + "Publisher_ID", + "Platform", + "DOI", + "Proprietary_ID", + grep ( /ISBN/, @{$specific_fields} ) ? ("ISBN") : (), + "Print_ISSN", + "Online_ISSN", + "URI", + + #"Data_Type", #TODO: Only if requested (?) + #"Section_Type", #TODO: Only if requested (?) + grep ( /YOP/, @{$specific_fields} ) ? ("YOP") : (), + grep ( /Access_Type/, @{$specific_fields} ) ? ("Access_Type") : (), + + #"Access_Method", #TODO: Only if requested (?) + "Metric_Type", + "Reporting_Period_Total", + +# @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true + @month_headings + ] + ); +} + +=head3 _get_usage_months + +Return report usage months. Formatted for column headings if $column_headings_formatting + +=cut + +sub _get_usage_months { + my ( $self, $header, $column_headings_formatting ) = @_; + + my @months = ( + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" + ); + + my @begin_date = map( $_->{Name} eq "Begin_Date" ? $_->{Value} : (), + @{ $header->{Report_Filters} } ); + my $begin_month = substr( $begin_date[0], 5, 2 ); + my $begin_year = substr( $begin_date[0], 0, 4 ); + + my @end_date = map( $_->{Name} eq "End_Date" ? $_->{Value} : (), + @{ $header->{Report_Filters} } ); + my $end_month = substr( $end_date[0], 5, 2 ); + my $end_year = substr( $end_date[0], 0, 4 ); + + my @month_headings = (); + while ( $begin_month <= $end_month || $begin_year < $end_year ) { + push( @month_headings, + $column_headings_formatting + ? $months[ $begin_month - 1 ] . " " . $begin_year + : $begin_year . "-" . $begin_month ); + $begin_month++; + if ( $begin_month > 12 ) { + $begin_month = 1; + $begin_year++; + } + $begin_month = "0" . $begin_month if length($begin_month) == 1; + } + + return @month_headings; +} + +=head3 get_report_type_specific_fields + +Returns the specific fields for a given report_type + +=cut + +sub get_report_type_specific_fields { + my ( $self, $report_type ) = @_; + + my %report_type_map = ( + "TR_B1" => [ 'YOP', 'ISBN' ], + "TR_B2" => [ 'YOP', 'ISBN' ], + "TR_B3" => [ 'YOP', 'Access_Type', 'ISBN' ], + "TR_J3" => ['Access_Type'], + "TR_J4" => ['YOP'], + ); + + return $report_type_map{$report_type}; + +} + =head3 _type =cut diff --git a/Koha/ERM/UsageDatabase.pm b/Koha/ERM/UsageDatabase.pm new file mode 100644 index 0000000000..628686a9a9 --- /dev/null +++ b/Koha/ERM/UsageDatabase.pm @@ -0,0 +1,120 @@ +package Koha::ERM::UsageDatabase; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + +use base qw(Koha::Object); + +use Koha::ERM::YearlyUsage; +use Koha::ERM::YearlyUsages; +use Koha::ERM::MonthlyUsage; +use Koha::ERM::MonthlyUsages; + +=head1 NAME + +Koha::ERM::UsageDatabase - Koha ErmUsageDatabaseObject class + +=head1 API + +=head2 Class Methods +=head3 erm_usage_muses + +Method to embed erm_usage_muses to databases for report formatting + +=cut + +sub erm_usage_muses { + my ($self) = @_; + my $usage_mus_rs = $self->_result->erm_usage_muses; + return Koha::ERM::MonthlyUsages->_new_from_dbic($usage_mus_rs); +} + +=head3 erm_usage_yuses + +Method to embed erm_usage_yuses to databases for report formatting + +=cut + +sub erm_usage_yuses { + my ($self) = @_; + my $usage_yus_rs = $self->_result->erm_usage_yuses; + return Koha::ERM::YearlyUsages->_new_from_dbic($usage_yus_rs); +} + +=head3 yearly_usages + +Getter/setter for yearly_usages for this database +Skips adding yearly_usage if it already exists + +=cut + +sub yearly_usages { + my ( $self, $yearly_usages, $job_callbacks ) = @_; + + if ($yearly_usages) { + for my $yearly_usage (@$yearly_usages) { + if ( $self->yearly_usages()->search($yearly_usage)->last ) { + $job_callbacks->{report_info_callback}->('skipped_yus') + if $job_callbacks; + next; + } + $job_callbacks->{report_info_callback}->('added_yus') + if $job_callbacks; + Koha::ERM::YearlyUsage->new($yearly_usage)->store; + } + } + my $yearly_usages_rs = $self->_result->erm_usage_yuses; + return Koha::ERM::YearlyUsages->_new_from_dbic($yearly_usages_rs); +} + +=head3 monthly_usages + +Getter/setter for monthly_usages for this database +Skips adding monthly_usage if it already exists + +=cut + +sub monthly_usages { + my ( $self, $monthly_usages, $job_callbacks ) = @_; + + if ($monthly_usages) { + for my $monthly_usage (@$monthly_usages) { + if ( $self->monthly_usages()->search($monthly_usage)->last ) { + $job_callbacks->{report_info_callback}->('skipped_mus') + if $job_callbacks; + next; + } + $job_callbacks->{report_info_callback}->('added_mus') + if $job_callbacks; + Koha::ERM::MonthlyUsage->new($monthly_usage)->store; + } + } + my $monthly_usages_rs = $self->_result->erm_usage_muses; + return Koha::ERM::MonthlyUsages->_new_from_dbic($monthly_usages_rs); +} + +=head2 Internal methods + +=head3 _type + +=cut + +sub _type { + return 'ErmUsageDatabase'; +} + +1; diff --git a/Koha/ERM/UsageDatabases.pm b/Koha/ERM/UsageDatabases.pm new file mode 100644 index 0000000000..f9c0fe2fee --- /dev/null +++ b/Koha/ERM/UsageDatabases.pm @@ -0,0 +1,52 @@ +package Koha::ERM::UsageDatabases; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + +use Koha::Database; + +use Koha::ERM::UsageDatabase; + +use base qw(Koha::Objects); + +=head1 NAME + +Koha::ERM::UsageDatabases- Koha ErmUsageDatabase Object set class + +=head1 API + +=head2 Class Methods + +=cut + +=head3 type + +=cut + +sub _type { + return 'ErmUsageDatabase'; +} + +=head3 object_class + +=cut + +sub object_class { + return 'Koha::ERM::UsageDatabase'; +} + +1; diff --git a/Koha/ERM/UsageItem.pm b/Koha/ERM/UsageItem.pm new file mode 100644 index 0000000000..b187cb3db0 --- /dev/null +++ b/Koha/ERM/UsageItem.pm @@ -0,0 +1,120 @@ +package Koha::ERM::UsageItem; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + +use base qw(Koha::Object); + +use Koha::ERM::YearlyUsage; +use Koha::ERM::YearlyUsages; +use Koha::ERM::MonthlyUsage; +use Koha::ERM::MonthlyUsages; + +=head1 NAME + +Koha::ERM::UsageItem - Koha ErmUsageItemObject class + +=head1 API + +=head2 Class Methods +=head3 erm_usage_muses + +Method to embed erm_usage_muses to items for report formatting + +=cut + +sub erm_usage_muses { + my ($self) = @_; + my $usage_mus_rs = $self->_result->erm_usage_muses; + return Koha::ERM::MonthlyUsages->_new_from_dbic($usage_mus_rs); +} + +=head3 erm_usage_yuses + +Method to embed erm_usage_yuses to items for report formatting + +=cut + +sub erm_usage_yuses { + my ($self) = @_; + my $usage_yus_rs = $self->_result->erm_usage_yuses; + return Koha::ERM::YearlyUsages->_new_from_dbic($usage_yus_rs); +} + +=head3 yearly_usages + +Getter/setter for yearly_usages for this item +Skips adding yearly_usage if it already exists + +=cut + +sub yearly_usages { + my ( $self, $yearly_usages, $job_callbacks ) = @_; + + if ($yearly_usages) { + for my $yearly_usage (@$yearly_usages) { + if ( $self->yearly_usages()->search($yearly_usage)->last ) { + $job_callbacks->{report_info_callback}->('skipped_yus') + if $job_callbacks; + next; + } + $job_callbacks->{report_info_callback}->('added_yus') + if $job_callbacks; + Koha::ERM::YearlyUsage->new($yearly_usage)->store; + } + } + my $yearly_usages_rs = $self->_result->erm_usage_yuses; + return Koha::ERM::YearlyUsages->_new_from_dbic($yearly_usages_rs); +} + +=head3 monthly_usages + +Getter/setter for monthly_usages for this item +Skips adding monthly_usage if it already exists + +=cut + +sub monthly_usages { + my ( $self, $monthly_usages, $job_callbacks ) = @_; + + if ($monthly_usages) { + for my $monthly_usage (@$monthly_usages) { + if ( $self->monthly_usages()->search($monthly_usage)->last ) { + $job_callbacks->{report_info_callback}->('skipped_mus') + if $job_callbacks; + next; + } + $job_callbacks->{report_info_callback}->('added_mus') + if $job_callbacks; + Koha::ERM::MonthlyUsage->new($monthly_usage)->store; + } + } + my $monthly_usages_rs = $self->_result->erm_usage_muses; + return Koha::ERM::MonthlyUsages->_new_from_dbic($monthly_usages_rs); +} + +=head2 Internal methods + +=head3 _type + +=cut + +sub _type { + return 'ErmUsageItem'; +} + +1; diff --git a/Koha/ERM/UsageItems.pm b/Koha/ERM/UsageItems.pm new file mode 100644 index 0000000000..bae41a6f2a --- /dev/null +++ b/Koha/ERM/UsageItems.pm @@ -0,0 +1,52 @@ +package Koha::ERM::UsageItems; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + +use Koha::Database; + +use Koha::ERM::UsageItem; + +use base qw(Koha::Objects); + +=head1 NAME + +Koha::ERM::UsageItems- Koha ErmUsageItem Object set class + +=head1 API + +=head2 Class Methods + +=cut + +=head3 type + +=cut + +sub _type { + return 'ErmUsageItem'; +} + +=head3 object_class + +=cut + +sub object_class { + return 'Koha::ERM::UsageItem'; +} + +1; diff --git a/Koha/ERM/UsagePlatform.pm b/Koha/ERM/UsagePlatform.pm new file mode 100644 index 0000000000..947c34cf4b --- /dev/null +++ b/Koha/ERM/UsagePlatform.pm @@ -0,0 +1,120 @@ +package Koha::ERM::UsagePlatform; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + +use base qw(Koha::Object); + +use Koha::ERM::YearlyUsage; +use Koha::ERM::YearlyUsages; +use Koha::ERM::MonthlyUsage; +use Koha::ERM::MonthlyUsages; + +=head1 NAME + +Koha::ERM::UsagePlatform - Koha ErmUsagePlatform Object class + +=head1 API + +=head2 Class Methods +=head3 erm_usage_muses + +Method to embed erm_usage_muses to platforms for report formatting + +=cut + +sub erm_usage_muses { + my ($self) = @_; + my $usage_mus_rs = $self->_result->erm_usage_muses; + return Koha::ERM::MonthlyUsages->_new_from_dbic($usage_mus_rs); +} + +=head3 erm_usage_yuses + +Method to embed erm_usage_yuses to platforms for report formatting + +=cut + +sub erm_usage_yuses { + my ($self) = @_; + my $usage_yus_rs = $self->_result->erm_usage_yuses; + return Koha::ERM::YearlyUsages->_new_from_dbic($usage_yus_rs); +} + +=head3 yearly_usages + +Getter/setter for yearly_usages for this platform +Skips adding yearly_usage if it already exists + +=cut + +sub yearly_usages { + my ( $self, $yearly_usages, $job_callbacks ) = @_; + + if ($yearly_usages) { + for my $yearly_usage (@$yearly_usages) { + if ( $self->yearly_usages()->search($yearly_usage)->last ) { + $job_callbacks->{report_info_callback}->('skipped_yus') + if $job_callbacks; + next; + } + $job_callbacks->{report_info_callback}->('added_yus') + if $job_callbacks; + Koha::ERM::YearlyUsage->new($yearly_usage)->store; + } + } + my $yearly_usages_rs = $self->_result->erm_usage_yuses; + return Koha::ERM::YearlyUsages->_new_from_dbic($yearly_usages_rs); +} + +=head3 monthly_usages + +Getter/setter for monthly_usages for this platform +Skips adding monthly_usage if it already exists + +=cut + +sub monthly_usages { + my ( $self, $monthly_usages, $job_callbacks ) = @_; + + if ($monthly_usages) { + for my $monthly_usage (@$monthly_usages) { + if ( $self->monthly_usages()->search($monthly_usage)->last ) { + $job_callbacks->{report_info_callback}->('skipped_mus') + if $job_callbacks; + next; + } + $job_callbacks->{report_info_callback}->('added_mus') + if $job_callbacks; + Koha::ERM::MonthlyUsage->new($monthly_usage)->store; + } + } + my $monthly_usages_rs = $self->_result->erm_usage_muses; + return Koha::ERM::MonthlyUsages->_new_from_dbic($monthly_usages_rs); +} + +=head2 Internal methods + +=head3 _type + +=cut + +sub _type { + return 'ErmUsagePlatform'; +} + +1; diff --git a/Koha/ERM/UsagePlatforms.pm b/Koha/ERM/UsagePlatforms.pm new file mode 100644 index 0000000000..f26158d531 --- /dev/null +++ b/Koha/ERM/UsagePlatforms.pm @@ -0,0 +1,52 @@ +package Koha::ERM::UsagePlatforms; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + +use Koha::Database; + +use Koha::ERM::UsagePlatform; + +use base qw(Koha::Objects); + +=head1 NAME + +Koha::ERM::UsagePlatforms- Koha ErmUsagePlatform Object set class + +=head1 API + +=head2 Class Methods + +=cut + +=head3 type + +=cut + +sub _type { + return 'ErmUsagePlatform'; +} + +=head3 object_class + +=cut + +sub object_class { + return 'Koha::ERM::UsagePlatform'; +} + +1; diff --git a/Koha/ERM/UsageTitle.pm b/Koha/ERM/UsageTitle.pm index 78ef910fd9..e8d607c141 100644 --- a/Koha/ERM/UsageTitle.pm +++ b/Koha/ERM/UsageTitle.pm @@ -38,7 +38,7 @@ Method to embed erm_usage_muses to titles for report formatting =cut sub erm_usage_muses { - my ( $self ) = @_; + my ($self) = @_; my $usage_mus_rs = $self->_result->erm_usage_muses; return Koha::ERM::MonthlyUsages->_new_from_dbic($usage_mus_rs); } @@ -50,11 +50,63 @@ Method to embed erm_usage_yuses to titles for report formatting =cut sub erm_usage_yuses { - my ( $self ) = @_; + my ($self) = @_; my $usage_yus_rs = $self->_result->erm_usage_yuses; return Koha::ERM::YearlyUsages->_new_from_dbic($usage_yus_rs); } +=head3 yearly_usages + +Getter/setter for yearly_usages for this title +Skips adding yearly_usage if it already exists + +=cut + +sub yearly_usages { + my ( $self, $yearly_usages, $job_callbacks ) = @_; + + if ($yearly_usages) { + for my $yearly_usage (@$yearly_usages) { + if ( $self->yearly_usages()->search($yearly_usage)->last ) { + $job_callbacks->{report_info_callback}->('skipped_yus') + if $job_callbacks; + next; + } + $job_callbacks->{report_info_callback}->('added_yus') + if $job_callbacks; + Koha::ERM::YearlyUsage->new($yearly_usage)->store; + } + } + my $yearly_usages_rs = $self->_result->erm_usage_yuses; + return Koha::ERM::YearlyUsages->_new_from_dbic($yearly_usages_rs); +} + +=head3 monthly_usages + +Getter/setter for monthly_usages for this title +Skips adding monthly_usage if it already exists + +=cut + +sub monthly_usages { + my ( $self, $monthly_usages, $job_callbacks ) = @_; + + if ($monthly_usages) { + for my $monthly_usage (@$monthly_usages) { + if ( $self->monthly_usages()->search($monthly_usage)->last ) { + $job_callbacks->{report_info_callback}->('skipped_mus') + if $job_callbacks; + next; + } + $job_callbacks->{report_info_callback}->('added_mus') + if $job_callbacks; + Koha::ERM::MonthlyUsage->new($monthly_usage)->store; + } + } + my $monthly_usages_rs = $self->_result->erm_usage_muses; + return Koha::ERM::MonthlyUsages->_new_from_dbic($monthly_usages_rs); +} + =head2 Internal methods =head3 _type diff --git a/Koha/REST/V1/ERM/CounterFiles.pm b/Koha/REST/V1/ERM/CounterFiles.pm index d7e700dc95..b19afd497e 100644 --- a/Koha/REST/V1/ERM/CounterFiles.pm +++ b/Koha/REST/V1/ERM/CounterFiles.pm @@ -57,9 +57,11 @@ sub get { return try { my $counter_file_id = $c->validation->param('erm_counter_files_id'); - my $counter_file = $c->objects->find( Koha::ERM::CounterFiles->search, $counter_file_id ); - unless ($counter_file) { + # Do not use $c->objects->find here, we need the file_content + my $counter_file = Koha::ERM::CounterFiles->find($counter_file_id); + + if ( !$counter_file ) { return $c->render( status => 404, openapi => { error => "Counter file not found" } diff --git a/Koha/REST/V1/ERM/UsageDataProviders.pm b/Koha/REST/V1/ERM/UsageDataProviders.pm index ea969e7946..7d4293a28d 100644 --- a/Koha/REST/V1/ERM/UsageDataProviders.pm +++ b/Koha/REST/V1/ERM/UsageDataProviders.pm @@ -303,4 +303,32 @@ sub delete { }; } +=head3 run + +=cut + +sub run { + my $c = shift->openapi->valid_input or return; + + my $udprovider = Koha::ERM::UsageDataProviders->find( $c->validation->param('erm_usage_data_provider_id') ); + + unless ($udprovider) { + return $c->render( + status => 404, + openapi => { error => "Usage data provider not found" } + ); + } + + return try { + my $jobs = $udprovider->run; + + return $c->render( + status => 200, + openapi => { jobs => [ @{$jobs} ] } + ); + } + catch { + $c->unhandled_exception($_); + }; +} 1; diff --git a/Koha/Schema/Result/ErmUsageDataProvider.pm b/Koha/Schema/Result/ErmUsageDataProvider.pm index 259a685d3a..de4ec8877b 100644 --- a/Koha/Schema/Result/ErmUsageDataProvider.pm +++ b/Koha/Schema/Result/ErmUsageDataProvider.pm @@ -228,6 +228,40 @@ __PACKAGE__->has_many( { cascade_copy => 0, cascade_delete => 0 }, ); +=head2 erm_usage_databases + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "erm_usage_databases", + "Koha::Schema::Result::ErmUsageDatabase", + { + "foreign.usage_data_provider_id" => "self.erm_usage_data_provider_id", + }, + { cascade_copy => 0, cascade_delete => 0 }, +); + +=head2 erm_usage_items + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "erm_usage_items", + "Koha::Schema::Result::ErmUsageItem", + { + "foreign.usage_data_provider_id" => "self.erm_usage_data_provider_id", + }, + { cascade_copy => 0, cascade_delete => 0 }, +); + =head2 erm_usage_muses Type: has_many @@ -245,6 +279,23 @@ __PACKAGE__->has_many( { cascade_copy => 0, cascade_delete => 0 }, ); +=head2 erm_usage_platforms + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "erm_usage_platforms", + "Koha::Schema::Result::ErmUsagePlatform", + { + "foreign.usage_data_provider_id" => "self.erm_usage_data_provider_id", + }, + { cascade_copy => 0, cascade_delete => 0 }, +); + =head2 erm_usage_titles Type: has_many @@ -280,8 +331,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2023-03-16 17:38:56 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:XdI3SAA+QWQsWyQdoS0EEQ +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2023-07-26 14:35:34 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:AVt5SSRe/g6EmvBtASWHPA # __PACKAGE__->add_columns( # '+active' => { is_boolean => 1 } diff --git a/Koha/Schema/Result/ErmUsageDatabase.pm b/Koha/Schema/Result/ErmUsageDatabase.pm new file mode 100644 index 0000000000..73df6f1b75 --- /dev/null +++ b/Koha/Schema/Result/ErmUsageDatabase.pm @@ -0,0 +1,156 @@ +use utf8; +package Koha::Schema::Result::ErmUsageDatabase; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Koha::Schema::Result::ErmUsageDatabase + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("erm_usage_databases"); + +=head1 ACCESSORS + +=head2 database_id + + data_type: 'integer' + is_auto_increment: 1 + is_nullable: 0 + +primary key + +=head2 database + + data_type: 'varchar' + is_nullable: 1 + size: 255 + +item title + +=head2 platform + + data_type: 'varchar' + is_nullable: 1 + size: 24 + +database platform + +=head2 publisher + + data_type: 'varchar' + is_nullable: 1 + size: 24 + +Publisher for the database + +=head2 publisher_id + + data_type: 'varchar' + is_nullable: 1 + size: 24 + +Publisher ID for the database + +=head2 usage_data_provider_id + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 0 + +data provider the database is harvested by + +=cut + +__PACKAGE__->add_columns( + "database_id", + { data_type => "integer", is_auto_increment => 1, is_nullable => 0 }, + "database", + { data_type => "varchar", is_nullable => 1, size => 255 }, + "platform", + { data_type => "varchar", is_nullable => 1, size => 24 }, + "publisher", + { data_type => "varchar", is_nullable => 1, size => 24 }, + "publisher_id", + { data_type => "varchar", is_nullable => 1, size => 24 }, + "usage_data_provider_id", + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("database_id"); + +=head1 RELATIONS + +=head2 erm_usage_muses + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "erm_usage_muses", + "Koha::Schema::Result::ErmUsageMus", + { "foreign.database_id" => "self.database_id" }, + { cascade_copy => 0, cascade_delete => 0 }, +); + +=head2 erm_usage_yuses + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "erm_usage_yuses", + "Koha::Schema::Result::ErmUsageYus", + { "foreign.database_id" => "self.database_id" }, + { cascade_copy => 0, cascade_delete => 0 }, +); + +=head2 usage_data_provider + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "usage_data_provider", + "Koha::Schema::Result::ErmUsageDataProvider", + { erm_usage_data_provider_id => "usage_data_provider_id" }, + { is_deferrable => 1, on_delete => "CASCADE", on_update => "CASCADE" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2023-07-26 11:45:56 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:N/qxjz0Jz0NOI6lFVkUXhg + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration +1; diff --git a/Koha/Schema/Result/ErmUsageItem.pm b/Koha/Schema/Result/ErmUsageItem.pm new file mode 100644 index 0000000000..90f9345d52 --- /dev/null +++ b/Koha/Schema/Result/ErmUsageItem.pm @@ -0,0 +1,146 @@ +use utf8; +package Koha::Schema::Result::ErmUsageItem; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Koha::Schema::Result::ErmUsageItem + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("erm_usage_items"); + +=head1 ACCESSORS + +=head2 item_id + + data_type: 'integer' + is_auto_increment: 1 + is_nullable: 0 + +primary key + +=head2 item + + data_type: 'varchar' + is_nullable: 1 + size: 255 + +item title + +=head2 platform + + data_type: 'varchar' + is_nullable: 1 + size: 80 + +item platform + +=head2 publisher + + data_type: 'varchar' + is_nullable: 1 + size: 80 + +Publisher for the item + +=head2 usage_data_provider_id + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 0 + +data provider the database is harvested by + +=cut + +__PACKAGE__->add_columns( + "item_id", + { data_type => "integer", is_auto_increment => 1, is_nullable => 0 }, + "item", + { data_type => "varchar", is_nullable => 1, size => 255 }, + "platform", + { data_type => "varchar", is_nullable => 1, size => 80 }, + "publisher", + { data_type => "varchar", is_nullable => 1, size => 80 }, + "usage_data_provider_id", + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("item_id"); + +=head1 RELATIONS + +=head2 erm_usage_muses + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "erm_usage_muses", + "Koha::Schema::Result::ErmUsageMus", + { "foreign.item_id" => "self.item_id" }, + { cascade_copy => 0, cascade_delete => 0 }, +); + +=head2 erm_usage_yuses + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "erm_usage_yuses", + "Koha::Schema::Result::ErmUsageYus", + { "foreign.item_id" => "self.item_id" }, + { cascade_copy => 0, cascade_delete => 0 }, +); + +=head2 usage_data_provider + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "usage_data_provider", + "Koha::Schema::Result::ErmUsageDataProvider", + { erm_usage_data_provider_id => "usage_data_provider_id" }, + { is_deferrable => 1, on_delete => "CASCADE", on_update => "CASCADE" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2023-08-02 12:15:06 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ovoxM438smZou+pFm85igw + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration +1; diff --git a/Koha/Schema/Result/ErmUsageMus.pm b/Koha/Schema/Result/ErmUsageMus.pm index 3563bc02d8..bb34b77101 100644 --- a/Koha/Schema/Result/ErmUsageMus.pm +++ b/Koha/Schema/Result/ErmUsageMus.pm @@ -39,6 +39,30 @@ primary key item title id number +=head2 platform_id + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 1 + +platform id number + +=head2 database_id + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 1 + +database id number + +=head2 item_id + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 1 + +item id number + =head2 usage_data_provider_id data_type: 'integer' @@ -76,6 +100,14 @@ usage count for the title metric type for the usage statistic +=head2 access_type + + data_type: 'varchar' + is_nullable: 1 + size: 50 + +access type for the usage statistic + =head2 report_type data_type: 'varchar' @@ -91,6 +123,12 @@ __PACKAGE__->add_columns( { data_type => "integer", is_auto_increment => 1, is_nullable => 0 }, "title_id", { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, + "platform_id", + { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, + "database_id", + { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, + "item_id", + { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, "usage_data_provider_id", { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, "year", @@ -101,6 +139,8 @@ __PACKAGE__->add_columns( { data_type => "integer", is_nullable => 1 }, "metric_type", { data_type => "varchar", is_nullable => 1, size => 50 }, + "access_type", + { data_type => "varchar", is_nullable => 1, size => 50 }, "report_type", { data_type => "varchar", is_nullable => 1, size => 50 }, ); @@ -119,6 +159,66 @@ __PACKAGE__->set_primary_key("monthly_usage_summary_id"); =head1 RELATIONS +=head2 database + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "database", + "Koha::Schema::Result::ErmUsageDatabase", + { database_id => "database_id" }, + { + is_deferrable => 1, + join_type => "LEFT", + on_delete => "CASCADE", + on_update => "CASCADE", + }, +); + +=head2 item + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "item", + "Koha::Schema::Result::ErmUsageItem", + { item_id => "item_id" }, + { + is_deferrable => 1, + join_type => "LEFT", + on_delete => "CASCADE", + on_update => "CASCADE", + }, +); + +=head2 platform + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "platform", + "Koha::Schema::Result::ErmUsagePlatform", + { platform_id => "platform_id" }, + { + is_deferrable => 1, + join_type => "LEFT", + on_delete => "CASCADE", + on_update => "CASCADE", + }, +); + =head2 title Type: belongs_to @@ -160,8 +260,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2023-04-25 13:49:06 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:QarVvHaFDYvZtUt2brKM2w +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2023-08-02 15:57:07 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:MY7HLGZQmotzgmOkz2OWWg sub koha_object_class { diff --git a/Koha/Schema/Result/ErmUsagePlatform.pm b/Koha/Schema/Result/ErmUsagePlatform.pm new file mode 100644 index 0000000000..610b83a247 --- /dev/null +++ b/Koha/Schema/Result/ErmUsagePlatform.pm @@ -0,0 +1,133 @@ +use utf8; +package Koha::Schema::Result::ErmUsagePlatform; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Koha::Schema::Result::ErmUsagePlatform + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("erm_usage_platforms"); + +=head1 ACCESSORS + +=head2 platform_id + + data_type: 'integer' + is_auto_increment: 1 + is_nullable: 0 + +primary key + +=head2 platform + + data_type: 'varchar' + is_nullable: 1 + size: 255 + +item title + +=head2 usage_data_provider_id + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 0 + +data provider the platform is harvested by + +=cut + +__PACKAGE__->add_columns( + "platform_id", + { data_type => "integer", is_auto_increment => 1, is_nullable => 0 }, + "platform", + { data_type => "varchar", is_nullable => 1, size => 255 }, + "usage_data_provider_id", + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("platform_id"); + +=head1 RELATIONS + +=head2 erm_usage_muses + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "erm_usage_muses", + "Koha::Schema::Result::ErmUsageMus", + { "foreign.platform_id" => "self.platform_id" }, + { cascade_copy => 0, cascade_delete => 0 }, +); + +=head2 erm_usage_yuses + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "erm_usage_yuses", + "Koha::Schema::Result::ErmUsageYus", + { "foreign.platform_id" => "self.platform_id" }, + { cascade_copy => 0, cascade_delete => 0 }, +); + +=head2 usage_data_provider + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "usage_data_provider", + "Koha::Schema::Result::ErmUsageDataProvider", + { erm_usage_data_provider_id => "usage_data_provider_id" }, + { is_deferrable => 1, on_delete => "CASCADE", on_update => "CASCADE" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2023-07-24 16:30:43 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:7Qu5XYiD2K+Pn590ZoTs7Q + + +sub koha_object_class { + 'Koha::ERM::UsagePlatform'; +} + +sub koha_objects_class { + 'Koha::ERM::UsagePlatforms'; +} + +1; diff --git a/Koha/Schema/Result/ErmUsageTitle.pm b/Koha/Schema/Result/ErmUsageTitle.pm index 0bbee58cab..f86750ba48 100644 --- a/Koha/Schema/Result/ErmUsageTitle.pm +++ b/Koha/Schema/Result/ErmUsageTitle.pm @@ -79,6 +79,38 @@ Online ISSN number for the title URI number for the title +=head2 publisher + + data_type: 'varchar' + is_nullable: 1 + size: 24 + +Publisher for the title + +=head2 publisher_id + + data_type: 'varchar' + is_nullable: 1 + size: 24 + +Publisher ID for the title + +=head2 yop + + data_type: 'varchar' + is_nullable: 1 + size: 24 + +year of publication of the title + +=head2 isbn + + data_type: 'varchar' + is_nullable: 1 + size: 24 + +ISBN of the title + =cut __PACKAGE__->add_columns( @@ -96,6 +128,14 @@ __PACKAGE__->add_columns( { data_type => "varchar", is_nullable => 1, size => 24 }, "title_uri", { data_type => "varchar", is_nullable => 1, size => 24 }, + "publisher", + { data_type => "varchar", is_nullable => 1, size => 24 }, + "publisher_id", + { data_type => "varchar", is_nullable => 1, size => 24 }, + "yop", + { data_type => "varchar", is_nullable => 1, size => 24 }, + "isbn", + { data_type => "varchar", is_nullable => 1, size => 24 }, ); =head1 PRIMARY KEY @@ -158,8 +198,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2023-03-15 09:43:08 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:IfPJJVTctPmRyVYt0DUbYA +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2023-08-02 16:27:18 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:2ChttNyhXjVRUWg0nCfg8A sub koha_object_class { diff --git a/Koha/Schema/Result/ErmUsageYus.pm b/Koha/Schema/Result/ErmUsageYus.pm index 07f3d09cd5..c48811c648 100644 --- a/Koha/Schema/Result/ErmUsageYus.pm +++ b/Koha/Schema/Result/ErmUsageYus.pm @@ -39,6 +39,30 @@ primary key item title id number +=head2 platform_id + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 1 + +platform id number + +=head2 database_id + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 1 + +database id number + +=head2 item_id + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 1 + +item id number + =head2 usage_data_provider_id data_type: 'integer' @@ -69,6 +93,14 @@ usage count for the title metric type for the usage statistic +=head2 access_type + + data_type: 'varchar' + is_nullable: 1 + size: 50 + +access type for the usage statistic + =head2 report_type data_type: 'varchar' @@ -84,6 +116,12 @@ __PACKAGE__->add_columns( { data_type => "integer", is_auto_increment => 1, is_nullable => 0 }, "title_id", { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, + "platform_id", + { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, + "database_id", + { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, + "item_id", + { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, "usage_data_provider_id", { data_type => "integer", is_foreign_key => 1, is_nullable => 1 }, "year", @@ -92,6 +130,8 @@ __PACKAGE__->add_columns( { data_type => "integer", is_nullable => 1 }, "metric_type", { data_type => "varchar", is_nullable => 1, size => 50 }, + "access_type", + { data_type => "varchar", is_nullable => 1, size => 50 }, "report_type", { data_type => "varchar", is_nullable => 1, size => 50 }, ); @@ -110,6 +150,66 @@ __PACKAGE__->set_primary_key("yearly_usage_summary_id"); =head1 RELATIONS +=head2 database + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "database", + "Koha::Schema::Result::ErmUsageDatabase", + { database_id => "database_id" }, + { + is_deferrable => 1, + join_type => "LEFT", + on_delete => "CASCADE", + on_update => "CASCADE", + }, +); + +=head2 item + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "item", + "Koha::Schema::Result::ErmUsageItem", + { item_id => "item_id" }, + { + is_deferrable => 1, + join_type => "LEFT", + on_delete => "CASCADE", + on_update => "CASCADE", + }, +); + +=head2 platform + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "platform", + "Koha::Schema::Result::ErmUsagePlatform", + { platform_id => "platform_id" }, + { + is_deferrable => 1, + join_type => "LEFT", + on_delete => "CASCADE", + on_update => "CASCADE", + }, +); + =head2 title Type: belongs_to @@ -151,8 +251,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07049 @ 2023-04-25 13:49:06 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zXTpOQQ2kTmGzzOSohg0YA +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2023-08-02 15:57:07 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:B439wdXto/YBcahGqalurg sub koha_object_class { diff --git a/api/v1/swagger/definitions/erm_usage_mus.yaml b/api/v1/swagger/definitions/erm_usage_mus.yaml index 585fb29570..dad31e1c00 100644 --- a/api/v1/swagger/definitions/erm_usage_mus.yaml +++ b/api/v1/swagger/definitions/erm_usage_mus.yaml @@ -10,6 +10,21 @@ properties: type: - integer - "null" + platform_id: + description: platform_id of the monthly usage summary + type: + - integer + - "null" + database_id: + description: database_id of the monthly usage summary + type: + - integer + - "null" + item_id: + description: item_id of the monthly usage summary + type: + - integer + - "null" usage_data_provider_id: description: usage_data_provider_id of the monthly usage summary type: @@ -35,6 +50,11 @@ properties: type: - string - "null" + access_type: + description: access type of the monthly usage summary + type: + - string + - "null" report_type: description: report type of the monthly usage summary type: diff --git a/api/v1/swagger/definitions/erm_usage_title.yaml b/api/v1/swagger/definitions/erm_usage_title.yaml index fcc2e01b67..0cea227ab5 100644 --- a/api/v1/swagger/definitions/erm_usage_title.yaml +++ b/api/v1/swagger/definitions/erm_usage_title.yaml @@ -43,6 +43,26 @@ properties: type: - string - "null" + publisher: + description: publisher of the title data + type: + - string + - "null" + publisher_id: + description: publisher_id of the title data + type: + - string + - "null" + yop: + description: year of publication of the title + type: + - string + - "null" + isbn: + description: isbn of the title + type: + - string + - "null" erm_usage_muses: type: array description: usage mus diff --git a/api/v1/swagger/definitions/erm_usage_yus.yaml b/api/v1/swagger/definitions/erm_usage_yus.yaml index 4504aa21c7..0eb3562798 100644 --- a/api/v1/swagger/definitions/erm_usage_yus.yaml +++ b/api/v1/swagger/definitions/erm_usage_yus.yaml @@ -10,6 +10,21 @@ properties: type: - integer - "null" + platform_id: + description: platform_id of the yearly usage summary + type: + - integer + - "null" + database_id: + description: database_id of the yearly usage summary + type: + - integer + - "null" + item_id: + description: item_id of the yearly usage summary + type: + - integer + - "null" usage_data_provider_id: description: usage_data_provider_id of the yearly usage summary type: @@ -30,6 +45,11 @@ properties: type: - string - "null" + access_type: + description: access type of the yearly usage summary + type: + - string + - "null" report_type: description: report type of the yearly usage summary type: diff --git a/api/v1/swagger/paths/erm_counter_files.yaml b/api/v1/swagger/paths/erm_counter_files.yaml index fde2a8b071..ec7ec071b1 100644 --- a/api/v1/swagger/paths/erm_counter_files.yaml +++ b/api/v1/swagger/paths/erm_counter_files.yaml @@ -1,4 +1,49 @@ --- +"/erm/counter_files/{erm_counter_files_id}/file/content": + get: + x-mojo-to: ERM::CounterFiles#get + operationId: downloadCounterFile + tags: + - counter_file + summary: Download Counter file + produces: + - application/octet-stream + parameters: + - description: Case insensitive search on erm_counter_files_id + in: path + name: erm_counter_files_id + required: true + type: integer + responses: + 200: + description: A counter file + schema: + type: file + 401: + description: Authentication required + schema: + $ref: "../swagger.yaml#/definitions/error" + 403: + description: Access forbidden + schema: + $ref: "../swagger.yaml#/definitions/error" + 404: + description: Ressource not found + schema: + $ref: "../swagger.yaml#/definitions/error" + 500: + description: |- + Internal server error. Possible `error_code` attribute values: + * `internal_server_error` + schema: + $ref: "../swagger.yaml#/definitions/error" + 503: + description: Under maintenance + schema: + $ref: "../swagger.yaml#/definitions/error" + x-koha-authorization: + permissions: + erm: 1 /erm/counter_files: get: x-mojo-to: ERM::CounterFiles#list diff --git a/api/v1/swagger/paths/erm_usage_data_providers.yaml b/api/v1/swagger/paths/erm_usage_data_providers.yaml index 18c79ae105..2ad45858ab 100644 --- a/api/v1/swagger/paths/erm_usage_data_providers.yaml +++ b/api/v1/swagger/paths/erm_usage_data_providers.yaml @@ -357,6 +357,56 @@ description: under maintenance schema: $ref: "../swagger.yaml#/definitions/error" + x-koha-authorization: + permissions: + erm: 1 +"/erm/usage_data_providers/{erm_usage_data_provider_id}/run": + get: + x-mojo-to: ERM::UsageDataProviders#run + operationId: runUsageDataProviderHarvester + tags: + - usage data provider harvester + summary: Run this data provider's harvester + produces: + - application/json + parameters: + - $ref: "../swagger.yaml#/parameters/erm_usage_data_provider_id_pp" + - description: begin_date to run harvest + in: query + name: begin_date + required: true + type: string + format: date + - description: end_date to run harvest + in: query + name: end_date + required: true + type: string + format: date + responses: + 200: + description: Successful harvest run + schema: + items: + $ref: "../swagger.yaml#/definitions/erm_usage_data_provider" + 400: + description: Bad request + schema: + $ref: "../swagger.yaml#/definitions/error" + 403: + description: Access forbidden + schema: + $ref: "../swagger.yaml#/definitions/error" + 500: + description: |- + Internal server error. Possible `error_code` attribute values: + * `internal_server_error` + schema: + $ref: "../swagger.yaml#/definitions/error" + 503: + description: Under maintenance + schema: + $ref: "../swagger.yaml#/definitions/error" x-koha-authorization: permissions: erm: 1 \ No newline at end of file diff --git a/api/v1/swagger/paths/erm_usage_titles.yaml b/api/v1/swagger/paths/erm_usage_titles.yaml index dc59cd2c59..9e553de098 100644 --- a/api/v1/swagger/paths/erm_usage_titles.yaml +++ b/api/v1/swagger/paths/erm_usage_titles.yaml @@ -44,6 +44,26 @@ name: title_uri required: false type: string + - description: Case insensitive search on usage_title publisher + in: query + name: publisher + required: false + type: string + - description: Case insensitive search on usage_title publisher_id + in: query + name: publisher_id + required: false + type: string + - description: Case insensitive search on usage_title yop + in: query + name: yop + required: false + type: string + - description: Case insensitive search on usage_title isbn + in: query + name: isbn + required: false + type: string - name: x-koha-embed in: header required: false diff --git a/api/v1/swagger/swagger.yaml b/api/v1/swagger/swagger.yaml index 162e6b5d66..387f795e3a 100644 --- a/api/v1/swagger/swagger.yaml +++ b/api/v1/swagger/swagger.yaml @@ -291,6 +291,8 @@ paths: $ref: ./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers "/erm/usage_data_providers/{erm_usage_data_provider_id}": $ref: "./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers~1{erm_usage_data_provider_id}" + "/erm/usage_data_providers/{erm_usage_data_provider_id}/run": + $ref: "./paths/erm_usage_data_providers.yaml#/~1erm~1usage_data_providers~1{erm_usage_data_provider_id}~1run" /erm/usage_titles: $ref: ./paths/erm_usage_titles.yaml#/~1erm~1usage_titles /erm/users: diff --git a/installer/data/mysql/atomicupdate/erm_usage_statistics_tables.pl b/installer/data/mysql/atomicupdate/erm_usage_statistics_tables.pl index c317fe781c..255294249d 100644 --- a/installer/data/mysql/atomicupdate/erm_usage_statistics_tables.pl +++ b/installer/data/mysql/atomicupdate/erm_usage_statistics_tables.pl @@ -92,6 +92,10 @@ return { `print_issn` varchar(24) DEFAULT NULL COMMENT 'Print ISSN number for the title', `online_issn` varchar(24) DEFAULT NULL COMMENT 'Online ISSN number for the title', `title_uri` varchar(24) DEFAULT NULL COMMENT 'URI number for the title', + `publisher` varchar(24) DEFAULT NULL COMMENT 'Publisher for the title', + `publisher_id` varchar(24) DEFAULT NULL COMMENT 'Publisher ID for the title', + `yop` varchar(24) DEFAULT NULL COMMENT 'year of publication of the title', + `isbn` varchar(24) DEFAULT NULL COMMENT 'ISBN of the title', PRIMARY KEY (`title_id`), CONSTRAINT `erm_usage_titles_ibfk_1` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; @@ -103,21 +107,87 @@ return { say $out "erm_usage_titles table already exists - skipping to next table"; } + unless ( TableExists('erm_usage_platforms') ) { + $dbh->do( + q{ + CREATE TABLE `erm_usage_platforms` ( + `platform_id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key', + `platform` varchar(255) DEFAULT NULL COMMENT 'item title', + `usage_data_provider_id` int(11) NOT NULL COMMENT 'data provider the platform is harvested by', + PRIMARY KEY (`platform_id`), + CONSTRAINT `erm_usage_platforms_ibfk_1` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + } + ); + + say $out "Added new table erm_usage_platforms"; + } else { + say $out "erm_usage_platforms table already exists - skipping to next table"; + } + + unless ( TableExists('erm_usage_databases') ) { + $dbh->do( + q{ + CREATE TABLE `erm_usage_databases` ( + `database_id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key', + `database` varchar(255) DEFAULT NULL COMMENT 'item title', + `platform` varchar(24) DEFAULT NULL COMMENT 'database platform', + `publisher` varchar(24) DEFAULT NULL COMMENT 'Publisher for the database', + `publisher_id` varchar(24) DEFAULT NULL COMMENT 'Publisher ID for the database', + `usage_data_provider_id` int(11) NOT NULL COMMENT 'data provider the database is harvested by', + PRIMARY KEY (`database_id`), + CONSTRAINT `erm_usage_databases_ibfk_1` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + } + ); + + say $out "Added new table erm_usage_databases"; + } else { + say $out "erm_usage_databases table already exists - skipping to next table"; + } + + unless ( TableExists('erm_usage_items') ) { + $dbh->do( + q{ + CREATE TABLE `erm_usage_items` ( + `item_id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key', + `item` varchar(255) DEFAULT NULL COMMENT 'item title', + `platform` varchar(80) DEFAULT NULL COMMENT 'item platform', + `publisher` varchar(80) DEFAULT NULL COMMENT 'Publisher for the item', + `usage_data_provider_id` int(11) NOT NULL COMMENT 'data provider the database is harvested by', + PRIMARY KEY (`item_id`), + CONSTRAINT `erm_usage_items_ibfk_1` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + } + ); + + say $out "Added new table erm_usage_items"; + } else { + say $out "erm_usage_items table already exists - skipping to next table"; + } + unless( TableExists( 'erm_usage_mus')) { $dbh->do( q{ CREATE TABLE `erm_usage_mus` ( `monthly_usage_summary_id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key', `title_id` int(11) DEFAULT NULL COMMENT 'item title id number', + `platform_id` int(11) DEFAULT NULL COMMENT 'platform id number', + `database_id` int(11) DEFAULT NULL COMMENT 'database id number', + `item_id` int(11) DEFAULT NULL COMMENT 'item id number', `usage_data_provider_id` int(11) DEFAULT NULL COMMENT 'item title id number', `year` int(4) DEFAULT NULL COMMENT 'year of usage statistics', `month` int(2) DEFAULT NULL COMMENT 'month of usage statistics', `usage_count` int(11) DEFAULT NULL COMMENT 'usage count for the title', `metric_type` varchar(50) DEFAULT NULL COMMENT 'metric type for the usage statistic', + `access_type` varchar(50) DEFAULT NULL COMMENT 'access type for the usage statistic', `report_type` varchar(50) DEFAULT NULL COMMENT 'report type for the usage statistic', PRIMARY KEY (`monthly_usage_summary_id`), CONSTRAINT `erm_usage_mus_ibfk_1` FOREIGN KEY (`title_id`) REFERENCES `erm_usage_titles` (`title_id`) ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT `erm_usage_mus_ibfk_2` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE + CONSTRAINT `erm_usage_mus_ibfk_2` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `erm_usage_mus_ibfk_3` FOREIGN KEY (`platform_id`) REFERENCES `erm_usage_platforms` (`platform_id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `erm_usage_mus_ibfk_4` FOREIGN KEY (`database_id`) REFERENCES `erm_usage_databases` (`database_id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `erm_usage_mus_ibfk_5` FOREIGN KEY (`item_id`) REFERENCES `erm_usage_items` (`item_id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; } ); @@ -133,14 +203,21 @@ return { CREATE TABLE `erm_usage_yus` ( `yearly_usage_summary_id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key', `title_id` int(11) DEFAULT NULL COMMENT 'item title id number', + `platform_id` int(11) DEFAULT NULL COMMENT 'platform id number', + `database_id` int(11) DEFAULT NULL COMMENT 'database id number', + `item_id` int(11) DEFAULT NULL COMMENT 'item id number', `usage_data_provider_id` int(11) DEFAULT NULL COMMENT 'item title id number', `year` int(4) DEFAULT NULL COMMENT 'year of usage statistics', `totalcount` int(11) DEFAULT NULL COMMENT 'usage count for the title', `metric_type` varchar(50) DEFAULT NULL COMMENT 'metric type for the usage statistic', + `access_type` varchar(50) DEFAULT NULL COMMENT 'access type for the usage statistic', `report_type` varchar(50) DEFAULT NULL COMMENT 'report type for the usage statistic', PRIMARY KEY (`yearly_usage_summary_id`), CONSTRAINT `erm_usage_yus_ibfk_1` FOREIGN KEY (`title_id`) REFERENCES `erm_usage_titles` (`title_id`) ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT `erm_usage_yus_ibfk_2` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE + CONSTRAINT `erm_usage_yus_ibfk_2` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `erm_usage_yus_ibfk_3` FOREIGN KEY (`platform_id`) REFERENCES `erm_usage_platforms` (`platform_id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `erm_usage_yus_ibfk_4` FOREIGN KEY (`database_id`) REFERENCES `erm_usage_databases` (`database_id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `erm_usage_yus_ibfk_5` FOREIGN KEY (`item_id`) REFERENCES `erm_usage_items` (`item_id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; } ); diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index c74fe9f20b..83cb89387a 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -3162,15 +3162,22 @@ DROP TABLE IF EXISTS `erm_usage_mus`; CREATE TABLE `erm_usage_mus` ( `monthly_usage_summary_id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key', `title_id` int(11) DEFAULT NULL COMMENT 'item title id number', + `platform_id` int(11) DEFAULT NULL COMMENT 'platform id number', + `database_id` int(11) DEFAULT NULL COMMENT 'database id number', + `item_id` int(11) DEFAULT NULL COMMENT 'item id number', `usage_data_provider_id` int(11) DEFAULT NULL COMMENT 'item title id number', `year` int(4) DEFAULT NULL COMMENT 'year of usage statistics', `month` int(2) DEFAULT NULL COMMENT 'month of usage statistics', `usage_count` int(11) DEFAULT NULL COMMENT 'usage count for the title', `metric_type` varchar(50) DEFAULT NULL COMMENT 'metric type for the usage statistic', + `access_type` varchar(50) DEFAULT NULL COMMENT 'access type for the usage statistic', `report_type` varchar(50) DEFAULT NULL COMMENT 'report type for the usage statistic', PRIMARY KEY (`monthly_usage_summary_id`), CONSTRAINT `erm_usage_mus_ibfk_1` FOREIGN KEY (`title_id`) REFERENCES `erm_usage_titles` (`title_id`) ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT `erm_usage_mus_ibfk_2` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE + CONSTRAINT `erm_usage_mus_ibfk_2` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `erm_usage_mus_ibfk_3` FOREIGN KEY (`platform_id`) REFERENCES `erm_usage_platforms` (`platform_id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `erm_usage_mus_ibfk_4` FOREIGN KEY (`database_id`) REFERENCES `erm_usage_databases` (`database_id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `erm_usage_mus_ibfk_5` FOREIGN KEY (`item_id`) REFERENCES `erm_usage_items` (`item_id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; @@ -3189,11 +3196,68 @@ CREATE TABLE `erm_usage_titles` ( `print_issn` varchar(24) DEFAULT NULL COMMENT 'Print ISSN number for the title', `online_issn` varchar(24) DEFAULT NULL COMMENT 'Online ISSN number for the title', `title_uri` varchar(24) DEFAULT NULL COMMENT 'URI number for the title', + `publisher` varchar(24) DEFAULT NULL COMMENT 'Publisher for the title', + `publisher_id` varchar(24) DEFAULT NULL COMMENT 'Publisher ID for the title', + `yop` varchar(24) DEFAULT NULL COMMENT 'year of publication of the title', + `isbn` varchar(24) DEFAULT NULL COMMENT 'ISBN of the title', PRIMARY KEY (`title_id`), CONSTRAINT `erm_usage_titles_ibfk_1` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; +-- +-- Table structure for table `erm_usage_platforms` +-- + +DROP TABLE IF EXISTS `erm_usage_platforms`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `erm_usage_platforms` ( + `platform_id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key', + `platform` varchar(255) DEFAULT NULL COMMENT 'item title', + `usage_data_provider_id` int(11) NOT NULL COMMENT 'data provider the platform is harvested by', + PRIMARY KEY (`platform_id`), + CONSTRAINT `erm_usage_platforms_ibfk_1` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `erm_usage_databases` +-- + +DROP TABLE IF EXISTS `erm_usage_databases`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `erm_usage_databases` ( + `database_id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key', + `database` varchar(255) DEFAULT NULL COMMENT 'item title', + `platform` varchar(24) DEFAULT NULL COMMENT 'database platform', + `publisher` varchar(24) DEFAULT NULL COMMENT 'Publisher for the database', + `publisher_id` varchar(24) DEFAULT NULL COMMENT 'Publisher ID for the database', + `usage_data_provider_id` int(11) NOT NULL COMMENT 'data provider the database is harvested by', + PRIMARY KEY (`database_id`), + CONSTRAINT `erm_usage_databases_ibfk_1` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Table structure for table `erm_usage_items` +-- + +DROP TABLE IF EXISTS `erm_usage_items`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `erm_usage_items` ( + `item_id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key', + `item` varchar(255) DEFAULT NULL COMMENT 'item title', + `platform` varchar(80) DEFAULT NULL COMMENT 'item platform', + `publisher` varchar(80) DEFAULT NULL COMMENT 'Publisher for the item', + `usage_data_provider_id` int(11) NOT NULL COMMENT 'data provider the database is harvested by', + PRIMARY KEY (`item_id`), + CONSTRAINT `erm_usage_items_ibfk_1` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +/*!40101 SET character_set_client = @saved_cs_client */; + -- -- Table structure for table `erm_usage_yus` -- @@ -3204,14 +3268,21 @@ DROP TABLE IF EXISTS `erm_usage_yus`; CREATE TABLE `erm_usage_yus` ( `yearly_usage_summary_id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'primary key', `title_id` int(11) DEFAULT NULL COMMENT 'item title id number', + `platform_id` int(11) DEFAULT NULL COMMENT 'platform id number', + `database_id` int(11) DEFAULT NULL COMMENT 'database id number', + `item_id` int(11) DEFAULT NULL COMMENT 'item id number', `usage_data_provider_id` int(11) DEFAULT NULL COMMENT 'item title id number', `year` int(4) DEFAULT NULL COMMENT 'year of usage statistics', `totalcount` int(11) DEFAULT NULL COMMENT 'usage count for the title', `metric_type` varchar(50) DEFAULT NULL COMMENT 'metric type for the usage statistic', + `access_type` varchar(50) DEFAULT NULL COMMENT 'access type for the usage statistic', `report_type` varchar(50) DEFAULT NULL COMMENT 'report type for the usage statistic', PRIMARY KEY (`yearly_usage_summary_id`), CONSTRAINT `erm_usage_yus_ibfk_1` FOREIGN KEY (`title_id`) REFERENCES `erm_usage_titles` (`title_id`) ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT `erm_usage_yus_ibfk_2` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE + CONSTRAINT `erm_usage_yus_ibfk_2` FOREIGN KEY (`usage_data_provider_id`) REFERENCES `erm_usage_data_providers` (`erm_usage_data_provider_id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `erm_usage_yus_ibfk_3` FOREIGN KEY (`platform_id`) REFERENCES `erm_usage_platforms` (`platform_id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `erm_usage_yus_ibfk_4` FOREIGN KEY (`database_id`) REFERENCES `erm_usage_databases` (`database_id`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `erm_usage_yus_ibfk_5` FOREIGN KEY (`item_id`) REFERENCES `erm_usage_items` (`item_id`) ON DELETE CASCADE ON UPDATE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc new file mode 100644 index 0000000000..12a84ed00a --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/background_jobs/erm_sushi_harvester.inc @@ -0,0 +1,113 @@ +[% USE Koha %] + +[% BLOCK report %] + [% SET report = job.report %] + [% IF report %] +
+ [% IF job.status == 'finished' %] + + + + + + + + + + + + + + + [% IF report.report_type.match( '^PR' ) %] + + [% ELSIF report.report_type.match( '^DR' ) %] + + [% ELSIF report.report_type.match( '^IR' ) %] + + [% ELSIF report.report_type.match( '^TR' ) %] + + [% END %] + + + + + + + + + + + + + + + + + + +
Usage data provider[% report.ud_provider_name | html %]
Report type[% report.report_type | html %]
Report rows processed[% job.progress %]
Platforms processedDatabases processedItems processedTitles processed[% report.total_records | html %]
Monthly usages skipped[% report.us_report_info.skipped_mus | html %]
Monthly usages added[% report.us_report_info.added_mus | html %]
Yearly usages skipped[% report.us_report_info.skipped_yus | html %]
Yearly usages added[% report.us_report_info.added_yus | html %]
+ [% ELSIF job.status == 'started' %] +

+ [% END %] + [% IF job.status != 'finished' %] + [% INCLUDE "job_progress.inc" job_id=job.id %] + [% END %] + [% IF job.status == 'cancelled' %] +

The job has been cancelled before it finished.

+ [% END %] +
+ [% END %] +[% END %] + +[% Asset.js("js/job_progress.js") | $raw %] + +[% BLOCK detail %] + [% FOR m IN job.messages %] +
+ [% IF m.type == 'success' %] + + [% ELSIF m.type == 'warning' %] + + [% ELSIF m.type == 'error' %] + + [% END %] + [% SWITCH m.code %] + [% CASE 'object_already_exists' %] + [% m.title %] already exists in this data provider and was not created. New usage statistics were verified anyway + [% CASE 'object_added' %] + [% m.title %] successfully added + [% END %] +
+ [% END %] +[% END %] + [% Asset.js("js/job_progress.js") | $raw %] + +[% BLOCK js %] + [% INCLUDE 'str/job_progress.inc' job_id=job.id %] + [% INCLUDE 'str/erm_sushi_harvester.inc' %] + +[% END %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/str/erm_sushi_harvester.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/str/erm_sushi_harvester.inc new file mode 100644 index 0000000000..8d6db83524 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/str/erm_sushi_harvester.inc @@ -0,0 +1,5 @@ + + diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt index 49708ae610..5ed845b13e 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/background_jobs.tt @@ -201,6 +201,10 @@ '_id': 'batch_item_record_deletion', '_str': _("Batch item record deletion") }, + { + '_id': 'erm_sushi_harvester', + '_str': _("ERM Usage Statistics SUSHI Harvester") + }, { '_id': 'batch_hold_cancel', '_str': _("Batch hold cancellation") diff --git a/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/UsageStatisticsDataProvidersList.vue b/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/UsageStatisticsDataProvidersList.vue index 847023aad9..acced92ce0 100644 --- a/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/UsageStatisticsDataProvidersList.vue +++ b/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/UsageStatisticsDataProvidersList.vue @@ -197,16 +197,18 @@ export default { () => { const client = APIClient.erm // TODO: below begin_date and end_date need to be dynamic, or are they needed at all? + // FIXME: this is not even being used atm, API backend is using begin_date+end_date from data provider in database client.usage_data_providers - .run(id, "2022-06-01", "2022-12-31") + .run(id, "2022-01-01", "2023-06-15") .then( success => { - this.setMessage( - this.$__( - "Ran harvester for usage data provider %s" - ).format(name), - true - ) + let message = "" + success.jobs.forEach((job, i) => { + message += this.$__( + '
  • Job for report type %s has been queued, click here to check its progress.
    • ' + ).format(job.report_type, job.job_id) + }) + this.setMessage(message, true) }, error => {} ) diff --git a/koha-tmpl/intranet-tmpl/prog/js/vue/fetch/erm-api-client.js b/koha-tmpl/intranet-tmpl/prog/js/vue/fetch/erm-api-client.js index 8fc8438cc7..e2c233fa83 100644 --- a/koha-tmpl/intranet-tmpl/prog/js/vue/fetch/erm-api-client.js +++ b/koha-tmpl/intranet-tmpl/prog/js/vue/fetch/erm-api-client.js @@ -276,6 +276,10 @@ export class ERMAPIClient extends HttpClient { endpoint: "usage_data_providers/" + id, body: usage_data_provider, }), + run: (id, begin_date, end_date) => + this.get({ + endpoint: "usage_data_providers/" + id + "/run?begin_date="+ begin_date + "&end_date=" + end_date, + }), //count: () => this.count("usage_data_providers"), //TODO: Implement count method }; } diff --git a/misc/cronjobs/erm_run_harvester.pl b/misc/cronjobs/erm_run_harvester.pl new file mode 100755 index 0000000000..c3712138ab --- /dev/null +++ b/misc/cronjobs/erm_run_harvester.pl @@ -0,0 +1,129 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Copyright (C) 2022 PTFS Europe +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; +use Getopt::Long qw( GetOptions ); +use Koha::DateUtils qw( dt_from_string ); +use POSIX; + +use Koha::Script; +use Koha::ERM::UsageDataProviders; + +# Command line option values +my $get_help = 0; +my $begin_date = 0; +my $end_date = 0; +my $dry_run = 0; +my $debug = 0; + +my $options = GetOptions( + 'h|help' => \$get_help, + 'begin-date=s' => \$begin_date, + 'end-date=s' => \$end_date, + 'dry-run' => \$dry_run, + 'debug' => \$debug +); + +if ($get_help) { + get_help(); + exit 1; +} + +my $udproviders = Koha::ERM::UsageDataProviders->search( { active => 1 } ); +unless ( scalar @{ $udproviders->as_list() } ) { + die "ERROR: No usage data providers found."; +} + +unless ( $begin_date ) { + die "ERROR: Please specify a begin-date"; +} + +debug_msg("Dry run: Harvests will not be enqueued"); +while ( my $udprovider = $udproviders->next ) { + debug_msg( + sprintf( + "Processing usage data provider #%d - %s", $udprovider->erm_usage_data_provider_id, $udprovider->name + ) + ); + + my $harvest_begin_date = dt_from_string($begin_date); + my $harvest_end_date = $end_date || dt_from_string(); + + if ( $harvest_begin_date > $harvest_end_date ) { + die sprintf( + "ERROR: begin-date must be earlier than end-date. Begin is %s and end date is %s", + $harvest_begin_date->ymd, + $harvest_end_date->ymd, + ); + } + + if ( !$dry_run ) { + my $job_ids = $udprovider->enqueue_sushi_harvest_jobs( + { + begin_date => $harvest_begin_date->ymd, + end_date => $harvest_end_date->ymd + } + ); + my $report_type_jobs = join ", ", map { $_->{report_type} . ': Job ID #' . $_->{job_id} } @{$job_ids}; + + debug_msg( + sprintf( + " - Harvest job enqueued (yyyy-mm-dd):\n - Begin date: %s \n - End date: %s \n - %s", + $harvest_begin_date->ymd, + $harvest_end_date->ymd, + $report_type_jobs + ) + ); + } + + +} + +sub debug_msg { + my ($msg) = @_; + + if ( !$debug ) { + return; + } + + if ( ref $msg eq 'HASH' ) { + use Data::Dumper; + $msg = Dumper $msg; + } + print STDERR "$msg\n"; +} + +sub get_help { + print <<"HELP"; +$0: Run a SUSHI harvesting for a ERM usage data provider + +This script will run the SUSHI harvesting for usage data providers + +Parameters: + --help or -h get help + --begin-date begin date for the harvest in yyyy-mm-dd format (e.g.: '2023-08-21') + --end-date end date for the harvest in yyyy-mm-dd format (e.g.: '2023-08-21') + --dry-run only produce a run report, without actually doing anything permanent + --debug print additional debugging info during run + +Usage example: +./misc/cronjobs/erm_run_harvester.pl --begin-date 2023-06-21 --debug + +HELP +} -- 2.39.5