From e8763a720c96bbad3e700302845c58f952bd0280 Mon Sep 17 00:00:00 2001 From: Matt Blenkinsop Date: Tue, 30 Apr 2024 10:27:48 +0000 Subject: [PATCH] Bug 34788: Allow import of file with additional columns This patch allows a file with additional columns to be imported. When the file is submitted, the system will enqueue the background job and send back information to the UI that certain columns have been ignored. The data will stil l import as normal but only the standard KBART columns will be parsed and imported Signed-off-by: Clemens Tubach Signed-off-by: Nick Clemens Signed-off-by: Katrin Fischer --- Koha/BackgroundJob/ImportKBARTFile.pm | 17 +++- Koha/REST/V1/ERM/EHoldings/Titles/Local.pm | 11 ++- .../swagger/paths/erm_eholdings_titles.yaml | 2 + .../ERM/EHoldingsLocalTitlesKBARTImport.vue | 27 +++--- .../Koha/BackgroundJob/ImportKBARTFile.t | 87 ++++++++++++++++++- 5 files changed, 122 insertions(+), 22 deletions(-) diff --git a/Koha/BackgroundJob/ImportKBARTFile.pm b/Koha/BackgroundJob/ImportKBARTFile.pm index 64e757d25e..398476e3f5 100644 --- a/Koha/BackgroundJob/ImportKBARTFile.pm +++ b/Koha/BackgroundJob/ImportKBARTFile.pm @@ -77,6 +77,7 @@ sub process { try { my $column_headers = $args->{column_headers}; + my $invalid_columns = $args->{invalid_columns}; my $rows = $args->{rows}; my $package_id = $args->{package_id}; my $create_linked_biblio = $args->{create_linked_biblio}; @@ -95,7 +96,7 @@ sub process { foreach my $row ( @{$rows} ) { next if !$row; - my $new_title = create_title_hash_from_line_data( $row, $column_headers ); + my $new_title = create_title_hash_from_line_data( $row, $column_headers, $invalid_columns ); my $title_match = check_for_matching_title($new_title); if ($title_match) { @@ -245,11 +246,13 @@ sub read_file { =head3 create_title_hash_from_line_data Takes a line and creates a hash of the values mapped to the column headings +Only accepts fields that are in the list of permitted KBART fields, other fields are ignored +(This is identified to the user on the background job status page) =cut sub create_title_hash_from_line_data { - my ( $row, $column_headers ) = @_; + my ( $row, $column_headers, $invalid_columns ) = @_; my %new_title; @@ -262,6 +265,11 @@ sub create_title_hash_from_line_data { $new_title{publication_title} =~ s/^"|"$//g; } + # Remove any additional columns + foreach my $invalid_column ( @$invalid_columns ) { + delete $new_title{$invalid_column}; + } + return \%new_title; } @@ -281,6 +289,9 @@ sub check_for_matching_title { # Use external_id in case title exists for a different provider, we want to add it for the new provider $match_parameters->{external_id} = $title->{title_id} if $title->{title_id}; + # We should also check the date_first_issue_online for serial publications + $match_parameters->{date_first_issue_online} = $title->{date_first_issue_online} if $title->{date_first_issue_online}; + # If no match parameters are provided in the file we should add the new title return 0 if !%$match_parameters; @@ -426,7 +437,7 @@ sub is_file_too_large { =head3 rescue_EBSCO_files -EBSCO have an incorrect spelling of "preceding_publication_title_id" in all of their KBART files ("preceeding" instead of "preceding"). +EBSCO have an incorrect spelling for "preceding_publication_title_id" in all of their KBART files ("preceeding" instead of "preceding"). This means all of their KBART files fail to import using the current methodology. There is no simple way of finding out who the vendor is before importing so all KBART files from any vendor are going to have to be checked for this spelling and corrected. diff --git a/Koha/REST/V1/ERM/EHoldings/Titles/Local.pm b/Koha/REST/V1/ERM/EHoldings/Titles/Local.pm index 775fb89da6..5e363bc05b 100644 --- a/Koha/REST/V1/ERM/EHoldings/Titles/Local.pm +++ b/Koha/REST/V1/ERM/EHoldings/Titles/Local.pm @@ -294,20 +294,19 @@ sub import_from_kbart_file { # Check that the column headers in the file match the standardised KBART phase II columns # If not, return a warning + my $warnings = {}; my @valid_headers = Koha::BackgroundJob::ImportKBARTFile::get_valid_headers(); foreach my $header (@$column_headers) { if ( !grep { $_ eq $header } @valid_headers ) { - $header = 'Empty column - please remove' if $header eq ''; + $header = 'Empty column' if $header eq ''; push @invalid_columns, $header; } } - return $c->render( - status => 201, - openapi => { invalid_columns => \@invalid_columns, valid_columns => \@valid_headers, invalid_filetype => 0 } - ) if scalar(@invalid_columns) > 0; + $warnings->{invalid_columns} = \@invalid_columns if scalar(@invalid_columns) > 0; my $params = { column_headers => $column_headers, + invalid_columns => \@invalid_columns, rows => $rows, package_id => $package_id, file_name => $file->{filename}, @@ -342,7 +341,7 @@ sub import_from_kbart_file { return $c->render( status => 201, - openapi => { job_ids => \@job_ids } + openapi => { job_ids => \@job_ids, warnings => $warnings } ); } catch { $c->unhandled_exception($_); diff --git a/api/v1/swagger/paths/erm_eholdings_titles.yaml b/api/v1/swagger/paths/erm_eholdings_titles.yaml index 7e4e60411e..7cc31159f5 100644 --- a/api/v1/swagger/paths/erm_eholdings_titles.yaml +++ b/api/v1/swagger/paths/erm_eholdings_titles.yaml @@ -533,6 +533,8 @@ type: array invalid_filetype: type: integer + warnings: + type: object additionalProperties: false 400: description: Bad parameter diff --git a/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/EHoldingsLocalTitlesKBARTImport.vue b/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/EHoldingsLocalTitlesKBARTImport.vue index 401732ce35..6a75b2118b 100644 --- a/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/EHoldingsLocalTitlesKBARTImport.vue +++ b/koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/EHoldingsLocalTitlesKBARTImport.vue @@ -28,6 +28,7 @@ :id="`import_file`" :name="`import_file`" required + ref="fileLoader" />
  • @@ -80,9 +81,11 @@ import ButtonSubmit from "../ButtonSubmit.vue" import { APIClient } from "../../fetch/api-client.js" import { setMessage, setWarning } from "../../messages" +import { ref } from "vue" export default { data() { + const fileLoader = ref() return { file: { filename: null, @@ -91,6 +94,7 @@ export default { packages: [], package_id: null, create_linked_biblio: false, + fileLoader, } }, beforeCreate() { @@ -131,29 +135,30 @@ export default { if (success.job_ids) { if (success.job_ids.length > 1) { message += this.$__( - "
  • Your file was too large to process in one job, the file has been split into %s jobs to meet the maximum size limits.
    • " + "

    Your file was too large to process in one job, the file has been split into %s jobs to meet the maximum size limits.

    " ).format(success.job_ids.length) } success.job_ids.forEach((job, i) => { message += this.$__( - '
  • Job for uploaded file %s has been queued, click here to check its progress.
    • ' + '
  • Job %s for uploaded file has been queued, click here to check its progress.
    • ' ).format(i + 1, job) }) setMessage(message, true) } - if (success.invalid_columns) { + if (success.warnings.invalid_columns) { message += this.$__( - "

    Invalid columns were detected in your report, please check the list below:

    " + "

    Information:

    " ) - success.invalid_columns.forEach(column => { - message += this.$__( - `
  • %s
    • ` - ).format(column) + message += this.$__( + "

    Additional columns were detected in your report, please see the list below:

    " + ) + success.warnings.invalid_columns.forEach(column => { + message += this.$__(`
  • %s
    • `).format(column) }) message += this.$__( - '

    For a list of compliant column headers, please click here

    ' + "

    The data in these columns will not be imported.

    " ) - setWarning(message) + setMessage(message) } if (success.invalid_filetype) { message += this.$__( @@ -174,6 +179,8 @@ export default { } this.package_id = null this.create_linked_biblio = false + this.$refs.fileLoader.files = null + this.$refs.fileLoader.value = null }, }, components: { diff --git a/t/db_dependent/Koha/BackgroundJob/ImportKBARTFile.t b/t/db_dependent/Koha/BackgroundJob/ImportKBARTFile.t index 57e68e0b98..a3486a7ed0 100755 --- a/t/db_dependent/Koha/BackgroundJob/ImportKBARTFile.t +++ b/t/db_dependent/Koha/BackgroundJob/ImportKBARTFile.t @@ -17,7 +17,7 @@ use Modern::Perl; -use Test::More tests => 6; +use Test::More tests => 7; use Test::MockModule; use Koha::Database; @@ -128,11 +128,92 @@ Nature Astronomy 2397-3366 2017-01 1 1 https://www.nature.com/natastron 4bb }; my ( $column_headers, $lines ) = Koha::BackgroundJob::ImportKBARTFile::read_file($file); + my @invalid_columns; my $title_from_line1 = - Koha::BackgroundJob::ImportKBARTFile::create_title_hash_from_line_data( @{$lines}[0], $column_headers ); + Koha::BackgroundJob::ImportKBARTFile::create_title_hash_from_line_data( @{$lines}[0], $column_headers, \@invalid_columns ); my $title_from_line2 = - Koha::BackgroundJob::ImportKBARTFile::create_title_hash_from_line_data( @{$lines}[1], $column_headers ); + Koha::BackgroundJob::ImportKBARTFile::create_title_hash_from_line_data( @{$lines}[1], $column_headers, \@invalid_columns ); + + my $line1_match = { + 'coverage_depth' => 'fulltext', + 'date_monograph_published_print' => '', + 'date_first_issue_online' => '2015-01', + 'date_last_issue_online' => '', + 'coverage_notes' => 'Hybrid (Open Choice)', + 'first_editor' => '', + 'date_monograph_published_online' => '', + 'preceding_publication_title_id' => '', + 'num_last_issue_online' => '', + 'embargo_info' => '', + 'access_type' => 'P', + 'num_first_issue_online' => '1', + 'online_identifier' => '2055-0278', + 'title_url' => 'https://www.nature.com/nplants', + 'monograph_volume' => '', + 'first_author' => '', + 'parent_publication_title_id' => '', + 'num_last_vol_online' => '', + 'publication_title' => 'Nature Plants', + 'num_first_vol_online' => '1', + 'print_identifier' => '', + 'publisher_name' => 'Nature Publishing Group UK', + 'title_id' => '4aaa7', + 'publication_type' => 'serial', + 'monograph_edition' => '' + }; + my $line2_match = { + 'date_monograph_published_online' => '', + 'num_first_vol_online' => '1', + 'num_last_issue_online' => '', + 'preceding_publication_title_id' => '', + 'title_url' => 'https://www.nature.com/natastron', + 'online_identifier' => '2397-3366', + 'print_identifier' => '', + 'num_last_vol_online' => '', + 'embargo_info' => '', + 'parent_publication_title_id' => '', + 'publisher_name' => 'Nature Publishing Group UK', + 'date_first_issue_online' => '2017-01', + 'monograph_volume' => '', + 'monograph_edition' => '', + 'access_type' => 'P', + 'first_author' => '', + 'num_first_issue_online' => '1', + 'first_editor' => '', + 'publication_title' => 'Nature Astronomy', + 'date_monograph_published_print' => '', + 'publication_type' => 'serial', + 'title_id' => '4bbb0', + 'coverage_depth' => 'fulltext', + 'coverage_notes' => 'Hybrid (Open Choice)', + 'date_last_issue_online' => '' + }; + + is_deeply( $title_from_line1, $line1_match, 'Title hash created correctly' ); + is_deeply( $title_from_line2, $line2_match, 'Title hash created correctly' ); +}; + +subtest 'create_title_hash_from_line_data with invalid columns using csv' => sub { + + plan tests => 2; + + my $file = { + filename => 'Test_file.csv', + file_content => encode_base64( + 'publication_title,print_identifier,online_identifier,date_first_issue_online,num_first_vol_online,num_first_issue_online,date_last_issue_online,num_last_vol_online,num_last_issue_online,title_url,first_author,title_id,embargo_info,coverage_depth,coverage_notes,publisher_name,publication_type,date_monograph_published_print,date_monograph_published_online,monograph_volume,monograph_edition,first_editor,parent_publication_title_id,preceding_publication_title_id,access_type,invalid_column +Nature Plants,,2055-0278,2015-01,1,1,,,,https://www.nature.com/nplants,,4aaa7,,fulltext,Hybrid (Open Choice),Nature Publishing Group UK,serial,,,,,,,,P,invalid_column_data +Nature Astronomy,,2397-3366,2017-01,1,1,,,,https://www.nature.com/natastron,,4bbb0,,fulltext,Hybrid (Open Choice),Nature Publishing Group UK,serial,,,,,,,,P,invalid_column_data' + ) + }; + + my ( $column_headers, $lines ) = Koha::BackgroundJob::ImportKBARTFile::read_file($file); + my @invalid_columns = ('invalid_column'); + + my $title_from_line1 = + Koha::BackgroundJob::ImportKBARTFile::create_title_hash_from_line_data( @{$lines}[0], $column_headers, \@invalid_columns ); + my $title_from_line2 = + Koha::BackgroundJob::ImportKBARTFile::create_title_hash_from_line_data( @{$lines}[1], $column_headers, \@invalid_columns ); my $line1_match = { 'coverage_depth' => 'fulltext', -- 2.39.5