Koha-community/Koha
15
7
Fork 0
Code Issues Releases Wiki Activity

Bug 34788: Allow import of file with additional columns

Browse source 
This patch allows a file with additional columns to be imported. When the file is submitted, the system will enqueue the background job and send back information to the UI that certain columns have been ignored. The data will stil l import as normal but only the standard KBART columns will be parsed and imported

Signed-off-by: Clemens Tubach <clemens.tubach@kit.edu>
Signed-off-by: Nick Clemens <nick@bywatersolutions.com>
Signed-off-by: Katrin Fischer <katrin.fischer@bsz-bw.de>
This commit is contained in:
Matt Blenkinsop 2024-04-30 10:27:48 +00:00 committed by Katrin Fischer
parent d31f453186
commit e8763a720c
Signed by: kfischer
GPG key ID: 0EF6E2C03357A834
5 changed files with 122 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
Koha/BackgroundJob/ImportKBARTFile.pm
View file

@ -77,6 +77,7 @@ sub process {
try {
my $column_headers = $args->{column_headers};
my $invalid_columns = $args->{invalid_columns};
my $rows = $args->{rows};
my $package_id = $args->{package_id};
my $create_linked_biblio = $args->{create_linked_biblio};
@ -95,7 +96,7 @@ sub process {
foreach my $row ( @{$rows} ) {
next if !$row;
my $new_title = create_title_hash_from_line_data( $row, $column_headers );
my $new_title = create_title_hash_from_line_data( $row, $column_headers, $invalid_columns );
my $title_match = check_for_matching_title($new_title);
if ($title_match) {
@ -245,11 +246,13 @@ sub read_file {
=head3 create_title_hash_from_line_data
Takes a line and creates a hash of the values mapped to the column headings
Only accepts fields that are in the list of permitted KBART fields, other fields are ignored
(This is identified to the user on the background job status page)
=cut
sub create_title_hash_from_line_data {
my ( $row, $column_headers ) = @_;
my ( $row, $column_headers, $invalid_columns ) = @_;
my %new_title;
@ -262,6 +265,11 @@ sub create_title_hash_from_line_data {
$new_title{publication_title} =~ s/^"|"$//g;
}
# Remove any additional columns
foreach my $invalid_column ( @$invalid_columns ) {
delete $new_title{$invalid_column};
}
return \%new_title;
}
@ -281,6 +289,9 @@ sub check_for_matching_title {
# Use external_id in case title exists for a different provider, we want to add it for the new provider
$match_parameters->{external_id} = $title->{title_id} if $title->{title_id};
# We should also check the date_first_issue_online for serial publications
$match_parameters->{date_first_issue_online} = $title->{date_first_issue_online} if $title->{date_first_issue_online};
# If no match parameters are provided in the file we should add the new title
return 0 if !%$match_parameters;
@ -426,7 +437,7 @@ sub is_file_too_large {
=head3 rescue_EBSCO_files
EBSCO have an incorrect spelling of "preceding_publication_title_id" in all of their KBART files ("preceeding" instead of "preceding").
EBSCO have an incorrect spelling for "preceding_publication_title_id" in all of their KBART files ("preceeding" instead of "preceding").
This means all of their KBART files fail to import using the current methodology.
There is no simple way of finding out who the vendor is before importing so all KBART files from any vendor are going to have to be checked for this spelling and corrected.

11
Koha/REST/V1/ERM/EHoldings/Titles/Local.pm
View file

@ -294,20 +294,19 @@ sub import_from_kbart_file {
# Check that the column headers in the file match the standardised KBART phase II columns
# If not, return a warning
my $warnings = {};
my @valid_headers = Koha::BackgroundJob::ImportKBARTFile::get_valid_headers();
foreach my $header (@$column_headers) {
if ( !grep { $_ eq $header } @valid_headers ) {
$header = 'Empty column - please remove' if $header eq '';
$header = 'Empty column' if $header eq '';
push @invalid_columns, $header;
}
}
return $c->render(
status => 201,
openapi => { invalid_columns => \@invalid_columns, valid_columns => \@valid_headers, invalid_filetype => 0 }
) if scalar(@invalid_columns) > 0;
$warnings->{invalid_columns} = \@invalid_columns if scalar(@invalid_columns) > 0;
my $params = {
column_headers => $column_headers,
invalid_columns => \@invalid_columns,
rows => $rows,
package_id => $package_id,
file_name => $file->{filename},
@ -342,7 +341,7 @@ sub import_from_kbart_file {
return $c->render(
status => 201,
openapi => { job_ids => \@job_ids }
openapi => { job_ids => \@job_ids, warnings => $warnings }
);
} catch {
$c->unhandled_exception($_);

2
api/v1/swagger/paths/erm_eholdings_titles.yaml
View file

@ -533,6 +533,8 @@
type: array
invalid_filetype:
type: integer
warnings:
type: object
additionalProperties: false
400:
description: Bad parameter

27
koha-tmpl/intranet-tmpl/prog/js/vue/components/ERM/EHoldingsLocalTitlesKBARTImport.vue
View file

@ -28,6 +28,7 @@
:id="`import_file`"
:name="`import_file`"
required
ref="fileLoader"
/>
</li>
<li>
@ -80,9 +81,11 @@
import ButtonSubmit from "../ButtonSubmit.vue"
import { APIClient } from "../../fetch/api-client.js"
import { setMessage, setWarning } from "../../messages"
import { ref } from "vue"
export default {
data() {
const fileLoader = ref()
return {
file: {
filename: null,
@ -91,6 +94,7 @@ export default {
packages: [],
package_id: null,
create_linked_biblio: false,
fileLoader,
}
},
beforeCreate() {
@ -131,29 +135,30 @@ export default {
if (success.job_ids) {
if (success.job_ids.length > 1) {
message += this.$__(
"<li>Your file was too large to process in one job, the file has been split into %s jobs to meet the maximum size limits.</li>"
"<p style='font-weight: normal; font-size: medium; margin-top: 1em;'>Your file was too large to process in one job, the file has been split into %s jobs to meet the maximum size limits.</p>"
).format(success.job_ids.length)
}
success.job_ids.forEach((job, i) => {
message += this.$__(
'<li>Job for uploaded file %s has been queued, <a href="/cgi-bin/koha/admin/background_jobs.pl?op=view&id=%s" target="_blank">click here</a> to check its progress.</li>'
'<li>Job %s for uploaded file has been queued, <a href="/cgi-bin/koha/admin/background_jobs.pl?op=view&id=%s" target="_blank">click here</a> to check its progress.</li>'
).format(i + 1, job)
})
setMessage(message, true)
}
if (success.invalid_columns) {
if (success.warnings.invalid_columns) {
message += this.$__(
"<p>Invalid columns were detected in your report, please check the list below:</p>"
"<p style='font-weight: normal; font-size: medium; margin-top: 1em;'>Information:</p>"
)
success.invalid_columns.forEach(column => {
message += this.$__(
`<li style="font-weight: normal; font-size: medium;">%s</li>`
).format(column)
message += this.$__(
"<p>Additional columns were detected in your report, please see the list below:</p>"
)
success.warnings.invalid_columns.forEach(column => {
message += this.$__(`<li>%s</li>`).format(column)
})
message += this.$__(
'<p style="margin-top: 1em;">For a list of compliant column headers, please click <a target="_blank" href="https://groups.niso.org/higherlogic/ws/public/download/16900/RP-9-2014_KBART.pdf" />here</p>'
"<p style='margin-top: 0.1em;'>The data in these columns will not be imported.</p>"
)
setWarning(message)
setMessage(message)
}
if (success.invalid_filetype) {
message += this.$__(
@ -174,6 +179,8 @@ export default {
}
this.package_id = null
this.create_linked_biblio = false
this.$refs.fileLoader.files = null
this.$refs.fileLoader.value = null
},
},
components: {

87
t/db_dependent/Koha/BackgroundJob/ImportKBARTFile.t
View file

@ -17,7 +17,7 @@
use Modern::Perl;
use Test::More tests => 6;
use Test::More tests => 7;
use Test::MockModule;
use Koha::Database;
@ -128,11 +128,92 @@ Nature Astronomy 2397-3366 2017-01 1 1 https://www.nature.com/natastron 4bb
};
my ( $column_headers, $lines ) = Koha::BackgroundJob::ImportKBARTFile::read_file($file);
my @invalid_columns;
my $title_from_line1 =
Koha::BackgroundJob::ImportKBARTFile::create_title_hash_from_line_data( @{$lines}[0], $column_headers );
Koha::BackgroundJob::ImportKBARTFile::create_title_hash_from_line_data( @{$lines}[0], $column_headers, \@invalid_columns );
my $title_from_line2 =
Koha::BackgroundJob::ImportKBARTFile::create_title_hash_from_line_data( @{$lines}[1], $column_headers );
Koha::BackgroundJob::ImportKBARTFile::create_title_hash_from_line_data( @{$lines}[1], $column_headers, \@invalid_columns );
my $line1_match = {
'coverage_depth' => 'fulltext',
'date_monograph_published_print' => '',
'date_first_issue_online' => '2015-01',
'date_last_issue_online' => '',
'coverage_notes' => 'Hybrid (Open Choice)',
'first_editor' => '',
'date_monograph_published_online' => '',
'preceding_publication_title_id' => '',
'num_last_issue_online' => '',
'embargo_info' => '',
'access_type' => 'P',
'num_first_issue_online' => '1',
'online_identifier' => '2055-0278',
'title_url' => 'https://www.nature.com/nplants',
'monograph_volume' => '',
'first_author' => '',
'parent_publication_title_id' => '',
'num_last_vol_online' => '',
'publication_title' => 'Nature Plants',
'num_first_vol_online' => '1',
'print_identifier' => '',
'publisher_name' => 'Nature Publishing Group UK',
'title_id' => '4aaa7',
'publication_type' => 'serial',
'monograph_edition' => ''
};
my $line2_match = {
'date_monograph_published_online' => '',
'num_first_vol_online' => '1',
'num_last_issue_online' => '',
'preceding_publication_title_id' => '',
'title_url' => 'https://www.nature.com/natastron',
'online_identifier' => '2397-3366',
'print_identifier' => '',
'num_last_vol_online' => '',
'embargo_info' => '',
'parent_publication_title_id' => '',
'publisher_name' => 'Nature Publishing Group UK',
'date_first_issue_online' => '2017-01',
'monograph_volume' => '',
'monograph_edition' => '',
'access_type' => 'P',
'first_author' => '',
'num_first_issue_online' => '1',
'first_editor' => '',
'publication_title' => 'Nature Astronomy',
'date_monograph_published_print' => '',
'publication_type' => 'serial',
'title_id' => '4bbb0',
'coverage_depth' => 'fulltext',
'coverage_notes' => 'Hybrid (Open Choice)',
'date_last_issue_online' => ''
};
is_deeply( $title_from_line1, $line1_match, 'Title hash created correctly' );
is_deeply( $title_from_line2, $line2_match, 'Title hash created correctly' );
};
subtest 'create_title_hash_from_line_data with invalid columns using csv' => sub {
plan tests => 2;
my $file = {
filename => 'Test_file.csv',
file_content => encode_base64(
'publication_title,print_identifier,online_identifier,date_first_issue_online,num_first_vol_online,num_first_issue_online,date_last_issue_online,num_last_vol_online,num_last_issue_online,title_url,first_author,title_id,embargo_info,coverage_depth,coverage_notes,publisher_name,publication_type,date_monograph_published_print,date_monograph_published_online,monograph_volume,monograph_edition,first_editor,parent_publication_title_id,preceding_publication_title_id,access_type,invalid_column
Nature Plants,,2055-0278,2015-01,1,1,,,,https://www.nature.com/nplants,,4aaa7,,fulltext,Hybrid (Open Choice),Nature Publishing Group UK,serial,,,,,,,,P,invalid_column_data
Nature Astronomy,,2397-3366,2017-01,1,1,,,,https://www.nature.com/natastron,,4bbb0,,fulltext,Hybrid (Open Choice),Nature Publishing Group UK,serial,,,,,,,,P,invalid_column_data'
)
};
my ( $column_headers, $lines ) = Koha::BackgroundJob::ImportKBARTFile::read_file($file);
my @invalid_columns = ('invalid_column');
my $title_from_line1 =
Koha::BackgroundJob::ImportKBARTFile::create_title_hash_from_line_data( @{$lines}[0], $column_headers, \@invalid_columns );
my $title_from_line2 =
Koha::BackgroundJob::ImportKBARTFile::create_title_hash_from_line_data( @{$lines}[1], $column_headers, \@invalid_columns );
my $line1_match = {
'coverage_depth' => 'fulltext',