1 package Koha::BackgroundJob::ImportKBARTFile;
3 # This file is part of Koha.
5 # Koha is free software; you can redistribute it and/or modify it
6 # under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 3 of the License, or
8 # (at your option) any later version.
10 # Koha is distributed in the hope that it will be useful, but
11 # WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with Koha; if not, see <http://www.gnu.org/licenses>.
19 use JSON qw( decode_json encode_json );
20 use Try::Tiny qw( catch try );
21 use MIME::Base64 qw( decode_base64 );
22 use POSIX qw( floor );
26 use Koha::ERM::EHoldings::Titles;
28 use base 'Koha::BackgroundJob';
32 ImportKBARTFile - Create new eHoldings titles from a KBART file
34 This is a subclass of Koha::BackgroundJob.
42 Define the job type of this job.
47 return 'import_from_kbart_file';
57 my ( $self, $args ) = @_;
59 if ( $self->status eq 'cancelled' ) {
65 my $titles_imported = 0;
66 my $duplicate_titles = 0;
67 my $failed_imports = 0;
69 my $file_name = $args->{file}->{filename};
71 duplicates_found => undef,
72 titles_imported => undef,
73 file_name => $file_name,
75 failed_imports => undef
79 my $file = $args->{file};
80 my ( $column_headers, $lines ) = format_file($file);
82 if ( scalar( @{$lines} ) == 0 ) {
86 error_message => 'No valid lines were found in this file. Please check the file formatting.',
88 $self->status('failed')->store;
91 $self->size( scalar( @{$lines} ) )->store;
92 $total_lines = scalar( @{$lines} );
94 foreach my $line ( @{$lines} ) {
96 my $new_title = create_title_hash_from_line_data( $line, $column_headers );
97 my $title_match = Koha::ERM::EHoldings::Titles->search( { external_id => $new_title->{title_id} } )->count;
102 code => 'title_already_exists',
104 error_message => undef,
105 title => $new_title->{publication_title}
109 my $formatted_title = format_title($new_title);
110 if ( !$formatted_title->{publication_title} ) {
112 code => 'title_failed',
114 error_message => "No publication_title found for title_id: ",
115 title => '(Unknown)',
116 title_id => $formatted_title->{external_id}
120 my $imported_title = Koha::ERM::EHoldings::Title->new($formatted_title)->store;
122 # No need to add a message for a successful import,
123 # files could have 1000s of titles which will lead to lots of messages in background_job->data
124 $titles_imported++ if $imported_title;
129 code => 'title_failed',
131 error_message => $_->{msg},
132 title => $new_title->{publication_title}
139 $report->{duplicates_found} = $duplicate_titles;
140 $report->{titles_imported} = $titles_imported;
141 $report->{total_lines} = $total_lines;
142 $report->{failed_imports} = $failed_imports;
144 my $data = $self->decoded_data;
145 $data->{messages} = \@messages;
146 $data->{report} = $report;
148 # Remove the file content as this is no longer needed and can be very large
149 $data->{file}->{file_content} = undef;
151 $self->finish($data);
164 my ( $self, $args ) = @_;
166 return unless exists $args->{file};
168 $self->SUPER::enqueue(
172 job_queue => 'long_tasks',
179 Formats a title to fit the names of the database fields in Koha
181 Kbart field "title_id" = "external_id" in Koha
182 Kbart field "coverage_notes" = "notes" in Koha
189 $title->{external_id} = $title->{title_id};
190 delete $title->{title_id};
192 # Some files appear to use coverage_notes instead of "notes" as in the KBART standard
193 if ( $title->{coverage_notes} ) {
194 $title->{notes} = $title->{coverage_notes};
195 delete $title->{coverage_notes};
203 Formats a file to provide report headers and lines to be processed
210 my $file_content = decode_base64( $file->{file_content} );
211 $file_content =~ s/\n/\r/g;
212 my @lines = split /\r/, $file_content;
213 my @column_headers = split /\t/, $lines[0];
214 shift @lines; # Remove headers row
215 my @remove_null_lines = grep $_ ne '', @lines;
217 return ( \@column_headers, \@remove_null_lines );
220 =head3 create_title_hash_from_line_data
222 Takes a line and creates a hash of the values mapped to the column headings
226 sub create_title_hash_from_line_data {
227 my ( $line, $column_headers ) = @_;
230 my @values = split /\t/, $line;
232 @new_title{ @{$column_headers} } = @values;
237 =head3 get_valid_headers
239 Returns a list of permitted headers in a KBART phase II file
243 sub get_valid_headers {
248 'date_first_issue_online',
249 'num_first_vol_online',
250 'num_first_issue_online',
251 'date_last_issue_online',
252 'num_last_vol_online',
253 'num_last_issue_online',
262 'date_monograph_published_print',
263 'date_monograph_published_online',
267 'parent_publication_title_id',
268 'preceding_publication_title_id',
274 =head3 calculate_chunked_file_size
276 Calculates average line size to work out how many lines to chunk a large file into
277 Knocks 10% off the final result to give some margin for error
281 sub calculate_chunked_file_size {
282 my ( $file_size, $max_allowed_packet, $number_of_lines ) = @_;
284 my $average_line_size = $file_size / $number_of_lines;
285 my $lines_possible = $max_allowed_packet / $average_line_size;
286 my $moderated_value = floor( $lines_possible * 0.9 );
287 return $moderated_value;