4 # This file is part of Koha.
6 # Koha is free software; you can redistribute it and/or modify it
7 # under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # Koha is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with Koha; if not, see <http://www.gnu.org/licenses>.
21 use List::MoreUtils qw(uniq);
25 use C4::AuthoritiesMarc; # GetAuthority
26 use C4::Biblio; # GetMarcBiblio
27 use C4::Branch; # GetBranches
29 use C4::Koha; # GetItemTypes
36 my $dont_export_items;
42 my $op = $query->param("op") || '';
43 my $filename = $query->param("filename") || 'koha.mrc';
44 my $dbh = C4::Context->dbh;
45 my $marcflavour = C4::Context->preference("marcflavour");
46 my $output_format = $query->param("format") || $query->param("output_format") || 'iso2709';
48 # Checks if the script is called from commandline
49 my $commandline = not defined $ENV{GATEWAY_INTERFACE};
56 'format=s' => \$output_format,
57 'date=s' => \$timestamp,
58 'dont_export_items' => \$dont_export_items,
59 'deleted_barcodes' => \$deleted_barcodes,
61 'filename=s' => \$filename,
62 'record-type=s' => \$record_type,
63 'id_list_file=s' => \$id_list_file,
69 export.pl [--format=format] [--date=date] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
72 --format=FORMAT FORMAT is either 'xml' or 'marc' (default)
74 --date=DATE DATE should be entered as the 'dateformat' syspref is
75 set (dd/mm/yyyy for metric, yyyy-mm-dd for iso,
76 mm/dd/yyyy for us) records exported are the ones that
77 have been modified since DATE
79 --record-type=TYPE TYPE is 'bibs' or 'auths'
81 --deleted_barcodes If used, a list of barcodes of items deleted since DATE
82 is produced (or from all deleted items if no date is
83 specified). Used only if TYPE is 'bibs'
85 --clean removes NSE/NSB
87 --id_list_file=PATH PATH is a path to a file containing a list of
88 IDs (biblionumber or authid) with one ID per line.
89 This list works as a filter; it is compatible with
90 other parameters for selecting records
95 # Default parameters values :
97 $dont_export_items ||= 0;
98 $deleted_barcodes ||= 0;
100 $record_type ||= "bibs";
104 open STDOUT, '>', $filename if $filename;
109 $op = $query->param("op") || '';
110 $filename = $query->param("filename") || 'koha.mrc';
111 $filename =~ s/(\r|\n)//;
115 # Default value for output_format is 'iso2709'
116 $output_format ||= 'iso2709';
117 # Retrocompatibility for the format parameter
118 $output_format = 'iso2709' if $output_format eq 'marc';
120 my ( $template, $loggedinuser, $cookie, $flags ) = get_template_and_user(
122 template_name => "tools/export.tt",
125 authnotrequired => $commandline,
126 flagsrequired => { tools => 'export_catalog' },
131 my $limit_ind_branch =
132 ( C4::Context->preference('IndependentBranches')
133 && C4::Context->userenv
134 && !C4::Context->IsSuperLibrarian()
135 && C4::Context->userenv->{branch} ) ? 1 : 0;
137 my @branch = $query->param("branch");
138 if ( C4::Context->preference("IndependentBranches")
139 && C4::Context->userenv
140 && !C4::Context->IsSuperLibrarian() )
142 @branch = ( C4::Context->userenv->{'branch'} );
144 # if stripping nonlocal items, use loggedinuser's branch
145 my $localbranch = C4::Context->userenv->{'branch'};
147 my %branchmap = map { $_ => 1 } @branch; # for quick lookups
149 my $backupdir = C4::Context->config('backupdir');
151 if ( $op eq "export" ) {
152 if ( $output_format eq "iso2709" or $output_format eq "xml" ) {
153 my $charset = 'utf-8';
154 my $mimetype = 'application/octet-stream';
155 binmode STDOUT, ':encoding(UTF-8)';
156 if ( $filename =~ m/\.gz$/ ) {
157 $mimetype = 'application/x-gzip';
161 elsif ( $filename =~ m/\.bz2$/ ) {
162 $mimetype = 'application/x-bzip2';
166 print $query->header(
168 -charset => $charset,
169 -attachment => $filename
170 ) unless ($commandline);
172 $record_type = $query->param("record_type") unless ($commandline);
173 my $export_remove_fields = $query->param("export_remove_fields");
174 my @biblionumbers = $query->param("biblionumbers");
175 my @itemnumbers = $query->param("itemnumbers");
180 my $StartingBiblionumber = $query->param("StartingBiblionumber");
181 my $EndingBiblionumber = $query->param("EndingBiblionumber");
182 my $itemtype = $query->param("itemtype");
183 my $start_callnumber = $query->param("start_callnumber");
184 my $end_callnumber = $query->param("end_callnumber");
185 $timestamp = ($timestamp) ? C4::Dates->new($timestamp) : ''
187 my $start_accession =
188 ( $query->param("start_accession") )
189 ? C4::Dates->new( $query->param("start_accession") )
192 ( $query->param("end_accession") )
193 ? C4::Dates->new( $query->param("end_accession") )
195 $dont_export_items = $query->param("dont_export_item")
196 unless ($commandline);
198 my $strip_nonlocal_items = $query->param("strip_nonlocal_items");
200 my $biblioitemstable =
201 ( $commandline and $deleted_barcodes )
202 ? 'deletedbiblioitems'
205 ( $commandline and $deleted_barcodes )
209 my $starting_authid = $query->param('starting_authid');
210 my $ending_authid = $query->param('ending_authid');
211 my $authtype = $query->param('authtype');
214 open $filefh,"<", $id_list_file or die "cannot open $id_list_file: $!" if $id_list_file;
216 $filefh = $query->upload("id_list_file");
220 while (my $number=<$filefh>){
221 $number=~s/[\r\n]*$//;
222 $id_filter{$number}=1 if $number=~/^\d+$/;
226 if ( $record_type eq 'bibs' and not @biblionumbers ) {
229 # Specific query when timestamp is used
230 # Actually it's used only with CLI and so all previous filters
232 # If one day timestamp is used via the web interface, this part will
233 # certainly have to be rewrited
234 my ( $query, $params ) = construct_query(
236 recordtype => $record_type,
237 timestamp => $timestamp,
238 biblioitemstable => $biblioitemstable,
242 @sql_params = @$params;
246 my ( $query, $params ) = construct_query(
248 recordtype => $record_type,
249 biblioitemstable => $biblioitemstable,
250 itemstable => $itemstable,
251 StartingBiblionumber => $StartingBiblionumber,
252 EndingBiblionumber => $EndingBiblionumber,
254 start_callnumber => $start_callnumber,
255 end_callnumber => $end_callnumber,
256 start_accession => $start_accession,
257 end_accession => $end_accession,
258 itemtype => $itemtype,
262 @sql_params = @$params;
265 elsif ( $record_type eq 'auths' ) {
266 my ( $query, $params ) = construct_query(
268 recordtype => $record_type,
269 starting_authid => $starting_authid,
270 ending_authid => $ending_authid,
271 authtype => $authtype,
275 @sql_params = @$params;
278 elsif ( $record_type eq 'db' ) {
279 my $successful_export;
280 if ( $flags->{superlibrarian}
281 && C4::Context->config('backup_db_via_tools') )
283 $successful_export = download_backup(
285 directory => "$backupdir",
287 filename => "$filename"
291 unless ($successful_export) {
292 my $remotehost = $query->remote_host();
293 $remotehost =~ s/(\n|\r)//;
295 "A suspicious attempt was made to download the db at '$filename' by someone at "
296 . $remotehost . "\n";
300 elsif ( $record_type eq 'conf' ) {
301 my $successful_export;
302 if ( $flags->{superlibrarian}
303 && C4::Context->config('backup_conf_via_tools') )
305 $successful_export = download_backup(
307 directory => "$backupdir",
309 filename => "$filename"
313 unless ($successful_export) {
314 my $remotehost = $query->remote_host();
315 $remotehost =~ s/(\n|\r)//;
317 "A suspicious attempt was made to download the configuration at '$filename' by someone at "
318 . $remotehost . "\n";
322 elsif (@biblionumbers) {
323 push @recordids, (@biblionumbers);
327 # Someone is trying to mess us up
330 unless (@biblionumbers) {
331 my $sth = $dbh->prepare($sql_query);
332 $sth->execute(@sql_params);
333 push @recordids, map {
335 } @{ $sth->fetchall_arrayref };
336 @recordids = grep { exists($id_filter{$_}) } @recordids if scalar(%id_filter);
339 my $xml_header_written = 0;
340 for my $recordid ( uniq @recordids ) {
341 if ($deleted_barcodes) {
343 SELECT DISTINCT barcode
345 WHERE deleteditems.biblionumber = ?
347 my $sth = $dbh->prepare($q);
348 $sth->execute($recordid);
349 while ( my $row = $sth->fetchrow_array ) {
355 if ( $record_type eq 'bibs' ) {
356 $record = eval { GetMarcBiblio($recordid); };
359 next if not defined $record;
360 C4::Biblio::EmbedItemsInMarcBiblio( $record, $recordid,
362 unless $dont_export_items;
363 if ( $strip_nonlocal_items
365 || $dont_export_items )
367 my ( $homebranchfield, $homebranchsubfield ) =
368 GetMarcFromKohaField( 'items.homebranch', '' );
369 for my $itemfield ( $record->field($homebranchfield) ) {
370 $record->delete_field($itemfield)
371 if ( $dont_export_items
372 || $localbranch ne $itemfield->subfield(
373 $homebranchsubfield) );
377 elsif ( $record_type eq 'auths' ) {
378 $record = C4::AuthoritiesMarc::GetAuthority($recordid);
379 next if not defined $record;
382 if ($export_remove_fields) {
383 for my $f ( split / /, $export_remove_fields ) {
384 if ( $f =~ m/^(\d{3})(.)?$/ ) {
385 my ( $field, $subfield ) = ( $1, $2 );
387 # skip if this record doesn't have this field
388 if ( defined $record->field($field) ) {
389 if ( defined $subfield ) {
390 my @tags = $record->field($field);
391 foreach my $t (@tags) {
392 $t->delete_subfields($subfield);
396 $record->delete_fields($record->field($field));
402 RemoveAllNsb($record) if ($clean);
403 if ( $output_format eq "xml" ) {
404 unless ($xml_header_written) {
405 MARC::File::XML->default_record_format(
407 $marcflavour eq 'UNIMARC'
408 && $record_type eq 'auths'
409 ) ? 'UNIMARCAUTH' : $marcflavour
411 print MARC::File::XML::header();
413 $xml_header_written = 1;
415 print MARC::File::XML::record($record);
419 my $errorcount_on_decode = eval { scalar(MARC::File::USMARC->decode( $record->as_usmarc )->warnings()) };
420 if ($errorcount_on_decode or $@){
422 warn "record (number $recordid) is invalid and therefore not exported because its reopening generates warnings above";
425 print $record->as_usmarc();
429 if ($xml_header_written) {
430 print MARC::File::XML::footer();
436 elsif ( $output_format eq "csv" ) {
437 my @biblionumbers = uniq $query->param("biblionumbers");
438 my @itemnumbers = $query->param("itemnumbers");
440 marc2csv( \@biblionumbers,
441 GetCsvProfileId( C4::Context->preference('ExportWithCsvProfile') ),
443 print $query->header(
444 -type => 'application/octet-stream',
445 -'Content-Transfer-Encoding' => 'binary',
446 -attachment => "export.csv"
455 my $itemtypes = GetItemTypes;
457 foreach my $thisitemtype ( sort keys %$itemtypes ) {
459 value => $thisitemtype,
460 description => $itemtypes->{$thisitemtype}->{'description'},
462 push @itemtypesloop, \%row;
464 my $branches = GetBranches($limit_ind_branch);
467 sort { $branches->{$a}->{branchname} cmp $branches->{$b}->{branchname} }
473 value => $thisbranch,
474 selected => %branchmap ? $branchmap{$thisbranch} : 1,
475 branchname => $branches->{$thisbranch}->{'branchname'},
479 my $authtypes = getauthtypes;
481 foreach my $thisauthtype ( sort keys %$authtypes ) {
482 next unless $thisauthtype;
484 value => $thisauthtype,
485 description => $authtypes->{$thisauthtype}->{'authtypetext'},
487 push @authtypesloop, \%row;
490 if ( $flags->{superlibrarian}
491 && C4::Context->config('backup_db_via_tools')
495 $template->{VARS}->{'allow_db_export'} = 1;
496 $template->{VARS}->{'dbfiles'} = getbackupfilelist(
497 { directory => "$backupdir", extension => 'sql' } );
500 if ( $flags->{superlibrarian}
501 && C4::Context->config('backup_conf_via_tools')
505 $template->{VARS}->{'allow_conf_export'} = 1;
506 $template->{VARS}->{'conffiles'} = getbackupfilelist(
507 { directory => "$backupdir", extension => 'tar' } );
511 branchloop => \@branchloop,
512 itemtypeloop => \@itemtypesloop,
513 authtypeloop => \@authtypesloop,
514 export_remove_fields => C4::Context->preference("ExportRemoveFields"),
517 output_html_with_http_headers $query, $cookie, $template->output;
520 sub construct_query {
523 my ( $sql_query, @sql_params );
525 if ( $params->{recordtype} eq "bibs" ) {
526 if ( $params->{timestamp} ) {
527 my $biblioitemstable = $params->{biblioitemstable};
530 FROM $biblioitemstable
531 LEFT JOIN items USING(biblionumber)
532 WHERE $biblioitemstable.timestamp >= ?
533 OR items.timestamp >= ?
536 FROM $biblioitemstable
537 LEFT JOIN deleteditems USING(biblionumber)
538 WHERE $biblioitemstable.timestamp >= ?
539 OR deleteditems.timestamp >= ?
541 my $ts = $timestamp->output('iso');
542 @sql_params = ( $ts, $ts, $ts, $ts );
545 my $biblioitemstable = $params->{biblioitemstable};
546 my $itemstable = $params->{itemstable};
547 my $StartingBiblionumber = $params->{StartingBiblionumber};
548 my $EndingBiblionumber = $params->{EndingBiblionumber};
549 my @branch = @{ $params->{branch} };
550 my $start_callnumber = $params->{start_callnumber};
551 my $end_callnumber = $params->{end_callnumber};
552 my $start_accession = $params->{start_accession};
553 my $end_accession = $params->{end_accession};
554 my $itemtype = $params->{itemtype};
561 || ( $itemtype && C4::Context->preference('item-level_itypes') );
562 $sql_query = $items_filter
563 ? "SELECT DISTINCT $biblioitemstable.biblionumber
564 FROM $biblioitemstable JOIN $itemstable
565 USING (biblionumber) WHERE 1"
566 : "SELECT $biblioitemstable.biblionumber FROM $biblioitemstable WHERE biblionumber >0 ";
568 if ($StartingBiblionumber) {
569 $sql_query .= " AND $biblioitemstable.biblionumber >= ? ";
570 push @sql_params, $StartingBiblionumber;
573 if ($EndingBiblionumber) {
574 $sql_query .= " AND $biblioitemstable.biblionumber <= ? ";
575 push @sql_params, $EndingBiblionumber;
579 $sql_query .= " AND homebranch IN (".join(',',map({'?'} @branch)).")";
580 push @sql_params, @branch;
583 if ($start_callnumber) {
584 $sql_query .= " AND itemcallnumber >= ? ";
585 push @sql_params, $start_callnumber;
588 if ($end_callnumber) {
589 $sql_query .= " AND itemcallnumber <= ? ";
590 push @sql_params, $end_callnumber;
592 if ($start_accession) {
593 $sql_query .= " AND dateaccessioned >= ? ";
594 push @sql_params, $start_accession->output('iso');
597 if ($end_accession) {
598 $sql_query .= " AND dateaccessioned <= ? ";
599 push @sql_params, $end_accession->output('iso');
604 ( C4::Context->preference('item-level_itypes') )
605 ? " AND items.itype = ? "
606 : " AND biblioitems.itemtype = ?";
607 push @sql_params, $itemtype;
611 elsif ( $params->{recordtype} eq "auths" ) {
612 if ( $params->{timestamp} ) {
617 my $starting_authid = $params->{starting_authid};
618 my $ending_authid = $params->{ending_authid};
619 my $authtype = $params->{authtype};
621 "SELECT DISTINCT auth_header.authid FROM auth_header WHERE 1";
623 if ($starting_authid) {
624 $sql_query .= " AND auth_header.authid >= ? ";
625 push @sql_params, $starting_authid;
628 if ($ending_authid) {
629 $sql_query .= " AND auth_header.authid <= ? ";
630 push @sql_params, $ending_authid;
634 $sql_query .= " AND auth_header.authtypecode = ? ";
635 push @sql_params, $authtype;
639 return ( $sql_query, \@sql_params );
642 sub getbackupfilelist {
644 my $directory = $args->{directory};
645 my $extension = $args->{extension};
648 if ( opendir( my $dir, $directory ) ) {
649 while ( my $file = readdir($dir) ) {
650 next unless ( $file =~ m/\.$extension(\.(gz|bz2|xz))?/ );
652 if ( -f "$directory/$file" && -r "$directory/$file" );
659 sub download_backup {
661 my $directory = $args->{directory};
662 my $extension = $args->{extension};
663 my $filename = $args->{filename};
665 return unless ( $directory && -d $directory );
666 return unless ( $filename =~ m/\.$extension(\.(gz|bz2|xz))?$/ );
667 return if ( $filename =~ m#/# );
668 $filename = "$directory/$filename";
669 return unless ( -f $filename && -r $filename );
670 return unless ( open( my $dump, '<', $filename ) );
673 while ( read( $dump, my $data, 64 * 1024 ) ) {