2 # Import an iso2709 file into Koha 3
8 # find Koha's Perl modules
9 # test carefully before changing this
11 eval { require "$FindBin::Bin/../kohalib.pl" };
15 use MARC::File::USMARC;
25 use Unicode::Normalize;
26 use Time::HiRes qw(gettimeofday);
30 binmode(STDOUT, ":utf8");
32 my ( $input_marc_file, $number, $offset) = ('',0,0);
33 my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format);
34 my ($sourcetag,$sourcesubfield,$idmapfl);
39 'commit:f' => \$commit,
40 'file:s' => \$input_marc_file,
42 'o|offset:f' => \$offset,
45 't' => \$test_parameter,
46 's' => \$skip_marc8_conversion,
47 'c:s' => \$char_encoding,
52 'y:s' => \$sourcesubfield,
53 'idmap:s' => \$idmapfl,
56 if ($version || ($input_marc_file eq '')) {
58 Small script to import bibliographic records into Koha.
61 h this version/help screen
62 file /path/to/file/to/dump: the file to import
63 v verbose mode. 1 means "some infos", 2 means "MARC dumping"
64 fk Turn off foreign key checks during import.
65 n the number of records to import. If missing, all the file is imported
66 o file offset before importing, ie number of records to skip.
67 commit the number of records to wait before performing a 'commit' operation
68 t test mode: parses the file, saying what he would do, but doing nothing.
69 s skip automatic conversion of MARC-8 to UTF-8. This option is
70 provided for debugging.
71 c the characteristic MARC flavour. At the moment, only MARC21 and
72 UNIMARC are supported. MARC21 by default.
73 d delete EVERYTHING related to biblio in koha-DB before import. Tables:
74 biblio, biblioitems, titems
75 m format, MARCXML or ISO2709 (defaults to ISO2709)
76 x source bib tag for reporting the source bib number
77 y source subfield for reporting the source bib number
78 idmap file for the koha bib and source id
80 IMPORTANT: don't use this script before you've entered and checked your MARC
81 parameters tables twice (or more!). Otherwise, the import won't work
82 correctly and you will get invalid data.
85 \$ export KOHA_CONF=/etc/koha.conf
86 \$ perl misc/migration_tools/bulkmarcimport.pl -d -commit 1000 \\
87 -file /home/jmf/koha.mrc -n 3000
93 if (defined $idmapfl) {
94 open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
97 if ((not defined $sourcesubfield) && (not defined $sourcetag)){
102 my $dbh = C4::Context->dbh;
104 # save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import &
105 # will create problems in the action_logs table, that can't handle more than 1 entry per second per user.
106 my $CataloguingLog = C4::Context->preference('CataloguingLog');
107 $dbh->do("UPDATE systempreferences SET value=0 WHERE variable='CataloguingLog'");
110 $dbh->do("SET FOREIGN_KEY_CHECKS = 0");
115 print "deleting biblios\n";
116 $dbh->do("truncate biblio");
117 $dbh->do("truncate biblioitems");
118 $dbh->do("truncate items");
119 $dbh->do("truncate zebraqueue");
124 if ($test_parameter) {
125 print "TESTING MODE ONLY\n DOING NOTHING\n===============\n";
128 my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';
130 print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;
131 my $starttime = gettimeofday;
133 my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
134 if ($format =~ /XML/i) {
135 # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
136 # appears to try to convert incoming XML records from MARC-8
137 # to UTF-8. Setting the BinaryEncoding key turns that off
138 # TODO: see what happens to ISO-8859-1 XML files.
139 # TODO: determine if MARC::Batch can be fixed to handle
140 # XML records properly -- it probably should be
141 # be using a proper push or pull XML parser to
142 # extract the records, not using regexes to look
143 # for <record>.*</record>.
144 $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
145 $batch = MARC::Batch->new( 'XML', $fh );
147 $batch = MARC::Batch->new( 'USMARC', $fh );
149 $batch->warnings_off();
150 $batch->strict_off();
152 my $commitnum = $commit ? $commit : 50;
157 print "Skipping file offset: $offset records\n";
158 $batch->next() while ($offset--);
161 $dbh->{AutoCommit} = 0;
164 eval { $record = $batch->next() };
166 print "Bad MARC record: skipped\n";
167 # FIXME - because MARC::Batch->next() combines grabbing the next
168 # blob and parsing it into one operation, a correctable condition
169 # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
170 # from because we don't have access to the original blob. Note
171 # that the staging import can deal with this condition (via
172 # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
175 last unless ( $record );
178 print "\r$i" unless $i % 100;
180 if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
181 # FIXME update condition
182 my ($guessed_charset, $charset_errors);
183 ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour);
184 if ($guessed_charset eq 'failed') {
185 warn "ERROR: failed to perform character conversion for record $i\n";
190 unless ($test_parameter) {
191 my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
192 eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, '', { defer_marc_save => 1 }) };
194 warn "ERROR: Adding biblio $biblionumber failed: $@\n";
197 if (defined $idmapfl) {
198 if ($sourcetag lt '010'){
199 if ($record->field($sourcetag)){
200 my $source = $record->field($sourcetag)->data();
201 printf(IDMAP "%s|%s\n",$source,$biblionumber);
204 my $source=$record->subfield($sourcetag,$sourcesubfield);
205 printf(IDMAP "%s|%s\n",$source,$biblionumber);
209 eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
211 warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
212 # if we failed because of an exception, assume that
213 # the MARC columns in biblioitems were not set.
214 ModBiblioMarc( $record, $biblionumber, '' );
217 if ($#{ $errors_ref } > -1) {
218 report_item_errors($biblionumber, $errors_ref);
221 $dbh->commit() if (0 == $i % $commitnum);
223 last if $i == $number;
229 $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
232 # restore CataloguingLog
233 $dbh->do("UPDATE systempreferences SET value=$CataloguingLog WHERE variable='CataloguingLog'");
235 my $timeneeded = gettimeofday - $starttime;
236 print "\n$i MARC records done in $timeneeded seconds\n";
240 sub report_item_errors {
241 my $biblionumber = shift;
242 my $errors_ref = shift;
244 foreach my $error (@{ $errors_ref }) {
245 my $msg = "Item not added (bib $biblionumber, item tag #$error->{'item_sequence'}, barcode $error->{'item_barcode'}): ";
246 my $error_code = $error->{'error_code'};
247 $error_code =~ s/_/ /g;
248 $msg .= "$error_code $error->{'error_information'}";