2 # small script that import an iso2709 file into koha 2.0
7 # find Koha's Perl modules
8 # test carefully before changing this
10 eval { require "$FindBin::Bin/../kohalib.pl" };
14 use MARC::File::USMARC;
15 # Uncomment the line below and use MARC::File::XML again when it works better.
17 # use MARC::File::XML;
22 # According to kados, an undocumented feature of setting MARC::Charset to
23 # ignore_errors(1) is that errors are not ignored. Instead of deleting the
24 # whole subfield when a character does not translate properly from MARC8 into
25 # UTF-8, just the problem characters are deleted. This should solve at least
26 # some of the fixme problems for fMARC8ToUTF8().
28 # Problems remain if there are MARC 21 records where 000/09 is set incorrectly.
30 # MARC::Charset->ignore_errors(1);
34 use Unicode::Normalize;
35 use Time::HiRes qw(gettimeofday);
37 binmode(STDOUT, ":utf8");
41 my ( $input_marc_file, $number) = ('',0);
42 my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off);
47 'commit:f' => \$commit,
48 'file:s' => \$input_marc_file,
52 't' => \$test_parameter,
53 's' => \$skip_marc8_conversion,
54 'c:s' => \$char_encoding,
59 # FIXME: Management of error conditions needed for record parsing problems
60 # and MARC8 character sets with mappings to Unicode not yet included in
61 # MARC::Charset. The real world rarity of these problems is not fully tested.
62 # Unmapped character sets will throw a warning currently and processing will
63 # continue with the error condition. A fairly trivial correction should
64 # address some record parsing and unmapped character set problems but I need
65 # time to implement a test and correction for undef subfields and revert to
66 # MARC8 if mappings are missing. -- thd
67 sub fMARC8ToUTF8($$) {
69 my ($verbose) = shift;
71 foreach my $field ($record->fields()) {
72 if ($field->is_control_field()) {
73 ; # do nothing -- control fields should not contain non-ASCII characters
76 my $fieldName = $field->tag();
77 my $indicator1Value = $field->indicator(1);
78 my $indicator2Value = $field->indicator(2);
79 foreach my $subfield ($field->subfields()) {
80 my $subfieldName = $subfield->[0];
81 my $subfieldValue = $subfield->[1];
82 my $utf8sf = MARC::Charset::marc8_to_utf8($subfieldValue);
83 unless (defined $utf8sf) {
84 # For now, we're being very strict about
85 # error during the MARC8 conversion, so return
86 # if there's a problem.
89 $subfieldValue = NFC($utf8sf); # Normalization Form C to assist
90 # some browswers (e.g., Firefox on OS X)
91 # that have issues with decomposed characters
94 # Alas, MARC::Field::update() does not work correctly.
95 ## push (@subfieldsArray, $subfieldName, $subfieldValue);
97 push @subfieldsArray, [$subfieldName, $subfieldValue];
100 # Alas, MARC::Field::update() does not work correctly.
102 # The first instance in the field of a of a repeated subfield
103 # overwrites the content from later instances with the content
104 # from the first instance.
105 ## $field->update(@subfieldsArray);
107 foreach my $subfieldRow(@subfieldsArray) {
108 my $subfieldName = $subfieldRow->[0];
109 $field->delete_subfields($subfieldName);
111 foreach my $subfieldRow(@subfieldsArray) {
112 $field->add_subfields(@$subfieldRow);
117 # Reading the indicator values again is not necessary.
118 # They were not converted.
119 # $indicator1Value = $field->indicator(1);
120 # $indicator2Value = $field->indicator(2);
121 # $indicator1Value =~ s/ /#/;
122 # $indicator2Value =~ s/ /#/;
123 print "\nCONVERTED TO UTF-8:\n" . $fieldName . ' ' .
126 foreach my $subfield ($field->subfields()) {
127 my $subfieldName = $subfield->[0];
128 my $subfieldValue = $subfield->[1];
129 print " \$" . $subfieldName . ' ' . $subfieldValue;
135 print "\n" if $verbose;
141 # must set Leader/09 to 'a' to indicate that
142 # record is now in UTF-8
143 my $leader = $record->leader();
144 substr($leader, 9, 1) = 'a';
145 $record->leader($leader);
147 $record->encoding('UTF-8');
152 if ($version || ($input_marc_file eq '')) {
154 small script to import an iso2709 file into Koha.
156 \th : this version/help screen
157 \tfile /path/to/file/to/dump : the file to import
158 \tv : verbose mode. 1 means "some infos", 2 means "MARC dumping"
159 \tfk : Turn off foreign key checks during import.
160 \tn : the number of records to import. If missing, all the file is imported
161 \tcommit : the number of records to wait before performing a 'commit' operation
162 \tt : test mode : parses the file, saying what he would do, but doing nothing.
163 \ts : skip automatic conversion of MARC-8 to UTF-8. This option is
164 \t provided for debugging.
165 \tc : the characteristic MARC flavour. At the moment, only MARC21 and UNIMARC
166 \tsupported. MARC21 by default.
167 \td : delete EVERYTHING related to biblio in koha-DB before import :tables :
168 \t\tbiblio, \tbiblioitems,\titems
169 IMPORTANT : don't use this script before you've entered and checked your MARC parameters tables twice (or more!).
170 Otherwise, the import won't work correctly and you will get invalid data.
173 \t\$ export KOHA_CONF=/etc/koha.conf
174 \t\$ perl misc/migration_tools/bulkmarcimport.pl -d -commit 1000 -file /home/jmf/koha.mrc -n 3000
180 my $dbh = C4::Context->dbh;
182 # save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import &
183 # will create problems in the action_logs table, that can't handle more than 1 entry per second per user.
184 my $CataloguingLog = C4::Context->preference('CataloguingLog');
185 $dbh->do("UPDATE systempreferences SET value=0 WHERE variable='CataloguingLog'");
188 print "deleting biblios\n";
189 $dbh->do("truncate biblio");
190 $dbh->do("truncate biblioitems");
191 $dbh->do("truncate items");
194 $dbh->do("SET FOREIGN_KEY_CHECKS = 0");
196 if ($test_parameter) {
197 print "TESTING MODE ONLY\n DOING NOTHING\n===============\n";
200 my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';
202 print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;
203 my $starttime = gettimeofday;
204 my $batch = MARC::Batch->new( 'USMARC', $input_marc_file );
205 $batch->warnings_off();
206 $batch->strict_off();
212 $commitnum = $commit;
216 my $dbh = C4::Context->dbh();
217 $dbh->{AutoCommit} = 0;
218 RECORD: while ( my $record = $batch->next() ) {
221 print "\r$i" unless $i % 100;
223 if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
224 unless (fMARC8ToUTF8($record, $verbose)) {
225 warn "ERROR: failed to perform character conversion for record $i\n";
230 unless ($test_parameter) {
231 my ( $bibid, $oldbibitemnum, $itemnumbers_ref, $errors_ref );
232 eval { ( $bibid, $oldbibitemnum, $itemnumbers_ref, $errors_ref ) = AddBiblioAndItems( $record, '' ); };
234 warn "ERROR: Adding biblio and or items $bibid failed: $@\n";
236 if ($#{ $errors_ref } > -1) {
237 report_item_errors($bibid, $errors_ref);
240 $dbh->commit() if (0 == $i % $commitnum);
242 last if $i == $number;
248 $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
251 # restore CataloguingLog
252 $dbh->do("UPDATE systempreferences SET value=$CataloguingLog WHERE variable='CataloguingLog'");
254 my $timeneeded = gettimeofday - $starttime;
255 print "$i MARC records done in $timeneeded seconds\n";
259 sub report_item_errors {
261 my $errors_ref = shift;
263 foreach my $error (@{ $errors_ref }) {
264 my $msg = "Item not added (bib $bibid, item tag #$error->{'item_sequence'}, barcode $error->{'item_barcode'}): ";
265 my $error_code = $error->{'error_code'};
266 $error_code =~ s/_/ /g;
267 $msg .= "$error_code $error->{'error_information'}";