6 use C4::AuthoritiesMarc;
10 # script that checks zebradir structure & create directories & mandatory files if needed
14 $|=1; # flushes output
16 # limit for database dumping
17 my $limit;# = "LIMIT 1";
33 $directory = "export" unless $directory;
36 my $biblioserverdir = C4::Context->zebraconfig('biblioserver')->{directory};
37 my $authorityserverdir = C4::Context->zebraconfig('authorityserver')->{directory};
39 my $kohadir = C4::Context->config('intranetdir');
40 my $dbh = C4::Context->dbh;
41 my ($biblionumbertagfield,$biblionumbertagsubfield) = &GetMarcFromKohaField("biblio.biblionumber","");
42 my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = &GetMarcFromKohaField("biblioitems.biblioitemnumber","");
44 print "some informations\n";
45 print "=================\n";
46 print "Zebra biblio directory =>$biblioserverdir\n";
47 print "Zebra authorities directory =>$authorityserverdir\n";
48 print "Koha directory =>$kohadir\n";
49 print "BIBLIONUMBER in : $biblionumbertagfield\$$biblionumbertagsubfield\n";
50 print "BIBLIOITEMNUMBER in : $biblioitemnumbertagfield\$$biblioitemnumbertagsubfield\n";
51 print "=================\n";
53 # creating zebra-biblios.cfg depending on system
56 # getting zebraidx directory
58 foreach (qw(/usr/local/bin/zebraidx
67 unless ($zebraidxdir) {
69 ERROR: could not find zebraidx directory
70 ERROR: Either zebra is not installed,
71 ERROR: or it's in a directory I don't checked.
72 ERROR: do a which zebraidx and edit this file to add the result you get
76 $zebraidxdir =~ s/\/bin\/.*//;
77 print "Info : zebra is in $zebraidxdir \n";
79 # getting modules directory
81 foreach (qw(/usr/local/lib/idzebra-2.0/modules/mod-grs-xml.so
82 /usr/local/lib/idzebra/modules/mod-grs-xml.so
83 /usr/lib/idzebra/modules/mod-grs-xml.so
84 /usr/lib/idzebra-2.0/modules/mod-grs-xml.so
91 unless ($modulesdir) {
93 ERROR: could not find mod-grs-xml.so directory
94 ERROR: Either zebra is not properly compiled (libxml2 is not setup and you don t have mod-grs-xml.so,
95 ERROR: or it's in a directory I don't checked.
96 ERROR: find where mod-grs-xml.so is and edit this file to add the result you get
100 $modulesdir =~ s/\/modules\/.*//;
101 print "Info: zebra modules dir : $modulesdir\n";
103 # getting tab directory
105 foreach (qw(/usr/local/share/idzebra/tab/explain.att
106 /usr/local/share/idzebra-2.0/tab/explain.att
107 /usr/share/idzebra/tab/explain.att
108 /usr/share/idzebra-2.0/tab/explain.att
117 ERROR: could not find explain.att directory
118 ERROR: Either zebra is not properly compiled,
119 ERROR: or it's in a directory I don't checked.
120 ERROR: find where explain.att is and edit this file to add the result you get
124 $tabdir =~ s/\/tab\/.*//;
125 print "Info: tab dir : $tabdir\n";
128 # AUTHORITIES creating directory structure
130 my $created_dir_or_file = 0;
132 print "====================\n";
133 print "checking directories & files for authorities\n";
134 print "====================\n";
135 unless (-d "$authorityserverdir") {
136 system("mkdir -p $authorityserverdir");
137 print "Info: created $authorityserverdir\n";
138 $created_dir_or_file++;
140 unless (-d "$authorityserverdir/lock") {
141 mkdir "$authorityserverdir/lock";
142 print "Info: created $authorityserverdir/lock\n";
143 $created_dir_or_file++;
145 unless (-d "$authorityserverdir/register") {
146 mkdir "$authorityserverdir/register";
147 print "Info: created $authorityserverdir/register\n";
148 $created_dir_or_file++;
150 unless (-d "$authorityserverdir/shadow") {
151 mkdir "$authorityserverdir/shadow";
152 print "Info: created $authorityserverdir/shadow\n";
153 $created_dir_or_file++;
155 unless (-d "$authorityserverdir/tab") {
156 mkdir "$authorityserverdir/tab";
157 print "Info: created $authorityserverdir/tab\n";
158 $created_dir_or_file++;
160 unless (-d "$authorityserverdir/key") {
161 mkdir "$authorityserverdir/key";
162 print "Info: created $authorityserverdir/key\n";
163 $created_dir_or_file++;
166 unless (-d "$authorityserverdir/etc") {
167 mkdir "$authorityserverdir/etc";
168 print "Info: created $authorityserverdir/etc\n";
169 $created_dir_or_file++;
173 # AUTHORITIES : copying mandatory files
175 # the record model, depending on marc flavour
176 unless (-f "$authorityserverdir/tab/record.abs") {
177 if (C4::Context->preference("marcflavour") eq "UNIMARC") {
178 system("cp -f $kohadir/misc/zebra/record_authorities_unimarc.abs $authorityserverdir/tab/record.abs");
179 print "Info: copied record.abs for UNIMARC\n";
181 system("cp -f $kohadir/misc/zebra/record_authorities_usmarc.abs $authorityserverdir/tab/record.abs");
182 print "Info: copied record.abs for USMARC\n";
184 $created_dir_or_file++;
186 unless (-f "$authorityserverdir/tab/sort-string-utf.chr") {
187 system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr $authorityserverdir/tab/sort-string-utf.chr");
188 print "Info: copied sort-string-utf.chr\n";
189 $created_dir_or_file++;
191 unless (-f "$authorityserverdir/tab/word-phrase-utf.chr") {
192 system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr $authorityserverdir/tab/word-phrase-utf.chr");
193 print "Info: copied word-phase-utf.chr\n";
194 $created_dir_or_file++;
196 unless (-f "$authorityserverdir/tab/auth1.att") {
197 system("cp -f $kohadir/misc/zebra/bib1_authorities.att $authorityserverdir/tab/auth1.att");
198 print "Info: copied auth1.att\n";
199 $created_dir_or_file++;
201 unless (-f "$authorityserverdir/tab/default.idx") {
202 system("cp -f $kohadir/misc/zebra/default.idx $authorityserverdir/tab/default.idx");
203 print "Info: copied default.idx\n";
204 $created_dir_or_file++;
207 unless (-f "$authorityserverdir/etc/ccl.properties") {
208 # system("cp -f $kohadir/misc/zebra/ccl.properties ".C4::Context->zebraconfig('authorityserver')->{ccl2rpn});
209 system("cp -f $kohadir/misc/zebra/ccl.properties $authorityserverdir/etc/ccl.properties");
210 print "Info: copied ccl.properties\n";
211 $created_dir_or_file++;
213 unless (-f "$authorityserverdir/etc/pqf.properties") {
214 # system("cp -f $kohadir/misc/zebra/pqf.properties ".C4::Context->zebraconfig('authorityserver')->{ccl2rpn});
215 system("cp -f $kohadir/misc/zebra/pqf.properties $authorityserverdir/etc/pqf.properties");
216 print "Info: copied pqf.properties\n";
217 $created_dir_or_file++;
221 # AUTHORITIES : copying mandatory files
223 unless (-f C4::Context->zebraconfig('authorityserver')->{config}) {
224 open ZD,">:utf8 ",C4::Context->zebraconfig('authorityserver')->{config};
226 # generated by KOHA/misc/migration_tools/rebuild_zebra.pl
227 profilePath:\${srcdir:-.}:$authorityserverdir/tab/:$tabdir/tab/:\${srcdir:-.}/tab/
230 # Files that describe the attribute sets supported.
235 modulePath:$modulesdir/modules/
236 # Specify record type
237 iso2709.recordType:grs.marcxml.record
239 recordId: (auth1,Local-Number)
245 lockDir: $authorityserverdir/lock
250 register: $authorityserverdir/register:4G
251 shadow: $authorityserverdir/shadow:4G
253 # Temp File area for result sets
254 setTmpDir: $authorityserverdir/tmp
256 # Temp File area for index program
257 keyTmpDir: $authorityserverdir/key
259 # Approx. Memory usage during indexing
263 print "Info: creating zebra-authorities.cfg\n";
264 $created_dir_or_file++;
267 if ($created_dir_or_file) {
268 print "Info: created : $created_dir_or_file directories & files\n";
270 print "Info: file & directories OK\n";
274 # exporting authorities
277 print "====================\n";
278 print "SKIPPING authorities export\n";
279 print "====================\n";
281 print "====================\n";
282 print "exporting authorities\n";
283 print "====================\n";
284 mkdir "$directory" unless (-d $directory);
285 mkdir "$directory/authorities" unless (-d "$directory/authorities");
286 open(OUT,">:utf8","$directory/authorities/authorities.iso2709") or die $!;
287 my $dbh=C4::Context->dbh;
289 $sth=$dbh->prepare("select authid from auth_header $limit");
292 while (my ($authid) = $sth->fetchrow) {
295 $record = GetAuthority($authid);
298 print " There was some pb getting authority : ".$authid."\n";
303 print "\r$i" unless ($i++ %100);
304 # remove leader length, that could be wrong, it will be calculated automatically by as_usmarc
305 # otherwise, if it's wron, zebra will fail miserabily (and never index what is after the failing record)
306 my $leader=$record->leader;
307 substr($leader,0,5)=' ';
308 substr($leader,10,7)='22 ';
309 $record->leader(substr($leader,0,24));
310 print OUT $record->as_usmarc();
316 # and reindexing everything
318 print "====================\n";
319 print "REINDEXING zebra\n";
320 print "====================\n";
321 system("zebraidx -c ".C4::Context->zebraconfig('authorityserver')->{config}." -g iso2709 -d authorities init") if ($reset);
322 system("zebraidx -c ".C4::Context->zebraconfig('authorityserver')->{config}." -g iso2709 -d authorities update $directory/authorities");
323 system("zebraidx -c ".C4::Context->zebraconfig('authorityserver')->{config}." -g iso2709 -d authorities commit");
325 print "skipping authorities\n";
327 #################################################################################################################
329 #################################################################################################################
332 print "====================\n";
333 print "checking directories & files for biblios\n";
334 print "====================\n";
337 # BIBLIOS : creating directory structure
339 unless (-d "$biblioserverdir") {
340 system("mkdir -p $biblioserverdir");
341 print "Info: created $biblioserverdir\n";
342 $created_dir_or_file++;
344 unless (-d "$biblioserverdir/lock") {
345 mkdir "$biblioserverdir/lock";
346 print "Info: created $biblioserverdir/lock\n";
347 $created_dir_or_file++;
349 unless (-d "$biblioserverdir/register") {
350 mkdir "$biblioserverdir/register";
351 print "Info: created $biblioserverdir/register\n";
352 $created_dir_or_file++;
354 unless (-d "$biblioserverdir/shadow") {
355 mkdir "$biblioserverdir/shadow";
356 print "Info: created $biblioserverdir/shadow\n";
357 $created_dir_or_file++;
359 unless (-d "$biblioserverdir/tab") {
360 mkdir "$biblioserverdir/tab";
361 print "Info: created $biblioserverdir/tab\n";
362 $created_dir_or_file++;
364 unless (-d "$biblioserverdir/key") {
365 mkdir "$biblioserverdir/key";
366 print "Info: created $biblioserverdir/key\n";
367 $created_dir_or_file++;
369 unless (-d "$biblioserverdir/etc") {
370 mkdir "$biblioserverdir/etc";
371 print "Info: created $biblioserverdir/etc\n";
372 $created_dir_or_file++;
376 # BIBLIOS : copying mandatory files
378 # the record model, depending on marc flavour
379 unless (-f "$biblioserverdir/tab/record.abs") {
380 if (C4::Context->preference("marcflavour") eq "UNIMARC") {
381 system("cp -f $kohadir/misc/zebra/record_biblios_unimarc.abs $biblioserverdir/tab/record.abs");
382 print "Info: copied record.abs for UNIMARC\n";
384 system("cp -f $kohadir/misc/zebra/record_biblios_usmarc.abs $biblioserverdir/tab/record.abs");
385 print "Info: copied record.abs for USMARC\n";
387 $created_dir_or_file++;
389 unless (-f "$biblioserverdir/tab/sort-string-utf.chr") {
390 system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr $biblioserverdir/tab/sort-string-utf.chr");
391 print "Info: copied sort-string-utf.chr\n";
392 $created_dir_or_file++;
394 unless (-f "$biblioserverdir/tab/word-phrase-utf.chr") {
395 system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr $biblioserverdir/tab/word-phrase-utf.chr");
396 print "Info: copied word-phase-utf.chr\n";
397 $created_dir_or_file++;
399 unless (-f "$biblioserverdir/tab/bib1.att") {
400 system("cp -f $kohadir/misc/zebra/bib1_biblios.att $biblioserverdir/tab/bib1.att");
401 print "Info: copied bib1.att\n";
402 $created_dir_or_file++;
404 unless (-f "$biblioserverdir/tab/default.idx") {
405 system("cp -f $kohadir/misc/zebra/default.idx $biblioserverdir/tab/default.idx");
406 print "Info: copied default.idx\n";
407 $created_dir_or_file++;
409 unless (-f "$biblioserverdir/etc/ccl.properties") {
410 # system("cp -f $kohadir/misc/zebra/ccl.properties ".C4::Context->zebraconfig('biblioserver')->{ccl2rpn});
411 system("cp -f $kohadir/misc/zebra/ccl.properties $biblioserverdir/etc/ccl.properties");
412 print "Info: copied ccl.properties\n";
413 $created_dir_or_file++;
415 unless (-f "$biblioserverdir/etc/pqf.properties") {
416 # system("cp -f $kohadir/misc/zebra/pqf.properties ".C4::Context->zebraconfig('biblioserver')->{ccl2rpn});
417 system("cp -f $kohadir/misc/zebra/pqf.properties $biblioserverdir/etc/pqf.properties");
418 print "Info: copied pqf.properties\n";
419 $created_dir_or_file++;
423 # BIBLIOS : copying mandatory files
425 unless (-f C4::Context->zebraconfig('biblioserver')->{config}) {
426 open ZD,">:utf8 ",C4::Context->zebraconfig('biblioserver')->{config};
428 # generated by KOHA/misc/migrtion_tools/rebuild_zebra.pl
429 profilePath:\${srcdir:-.}:$biblioserverdir/tab/:$tabdir/tab/:\${srcdir:-.}/tab/
432 # Files that describe the attribute sets supported.
437 modulePath:$modulesdir/modules/
438 # Specify record type
439 iso2709.recordType:grs.marcxml.record
441 recordId: (bib1,Local-Number)
447 lockDir: $biblioserverdir/lock
452 register: $biblioserverdir/register:4G
453 shadow: $biblioserverdir/shadow:4G
455 # Temp File area for result sets
456 setTmpDir: $biblioserverdir/tmp
458 # Temp File area for index program
459 keyTmpDir: $biblioserverdir/key
461 # Approx. Memory usage during indexing
465 print "Info: creating zebra-biblios.cfg\n";
466 $created_dir_or_file++;
469 if ($created_dir_or_file) {
470 print "Info: created : $created_dir_or_file directories & files\n";
472 print "Info: file & directories OK\n";
480 print "====================\n";
481 print "SKIPPING biblio export\n";
482 print "====================\n";
484 print "====================\n";
485 print "exporting biblios\n";
486 print "====================\n";
487 mkdir "$directory" unless (-d $directory);
488 mkdir "$directory/biblios" unless (-d "$directory/biblios");
489 open(OUT,">:utf8 ","$directory/biblios/export") or die $!;
490 my $dbh=C4::Context->dbh;
492 $sth=$dbh->prepare("select biblionumber from biblioitems order by biblionumber $limit");
495 while (my ($biblionumber) = $sth->fetchrow) {
498 $record = GetMarcBiblio($biblionumber);
501 print " There was some pb getting biblio : #".$biblionumber."\n";
504 # warn $record->as_formatted;
505 # die if $record->subfield('090','9') eq 11;
507 # check that biblionumber & biblioitemnumber are stored in the MARC record, otherwise, add them & update the biblioitems.marcxml data.
508 my $record_correct=1;
509 next unless $record->field($biblionumbertagfield);
510 if ($biblionumbertagfield eq '001') {
511 unless ($record->field($biblionumbertagfield)->data()) {
514 # if the field where biblionumber is already exist, just update it, otherwise create it
515 if ($record->field($biblionumbertagfield)) {
516 $field = $record->field($biblionumbertagfield);
517 $field->update($biblionumber);
520 $newfield = MARC::Field->new( $biblionumbertagfield, $biblionumber);
521 $record->append_fields($newfield);
525 unless ($record->subfield($biblionumbertagfield,$biblionumbertagsubfield)) {
528 # if the field where biblionumber is already exist, just update it, otherwise create it
529 if ($record->field($biblionumbertagfield)) {
530 $field = $record->field($biblionumbertagfield);
531 $field->add_subfields($biblionumbertagsubfield => $biblionumber);
534 $newfield = MARC::Field->new( $biblionumbertagfield,'','', $biblionumbertagsubfield => $biblionumber);
535 $record->append_fields($newfield);
538 # warn "FIXED BIBLIONUMBER".$record->as_formatted;
540 unless ($record->subfield($biblioitemnumbertagfield,$biblioitemnumbertagsubfield)) {
542 # warn "INCORRECT BIBLIOITEMNUMBER :".$record->as_formatted;
544 # if the field where biblionumber is already exist, just update it, otherwise create it
545 if ($record->field($biblioitemnumbertagfield)) {
546 $field = $record->field($biblioitemnumbertagfield);
547 if ($biblioitemnumbertagfield <10) {
548 $field->update($biblionumber);
550 $field->add_subfields($biblioitemnumbertagsubfield => $biblionumber);
554 if ($biblioitemnumbertagfield <10) {
555 $newfield = MARC::Field->new( $biblioitemnumbertagfield, $biblionumber);
557 $newfield = MARC::Field->new( $biblioitemnumbertagfield,'','', $biblioitemnumbertagsubfield => $biblionumber);
559 $record->insert_grouped_field($newfield);
561 # warn "FIXED BIBLIOITEMNUMBER".$record->as_formatted;
563 unless ($record_correct) {
564 my $update_xml = $dbh->prepare("update biblioitems set marcxml=? where biblionumber=?");
565 warn "UPDATING $biblionumber (missing biblionumber or biblioitemnumber in MARC record : ".$record->as_xml;
566 $update_xml->execute($record->as_xml,$biblionumber);
569 print "\r$i" unless ($i++ %100);
570 # remove leader length, that could be wrong, it will be calculated automatically by as_usmarc
571 # otherwise, if it's wron, zebra will fail miserabily (and never index what is after the failing record)
572 my $leader=$record->leader;
573 substr($leader,0,5)=' ';
574 substr($leader,10,7)='22 ';
575 $record->leader(substr($leader,0,24));
576 print OUT $record->as_usmarc();
582 # and reindexing everything
584 print "====================\n";
585 print "REINDEXING zebra\n";
586 print "====================\n";
587 system("zebraidx -g iso2709 -c ".C4::Context->zebraconfig('biblioserver')->{config}." -d biblios init") if ($reset);
588 system("zebraidx -g iso2709 -c ".C4::Context->zebraconfig('biblioserver')->{config}." -d biblios update $directory/biblios");
589 system("zebraidx -g iso2709 -c ".C4::Context->zebraconfig('biblioserver')->{config}." -d biblios commit");
591 print "skipping biblios\n";
594 print "====================\n";
596 print "====================\n";
598 print "NOTHING cleaned : the $directory has been kept. You can re-run this script with the -s parameter if you just want to rebuild zebra after changing the record.abs or another zebra config file\n";
600 system("rm -rf $directory");
601 print "directory $directory deleted\n";