#!/usr/bin/perl

use C4::Context;
use Getopt::Long;
use C4::Biblio;
use C4::AuthoritiesMarc;

use strict;
#use warnings; FIXME - Bug 2505
# 
# script that fills the nozebra table
#
#

$|=1; # flushes output

# limit for database dumping
my $limit;# = "LIMIT 100";
my $directory;
#my $skip_export;
#my $keep_export;
#my $reset;
#my $biblios;
my $authorities;
my $sysprefs;
my $commit;
my $want_help;

my $result = GetOptions(
    'd:s'      => \$directory,
#    'reset'      => \$reset,
#    's'        => \$skip_export,    # Not used and conflicts with 's' option some lines below for sysprefs!!!
#    'k'        => \$keep_export,
#    'b'        => \$biblios,
#    'a'        => \$authorities,
    's'        => \$sysprefs,  # rebuild 'NoZebraIndexes' syspref
    'h|help'        => \$want_help,
   'commit:f'    => \$commit,
    );

if (not $result or $want_help) {
    print_usage();
    exit 0;
}


sub print_usage {
    print <<_USAGE_;
$0: reindex MARC bibs and authorities if NOT using Zebra ("NoZebra").

Use this batch job to reindex all biblio and authority
records in your Koha database.  This job is useful
only if you are NOT using Zebra ('NoZebra'); if you are 
using the 'Zebra'mode, this job should NOT be used.

Parameters:
    -d                      Temporary directory for indexing.
                            If not specified, one is automatically
                            created.  The export directory
                            is automatically deleted unless
                            you supply the -k switch.

    -s                      Rebuild "NoZebraIndexes" System Preference 

    --help or -h            show this message.
_USAGE_
}   # END of print_usage sub


my $commitnum = 1000; 
$commitnum = $commit if ($commit) ;

$directory = "export" unless $directory;
my $dbh=C4::Context->dbh;
$dbh->do("update systempreferences set value=1 where variable='NoZebra'");

$dbh->do("truncate nozebra");

my %index = GetNoZebraIndexes();

if  (!%index || $sysprefs ) {
    if (C4::Context->preference('marcflavour') eq 'UNIMARC') {
        $dbh->do("UPDATE systempreferences SET value=\"'title' => '200a,200c,200d,200e,225a,225d,225e,225f,225h,225i,225v,500*,501*,503*,510*,512*,513*,514*,515*,516*,517*,518*,519*,520*,530*,531*,532*,540*,541*,545*,604t,610t,605a',
        'author' =>'200f,600a,601a,604a,700a,700b,700c,700d,700a,701b,701c,701d,702a,702b,702c,702d,710a,710b,710c,710d,711a,711b,711c,711d,712a,712b,712c,712d',
        'isbn' => '010a',
        'issn' => '011a',
        'biblionumber' =>'0909',
        'itemtype' => '200b',
        'language' => '101a',
        'publisher' => '210c',
        'date' => '210d',
        'note' => '300a,301a,302a,303a,304a,305a,306az,307a,308a,309a,310a,311a,312a,313a,314a,315a,316a,317a,318a,319a,320a,321a,322a,323a,324a,325a,326a,327a,328a,330a,332a,333a,336a,337a,345a',
        'Koha-Auth-Number' => '6009,6019,6029,6039,6049,6059,6069,6109,7009,7019,7029,7109,7119,7129',
        'subject' => '600*,601*,606*,610*',
        'dewey' => '676a',
        'host-item' => '995a,995c',\" where variable='NoZebraIndexes'");
        %index = GetNoZebraIndexes();
    } elsif (C4::Context->preference('marcflavour') eq 'MARC21') {
        $dbh->do("UPDATE systempreferences SET value=\"
'title' => '130a,210a,222a,240a,243a,245a,245b,246a,246b,247a,247b,250a,250b,440a,830a',
'author' => '100a,100b,100c,100d,110a,111a,111b,111c,111d,245c,700a,710a,711a,800a,810a,811a',
'isbn' => '020a',
'issn' => '022a',
'lccn' => '010a',
'biblionumber => '999c',
'itemtype' => '942c',
'publisher' => '260b',
'date' => '260c',
'note' => '500a, 501a,504a,505a,508a,511a,518a,520a,521a,522a,524a,526a,530a,533a,538a,541a,546a,555a,556a,562a,563a,583a,585a,582a',
'subject' => '600*,610*,611*,630*,650*,651*,653*,654*,655*,662*,690*',
'dewey' => '082',
'bc' => '952p',
'callnum' => '952o',
'an' => '6009,6109,6119',
'series' => 440*,490*,
'host-item' => '9529
'shelf' => '952c',
'collection' => '9528',
\"WHERE variable='NoZebraIndexes'");

        %index = GetNoZebraIndexes();
    }
}
$|=1;

$dbh->{AutoCommit} = 0;

print "***********************************\n";
print "***** building BIBLIO indexes *****\n";
print "***********************************\n";

my $sth;
$sth=$dbh->prepare("select biblionumber from biblioitems order by biblionumber $limit");
$sth->execute();
my $i=0;
my %result;
while (my ($biblionumber) = $sth->fetchrow) {
        $i++;
        print "\r$i";
        my  $record;
    eval{
            $record = GetMarcBiblio($biblionumber);
    };
    if($@){
            print "  There was some pb getting biblionumber : ".$biblionumber."\n";
            next;
    }
    next unless $record;
    # get title of the record (to store the 10 first letters with the index)
    my ($titletag,$titlesubfield) = GetMarcFromKohaField('biblio.title', '');
    my $title = lc($record->subfield($titletag,$titlesubfield));

    # remove blancks comma (that could cause problem when decoding the string for CQL retrieval) and regexp specific values
    $title =~ s/ |\.|,|;|\[|\]|\(|\)|\*|-|'|=|://g;
    # limit to 10 char, should be enough, and limit the DB size
    $title = substr($title,0,10);
    #parse each field
    foreach my $field ($record->fields()) {
        #parse each subfield
        next if $field->tag <10;
        foreach my $subfield ($field->subfields()) {
            my $tag = $field->tag();
            my $subfieldcode = $subfield->[0];
            my $indexed=0;
            # check each index to see if the subfield is stored somewhere
            # otherwise, store it in __RAW__ index
            foreach my $key (keys %index) {
                if ($index{$key} =~ /\Q$tag\E\*/ or $index{$key} =~ /\Q$tag$subfieldcode\E/) {
                    $indexed=1;
                    my $line= lc $subfield->[1];
                    # remove meaningless value in the field...
                    $line =~ s/-|\.|\?|,|;|!|'|\(|\)|\[|\]|{|}|"|<|>|&|\+|\*|\/|=|:/ /g;
                    # ... and split in words
                    foreach (split / /,$line) {
                        next unless $_; # skip  empty values (multiple spaces)
                        # remove any accented char
                        # if the entry is already here, improve weight
                        if ($result{$key}->{"$_"} =~ /$biblionumber,\Q$title\E\-(\d);/) {
                            my $weight=$1+1;
                            $result{$key}->{"$_"} =~ s/$biblionumber,\Q$title\E\-(\d);//;
                            $result{$key}->{"$_"} .= "$biblionumber,$title-$weight;";
                        # otherwise, create it, with weight=1
                        } else {
                            $result{$key}->{"$_"}.="$biblionumber,$title-1;";
                        }
                    }
                }
            }
            # the subfield is not indexed, store it in __RAW__ index anyway
            unless ($indexed) {
                my $line= lc $subfield->[1];
                $line =~ s/-|\.|\?|,|;|!|'|\(|\)|\[|\]|{|}|"|<|>|&|\+|\*|\/|=/ /g;
                foreach (split / /,$line) {
                        next unless $_;
#                     warn $record->as_formatted."$_ =>".$title;
                        if ($result{__RAW__}->{"$_"} =~ /$biblionumber,\Q$title\E\-(\d);/) {
                            my $weight=$1+1;
#                             $weight++;
                            $result{__RAW__}->{"$_"} =~ s/$biblionumber,\Q$title\E\-(\d);//;
                            $result{__RAW__}->{"$_"} .= "$biblionumber,$title-$weight;";
                        } else {
                            $result{__RAW__}->{"$_"}.="$biblionumber,$title-1;";
                        }
                }
            }
        }
    }
}


print "\nInserting records...\n";
$i=0;

my $commitnum = 100;
$dbh->{AutoCommit} = 0;

$sth = $dbh->prepare("INSERT INTO nozebra (server,indexname,value,biblionumbers) VALUES ('biblioserver',?,?,?)");
foreach my $key (keys %result) {
    foreach my $index (keys %{$result{$key}}) {
        if (length($result{$key}->{$index}) > 1000000) {
            print "very long index (".length($result{$key}->{$index}).")for $key / $index. update mySQL config file if you have an error just after this warning (max_paquet_size parameter)\n";
        }
        print "\r$i";
        $i++;
        $sth->execute($key,$index,$result{$key}->{$index});
        $dbh->commit() if (0 == $i % $commitnum);
    }
   $dbh->commit() if (0 == $i % $commitnum);
}
$dbh->commit;


print "\nbiblios done\n";

print "\n***********************************\n";
print "***** building AUTHORITIES indexes *****\n";
print "***********************************\n";

$sth=$dbh->prepare("select authid from auth_header order by authid $limit");
$sth->execute();
$i=0;
%result = ();
while (my ($authid) = $sth->fetchrow) {
    $i++;
    print "\r$i";
    my $record;
    eval{
        $record = GetAuthority($authid);
    };
    if($@){
        print "  There was some pb getting authnumber : ".$authid."\n";
        next;
    }
    
    my %index;
    # for authorities, the "title" is the $a mainentry
    my $authref = C4::AuthoritiesMarc::GetAuthType(C4::AuthoritiesMarc::GetAuthTypeCode($authid));

    warn "ERROR : authtype undefined for ".$record->as_formatted unless $authref;
    my $title = $record->subfield($authref->{auth_tag_to_report},'a');
    $index{'mainmainentry'}= $authref->{'auth_tag_to_report'}.'a';
    $index{'mainentry'}    = $authref->{'auth_tag_to_report'}.'*';
    $index{'auth_type'}    = '152b';

    # remove blancks comma (that could cause problem when decoding the string for CQL retrieval) and regexp specific values
    $title =~ s/ |\.|,|;|\[|\]|\(|\)|\*|-|'|:|=//g;
    $title = quotemeta $title;
    # limit to 10 char, should be enough, and limit the DB size
    $title = substr($title,0,10);
    #parse each field
    foreach my $field ($record->fields()) {
        #parse each subfield
        next if $field->tag <10;
        foreach my $subfield ($field->subfields()) {
            my $tag = $field->tag();
            my $subfieldcode = $subfield->[0];
            my $indexed=0;
            # check each index to see if the subfield is stored somewhere
            # otherwise, store it in __RAW__ index
            foreach my $key (keys %index) {
                if ($index{$key} =~ /$tag\*/ or $index{$key} =~ /$tag$subfieldcode/) {
                    $indexed=1;
                    my $line= lc $subfield->[1];
                    # remove meaningless value in the field...
                    $line =~ s/-|\.|\?|,|;|!|'|\(|\)|\[|\]|{|}|"|<|>|&|\+|\*|\/|=|:/ /g;
                    # ... and split in words
                    foreach (split / /,$line) {
                        next unless $_; # skip  empty values (multiple spaces)
                        # if the entry is already here, improve weight
                        if ($result{$key}->{"$_"} =~ /$authid,$title\-(\d);/) {
                            my $weight=$1+1;
                            $result{$key}->{"$_"} =~ s/$authid,$title\-(\d);//;
                            $result{$key}->{"$_"} .= "$authid,$title-$weight;";
                        # otherwise, create it, with weight=1
                        } else {
                            $result{$key}->{"$_"}.="$authid,$title-1;";
                        }
                    }
                }
            }
            # the subfield is not indexed, store it in __RAW__ index anyway
            unless ($indexed) {
                my $line= lc $subfield->[1];
                $line =~ s/-|\.|\?|,|;|!|'|\(|\)|\[|\]|{|}|"|<|>|&|\+|\*|\/|=/ /g;
                foreach (split / /,$line) {
                        next unless $_;
#                     warn $record->as_formatted."$_ =>".$title;
                        if ($result{__RAW__}->{"$_"} =~ /$authid,$title\-(\d);/) {
                            my $weight=$1+1;
#                             $weight++;
                            $result{__RAW__}->{"$_"} =~ s/$authid,$title\-(\d);//;
                            $result{__RAW__}->{"$_"} .= "$authid,$title-$weight;";
                        } else {
                            $result{__RAW__}->{"$_"}.="$authid,$title-1;";
                        }
                }
            }
        }
    }
}



print "\nInserting...\n";
$i=0;

my $commitnum = 100;
$dbh->{AutoCommit} = 0;
$sth = $dbh->prepare("INSERT INTO nozebra (server,indexname,value,biblionumbers) VALUES ('authorityserver',?,?,?)");
foreach my $key (keys %result) {
    foreach my $index (keys %{$result{$key}}) {
        if (length($result{$key}->{$index}) > 1000000) {
            print "very long index (".length($result{$key}->{$index}).")for $key / $index. update mySQL config file if you have an error just after this warning (max_paquet_size parameter)\n";
        }
        print "\r$i";
        $i++;
        $sth->execute($key,$index,$result{$key}->{$index});
        $dbh->commit() if (0 == $i % $commitnum);
    }
   $dbh->commit() if (0 == $i % $commitnum);
}
$dbh->commit;
print "\nauthorities done\n";