From c2066bcadc3b691b4cbf3684c72e9d57e5989d81 Mon Sep 17 00:00:00 2001 From: Kyle M Hall Date: Mon, 21 Jul 2014 13:50:08 -0400 Subject: [PATCH] Bug 12598: New misc/import_borrowers.pl command line tool Test Plan: 1) Apply this patch 2) Test importing patrons from command line, options are availble with --help. Signed-off-by: Bernardo Gonzalez Kriegel Tested with minimal csv (cardnumber,surname,firstname,categorycode,branchcode,password,userid) Overwrite does not change category or branch. Patrons are loaded, userid & password works Updated license to GPLv3 No other koha-qa errors. Signed-off-by: Benjamin Rokseth Signed-off-by: Kyle M Hall Bug 12598 - Tidy import_borrowers.pl Signed-off-by: Benjamin Rokseth Signed-off-by: Kyle M Hall Bug 12598 - Move importing code to a subroutine Signed-off-by: Benjamin Rokseth Signed-off-by: Kyle M Hall Bug 12598 - Update command line script to use patron import subroutine Signed-off-by: Benjamin Rokseth Signed-off-by: Kyle M Hall Bug 12598 [QA Followup] * Fix copyright on import_borrowers.pl * Changes -c --csv to -f --file * Adds -c --confirm option * Renames misc/import_borrowers.pl to misc/import_patrons.pl * Restore userid matchpoint option Signed-off-by: Kyle M Hall Bug 12598 - Fix merge to master. Backport 3 updates from latest import_borrowers.pl Signed-off-by: Kyle M Hall Bug 12598 - Started regression tests. Fix missing C4::Members::Attributes package Signed-off-by: Kyle M Hall Bug 12598 - More refactoring and regression tests in Koha::Patrons::Import Signed-off-by: Kyle M Hall Bug 12598 - Creating objects in misc/import_patrons.pl and tools/import_borrowers.pl Signed-off-by: Kyle M Hall Bug 12598 - Refactoring Koha::Patrons::Import includes bug fixed for critical date types and header column parsing Signed-off-by: Kyle M Hall Bug 12598 - Rebase + backport of 16426 plus fixing 16426 Signed-off-by: Kyle M Hall Bug 12598: catch warnings raised by import_patrons in tests Signed-off-by: Colin Campbell Signed-off-by: Jonathan Druart --- Koha/Patrons/Import.pm | 516 ++++++++++++++ .../prog/en/modules/tools/import_borrowers.tt | 430 ++++++------ misc/import_patrons.pl | 103 +++ t/db_dependent/Koha/Patrons/Import.t | 632 ++++++++++++++++++ tools/import_borrowers.pl | 336 ++-------- 5 files changed, 1545 insertions(+), 472 deletions(-) create mode 100644 Koha/Patrons/Import.pm create mode 100755 misc/import_patrons.pl create mode 100644 t/db_dependent/Koha/Patrons/Import.t diff --git a/Koha/Patrons/Import.pm b/Koha/Patrons/Import.pm new file mode 100644 index 0000000000..6b53c4bebc --- /dev/null +++ b/Koha/Patrons/Import.pm @@ -0,0 +1,516 @@ +package Koha::Patrons::Import; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Moo; +use namespace::clean; + +use Carp; +use Text::CSV; + +use C4::Members; +use C4::Branch; +use C4::Members::Attributes qw(:all); +use C4::Members::AttributeTypes; + +use Koha::DateUtils; + +=head1 NAME + +Koha::Patrons::Import - Perl Module containing import_patrons method exported from import_borrowers script. + +=head1 SYNOPSIS + +use Koha::Patrons::Import; + +=head1 DESCRIPTION + +This module contains one method for importing patrons in bulk. + +=head1 FUNCTIONS + +=head2 import_patrons + + my $return = Koha::Patrons::Import::import_patrons($params); + +Applies various checks and imports patrons in bulk from a csv file. + +Further pod documentation needed here. + +=cut + +has 'today_iso' => ( is => 'ro', lazy => 1, + default => sub { output_pref( { dt => dt_from_string(), dateonly => 1, dateformat => 'iso' } ); }, ); + +has 'text_csv' => ( is => 'rw', lazy => 1, + default => sub { Text::CSV->new( { binary => 1, } ); }, ); + +sub import_patrons { + my ($self, $params) = @_; + + my $handle = $params->{file}; + unless( $handle ) { carp('No file handle passed in!'); return; } + + my $matchpoint = $params->{matchpoint}; + my $defaults = $params->{defaults}; + my $ext_preserve = $params->{preserve_extended_attributes}; + my $overwrite_cardnumber = $params->{overwrite_cardnumber}; + my $extended = C4::Context->preference('ExtendedPatronAttributes'); + my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences'); + + my @columnkeys = $self->set_column_keys($extended); + my @feedback; + my @errors; + + my $imported = 0; + my $alreadyindb = 0; + my $overwritten = 0; + my $invalid = 0; + my $matchpoint_attr_type = $self->set_attribute_types({ extended => $extended, matchpoint => $matchpoint, }); + + # Use header line to construct key to column map + my %csvkeycol; + my $borrowerline = <$handle>; + my @csvcolumns = $self->prepare_columns({headerrow => $borrowerline, keycol => \%csvkeycol, errors => \@errors, }); + push(@feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) }); + + my @criticals = qw( surname ); # there probably should be others - rm branchcode && categorycode + LINE: while ( my $borrowerline = <$handle> ) { + my $line_number = $.; + my %borrower; + my @missing_criticals; + + my $status = $self->text_csv->parse($borrowerline); + my @columns = $self->text_csv->fields(); + if ( !$status ) { + push @missing_criticals, { badparse => 1, line => $line_number, lineraw => $borrowerline }; + } + elsif ( @columns == @columnkeys ) { + @borrower{@columnkeys} = @columns; + + # MJR: try to fill blanks gracefully by using default values + foreach my $key (@columnkeys) { + if ( $borrower{$key} !~ /\S/ ) { + $borrower{$key} = $defaults->{$key}; + } + } + } + else { + # MJR: try to recover gracefully by using default values + foreach my $key (@columnkeys) { + if ( defined( $csvkeycol{$key} ) and $columns[ $csvkeycol{$key} ] =~ /\S/ ) { + $borrower{$key} = $columns[ $csvkeycol{$key} ]; + } + elsif ( $defaults->{$key} ) { + $borrower{$key} = $defaults->{$key}; + } + elsif ( scalar grep { $key eq $_ } @criticals ) { + + # a critical field is undefined + push @missing_criticals, { key => $key, line => $., lineraw => $borrowerline }; + } + else { + $borrower{$key} = ''; + } + } + } + + # Check if borrower category code exists and if it matches to a known category. Pushing error to missing_criticals otherwise. + $self->check_borrower_category($borrower{categorycode}, $borrowerline, $line_number, \@missing_criticals); + + # Check if branch code exists and if it matches to a branch name. Pushing error to missing_criticals otherwise. + $self->check_branch_code($borrower{branchcode}, $borrowerline, $line_number, \@missing_criticals); + + # Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it. + $self->format_dates({borrower => \%borrower, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals, }); + + if (@missing_criticals) { + foreach (@missing_criticals) { + $_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF'; + $_->{surname} = $borrower{surname} || 'UNDEF'; + } + $invalid++; + ( 25 > scalar @errors ) and push @errors, { missing_criticals => \@missing_criticals }; + + # The first 25 errors are enough. Keeping track of 30,000+ would destroy performance. + next LINE; + } + + # Set patron attributes if extended. + my $patron_attributes = $self->set_patron_attributes($extended, $borrower{patron_attributes}, \@feedback); + if( $extended ) { delete $borrower{patron_attributes}; } # Not really a field in borrowers. + + # Default date enrolled and date expiry if not already set. + $borrower{dateenrolled} = $self->today_iso() unless $borrower{dateenrolled}; + $borrower{dateexpiry} = GetExpiryDate( $borrower{categorycode}, $borrower{dateenrolled} ) unless $borrower{dateexpiry}; + + my $borrowernumber; + my $member; + if ( defined($matchpoint) && ( $matchpoint eq 'cardnumber' ) && ( $borrower{'cardnumber'} ) ) { + $member = GetMember( 'cardnumber' => $borrower{'cardnumber'} ); + if ($member) { + $borrowernumber = $member->{'borrowernumber'}; + } + } + elsif ($extended) { + if ( defined($matchpoint_attr_type) ) { + foreach my $attr (@$patron_attributes) { + if ( $attr->{code} eq $matchpoint and $attr->{value} ne '' ) { + my @borrowernumbers = $matchpoint_attr_type->get_patrons( $attr->{value} ); + $borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1; + last; + } + } + } + } + + if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) { + push @errors, + { + invalid_cardnumber => 1, + borrowernumber => $borrowernumber, + cardnumber => $borrower{cardnumber} + }; + $invalid++; + next; + } + + # Check if the userid provided does not exist yet + if ( exists $borrower{userid} + and $borrower{userid} + and not Check_Userid( $borrower{userid}, $borrower{borrowernumber} ) ) { + push @errors, { duplicate_userid => 1, userid => $borrower{userid} }; + $invalid++; + next LINE; + } + + if ($borrowernumber) { + + # borrower exists + unless ($overwrite_cardnumber) { + $alreadyindb++; + push( + @feedback, + { + already_in_db => 1, + value => $borrower{'surname'} . ' / ' . $borrowernumber + } + ); + next LINE; + } + $borrower{'borrowernumber'} = $borrowernumber; + for my $col ( keys %borrower ) { + + # use values from extant patron unless our csv file includes this column or we provided a default. + # FIXME : You cannot update a field with a perl-evaluated false value using the defaults. + + # The password is always encrypted, skip it! + next if $col eq 'password'; + + unless ( exists( $csvkeycol{$col} ) || $defaults->{$col} ) { + $borrower{$col} = $member->{$col} if ( $member->{$col} ); + } + } + + unless ( ModMember(%borrower) ) { + $invalid++; + + push( + @errors, + { + name => 'lastinvalid', + value => $borrower{'surname'} . ' / ' . $borrowernumber + } + ); + next LINE; + } + if ( $borrower{debarred} ) { + + # Check to see if this debarment already exists + my $debarrments = GetDebarments( + { + borrowernumber => $borrowernumber, + expiration => $borrower{debarred}, + comment => $borrower{debarredcomment} + } + ); + + # If it doesn't, then add it! + unless (@$debarrments) { + AddDebarment( + { + borrowernumber => $borrowernumber, + expiration => $borrower{debarred}, + comment => $borrower{debarredcomment} + } + ); + } + } + if ($extended) { + if ($ext_preserve) { + my $old_attributes = GetBorrowerAttributes($borrowernumber); + $patron_attributes = extended_attributes_merge( $old_attributes, $patron_attributes ); + } + push @errors, { unknown_error => 1 } + unless SetBorrowerAttributes( $borrower{'borrowernumber'}, $patron_attributes, 'no_branch_limit' ); + } + $overwritten++; + push( + @feedback, + { + feedback => 1, + name => 'lastoverwritten', + value => $borrower{'surname'} . ' / ' . $borrowernumber + } + ); + } + else { + # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either. + # At least this is closer to AddMember than in members/memberentry.pl + if ( !$borrower{'cardnumber'} ) { + $borrower{'cardnumber'} = fixup_cardnumber(undef); + } + if ( $borrowernumber = AddMember(%borrower) ) { + + if ( $borrower{debarred} ) { + AddDebarment( + { + borrowernumber => $borrowernumber, + expiration => $borrower{debarred}, + comment => $borrower{debarredcomment} + } + ); + } + + if ($extended) { + SetBorrowerAttributes( $borrowernumber, $patron_attributes ); + } + + if ($set_messaging_prefs) { + C4::Members::Messaging::SetMessagingPreferencesFromDefaults( + { + borrowernumber => $borrowernumber, + categorycode => $borrower{categorycode} + } + ); + } + + $imported++; + push( + @feedback, + { + feedback => 1, + name => 'lastimported', + value => $borrower{'surname'} . ' / ' . $borrowernumber + } + ); + } + else { + $invalid++; + push @errors, { unknown_error => 1 }; + push( + @errors, + { + name => 'lastinvalid', + value => $borrower{'surname'} . ' / AddMember', + } + ); + } + } + } + + return { + feedback => \@feedback, + errors => \@errors, + imported => $imported, + overwritten => $overwritten, + already_in_db => $alreadyindb, + invalid => $invalid, + }; +} + +=head2 prepare_columns + + my @csvcolumns = $self->prepare_columns({headerrow => $borrowerline, keycol => \%csvkeycol, errors => \@errors, }); + +Returns an array of all column key and populates a hash of colunm key positions. + +=cut + +sub prepare_columns { + my ($self, $params) = @_; + + my $status = $self->text_csv->parse($params->{headerrow}); + unless( $status ) { + push( @{$params->{errors}}, { badheader => 1, line => 1, lineraw => $params->{headerrow} }); + return; + } + + my @csvcolumns = $self->text_csv->fields(); + my $col = 0; + foreach my $keycol (@csvcolumns) { + # columnkeys don't contain whitespace, but some stupid tools add it + $keycol =~ s/ +//g; + $params->{keycol}->{$keycol} = $col++; + } + + return @csvcolumns; +} + +=head2 set_attribute_types + + my $matchpoint_attr_type = $self->set_attribute_types({ extended => $extended, matchpoint => $matchpoint, }); + +Returns an attribute type based on matchpoint parameter. + +=cut + +sub set_attribute_types { + my ($self, $params) = @_; + + my $attribute_types; + if( $params->{extended} ) { + $attribute_types = C4::Members::AttributeTypes->fetch($params->{matchpoint}); + } + + return $attribute_types; +} + +=head2 set_column_keys + + my @columnkeys = set_column_keys($extended); + +Returns an array of borrowers' table columns. + +=cut + +sub set_column_keys { + my ($self, $extended) = @_; + + my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } C4::Members::columns(); + push( @columnkeys, 'patron_attributes' ) if $extended; + + return @columnkeys; +} + +=head2 set_patron_attributes + + my $patron_attributes = set_patron_attributes($extended, $borrower{patron_attributes}, $feedback); + +Returns a reference to array of hashrefs data structure as expected by SetBorrowerAttributes. + +=cut + +sub set_patron_attributes { + my ($self, $extended, $patron_attributes, $feedback) = @_; + + unless( $extended ) { return; } + unless( defined($patron_attributes) ) { return; } + + # Fixup double quotes in case we are passed smart quotes + $patron_attributes =~ s/\xe2\x80\x9c/"/g; + $patron_attributes =~ s/\xe2\x80\x9d/"/g; + + push (@$feedback, { feedback => 1, name => 'attribute string', value => $patron_attributes }); + + my $result = extended_attributes_code_value_arrayref($patron_attributes); + + return $result; +} + +=head2 check_branch_code + + check_branch_code($borrower{branchcode}, $borrowerline, $line_number, \@missing_criticals); + +Pushes a 'missing_criticals' error entry if no branch code or branch code does not map to a branch name. + +=cut + +sub check_branch_code { + my ($self, $branchcode, $borrowerline, $line_number, $missing_criticals) = @_; + + # No branch code + unless( $branchcode ) { + push (@$missing_criticals, { key => 'branchcode', line => $line_number, lineraw => $borrowerline, }); + return; + } + + # look for branch code + my $branch_name = GetBranchName( $branchcode ); + unless( $branch_name ) { + push (@$missing_criticals, { key => 'branchcode', line => $line_number, lineraw => $borrowerline, + value => $branchcode, branch_map => 1, }); + } +} + +=head2 check_borrower_category + + check_borrower_category($borrower{categorycode}, $borrowerline, $line_number, \@missing_criticals); + +Pushes a 'missing_criticals' error entry if no category code or category code does not map to a known category. + +=cut + +sub check_borrower_category { + my ($self, $categorycode, $borrowerline, $line_number, $missing_criticals) = @_; + + # No branch code + unless( $categorycode ) { + push (@$missing_criticals, { key => 'categorycode', line => $line_number, lineraw => $borrowerline, }); + return; + } + + # Looking for borrower category + my $category = GetBorrowercategory( $categorycode ); + unless( $category ) { + push (@$missing_criticals, { key => 'categorycode', line => $line_number, lineraw => $borrowerline, + value => $categorycode, category_map => 1, }); + } +} + +=head2 format_dates + + format_dates({borrower => \%borrower, lineraw => $lineraw, line => $line_number, missing_criticals => \@missing_criticals, }); + +Pushes a 'missing_criticals' error entry for each of the 3 date types dateofbirth, dateenrolled and dateexpiry if it can not +be formatted to the chosen date format. Populates the correctly formatted date otherwise. + +=cut + +sub format_dates { + my ($self, $params) = @_; + + foreach my $date_type (qw(dateofbirth dateenrolled dateexpiry)) { + my $tempdate = $params->{borrower}->{$date_type} or next(); + my $formatted_date = eval { output_pref( { dt => dt_from_string( $tempdate ), dateonly => 1, dateformat => 'iso' } ); }; + + if ($formatted_date) { + $params->{borrower}->{$date_type} = $formatted_date; + } else { + $params->{borrower}->{$date_type} = ''; + push (@{$params->{missing_criticals}}, { key => $date_type, line => $params->{line}, lineraw => $params->{lineraw}, bad_date => 1 }); + } + } +} + +1; + +=head1 AUTHOR + +Koha Team + +=cut diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt index 576356f079..7c53da864b 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt @@ -26,171 +26,254 @@

Import patrons

[% IF ( uploadborrowers ) %] -
Import results :
-
    -
  • [% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]
    • +
    Import results :
    +
      +
    • [% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]
      • [% IF imported and patronlistname %]
    • Patronlist with imported patrons: [% patronlistname %]
      • [% END %] -
    • [% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]
      • -
    • [% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]
      • -
    • [% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]
      • -
    • [% total %] records parsed
      • -
    • Back to Tools
      • -
    - [% IF ( FEEDBACK ) %] -

    -
    -
    Feedback:
    - -
    - [% END %] - [% IF ( ERRORS ) %] -

    -
    -
    Error analysis:
    -
      - [% FOREACH ERROR IN ERRORS %] - [% IF ( ERROR.badheader ) %]
    • Header row could not be parsed
      • [% END %] - [% FOREACH missing_critical IN ERROR.missing_criticals %] -
    • - Line [% missing_critical.line %] - [% IF ( missing_critical.badparse ) %] - could not be parsed! - [% ELSIF ( missing_critical.bad_date ) %] - has "[% missing_critical.key %]" in unrecognized format: "[% missing_critical.value %]" - [% ELSE %] - Critical field "[% missing_critical.key %]" - [% IF ( missing_critical.branch_map ) %]has unrecognized value "[% missing_critical.value %]" - [% ELSIF ( missing_critical.category_map ) %]has unrecognized value "[% missing_critical.value %]" - [% ELSE %]missing + + [% IF ( feedback ) %] +

      + +
      +
      Feedback:
      + +
      + [% END %] + + [% IF ( errors ) %] +

      + +
      +
      Error analysis:
      +
        + [% FOREACH e IN errors %] + [% IF ( e.badheader ) %]
      • Header row could not be parsed
        • [% END %] + + [% FOREACH missing_critical IN e.missing_criticals %] +
      • + Line [% missing_critical.line %] + + [% IF ( missing_critical.badparse ) %] + could not be parsed! + [% ELSIF ( missing_critical.bad_date ) %] + has "[% missing_critical.key %]" in unrecognized format: "[% missing_critical.value %]" + [% ELSE %] + Critical field "[% missing_critical.key %]" + + [% IF ( missing_critical.branch_map ) %] + has unrecognized value "[% missing_critical.value %]" + [% ELSIF ( missing_critical.category_map ) %] + has unrecognized value "[% missing_critical.value %]" + [% ELSE %] + missing + [% END %] + + (borrowernumber: [% missing_critical.borrowernumber %]; surname: [% missing_critical.surname %]). + [% END %] + +
        + [% missing_critical.lineraw %] +
        • + [% END %] + + [% IF e.invalid_cardnumber %] +
      • + Cardnumber [% e.cardnumber %] is not a valid cardnumber + [% IF e.borrowernumber %] (for patron with borrowernumber [% e.borrowernumber %])[% END %] +
        • + [% END %] + [% IF e.duplicate_userid %] +
      • + Userid [% e.userid %] is already used by another patron. +
        • + [% END %] [% END %] - (borrowernumber: [% missing_critical.borrowernumber %]; surname: [% missing_critical.surname %]). - [% END %] -
        [% missing_critical.lineraw %] - - [% END %] - [% IF ERROR.invalid_cardnumber %] -
      • - Cardnumber [% ERROR.cardnumber %] is not a valid cardnumber - [% IF ERROR.borrowernumber %] (for patron with borrowernumber [% ERROR.borrowernumber %])[% END %] -
        • - [% END %] - [% IF ERROR.duplicate_userid %] -
      • - Userid [% ERROR.userid %] is already used by another patron. -
        • - [% END %] +
      +
      [% END %] -
    -
    - [% END %] [% ELSE %] -
      -
    • Select a file to import into the borrowers table.
      • -
    • If a cardnumber exists in the table, you can choose whether to ignore the new one or overwrite the old one.
      • -
    -
    -
    -Import into the borrowers table -
      -
    1. - - -
      2. -
    2. - - - List name will be file name with timestamp -
      3. +
        +
      • Select a file to import into the borrowers table
        • +
      • If a cardnumber exists in the table, you can choose whether to ignore the new one or overwrite the old one.
        • +
      -
    -
    - Field to use for record matching -
      -
    1. - +
      2. + +
    2. + + + List name will be file name with timestamp +
      3. +
    +
    + +
    + Field to use for record matching +
      +
    1. + +
      2. +
    +
    + +
    + Default values + +
      + [% FOREACH borrower_field IN borrower_fields %] + + [% SWITCH borrower_field.field %] + [% CASE 'branchcode' %] +
    1. + + + [% borrower_field.field %] +
      2. + [% CASE 'categorycode' %] +
    2. + + + [% borrower_field.field %] +
      3. + [% CASE %] +
    3. + + + [% borrower_field.field %] +
      4. [% END %] - - -
    -
    -
    -Default values -
      -[% FOREACH borrower_field IN borrower_fields %] - [% SWITCH borrower_field.field %] - [% CASE 'branchcode' %] -
    1. - - [% borrower_field.field %] -
      2. - [% CASE 'categorycode' %] -
    2. - - + patron_attributes +
      3. + [% END %] + +
    +
    + +
    + If matching record is already in the borrowers table: + +
      +
    1. + +
      2. + +
    2. + +
      3. +
    +
    + + [% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %] +
    + Patron attributes + +
      +
    1. + +
      2. + +
    2. + +
      3. +
    +
    [% END %] - [% borrower_field.field %] - - [% CASE %] -
  • - - [% borrower_field.field %] -
    • - [% END %] -[% END %] -[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %] -
  • - - - patron_attributes -
    • + +
    +
    [% END %] - -
    - If matching record is already in the borrowers table: -
    1. - + +
+ +
+

Notes:

+
    +
  • The first line in the file must be a header row defining which columns you are supplying in the import file.
    • + +
  • Download a starter CSV file with all the columns here. Values are comma-separated.
    • + +
  • + OR choose which fields you want to supply from the following list: +
      +
    • + [% FOREACH columnkey IN borrower_fields %]'[% columnkey.field %]', [% END %] +
      • +
    • -
  • - + + [% IF ( ExtendedPatronAttributes ) %] +
  • + If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value. For example: INSTID:12345,LANG:fr or STARTDATE:January 1 2010,TRACK:Day. If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma: "STARTDATE:January 1, 2010","TRACK:Day". The second syntax would be required if the data might have a comma in it, like a date string. +
    • + [% END %] + +
  • + The fields 'branchcode' and 'categorycode' are required and must match valid entries in your database.
    • - - - [% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %] -
    - Patron attributes -
    1. - + +
    2. + 'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).
      3. -
    3. - + +
    4. + Date formats should match your system preference, and must be zero-padded, e.g. '01/02/2008'. Alternatively, +you can supply dates in ISO format (e.g., '2010-10-28').
      5. -
    +
[% END %]
@@ -199,44 +282,16 @@
[% END %] + +
+ + + + +
+ [% INCLUDE 'tools-menu.inc' %] +
-
-

Notes:

-
    -
  • Header: The first line in the file must be a header row defining which columns you are supplying in the import file.
    • -
  • Separator: Values are comma-separated.
    • -
  • Starter CSV: Koha provides a starter CSV with all the columns. - -
    • -
  • Field list: Alternatively, you can create your own CSV and choose which fields you want to supply from the following list: -
    • - [% FOREACH columnkey IN borrower_fields %]'[% columnkey.field %]', [% END %] -
    -
    • -[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %] -
  • Extended patron attributes: If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value. -
    • Example 1: INSTID:12345,LANG:fr
    • Example 2: STARTDATE:January 1 2010,TRACK:Day
    -If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma: -
    • Example 3: "STARTDATE:January 1, 2010","TRACK:Day"
    -The second syntax would be required if the data might have a comma in it, like a date string.
    • -[% END %] -
  • Required fields: The fields 'branchcode' and 'categorycode' are required and must match valid entries in your database.
    • -
  • Password: Values for the field 'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).
    • -
  • Date formats: Date values should match your system preference, and must be zero-padded. -
    • Example: '01/02/2008'
    -Alternatively, you can supply dates in ISO format. -
    • Example: '2010-10-28'
    -
    • -
- -
- - - -
-[% INCLUDE 'tools-menu.inc' %] -
- [% MACRO jsinclude BLOCK %] [% INCLUDE 'calendar.inc' %] @@ -249,5 +304,4 @@ Alternatively, you can supply dates in ISO format. }); [% END %] - [% INCLUDE 'intranet-bottom.inc' %] diff --git a/misc/import_patrons.pl b/misc/import_patrons.pl new file mode 100755 index 0000000000..de728a4cec --- /dev/null +++ b/misc/import_patrons.pl @@ -0,0 +1,103 @@ +#!/usr/bin/perl + +# Parts copyright 2014 ByWater Solutions +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + +use Getopt::Long; + +use Koha::Patrons::Import; +my $Import = Koha::Patrons::Import->new(); + +my $csv_file; +my $matchpoint; +my $overwrite_cardnumber; +my %defaults; +my $ext_preserve = 0; +my $confirm; +my $verbose = 0; +my $help; + +GetOptions( + 'c|confirm' => \$confirm, + 'f|file=s' => \$csv_file, + 'm|matchpoint=s' => \$matchpoint, + 'd|default=s' => \%defaults, + 'o|overwrite' => \$overwrite_cardnumber, + 'p|preserve-extended-atributes' => \$ext_preserve, + 'v|verbose+' => \$verbose, + 'h|help|?' => \$help, +); + +print_help() if ( $help || !$csv_file || !$matchpoint || !$confirm ); + +my $handle; +open( $handle, "<", $csv_file ) or die $!; + +my $return = $Import->import_patrons( + { + file => $handle, + defaults => \%defaults, + matchpoint => $matchpoint, + overwrite_cardnumber => $overwrite_cardnumber, + preserve_extended_attributes => $ext_preserve, + } +); + +my $feedback = $return->{feedback}; +my $errors = $return->{errors}; +my $imported = $return->{imported}; +my $overwritten = $return->{overwritten}; +my $alreadyindb = $return->{already_in_db}; +my $invalid = $return->{invalid}; + +if ($verbose) { + my $total = $imported + $alreadyindb + $invalid + $overwritten; + say q{}; + say "Import complete:"; + say "Imported: $imported"; + say "Overwritten: $overwritten"; + say "Skipped: $alreadyindb"; + say "Invalid: $invalid"; + say "Total: $total"; + say q{}; +} + +if ($verbose > 1 ) { + say "Errors:"; + say Data::Dumper::Dumper( $errors ); +} + +if ($verbose > 2 ) { + say "Feedback:"; + say Data::Dumper::Dumper( $feedback ); +} + +sub print_help { + print <<_USAGE_; +import_patrons.pl -c /path/to/patrons.csv -m cardnumber + -c --confirm Confirms you really want to import these patrons, otherwise prints this help + -f --file Path to the CSV file of patrons to import + -m --matchpoint Field on which to match incoming patrons to existing patrons + -d --default Set defaults to patron fields, repeatable e.g. --default branchcode=MPL --default categorycode=PT + -p --preserve-extended-atributes Retain extended patron attributes for existing patrons being overwritten + -o --overwrite Overwrite existing patrons with new data if a match is found + -v --verbose Be verbose +_USAGE_ + exit; +} diff --git a/t/db_dependent/Koha/Patrons/Import.t b/t/db_dependent/Koha/Patrons/Import.t new file mode 100644 index 0000000000..3c46ce2bf7 --- /dev/null +++ b/t/db_dependent/Koha/Patrons/Import.t @@ -0,0 +1,632 @@ +#!/usr/bin/perl + +# Copyright 2015 Koha Development team +# +# This file is part of Koha +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; +use Test::More tests => 124; +use Test::Warn; + +# To be replaced by t::lib::Mock +use Test::MockModule; +use Koha::Database; + +use File::Temp qw(tempfile tempdir); +my $temp_dir = tempdir('Koha_patrons_import_test_XXXX', CLEANUP => 1, TMPDIR => 1); + +use t::lib::TestBuilder; +my $builder = t::lib::TestBuilder->new; + +my $schema = Koha::Database->new->schema; +$schema->storage->txn_begin; + +# ########## Tests start here ############################# +# Given ... we can use the module +BEGIN { use_ok('Koha::Patrons::Import'); } + +my $patrons_import = new_ok('Koha::Patrons::Import'); + +subtest 'test_methods' => sub { + plan tests => 1; + + # Given ... we can reach the method(s) + my @methods = ('import_patrons', + 'set_attribute_types', + 'prepare_columns', + 'set_column_keys', + 'set_patron_attributes', + 'check_branch_code', + 'format_dates', + ); + can_ok('Koha::Patrons::Import', @methods); +}; + +subtest 'test_attributes' => sub { + plan tests => 1; + + my @attributes = ('today_iso', 'text_csv'); + can_ok('Koha::Patrons::Import', @attributes); +}; + +# Tests for Koha::Patrons::Import::import_patrons() +# Given ... nothing much. When ... Then ... +my $result; +warning_is { $result = $patrons_import->import_patrons(undef) } + { carped => 'No file handle passed in!' }, + " Koha::Patrons::Import->import_patrons carps if no file handle is passed"; +is($result, undef, 'Got the expected undef from import_patrons with nothing much'); + +# Given ... some params but no file handle. +my $params_0 = { some_stuff => 'random stuff', }; + +# When ... Then ... +my $result_0; +warning_is { $result_0 = $patrons_import->import_patrons($params_0) } + { carped => 'No file handle passed in!' }, + " Koha::Patrons::Import->import_patrons carps if no file handle is passed"; +is($result_0, undef, 'Got the expected undef from import_patrons with no file handle'); + +# Given ... a file handle to file with headers only. +my $ExtendedPatronAttributes = 0; +my $context = Test::MockModule->new('C4::Context'); # Necessary mocking for consistent test results. +$context->mock('preference', sub { my ($mod, $meth) = @_; + if ( $meth eq 'ExtendedPatronAttributes' ) { return $ExtendedPatronAttributes; } + if ( $meth eq 'dateformat' ) { return 'us'; } + }); + + +my $csv_headers = 'cardnumber,surname,firstname,title,othernames,initials,streetnumber,streettype,address,address2,city,state,zipcode,country,email,phone,mobile,fax,dateofbirth,branchcode,categorycode,dateenrolled,dateexpiry,userid,password'; +my $res_header = 'cardnumber, surname, firstname, title, othernames, initials, streetnumber, streettype, address, address2, city, state, zipcode, country, email, phone, mobile, fax, dateofbirth, branchcode, categorycode, dateenrolled, dateexpiry, userid, password'; +my $csv_one_line = '1000,Nancy,Jenkins,Dr,,NJ,78,Circle,Bunting,El Paso,Henderson,Texas,79984,United States,ajenkins0@sourceforge.net,7-(388)559-6763,3-(373)151-4471,8-(509)286-4001,10/16/1965,CPL,PT,12/28/2014,07/01/2015,jjenkins0,DPQILy'; + +my $filename_1 = make_csv($temp_dir, $csv_headers, $csv_one_line); +open(my $handle_1, "<", $filename_1) or die "cannot open < $filename_1: $!"; +my $params_1 = { file => $handle_1, }; + +# When ... +my $result_1 = $patrons_import->import_patrons($params_1); + +# Then ... +is($result_1->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons with no matchpoint defined'); +is(scalar @{$result_1->{errors}}, 0, 'Got the expected 0 size error array from import_patrons with no matchpoint defined'); + +is($result_1->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with no matchpoint defined'); +is($result_1->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with no matchpoint defined'); +is($result_1->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with no matchpoint defined'); + +is($result_1->{feedback}->[1]->{feedback}, 1, 'Got the expected second feedback from import_patrons with no matchpoint defined'); +is($result_1->{feedback}->[1]->{name}, 'lastimported', 'Got the expected last imported name from import_patrons with no matchpoint defined'); +like($result_1->{feedback}->[1]->{value}, qr/^Nancy \/ \d+/, 'Got the expected second header row value from import_patrons with no matchpoint defined'); + +is($result_1->{imported}, 1, 'Got the expected 1 imported result from import_patrons with no matchpoint defined'); +is($result_1->{invalid}, 0, 'Got the expected 0 invalid result from import_patrons with no matchpoint defined'); +is($result_1->{overwritten}, 0, 'Got the expected 0 overwritten result from import_patrons with no matchpoint defined'); + +# Given ... a valid file handle, a bad matchpoint resulting in invalid card number +my $filename_2 = make_csv($temp_dir, $csv_headers, $csv_one_line); +open(my $handle_2, "<", $filename_2) or die "cannot open < $filename_2: $!"; +my $params_2 = { file => $handle_2, matchpoint => 'SHOW_BCODE', }; + +# When ... +my $result_2 = $patrons_import->import_patrons($params_2); + +# Then ... +is($result_2->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons with invalid card number'); +is($result_2->{errors}->[0]->{borrowernumber}, undef, 'Got the expected undef borrower number from import patrons with invalid card number'); +is($result_2->{errors}->[0]->{cardnumber}, 1000, 'Got the expected 1000 card number from import patrons with invalid card number'); +is($result_2->{errors}->[0]->{invalid_cardnumber}, 1, 'Got the expected invalid card number from import patrons with invalid card number'); + +is($result_2->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with invalid card number'); +is($result_2->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with invalid card number'); +is($result_2->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with invalid card number'); + +is($result_2->{imported}, 0, 'Got the expected 0 imported result from import_patrons with invalid card number'); +is($result_2->{invalid}, 1, 'Got the expected 1 invalid result from import_patrons with invalid card number'); +is($result_2->{overwritten}, 0, 'Got the expected 0 overwritten result from import_patrons with invalid card number'); + +# Given ... valid file handle, good matchpoint but same input as prior test. +my $filename_3 = make_csv($temp_dir, $csv_headers, $csv_one_line); +open(my $handle_3, "<", $filename_3) or die "cannot open < $filename_3: $!"; +my $params_3 = { file => $handle_3, matchpoint => 'cardnumber', }; + +# When ... +my $result_3 = $patrons_import->import_patrons($params_3); + +# Then ... +is($result_3->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons with duplicate userid'); +is($result_3->{errors}->[0]->{duplicate_userid}, 1, 'Got the expected duplicate userid error from import patrons with duplicate userid'); +is($result_3->{errors}->[0]->{userid}, 'jjenkins0', 'Got the expected userid error from import patrons with duplicate userid'); + +is($result_3->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with duplicate userid'); +is($result_3->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with duplicate userid'); +is($result_3->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with duplicate userid'); + +is($result_3->{imported}, 0, 'Got the expected 0 imported result from import_patrons with duplicate userid'); +is($result_3->{invalid}, 1, 'Got the expected 1 invalid result from import_patrons with duplicate userid'); +is($result_3->{overwritten}, 0, 'Got the expected 0 overwritten result from import_patrons with duplicate userid'); + +# Given ... a new input and mocked C4::Context +$ExtendedPatronAttributes = 1; # Updates mocked C4::Preferences result. + +my $new_input_line = '1001,Donna,Sullivan,Mrs,Henry,DS,59,Court,Burrows,Reading,Salt Lake City,Pennsylvania,19605,United States,hsullivan1@purevolume.com,3-(864)009-3006,7-(291)885-8423,1-(879)095-5038,09/19/1970,LPL,PT,03/04/2015,07/01/2015,hsullivan1,8j6P6Dmap'; +my $filename_4 = make_csv($temp_dir, $csv_headers, $new_input_line); +open(my $handle_4, "<", $filename_4) or die "cannot open < $filename_4: $!"; +my $params_4 = { file => $handle_4, matchpoint => 'SHOW_BCODE', }; + +# When ... +my $result_4 = $patrons_import->import_patrons($params_4); + +# Then ... +is($result_4->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons with extended user'); +is(scalar @{$result_4->{errors}}, 0, 'Got the expected 0 size error array from import_patrons with extended user'); + +is($result_4->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons with extended user'); +is($result_4->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons with extended user'); +is($result_4->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons with extended user'); + +is($result_4->{feedback}->[1]->{feedback}, 1, 'Got the expected second feedback from import_patrons with extended user'); +is($result_4->{feedback}->[1]->{name}, 'attribute string', 'Got the expected attribute string from import_patrons with extended user'); +is($result_4->{feedback}->[1]->{value}, '', 'Got the expected second feedback value from import_patrons with extended user'); + +is($result_4->{feedback}->[2]->{feedback}, 1, 'Got the expected third feedback from import_patrons with extended user'); +is($result_4->{feedback}->[2]->{name}, 'lastimported', 'Got the expected last imported name from import_patrons with extended user'); +like($result_4->{feedback}->[2]->{value}, qr/^Donna \/ \d+/, 'Got the expected third feedback value from import_patrons with extended user'); + +is($result_4->{imported}, 1, 'Got the expected 1 imported result from import_patrons with extended user'); +is($result_4->{invalid}, 0, 'Got the expected 0 invalid result from import_patrons with extended user'); +is($result_4->{overwritten}, 0, 'Got the expected 0 overwritten result from import_patrons with extended user'); + +$context->unmock('preference'); + +# Given ... 3 new inputs. One with no branch code, one with unexpected branch code. +my $input_no_branch = '1002,Johnny,Reynolds,Mr,Patricia,JR,12,Hill,Kennedy,Saint Louis,Colorado Springs,Missouri,63131,United States,preynolds2@washington.edu,7-(925)314-9514,0-(315)973-8956,4-(510)556-2323,09/18/1967,,PT,05/07/2015,07/01/2015,preynolds2,K3HiDzl'; +my $input_good_branch = '1003,Linda,Richardson,Mr,Kimberly,LR,90,Place,Bayside,Atlanta,Erie,Georgia,31190,United States,krichardson3@pcworld.com,8-(035)185-0387,4-(796)518-3676,3-(644)960-3789,04/13/1954,RPL,PT,06/06/2015,07/01/2015,krichardson3,P3EO0MVRPXbM'; +my $input_na_branch = '1005,Ruth,Greene,Mr,Michael,RG,3,Avenue,Grim,Peoria,Jacksonville,Illinois,61614,United States,mgreene5@seesaa.net,3-(941)565-5752,1-(483)885-8138,4-(979)577-6908,02/09/1957,ZZZ,ST,04/02/2015,07/01/2015,mgreene5,or4ORT6JH'; + +my $filename_5 = make_csv($temp_dir, $csv_headers, $input_no_branch, $input_good_branch, $input_na_branch); +open(my $handle_5, "<", $filename_5) or die "cannot open < $filename_5: $!"; +my $params_5 = { file => $handle_5, matchpoint => 'cardnumber', }; + +# When ... +my $result_5 = $patrons_import->import_patrons($params_5); + +# Then ... +is($result_5->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons for branch tests'); + +is($result_5->{errors}->[0]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for branch tests'); +is($result_5->{errors}->[0]->{missing_criticals}->[0]->{key}, 'branchcode', 'Got the expected branch code key from import patrons for branch tests'); +is($result_5->{errors}->[0]->{missing_criticals}->[0]->{line}, 2, 'Got the expected 2 line number error from import patrons for branch tests'); +is($result_5->{errors}->[0]->{missing_criticals}->[0]->{lineraw}, $input_no_branch."\r\n", 'Got the expected lineraw error from import patrons for branch tests'); +is($result_5->{errors}->[0]->{missing_criticals}->[0]->{surname}, 'Johnny', 'Got the expected surname error from import patrons for branch tests'); + +is($result_5->{errors}->[1]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for branch tests'); +is($result_5->{errors}->[1]->{missing_criticals}->[0]->{branch_map}, 1, 'Got the expected 1 branchmap error from import patrons for branch tests'); +is($result_5->{errors}->[1]->{missing_criticals}->[0]->{key}, 'branchcode', 'Got the expected branch code key from import patrons for branch tests'); +is($result_5->{errors}->[1]->{missing_criticals}->[0]->{line}, 4, 'Got the expected 4 line number error from import patrons for branch tests'); +is($result_5->{errors}->[1]->{missing_criticals}->[0]->{lineraw}, $input_na_branch."\r\n", 'Got the expected lineraw error from import patrons for branch tests'); +is($result_5->{errors}->[1]->{missing_criticals}->[0]->{surname}, 'Ruth', 'Got the expected surname error from import patrons for branch tests'); +is($result_5->{errors}->[1]->{missing_criticals}->[0]->{value}, 'ZZZ', 'Got the expected ZZZ value error from import patrons for branch tests'); + +is($result_5->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for branch tests'); +is($result_5->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons for branch tests'); +is($result_5->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons for branch tests'); + +is($result_5->{feedback}->[1]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for branch tests'); +is($result_5->{feedback}->[1]->{name}, 'lastimported', 'Got the expected lastimported name from import_patrons for branch tests'); +like($result_5->{feedback}->[1]->{value}, qr/^Linda \/ \d+/, 'Got the expected last imported value from import_patrons with for branch tests'); + +is($result_5->{imported}, 1, 'Got the expected 1 imported result from import patrons for branch tests'); +is($result_5->{invalid}, 2, 'Got the expected 2 invalid result from import patrons for branch tests'); +is($result_5->{overwritten}, 0, 'Got the expected 0 overwritten result from import patrons for branch tests'); + +# Given ... 3 new inputs. One with no category code, one with unexpected category code. +my $input_no_category = '1006,Christina,Olson,Rev,Kimberly,CO,8,Avenue,Northridge,Lexington,Wilmington,Kentucky,40510,United States,kolson6@dropbox.com,7-(810)636-6048,1-(052)012-8984,8-(567)232-7818,03/26/1952,FFL,,09/07/2014,01/07/2015,kolson6,x5D3qGbLlptx'; +my $input_good_category = '1007,Peter,Peters,Mrs,Lawrence,PP,6,Trail,South,Oklahoma City,Topeka,Oklahoma,73135,United States,lpeters7@bandcamp.com,5-(992)205-9318,0-(732)586-9365,3-(448)146-7936,08/16/1983,PVL,T,03/24/2015,07/01/2015,lpeters7,Z19BrQ4'; +my $input_na_category = '1008,Emily,Richards,Ms,Judy,ER,73,Way,Kedzie,Fort Wayne,Phoenix,Indiana,46825,United States,jrichards8@arstechnica.com,5-(266)658-8957,3-(550)500-9107,7-(816)675-9822,08/09/1984,FFL,ZZ,11/09/2014,07/01/2015,jrichards8,D5PvU6H2R'; + +my $filename_6 = make_csv($temp_dir, $csv_headers, $input_no_category, $input_good_category, $input_na_category); +open(my $handle_6, "<", $filename_6) or die "cannot open < $filename_6: $!"; +my $params_6 = { file => $handle_6, matchpoint => 'cardnumber', }; + +# When ... +my $result_6 = $patrons_import->import_patrons($params_6); + +# Then ... +is($result_6->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons for category tests'); + +is($result_6->{errors}->[0]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for category tests'); +is($result_6->{errors}->[0]->{missing_criticals}->[0]->{key}, 'categorycode', 'Got the expected category code key from import patrons for category tests'); +is($result_6->{errors}->[0]->{missing_criticals}->[0]->{line}, 2, 'Got the expected 2 line number error from import patrons for category tests'); +is($result_6->{errors}->[0]->{missing_criticals}->[0]->{lineraw}, $input_no_category."\r\n", 'Got the expected lineraw error from import patrons for category tests'); +is($result_6->{errors}->[0]->{missing_criticals}->[0]->{surname}, 'Christina', 'Got the expected surname error from import patrons for category tests'); + +is($result_6->{errors}->[1]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for category tests'); +is($result_6->{errors}->[1]->{missing_criticals}->[0]->{category_map}, 1, 'Got the expected 1 category_map error from import patrons for category tests'); +is($result_6->{errors}->[1]->{missing_criticals}->[0]->{key}, 'categorycode', 'Got the expected category code key from import patrons for category tests'); +is($result_6->{errors}->[1]->{missing_criticals}->[0]->{line}, 4, 'Got the expected 4 line number error from import patrons for category tests'); +is($result_6->{errors}->[1]->{missing_criticals}->[0]->{lineraw}, $input_na_category."\r\n", 'Got the expected lineraw error from import patrons for category tests'); +is($result_6->{errors}->[1]->{missing_criticals}->[0]->{surname}, 'Emily', 'Got the expected surname error from import patrons for category tests'); +is($result_6->{errors}->[1]->{missing_criticals}->[0]->{value}, 'ZZ', 'Got the expected ZZ value error from import patrons for category tests'); + +is($result_6->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for category tests'); +is($result_6->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons for category tests'); +is($result_6->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons for category tests'); + +is($result_6->{feedback}->[1]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for category tests'); +is($result_6->{feedback}->[1]->{name}, 'lastimported', 'Got the expected lastimported name from import_patrons for category tests'); +like($result_6->{feedback}->[1]->{value}, qr/^Peter \/ \d+/, 'Got the expected last imported value from import_patrons with for category tests'); + +is($result_6->{imported}, 1, 'Got the expected 1 imported result from import patrons for category tests'); +is($result_6->{invalid}, 2, 'Got the expected 2 invalid result from import patrons for category tests'); +is($result_6->{overwritten}, 0, 'Got the expected 0 overwritten result from import patrons for category tests'); + +# Given ... 2 new inputs. One without dateofbirth, dateenrolled and dateexpiry values. +my $input_complete = '1009,Christina,Harris,Dr,Philip,CH,99,Street,Grayhawk,Baton Rouge,Dallas,Louisiana,70810,United States,pharris9@hp.com,9-(317)603-5513,7-(005)062-7593,8-(349)134-1627,06/19/1969,IPT,PT,04/09/2015,07/01/2015,pharris9,NcAhcvvnB'; +my $input_no_date = '1010,Ralph,Warren,Ms,Linda,RW,6,Way,Barby,Orlando,Albany,Florida,32803,United States,lwarrena@multiply.com,7-(579)753-7752,6-(847)086-7566,9-(122)729-8226,26/01/2001,LPL,T,25/01/2001,24/01/2001,lwarrena,tJ56RD4uV'; + +my $filename_7 = make_csv($temp_dir, $csv_headers, $input_complete, $input_no_date); +open(my $handle_7, "<", $filename_7) or die "cannot open < $filename_7: $!"; +my $params_7 = { file => $handle_7, matchpoint => 'cardnumber', }; + +# When ... +my $result_7 = $patrons_import->import_patrons($params_7); + +# Then ... +is($result_7->{already_in_db}, 0, 'Got the expected 0 already_in_db from import_patrons for dates tests'); +is(scalar @{$result_7->{errors}}, 1, 'Got the expected 1 error array size from import_patrons for dates tests'); +is(scalar @{$result_7->{errors}->[0]->{missing_criticals}}, 3, 'Got the expected 3 missing critical errors from import_patrons for dates tests'); + +is($result_7->{errors}->[0]->{missing_criticals}->[0]->{bad_date}, 1, 'Got the expected 1 bad_date error from import patrons for dates tests'); +is($result_7->{errors}->[0]->{missing_criticals}->[0]->{borrowernumber}, 'UNDEF', 'Got the expected undef borrower number error from import patrons for dates tests'); +is($result_7->{errors}->[0]->{missing_criticals}->[0]->{key}, 'dateofbirth', 'Got the expected dateofbirth key from import patrons for dates tests'); +is($result_7->{errors}->[0]->{missing_criticals}->[0]->{line}, 3, 'Got the expected 2 line number error from import patrons for dates tests'); +is($result_7->{errors}->[0]->{missing_criticals}->[0]->{lineraw}, $input_no_date."\r\n", 'Got the expected lineraw error from import patrons for dates tests'); +is($result_7->{errors}->[0]->{missing_criticals}->[0]->{surname}, 'Ralph', 'Got the expected surname error from import patrons for dates tests'); + +is($result_7->{errors}->[0]->{missing_criticals}->[1]->{key}, 'dateenrolled', 'Got the expected dateenrolled key from import patrons for dates tests'); +is($result_7->{errors}->[0]->{missing_criticals}->[2]->{key}, 'dateexpiry', 'Got the expected dateexpiry key from import patrons for dates tests'); + +is(scalar @{$result_7->{feedback}}, 2, 'Got the expected 2 feedback from import patrons for dates tests'); +is($result_7->{feedback}->[0]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for dates tests'); +is($result_7->{feedback}->[0]->{name}, 'headerrow', 'Got the expected header row name from import_patrons for dates tests'); +is($result_7->{feedback}->[0]->{value}, $res_header, 'Got the expected header row value from import_patrons for dates tests'); + +is($result_7->{feedback}->[1]->{feedback}, 1, 'Got the expected 1 feedback from import_patrons for dates tests'); +is($result_7->{feedback}->[1]->{name}, 'lastimported', 'Got the expected lastimported from import_patrons for dates tests'); +like($result_7->{feedback}->[1]->{value}, qr/^Christina \/ \d+/, 'Got the expected lastimported value from import_patrons for dates tests'); + +is($result_7->{imported}, 1, 'Got the expected 1 imported result from import patrons for dates tests'); +is($result_7->{invalid}, 1, 'Got the expected 1 invalid result from import patrons for dates tests'); +is($result_7->{overwritten}, 0, 'Got the expected 0 overwritten result from import patrons for dates tests'); + +subtest 'test_prepare_columns' => sub { + plan tests => 16; + + # Given ... no header row + my $headerrow_0; + my %csvkeycol_0; + my @errors_0; + + # When ... + my @csvcolumns_0 = $patrons_import->prepare_columns({headerrow => undef, keycol => \%csvkeycol_0, errors => \@errors_0, }); + + # Then ... + is(scalar @csvcolumns_0, 0, 'Got the expected empty column array from prepare columns with no header row'); + + is(scalar @errors_0, 1, 'Got the expected 1 entry in error array from prepare columns with no header row'); + is($errors_0[0]->{badheader}, 1, 'Got the expected 1 badheader from prepare columns with no header row'); + is($errors_0[0]->{line}, 1, 'Got the expected 1 line from prepare columns with no header row'); + is($errors_0[0]->{lineraw}, undef, 'Got the expected undef lineraw from prepare columns with no header row'); + + # Given ... a good header row with plenty of whitespaces + my $headerrow_1 = 'a, b , c, , d'; + my %csvkeycol_1; + my @errors_1; + + # When ... + my @csvcolumns_1 = $patrons_import->prepare_columns({headerrow => $headerrow_1, keycol => \%csvkeycol_1, errors => \@errors_1, }); + + # Then ... + is(scalar @csvcolumns_1, 5, 'Got the expected 5 column array from prepare columns'); + is($csvcolumns_1[0], 'a', 'Got the expected a header from prepare columns'); + is($csvcolumns_1[1], 'b', 'Got the expected b header from prepare columns'); + is($csvcolumns_1[2], 'c', 'Got the expected c header from prepare columns'); + is($csvcolumns_1[3], '', 'Got the expected empty header from prepare columns'); + is($csvcolumns_1[4], 'd', 'Got the expected d header from prepare columns'); + + is($csvkeycol_1{a}, 0, 'Got the expected 0 value for key a from prepare columns hash'); + is($csvkeycol_1{b}, 1, 'Got the expected 1 value for key b from prepare columns hash'); + is($csvkeycol_1{c}, 2, 'Got the expected 2 value for key c from prepare columns hash'); + is($csvkeycol_1{''}, 3, 'Got the expected 3 value for empty string key from prepare columns hash'); + is($csvkeycol_1{d}, 4, 'Got the expected 4 value for key d from prepare columns hash'); +}; + +subtest 'test_set_column_keys' => sub { + plan tests => 5; + + # Given ... nothing at all + # When ... Then ... + my $attr_type_0 = $patrons_import->set_attribute_types(undef); + is($attr_type_0, undef, 'Got the expected undef attribute type from set attribute types with nothing'); + + # Given ... extended but not matchpoint + my $params_1 = { extended => 1, matchpoint => undef, }; + + # When ... Then ... + my $attr_type_1 = $patrons_import->set_attribute_types($params_1); + is($attr_type_1, undef, 'Got the expected undef attribute type from set attribute types with no matchpoint'); + + # Given ... extended and unexpected matchpoint + my $params_2 = { extended => 1, matchpoint => 'unexpected', }; + + # When ... Then ... + my $attr_type_2 = $patrons_import->set_attribute_types($params_2); + is($attr_type_2, undef, 'Got the expected undef attribute type from set attribute types with unexpected matchpoint'); + + # Given ... + my $code_3 = 'SHOW_BCODE'; + my $params_3 = { extended => 1, matchpoint => $code_3, }; + + # When ... + my $attr_type_3 = $patrons_import->set_attribute_types($params_3); + + # Then ... + isa_ok($attr_type_3, 'C4::Members::AttributeTypes'); + is($attr_type_3->{code}, $code_3, 'Got the expected code attribute type from set attribute types'); +}; + +subtest 'test_set_column_keys' => sub { + plan tests => 2; + + # Given ... nothing at all + # When ... Then ... + my @columnkeys_0 = $patrons_import->set_column_keys(undef); + is(scalar @columnkeys_0, 66, 'Got the expected array size from set column keys with undef extended'); + + # Given ... extended. + my $extended = 1; + + # When ... Then ... + my @columnkeys_1 = $patrons_import->set_column_keys($extended); + is(scalar @columnkeys_1, 67, 'Got the expected array size from set column keys with extended'); +}; + +subtest 'test_set_patron_attributes' => sub { + plan tests => 13; + + # Given ... nothing at all + # When ... Then ... + my $result_0 = $patrons_import->set_patron_attributes(undef, undef, undef); + is($result_0, undef, 'Got the expected undef from set patron attributes with nothing'); + + # Given ... not extended. + my $extended_1 = 0; + + # When ... Then ... + my $result_1 = $patrons_import->set_patron_attributes($extended_1, undef, undef); + is($result_1, undef, 'Got the expected undef from set patron attributes with not extended'); + + # Given ... NO patrons attributes + my $extended_2 = 1; + my $patron_attributes_2 = undef; + my @feedback_2; + + # When ... + my $result_2 = $patrons_import->set_patron_attributes($extended_2, $patron_attributes_2, \@feedback_2); + + # Then ... + is($result_2, undef, 'Got the expected undef from set patron attributes with no patrons attributes'); + is(scalar @feedback_2, 0, 'Got the expected 0 size feedback array from set patron attributes with no patrons attributes'); + + # Given ... some patrons attributes + my $patron_attributes_3 = "homeroom:1150605,grade:01"; + my @feedback_3; + + # When ... + my $result_3 = $patrons_import->set_patron_attributes($extended_2, $patron_attributes_3, \@feedback_3); + + # Then ... + ok($result_3, 'Got some data back from set patron attributes'); + is($result_3->[0]->{code}, 'grade', 'Got the expected first code from set patron attributes'); + is($result_3->[0]->{value}, '01', 'Got the expected first value from set patron attributes'); + + is($result_3->[1]->{code}, 'homeroom', 'Got the expected second code from set patron attributes'); + is($result_3->[1]->{value}, 1150605, 'Got the expected second value from set patron attributes'); + + is(scalar @feedback_3, 1, 'Got the expected 1 array size from set patron attributes with extended user'); + is($feedback_3[0]->{feedback}, 1, 'Got the expected second feedback from set patron attributes with extended user'); + is($feedback_3[0]->{name}, 'attribute string', 'Got the expected attribute string from set patron attributes with extended user'); + is($feedback_3[0]->{value}, 'homeroom:1150605,grade:01', 'Got the expected feedback value from set patron attributes with extended user'); +}; + +subtest 'test_check_branch_code' => sub { + plan tests => 11; + + # Given ... no branch code. + my $borrowerline = 'some, line'; + my $line_number = 78; + my @missing_criticals = (); + + # When ... + $patrons_import->check_branch_code(undef, $borrowerline, $line_number, \@missing_criticals); + + # Then ... + is(scalar @missing_criticals, 1, 'Got the expected missing critical array size of 1 from check_branch_code with no branch code'); + + is($missing_criticals[0]->{key}, 'branchcode', 'Got the expected branchcode key from check_branch_code with no branch code'); + is($missing_criticals[0]->{line}, $line_number, 'Got the expected line number from check_branch_code with no branch code'); + is($missing_criticals[0]->{lineraw}, $borrowerline, 'Got the expected lineraw value from check_branch_code with no branch code'); + + # Given ... unknown branch code + my $branchcode_1 = 'unexpected'; + my $borrowerline_1 = 'some, line,'.$branchcode_1; + my $line_number_1 = 79; + my @missing_criticals_1 = (); + + # When ... + $patrons_import->check_branch_code($branchcode_1, $borrowerline_1, $line_number_1, \@missing_criticals_1); + + # Then ... + is(scalar @missing_criticals_1, 1, 'Got the expected missing critical array size of 1 from check_branch_code with unexpected branch code'); + + is($missing_criticals_1[0]->{branch_map}, 1, 'Got the expected 1 branch_map from check_branch_code with unexpected branch code'); + is($missing_criticals_1[0]->{key}, 'branchcode', 'Got the expected branchcode key from check_branch_code with unexpected branch code'); + is($missing_criticals_1[0]->{line}, $line_number_1, 'Got the expected line number from check_branch_code with unexpected branch code'); + is($missing_criticals_1[0]->{lineraw}, $borrowerline_1, 'Got the expected lineraw value from check_branch_code with unexpected branch code'); + is($missing_criticals_1[0]->{value}, $branchcode_1, 'Got the expected value from check_branch_code with unexpected branch code'); + + # Given ... a known branch code. Relies on database sample data + my $branchcode_2 = 'FFL'; + my $borrowerline_2 = 'some, line,'.$branchcode_2; + my $line_number_2 = 80; + my @missing_criticals_2 = (); + + # When ... + $patrons_import->check_branch_code($branchcode_2, $borrowerline_2, $line_number_2, \@missing_criticals_2); + + # Then ... + is(scalar @missing_criticals_2, 0, 'Got the expected missing critical array size of 0 from check_branch_code'); +}; + +subtest 'test_check_borrower_category' => sub { + plan tests => 11; + + # Given ... no category code. + my $borrowerline = 'some, line'; + my $line_number = 781; + my @missing_criticals = (); + + # When ... + $patrons_import->check_borrower_category(undef, $borrowerline, $line_number, \@missing_criticals); + + # Then ... + is(scalar @missing_criticals, 1, 'Got the expected missing critical array size of 1 from check_branch_code with no category code'); + + is($missing_criticals[0]->{key}, 'categorycode', 'Got the expected categorycode key from check_branch_code with no category code'); + is($missing_criticals[0]->{line}, $line_number, 'Got the expected line number from check_branch_code with no category code'); + is($missing_criticals[0]->{lineraw}, $borrowerline, 'Got the expected lineraw value from check_branch_code with no category code'); + + # Given ... unknown category code + my $categorycode_1 = 'unexpected'; + my $borrowerline_1 = 'some, line, line, '.$categorycode_1; + my $line_number_1 = 791; + my @missing_criticals_1 = (); + + # When ... + $patrons_import->check_borrower_category($categorycode_1, $borrowerline_1, $line_number_1, \@missing_criticals_1); + + # Then ... + is(scalar @missing_criticals_1, 1, 'Got the expected missing critical array size of 1 from check_branch_code with unexpected category code'); + + is($missing_criticals_1[0]->{category_map}, 1, 'Got the expected 1 category_map from check_branch_code with unexpected category code'); + is($missing_criticals_1[0]->{key}, 'categorycode', 'Got the expected branchcode key from check_branch_code with unexpected category code'); + is($missing_criticals_1[0]->{line}, $line_number_1, 'Got the expected line number from check_branch_code with unexpected category code'); + is($missing_criticals_1[0]->{lineraw}, $borrowerline_1, 'Got the expected lineraw value from check_branch_code with unexpected category code'); + is($missing_criticals_1[0]->{value}, $categorycode_1, 'Got the expected value from check_branch_code with unexpected category code'); + + # Given ... a known category code. Relies on database sample data. + my $categorycode_2 = 'T'; + my $borrowerline_2 = 'some, line,'.$categorycode_2; + my $line_number_2 = 801; + my @missing_criticals_2 = (); + + # When ... + $patrons_import->check_borrower_category($categorycode_2, $borrowerline_2, $line_number_2, \@missing_criticals_2); + + # Then ... + is(scalar @missing_criticals_2, 0, 'Got the expected missing critical array size of 0 from check_branch_code'); +}; + +subtest 'test_format_dates' => sub { + plan tests => 22; + + # Given ... no borrower data. + my $borrowerline = 'another line'; + my $line_number = 987; + my @missing_criticals = (); + my %borrower; + my $params = {borrower => \%borrower, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals, }; + + # When ... + $patrons_import->format_dates($params); + + # Then ... + ok( not(%borrower), 'Got the expected no borrower from format_dates with no dates'); + is(scalar @missing_criticals, 0, 'Got the expected missing critical array size of 0 from format_dates with no dates'); + + # Given ... some good dates + my @missing_criticals_1 = (); + my $dateofbirth_1 = '2016-05-03'; + my $dateenrolled_1 = '2016-05-04'; + my $dateexpiry_1 = '2016-05-06'; + my $borrower_1 = { dateofbirth => $dateofbirth_1, dateenrolled => $dateenrolled_1, dateexpiry => $dateexpiry_1, }; + my $params_1 = {borrower => $borrower_1, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals_1, }; + + # When ... + $patrons_import->format_dates($params_1); + + # Then ... + is($borrower_1->{dateofbirth}, $dateofbirth_1, 'Got the expected date of birth from format_dates with good dates'); + is($borrower_1->{dateenrolled}, $dateenrolled_1, 'Got the expected date of birth from format_dates with good dates'); + is($borrower_1->{dateexpiry}, $dateexpiry_1, 'Got the expected date of birth from format_dates with good dates'); + is(scalar @missing_criticals_1, 0, 'Got the expected missing critical array size of 0 from check_branch_code with good dates'); + + # Given ... some very bad dates + my @missing_criticals_2 = (); + my $dateofbirth_2 = '03-2016-05'; + my $dateenrolled_2 = '04-2016-05'; + my $dateexpiry_2 = '06-2016-05'; + my $borrower_2 = { dateofbirth => $dateofbirth_2, dateenrolled => $dateenrolled_2, dateexpiry => $dateexpiry_2, }; + my $params_2 = {borrower => $borrower_2, lineraw => $borrowerline, line => $line_number, missing_criticals => \@missing_criticals_2, }; + + # When ... + $patrons_import->format_dates($params_2); + + # Then ... + is($borrower_2->{dateofbirth}, '', 'Got the expected empty date of birth from format_dates with bad dates'); + is($borrower_2->{dateenrolled}, '', 'Got the expected emptydate of birth from format_dates with bad dates'); + is($borrower_2->{dateexpiry}, '', 'Got the expected empty date of birth from format_dates with bad dates'); + + is(scalar @missing_criticals_2, 3, 'Got the expected missing critical array size of 3 from check_branch_code with bad dates'); + is($missing_criticals_2[0]->{bad_date}, 1, 'Got the expected first bad date flag from check_branch_code with bad dates'); + is($missing_criticals_2[0]->{key}, 'dateofbirth', 'Got the expected dateofbirth key from check_branch_code with bad dates'); + is($missing_criticals_2[0]->{line}, $line_number, 'Got the expected first line from check_branch_code with bad dates'); + is($missing_criticals_2[0]->{lineraw}, $borrowerline, 'Got the expected first lineraw from check_branch_code with bad dates'); + + is($missing_criticals_2[1]->{bad_date}, 1, 'Got the expected second bad date flag from check_branch_code with bad dates'); + is($missing_criticals_2[1]->{key}, 'dateenrolled', 'Got the expected dateenrolled key from check_branch_code with bad dates'); + is($missing_criticals_2[1]->{line}, $line_number, 'Got the expected second line from check_branch_code with bad dates'); + is($missing_criticals_2[1]->{lineraw}, $borrowerline, 'Got the expected second lineraw from check_branch_code with bad dates'); + + is($missing_criticals_2[2]->{bad_date}, 1, 'Got the expected third bad date flag from check_branch_code with bad dates'); + is($missing_criticals_2[2]->{key}, 'dateexpiry', 'Got the expected dateexpiry key from check_branch_code with bad dates'); + is($missing_criticals_2[2]->{line}, $line_number, 'Got the expected third line from check_branch_code with bad dates'); + is($missing_criticals_2[2]->{lineraw}, $borrowerline, 'Got the expected third lineraw from check_branch_code with bad dates'); +}; + +# ###### Test utility ########### +sub make_csv { + my ($temp_dir, @lines) = @_; + + my ($fh, $filename) = tempfile( DIR => $temp_dir) or die $!; + print $fh $_."\r\n" foreach @lines; + close $fh or die $!; + + return $filename; +} + +1; diff --git a/tools/import_borrowers.pl b/tools/import_borrowers.pl index d17bc408bc..c49a0bcd91 100755 --- a/tools/import_borrowers.pl +++ b/tools/import_borrowers.pl @@ -38,12 +38,6 @@ use Modern::Perl; use C4::Auth; use C4::Output; -use C4::Context; -use C4::Members; -use C4::Members::Attributes qw(:all); -use C4::Members::AttributeTypes; -use C4::Members::Messaging; -use C4::Reports::Guided; use C4::Templates; use Koha::Patron::Debarments; use Koha::Patrons; @@ -53,34 +47,37 @@ use Koha::Libraries; use Koha::Patron::Categories; use Koha::List::Patron; +use Koha::Patrons::Import; +my $Import = Koha::Patrons::Import->new(); + use Text::CSV; + # Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: # ė # č use CGI qw ( -utf8 ); -my (@errors, @feedback); +my ( @errors, @feedback ); my $extended = C4::Context->preference('ExtendedPatronAttributes'); -my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences'); -my @columnkeys = Koha::Patrons->columns(); -@columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } @columnkeys; -if ($extended) { - push @columnkeys, 'patron_attributes'; -} + +my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } Koha::Patrons->columns(); +push( @columnkeys, 'patron_attributes' ) if $extended; my $input = CGI->new(); -our $csv = Text::CSV->new({binary => 1}); # binary needed for non-ASCII Unicode + #push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX -my ( $template, $loggedinuser, $cookie ) = get_template_and_user({ +my ( $template, $loggedinuser, $cookie ) = get_template_and_user( + { template_name => "tools/import_borrowers.tt", query => $input, type => "intranet", authnotrequired => 0, flagsrequired => { tools => 'import_patrons' }, debug => 1, -}); + } +); # get the patron categories and pass them to the template my @patron_categories = Koha::Patron::Categories->search_limited({}, {order_by => ['description']}); @@ -89,21 +86,22 @@ my $columns = C4::Templates::GetColumnDefs( $input )->{borrowers}; $columns = [ grep { $_->{field} ne 'borrowernumber' ? $_ : () } @$columns ]; $template->param( borrower_fields => $columns ); -if ($input->param('sample')) { +if ( $input->param('sample') ) { + our $csv = Text::CSV->new( { binary => 1 } ); # binary needed for non-ASCII Unicode print $input->header( - -type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? + -type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ? -attachment => 'patron_import.csv', ); $csv->combine(@columnkeys); print $csv->string, "\n"; exit 0; } + my $uploadborrowers = $input->param('uploadborrowers'); my $matchpoint = $input->param('matchpoint'); if ($matchpoint) { $matchpoint =~ s/^patron_attribute_//; } -my $overwrite_cardnumber = $input->param('overwrite_cardnumber'); #create a patronlist my $createpatronlist = $input->param('createpatronlist') || 0; @@ -120,287 +118,57 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { token => scalar $input->param('csrf_token'), }); - push @feedback, {feedback=>1, name=>'filename', value=>$uploadborrowers, filename=>$uploadborrowers}; - my $handle = $input->upload('uploadborrowers'); - my $uploadinfo = $input->uploadInfo($uploadborrowers); - foreach (keys %$uploadinfo) { - push @feedback, {feedback=>1, name=>$_, value=>$uploadinfo->{$_}, $_=>$uploadinfo->{$_}}; - } - - my $imported = 0; - my @imported_borrowers; - my $alreadyindb = 0; - my $overwritten = 0; - my $invalid = 0; - my $matchpoint_attr_type; + my $handle = $input->upload('uploadborrowers'); my %defaults = $input->Vars; - # use header line to construct key to column map - my $borrowerline = <$handle>; - my $status = $csv->parse($borrowerline); - ($status) or push @errors, {badheader=>1,line=>$., lineraw=>$borrowerline}; - my @csvcolumns = $csv->fields(); - my %csvkeycol; - my $col = 0; - foreach my $keycol (@csvcolumns) { - # columnkeys don't contain whitespace, but some stupid tools add it - $keycol =~ s/ +//g; - $csvkeycol{$keycol} = $col++; - } - #warn($borrowerline); - my $ext_preserve = $input->param('ext_preserve') || 0; - if ($extended) { - $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint); - } - - push @feedback, {feedback=>1, name=>'headerrow', value=>join(', ', @csvcolumns)}; - my $today = output_pref; - my @criticals = qw(surname branchcode categorycode); # there probably should be others - my @bad_dates; # I've had a few. - LINE: while ( my $borrowerline = <$handle> ) { - my %borrower; - my @missing_criticals; - my $patron_attributes; - my $status = $csv->parse($borrowerline); - my @columns = $csv->fields(); - if (! $status) { - push @missing_criticals, {badparse=>1, line=>$., lineraw=>$borrowerline}; - } elsif (@columns == @columnkeys) { - @borrower{@columnkeys} = @columns; - # MJR: try to fill blanks gracefully by using default values - foreach my $key (@columnkeys) { - if ($borrower{$key} !~ /\S/) { - $borrower{$key} = $defaults{$key}; - } - } - } else { - # MJR: try to recover gracefully by using default values - foreach my $key (@columnkeys) { - if (defined($csvkeycol{$key}) and $columns[$csvkeycol{$key}] =~ /\S/) { - $borrower{$key} = $columns[$csvkeycol{$key}]; - } elsif ( $defaults{$key} ) { - $borrower{$key} = $defaults{$key}; - } elsif ( scalar grep {$key eq $_} @criticals ) { - # a critical field is undefined - push @missing_criticals, {key=>$key, line=>$., lineraw=>$borrowerline}; - } else { - $borrower{$key} = ''; - } - } - } - #warn join(':',%borrower); - if ($borrower{categorycode}) { - push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1} - unless Koha::Patron::Categories->find($borrower{categorycode}); - } else { - push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline}; - } - if ($borrower{branchcode}) { - push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline, value=>$borrower{branchcode}, branch_map=>1} - unless Koha::Libraries->find($borrower{branchcode}); - } else { - push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline}; - } - if (@missing_criticals) { - foreach (@missing_criticals) { - $_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF'; - $_->{surname} = $borrower{surname} || 'UNDEF'; - } - $invalid++; - (25 > scalar @errors) and push @errors, {missing_criticals=>\@missing_criticals}; - # The first 25 errors are enough. Keeping track of 30,000+ would destroy performance. - next LINE; - } - if ($extended) { - my $attr_str = $borrower{patron_attributes}; - $attr_str =~ s/\xe2\x80\x9c/"/g; # fixup double quotes in case we are passed smart quotes - $attr_str =~ s/\xe2\x80\x9d/"/g; - push @feedback, {feedback=>1, name=>'attribute string', value=>$attr_str, filename=>$uploadborrowers}; - delete $borrower{patron_attributes}; # not really a field in borrowers, so we don't want to pass it to ModMember. - $patron_attributes = extended_attributes_code_value_arrayref($attr_str); - } - # Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it. - foreach (qw(dateofbirth dateenrolled dateexpiry)) { - my $tempdate = $borrower{$_} or next; - $tempdate = eval { output_pref( { dt => dt_from_string( $tempdate ), dateonly => 1, dateformat => 'iso' } ); }; - if ($tempdate) { - $borrower{$_} = $tempdate; - } else { - $borrower{$_} = ''; - push @missing_criticals, {key=>$_, line=>$. , lineraw=>$borrowerline, bad_date=>1}; - } - } - $borrower{dateenrolled} ||= $today; - $borrower{dateexpiry} ||= Koha::Patron::Categories->find( $borrower{categorycode} )->get_expiry_date( $borrower{dateenrolled} ); - my $borrowernumber; - my $member; - if ( ($matchpoint eq 'cardnumber') && ($borrower{'cardnumber'}) ) { - $member = Koha::Patrons->find( { cardnumber => $borrower{'cardnumber'} } ); - } elsif ( ($matchpoint eq 'userid') && ($borrower{'userid'}) ) { - $member = Koha::Patrons->find( { userid => $borrower{'userid'} } ); - } elsif ($extended) { - if (defined($matchpoint_attr_type)) { - foreach my $attr (@$patron_attributes) { - if ($attr->{code} eq $matchpoint and $attr->{value} ne '') { - my @borrowernumbers = $matchpoint_attr_type->get_patrons($attr->{value}); - $borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1; - last; - } - } - } - } - - if ($member) { - $member = $member->unblessed; - $borrowernumber = $member->{'borrowernumber'}; - } else { - $member = {}; - } - - if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) { - push @errors, { - invalid_cardnumber => 1, - borrowernumber => $borrowernumber, - cardnumber => $borrower{cardnumber} - }; - $invalid++; - next; + my $return = $Import->import_patrons( + { + file => $handle, + defaults => \%defaults, + matchpoint => $matchpoint, + overwrite_cardnumber => $input->param('overwrite_cardnumber'), + preserve_extended_attributes => $input->param('ext_preserve') || 0, } + ); - if ($borrowernumber) { - # borrower exists - unless ($overwrite_cardnumber) { - $alreadyindb++; - $template->param('lastalreadyindb'=>$borrower{'surname'}.' / '.$borrowernumber); - next LINE; - } - $borrower{'borrowernumber'} = $borrowernumber; - for my $col (keys %borrower) { - # use values from extant patron unless our csv file includes this column or we provided a default. - # FIXME : You cannot update a field with a perl-evaluated false value using the defaults. - - # The password is always encrypted, skip it! - next if $col eq 'password'; - - unless(exists($csvkeycol{$col}) || $defaults{$col}) { - $borrower{$col} = $member->{$col} if($member->{$col}) ; - } - } - - # Check if the userid provided does not exist yet - if ( exists $borrower{userid} - and $borrower{userid} - and not Check_Userid( $borrower{userid}, $borrower{borrowernumber} ) ) { - push @errors, { duplicate_userid => 1, userid => $borrower{userid} }; - $invalid++; - next LINE; - } - - unless (ModMember(%borrower)) { - $invalid++; - # until we have better error trapping, we have no way of knowing why ModMember errored out... - push @errors, {unknown_error => 1}; - $template->param('lastinvalid'=>$borrower{'surname'}.' / '.$borrowernumber); - next LINE; - } - - # Don't add a new restriction if the existing 'combined' restriction matches this one - if ( $borrower{debarred} && ( ( $borrower{debarred} ne $member->{debarred} ) || ( $borrower{debarredcomment} ne $member->{debarredcomment} ) ) ) { - # Check to see if this debarment already exists - my $debarrments = GetDebarments( - { - borrowernumber => $borrowernumber, - expiration => $borrower{debarred}, - comment => $borrower{debarredcomment} - } - ); - # If it doesn't, then add it! - unless (@$debarrments) { - AddDebarment( - { - borrowernumber => $borrowernumber, - expiration => $borrower{debarred}, - comment => $borrower{debarredcomment} - } - ); - } - } - - if ($extended) { - if ($ext_preserve) { - my $old_attributes = GetBorrowerAttributes($borrowernumber); - $patron_attributes = extended_attributes_merge($old_attributes, $patron_attributes); #TODO: expose repeatable options in template - } - push @errors, {unknown_error => 1} unless SetBorrowerAttributes($borrower{'borrowernumber'}, $patron_attributes, 'no_branch_limit' ); - } - $overwritten++; - $template->param('lastoverwritten'=>$borrower{'surname'}.' / '.$borrowernumber); - } else { - # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either. - # At least this is closer to AddMember than in members/memberentry.pl - if (!$borrower{'cardnumber'}) { - $borrower{'cardnumber'} = fixup_cardnumber(undef); - } - if ($borrowernumber = AddMember(%borrower)) { - - if ( $borrower{debarred} ) { - AddDebarment( - { - borrowernumber => $borrowernumber, - expiration => $borrower{debarred}, - comment => $borrower{debarredcomment} - } - ); - } - - if ($extended) { - SetBorrowerAttributes($borrowernumber, $patron_attributes); - } - - if ($set_messaging_prefs) { - C4::Members::Messaging::SetMessagingPreferencesFromDefaults({ borrowernumber => $borrowernumber, - categorycode => $borrower{categorycode} }); - } + my $feedback = $return->{feedback}; + my $errors = $return->{errors}; + my $imported = $return->{imported}; + my $overwritten = $return->{overwritten}; + my $alreadyindb = $return->{already_in_db}; + my $invalid = $return->{invalid}; - $imported++; - $template->param('lastimported'=>$borrower{'surname'}.' / '.$borrowernumber); - push @imported_borrowers, $borrowernumber; #for patronlist - } else { - $invalid++; - push @errors, {unknown_error => 1}; - $template->param('lastinvalid'=>$borrower{'surname'}.' / AddMember'); - } - } + my $uploadinfo = $input->uploadInfo($uploadborrowers); + foreach ( keys %$uploadinfo ) { + push @$feedback, { feedback => 1, name => $_, value => $uploadinfo->{$_}, $_ => $uploadinfo->{$_} }; } - if ( $imported && $createpatronlist ) { - my $patronlist = AddPatronList({ name => $patronlistname }); - AddPatronsToList({ list => $patronlist, borrowernumbers => \@imported_borrowers }); - $template->param('patronlistname' => $patronlistname); - } + push @$feedback, { feedback => 1, name => 'filename', value => $uploadborrowers, filename => $uploadborrowers }; - (@errors ) and $template->param( ERRORS=>\@errors ); - (@feedback) and $template->param(FEEDBACK=>\@feedback); $template->param( - 'uploadborrowers' => 1, - 'imported' => $imported, - 'overwritten' => $overwritten, - 'alreadyindb' => $alreadyindb, - 'invalid' => $invalid, - 'total' => $imported + $alreadyindb + $invalid + $overwritten, + uploadborrowers => 1, + errors => $errors, + feedback => $feedback, + imported => $imported, + overwritten => $overwritten, + alreadyindb => $alreadyindb, + invalid => $invalid, + total => $imported + $alreadyindb + $invalid + $overwritten, ); -} else { +} +else { if ($extended) { my @matchpoints = (); - my @attr_types = C4::Members::AttributeTypes::GetAttributeTypes(undef, 1); + my @attr_types = C4::Members::AttributeTypes::GetAttributeTypes( undef, 1 ); foreach my $type (@attr_types) { - my $attr_type = C4::Members::AttributeTypes->fetch($type->{code}); - if ($attr_type->unique_id()) { - push @matchpoints, { code => "patron_attribute_" . $attr_type->code(), description => $attr_type->description() }; + my $attr_type = C4::Members::AttributeTypes->fetch( $type->{code} ); + if ( $attr_type->unique_id() ) { + push @matchpoints, + { code => "patron_attribute_" . $attr_type->code(), description => $attr_type->description() }; } } - $template->param(matchpoints => \@matchpoints); + $template->param( matchpoints => \@matchpoints ); } $template->param( -- 2.39.5