From 497a2d8e266dcbf6b584b94495a35a2e5bd7f97b Mon Sep 17 00:00:00 2001 From: Joe Atzberger Date: Thu, 18 Sep 2008 19:02:45 -0500 Subject: [PATCH] Bug 2627: Allow Unicode data in import_borrowers, expand feedback on errors. Date fields are now checked against syspref and iso regexps before attempting to convert or insert them. The problem characters were non-ASCII diacriticals. Note: this may rely on improvements in the current 0.54 version of Text::CSV and Text::CSV_XS, rather than the Koha minimum of 0.01 and 0.32, respectively. Signed-off-by: Galen Charlton --- .../en/modules/tools/import_borrowers.tmpl | 59 +++++++++++++----- tools/import_borrowers.pl | 62 +++++++++++++++---- 2 files changed, 91 insertions(+), 30 deletions(-) diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tmpl b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tmpl index 0c5f5c5319..47dc3a05e9 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tmpl +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tmpl @@ -3,6 +3,7 @@ @@ -13,12 +14,11 @@
- -
-
-
-
-
+
+
+
+
+

Import Patrons

Import results :
@@ -30,6 +30,23 @@
  • records parsed
  • Back to Tools
    • + +

    +
    +
    Feedback:
    + +
    +

    @@ -38,11 +55,18 @@
  • Header row could not be parsed
    • -
  • +
  • + Line - Line could not be parsed! + could not be parsed! + + has "" in unrecognized format: "" - Critical field "" missing on line + Critical field "" + has unrecognized value "" + has unrecognized value "" + missing + (borrowernumber: ; surname: ).
    @@ -105,9 +129,10 @@
    • Download a starter CSV file with all the columns here. Values are comma-separated.
    • OR format your file in CSV format with the following fields:
      • -
      • +
        • '',
        +
      • If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a ':' should precede each value. For example: "INSTID:12345,LANG:fr". This @@ -119,13 +144,13 @@ means that if an input record has more than one attribute, the 'patron_attribute
      • Date formats should match your system preference, and must be zero-padded, e.g. '01/02/2008'.
      • You may optionally include a header row, defining which columns you are supplying in the import file.
      -
    • -
    -
    -
    -
    +
    +
    +
    +
    +
    -
    -
    +
    +
    diff --git a/tools/import_borrowers.pl b/tools/import_borrowers.pl index 5cdf3c15f5..662dfccfcf 100755 --- a/tools/import_borrowers.pl +++ b/tools/import_borrowers.pl @@ -34,6 +34,8 @@ # branchcode and categorycode need to be valid use strict; +use warnings; + use C4::Auth; use C4::Output; use C4::Dates qw(format_date_in_iso); @@ -44,9 +46,14 @@ use C4::Members::Attributes; use C4::Members::AttributeTypes; use Text::CSV; +# Text::CSV::Unicode, even in binary mode, fails to parse lines with these diacriticals: +# ė +# č + use CGI; +# use encoding 'utf8'; # don't do this -my @errors; +my (@errors, @feedback); my $extended = C4::Context->preference('ExtendedPatronAttributes'); my @columnkeys = C4::Members->columns; if ($extended) { @@ -55,7 +62,8 @@ if ($extended) { my $columnkeystpl = [ map { {'key' => $_} } grep {$_ ne 'borrowernumber' && $_ ne 'cardnumber'} @columnkeys ]; # ref. to array of hashrefs. my $input = CGI->new(); -my $csv = Text::CSV->new(); +my $csv = Text::CSV->new({binary => 1}); # binary needed for non-ASCII Unicode +# push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; my ( $template, $loggedinuser, $cookie ) = get_template_and_user({ template_name => "tools/import_borrowers.tmpl", @@ -77,8 +85,8 @@ if ($input->param('sample')) { print $csv->string, "\n"; exit 1; } -my $uploadborrowers = $input->param('uploadborrowers'); -my $matchpoint = $input->param('matchpoint'); +my $uploadborrowers = $input->param('uploadborrowers'); +my $matchpoint = $input->param('matchpoint'); if ($matchpoint) { $matchpoint =~ s/^patron_attribute_//; } @@ -89,6 +97,12 @@ $template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} ); ($extended) and $template->param(ExtendedPatronAttributes => 1); if ( $uploadborrowers && length($uploadborrowers) > 0 ) { + push @feedback, {feedback=>1, name=>'filename', value=>$uploadborrowers, filename=>$uploadborrowers}; + my $handle = $input->upload('uploadborrowers'); + my $uploadinfo = $input->uploadInfo($uploadborrowers); + foreach (keys %$uploadinfo) { + push @feedback, {feedback=>1, name=>$_, value=>$uploadinfo->{$_}, $_=>$uploadinfo->{$_}}; + } my $imported = 0; my $alreadyindb = 0; my $overwritten = 0; @@ -97,7 +111,7 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { my %defaults = $input->Vars; # use header line to construct key to column map - my $borrowerline = <$uploadborrowers>; + my $borrowerline = <$handle>; my $status = $csv->parse($borrowerline); ($status) or push @errors, {badheader=>1,line=>$., lineraw=>$borrowerline}; my @csvcolumns = $csv->fields(); @@ -113,9 +127,13 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint); } + push @feedback, {feedback=>1, name=>'headerrow', value=>join(', ', @csvcolumns)}; + my $today_iso = C4::Dates->new()->output('iso'); my @criticals = qw(cardnumber surname categorycode); # there probably should be others - my @errors; - LINE: while ( my $borrowerline = <$uploadborrowers> ) { + my @bad_dates; # I've had a few. + my $date_re = C4::Dates->new->regexp('syspref'); + my $iso_re = C4::Dates->new->regexp('iso'); + LINE: while ( my $borrowerline = <$handle> ) { my %borrower; my @missing_criticals; my $patron_attributes; @@ -141,8 +159,18 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { } } #warn join(':',%borrower); - push @missing_criticals, {key=>'categorycode' , line=>$. , lineraw=>$borrowerline } unless( GetBorrowercategory($borrower{categorycode}) ); - push @missing_criticals, {key=>'branchcode' , line=>$. , lineraw=>$borrowerline } unless( GetBranchName($borrower{branchcode}) ); + if ($borrower{categorycode}) { + push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline, value=>$borrower{categorycode}, category_map=>1} + unless GetBorrowercategory($borrower{categorycode}); + } else { + push @missing_criticals, {key=>'categorycode', line=>$. , lineraw=>$borrowerline}; + } + if ($borrower{branchcode}) { + push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline, value=>$borrower{branchcode}, branch_map=>1} + unless GetBranchName($borrower{branchcode}); + } else { + push @missing_criticals, {key=>'branchcode', line=>$. , lineraw=>$borrowerline}; + } if (@missing_criticals) { foreach (@missing_criticals) { $_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF'; @@ -162,12 +190,19 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { # FIXME error handling $patron_attributes = [ map { map { my @arr = split /:/, $_, 2; { code => $arr[0], value => $arr[1] } } $_ } @list ]; } - # FIXME date handling. Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it. + # Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it. foreach (qw(dateofbirth dateenrolled dateexpiry)) { my $tempdate = $borrower{$_} or next; - $borrower{$_} = format_date_in_iso($tempdate) || ''; + if ($tempdate =~ /$date_re/) { + $borrower{$_} = format_date_in_iso($tempdate); + } elsif ($tempdate =~ /$iso_re/) { + $borrower{$_} = $tempdate; + } else { + $borrower{$_} = ''; + push @missing_criticals, {key=>$_, line=>$. , lineraw=>$borrowerline, bad_date=>1}; + } } - $borrower{dateenrolled} = C4::Dates->new()->output('iso') unless $borrower{dateenrolled}; + $borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled}; $borrower{dateexpiry} = GetExpiryDate($borrower{categorycode},$borrower{dateenrolled}) unless $borrower{dateexpiry}; my $borrowernumber; my $member; @@ -231,7 +266,8 @@ if ( $uploadborrowers && length($uploadborrowers) > 0 ) { } } } - (@errors) and $template->param(ERRORS=>\@errors); + (@errors ) and $template->param( ERRORS=>\@errors ); + (@feedback) and $template->param(FEEDBACK=>\@feedback); $template->param( 'uploadborrowers' => 1, 'imported' => $imported, -- 2.39.5