Bug 6752: Be stricter with utf-8 encoding of output

use encoding(UTF-8) rather than utf-8 for stricter
encoding
Marking output as ':utf8' only flags the data as utf8
using :encoding(UTF-8) also checks it as valid utf-8
see binmode in perlfunc for more details
In accordance with the robustness principle input
filehandles have not been changed as code may make
the undocumented assumption that invalid utf-8 is present
in the imput
Fixes errors reported by t/00-testcritic.t
Where feasable some filehandles have been made lexical rather than
reusing global filehandle vars

Signed-off-by: Jonathan Druart <jonathan.druart@biblibre.com>
Signed-off-by: Paul Poulain <paul.poulain@biblibre.com>

admin/aqplan.pl

@@ -463,7 +463,7 @@ output_html_with_http_headers $input, $cookie, $template->output;
 sub _print_to_csv {
     my ( $header, $results ) = @_;
 
-    binmode STDOUT, ":encoding(UTF-8)";
+    binmode STDOUT, ':encoding(UTF-8)';
 
     my $csv = Text::CSV_XS->new(
         {   sep_char     => $del,

authorities/authorities-list.pl

@@ -4,7 +4,7 @@ use warnings;
 use C4::Context;
 use C4::AuthoritiesMarc;
 use utf8;
-use open qw( :std :utf8 );
+use open qw[ :std :encoding(utf8) ];
 
 my $dbh=C4::Context->dbh;
 my $datatypes_query = $dbh->prepare(<<ENDSQL);

misc/cronjobs/MARC21_parse_test.pl

@@ -24,7 +24,7 @@ use MARC::Record;
 use MARC::File::XML;
 use MARC::File::USMARC;
 
-use open OUT => ':utf8';
+use open OUT => ':encoding(UTF-8)';
 
 use Getopt::Long qw(:config auto_help auto_version);
 use Pod::Usage;

misc/cronjobs/overdue_notices.pl

@@ -323,7 +323,7 @@ if (@branchcodes) {
 # these are the fields that will be substituted into <<item.content>>
 my @item_content_fields = split( /,/, $itemscontent );
 
-binmode( STDOUT, ":utf8" );
+binmode STDOUT, ':encoding(UTF-8)';
 
 
 our $csv;       # the Text::CSV_XS object

misc/migration_tools/22_to_30/export_Authorities_xml.pl

@@ -23,7 +23,7 @@ $rq->execute;
 #ATTENTION : Mettre la base en utf8 auparavant.
 #BEWARE : Set database into utf8 before.
 while (my ($authid)=$rq->fetchrow){
-open FILEOUTPUT,">:utf8", "./$filename/$authid.xml" or die "unable to open $filename";
+open my $fileoutput, '>:encoding(UTF-8)', "./$filename/$authid.xml" or die "unable to open $filename";
   my $record=AUTHgetauthority($dbh,$authid);
   if (! utf8::is_utf8($record)) {
     utf8::decode($record);
@@ -44,7 +44,7 @@ open FILEOUTPUT,">:utf8", "./$filename/$authid.xml" or die "unable to open $file
      # } else {
 #    $record->encoding( 'UTF-8' );
 #  }
-  print FILEOUTPUT $record->as_xml();
-close FILEOUPUT;
+  print {$fileoutput}  $record->as_xml();
+close $fileoutput;
 
 }

misc/migration_tools/bulkmarcimport.pl

@@ -30,7 +30,7 @@ use Getopt::Long;
 use IO::File;
 use Pod::Usage;
 
-binmode(STDOUT, ":utf8");
+binmode STDOUT, ':encoding(UTF-8)';
 my ( $input_marc_file, $number, $offset) = ('',0,0);
 my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
 my ($sourcetag,$sourcesubfield,$idmapfl);

misc/migration_tools/rebuild_zebra.pl

@@ -309,7 +309,7 @@ sub export_marc_records_from_sth {
     my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_;
 
     my $num_exported = 0;
-    open (OUT, ">:utf8 ", "$directory/exported_records") or die $!;
+    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
     my $i = 0;
     my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
     while (my ($record_number) = $sth->fetchrow_array) {
@@ -337,7 +337,7 @@ sub export_marc_records_from_sth {
                 }
             }
             if ( $marcxml ) {
-                print OUT $marcxml if $marcxml;
+                print {$fh} $marcxml if $marcxml;
                 $num_exported++;
             }
             next;
@@ -350,7 +350,7 @@ sub export_marc_records_from_sth {
             # to care, though, at least if you're using the GRS-1 filter.  It does
             # care if you're using the DOM filter, which requires valid XML file(s).
             eval {
-                print OUT ($as_xml) ? $marc->as_xml_record(C4::Context->preference('marcflavour')) : $marc->as_usmarc();
+                print {$fh} ($as_xml) ? $marc->as_xml_record(C4::Context->preference('marcflavour')) : $marc->as_usmarc();
                 $num_exported++;
             };
             if ($@) {
@@ -359,7 +359,7 @@ sub export_marc_records_from_sth {
         }
     }
     print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
-    close OUT;
+    close $fh;
     return $num_exported;
 }
 
@@ -367,7 +367,7 @@ sub export_marc_records_from_list {
     my ($record_type, $entries, $directory, $as_xml, $noxml, $records_deleted) = @_;
 
     my $num_exported = 0;
-    open (OUT, ">:utf8 ", "$directory/exported_records") or die $!;
+    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
     my $i = 0;
 
     # Skip any deleted records. We check for this anyway, but this reduces error spam
@@ -384,12 +384,12 @@ sub export_marc_records_from_list {
             # strung together with no single root element.  zebraidx doesn't seem
             # to care, though, at least if you're using the GRS-1 filter.  It does
             # care if you're using the DOM filter, which requires valid XML file(s).
-            print OUT ($as_xml) ? $marc->as_xml_record(C4::Context->preference('marcflavour')) : $marc->as_usmarc();
+            print {$fh} ($as_xml) ? $marc->as_xml_record(C4::Context->preference('marcflavour')) : $marc->as_usmarc();
             $num_exported++;
         }
     }
     print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
-    close OUT;
+    close $fh;
     return $num_exported;
 }
 
@@ -397,7 +397,7 @@ sub generate_deleted_marc_records {
     my ($record_type, $entries, $directory, $as_xml) = @_;
 
     my $records_deleted = {};
-    open (OUT, ">:utf8 ", "$directory/exported_records") or die $!;
+    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
     my $i = 0;
     foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
         print "\r$i" unless ($i++ %100 or !$verbose_logging);
@@ -413,12 +413,12 @@ sub generate_deleted_marc_records {
             fix_unimarc_100($marc);
         }
 
-        print OUT ($as_xml) ? $marc->as_xml_record(C4::Context->preference("marcflavour")) : $marc->as_usmarc();
+        print {$fh} ($as_xml) ? $marc->as_xml_record(C4::Context->preference("marcflavour")) : $marc->as_usmarc();
 
         $records_deleted->{$record_number} = 1;
     }
     print "\nRecords exported: $i\n" if ( $verbose_logging );
-    close OUT;
+    close $fh;
     return $records_deleted;
     
 
@@ -824,8 +824,8 @@ if ($authorities) {
     # AUTHORITIES : copying mandatory files
     #
     unless (-f C4::Context->zebraconfig('authorityserver')->{config}) {
-    open ZD,">:utf8 ",C4::Context->zebraconfig('authorityserver')->{config};
-    print ZD "
+    open my $zd, '>:encoding(UTF-8)' ,C4::Context->zebraconfig('authorityserver')->{config};
+    print {$zd} "
 # generated by KOHA/misc/migration_tools/rebuild_zebra.pl 
 profilePath:\${srcdir:-.}:$authorityserverdir/tab/:$tabdir/tab/:\${srcdir:-.}/tab/
 
@@ -969,8 +969,8 @@ if ($biblios) {
     # BIBLIOS : copying mandatory files
     #
     unless (-f C4::Context->zebraconfig('biblioserver')->{config}) {
-    open ZD,">:utf8 ",C4::Context->zebraconfig('biblioserver')->{config};
-    print ZD "
+    open my $zd, '>:encoding(UTF-8)', C4::Context->zebraconfig('biblioserver')->{config};
+    print {$zd} "
 # generated by KOHA/misc/migrtion_tools/rebuild_zebra.pl 
 profilePath:\${srcdir:-.}:$biblioserverdir/tab/:$tabdir/tab/:\${srcdir:-.}/tab/
 

misc/sax_parser_test.pl

@@ -9,7 +9,7 @@ use Encode;
 my $parser = XML::SAX::ParserFactory->parser(
 Handler => MySAXHandler->new
 );
-binmode STDOUT, ":utf8";
+binmode STDOUT, ':encoding(UTF-8)';
 print "\x{65}\x{301}\n";
 $parser->parse_string(encode_utf8("<xml>\x{65}\x{301}</xml>"));
 $parser->parse_string("<xml>\xEF\xBB\xBF\x{65}\x{301}</xml>");

misc/translator/xgettext.pl

@@ -351,7 +351,7 @@ if (defined $output && $output ne '-') {
     print STDERR "$0: Outputting to STDOUT...\n" if $verbose_p;
     open(OUTPUT, ">&STDOUT");
 }
-#binmode( OUTPUT, ":utf8" );
+binmode OUTPUT, ':encoding(UTF-8)';
 
 if (defined $files_from) {
     print STDERR "$0: Opening input file list \"$files_from\"\n" if $verbose_p;

opac/ilsdi.pl

@@ -228,7 +228,7 @@ if ( $service and any { $service eq $_ } @services ) {
 }
 
 # Output XML by passing the hashref to XMLOut
-binmode(STDOUT, ":utf8");
+binmode STDOUT, ':encoding(UTF-8)';
 print CGI::header('-type'=>'text/xml', '-charset'=>'utf-8');
 print XMLout(
     $out,

opac/oai.pl

@@ -41,7 +41,7 @@ else {
     );
 }
 
-binmode( STDOUT, ":utf8" );
+binmode STDOUT, ':encoding(UTF-8)';
 my $repository = C4::OAI::Repository->new();
 
 # __END__ Main Prog

reports/guided_reports.pl

@@ -539,7 +539,7 @@ elsif ($phase eq 'Run this report'){
 }
 
 elsif ($phase eq 'Export'){
-    binmode STDOUT, ':utf8';
+    binmode STDOUT, ':encoding(UTF-8)';
 
 	# export results to tab separated text or CSV
 	my $sql    = $input->param('sql');  # FIXME: use sql from saved report ID#, not new user-supplied SQL!

reports/serials_stats.pl

@@ -103,7 +103,7 @@ if($do_it){
         $template->param(datas => \@datas,
                          do_it => 1);
     }else{
-        binmode STDOUT, ':utf8';
+        binmode STDOUT, ':encoding(UTF-8)';
         print $input->header(-type => 'application/vnd.sun.xml.calc',
                          -encoding => 'utf-8',
                              -name => "$basename.csv",

svc/bib

@@ -27,7 +27,7 @@ use C4::Biblio;
 use XML::Simple;
 
 my $query = new CGI;
-binmode STDOUT, ":utf8";
+binmode STDOUT, ':encoding(UTF-8)';
 
 my ($status, $cookie, $sessionID) = check_api_auth($query, { editcatalogue => 'edit_catalogue'} );
 unless ($status eq "ok") {

svc/new_bib

@@ -28,7 +28,7 @@ use XML::Simple;
 use C4::Charset;
 
 my $query = new CGI;
-binmode STDOUT, ":utf8";
+binmode STDOUT, ':encoding(UTF-8)';
 
 my ($status, $cookie, $sessionID) = check_api_auth($query, { editcatalogue => 'edit_catalogue'} );
 unless ($status eq "ok") {

t/db_dependent/lib/KohaTest.pm

@@ -625,11 +625,11 @@ sub reindex_marc {
         mkdir "$directory/$record_type";
         my $sth = $dbh->prepare($record_type eq "biblio" ? "SELECT marc FROM biblioitems" : "SELECT marc FROM auth_header");
         $sth->execute();
-        open OUT, ">:utf8", "$directory/$record_type/records";
+        open my $out, '>:encoding(UTF-8)', "$directory/$record_type/records";
         while (my ($blob) = $sth->fetchrow_array) {
-            print OUT $blob;
+            print {$out} $blob;
         }
-        close OUT;
+        close $out;
         my $zebra_server = "${record_type}server";
         my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
         my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};

tools/export.pl

@@ -56,7 +56,7 @@ my ($template, $loggedinuser, $cookie)
 	}
 
 if ($op eq "export") {
-    binmode(STDOUT,":utf8");
+    binmode STDOUT, ':encoding(UTF-8)';
 	print $query->header(   -type => 'application/octet-stream', 
                             -charset => 'utf-8',
                             -attachment=>$filename);