diff --git a/C4/Auth.pm b/C4/Auth.pm index 837ff12a57..ca061b5284 100644 --- a/C4/Auth.pm +++ b/C4/Auth.pm @@ -19,7 +19,6 @@ package C4::Auth; use strict; use warnings; - use Digest::MD5 qw(md5_base64); use Storable qw(thaw freeze); use URI::Escape; @@ -675,7 +674,6 @@ sub checkauth { $ip = $session->param('ip'); $lasttime = $session->param('lasttime'); $userid = $session->param('id'); - utf8::decode($userid); $sessiontype = $session->param('sessiontype') || ''; } if ( ( ($query->param('koha_login_context')) && ($query->param('userid') ne $session->param('id')) ) @@ -751,7 +749,6 @@ sub checkauth { -HttpOnly => 1 ); $userid = $query->param('userid'); - utf8::decode($userid); if ( ( $cas && $query->param('ticket') ) || $userid || ( my $pki_field = C4::Context->preference('AllowPKIAuth') ) ne @@ -1068,10 +1065,14 @@ sub checkauth { LibraryName => C4::Context->preference("LibraryName"), ); $template->param( %info ); - - require C4::Output; - C4::Output::output_html_with_http_headers( $query, $cookie, - $template->output); +# $cookie = $query->cookie(CGISESSID => $session->id +# ); + print $query->header( + -type => 'text/html', + -charset => 'utf-8', + -cookie => $cookie + ), + $template->output; safe_exit; } diff --git a/C4/Context.pm b/C4/Context.pm index fee87520c9..7709f7effb 100644 --- a/C4/Context.pm +++ b/C4/Context.pm @@ -1104,22 +1104,7 @@ set_userenv is called in Auth.pm #' sub set_userenv { - my ( - $usernum, $userid, $usercnum, $userfirstname, - $usersurname, $userbranch, $branchname, $userflags, - $emailaddress, $branchprinter, $persona - ) = @_; - for ( - $usernum, $userid, $usercnum, $userfirstname, - $usersurname, $userbranch, $branchname, $userflags, - $emailaddress, $branchprinter, $persona - ) - { - utf8::decode($_) if $_; - } - - # CGI::Session doesn't handle utf-8, so we decode it here - + my ($usernum, $userid, $usercnum, $userfirstname, $usersurname, $userbranch, $branchname, $userflags, $emailaddress, $branchprinter, $persona)= @_; my $var=$context->{"activeuser"} || ''; my $cell = { "number" => $usernum, diff --git a/C4/ItemType.pm b/C4/ItemType.pm index fffa0a280d..648cff95f5 100644 --- a/C4/ItemType.pm +++ b/C4/ItemType.pm @@ -81,7 +81,7 @@ sub all { for ( @{$dbh->selectall_arrayref( "SELECT * FROM itemtypes ORDER BY description", { Slice => {} })} ) { -# utf8::encode($_->{description}); + utf8::encode($_->{description}); push @itypes, $class->new($_); } return @itypes; diff --git a/C4/Output.pm b/C4/Output.pm index 63acf1453e..458b92f040 100644 --- a/C4/Output.pm +++ b/C4/Output.pm @@ -298,12 +298,17 @@ sub output_with_http_headers { $options->{'Content-Script-Type'} = 'text/javascript'; } +# We can't encode here, that will double encode our templates, and xslt +# We need to fix the encoding as it comes out of the database, or when we pass the variables to templates + +# utf8::encode($data) if utf8::is_utf8($data); + $data =~ s/\&\;amp\; /\&\; /g; - utf8::encode($data); print $query->header($options), $data; } sub output_html_with_http_headers { + binmode( STDOUT, ":encoding(utf8)" ); my ( $query, $cookie, $data, $status ) = @_; output_with_http_headers( $query, $cookie, $data, 'html', $status ); } diff --git a/C4/Search.pm b/C4/Search.pm index 8261e0a30b..39bbbc54f4 100644 --- a/C4/Search.pm +++ b/C4/Search.pm @@ -484,7 +484,6 @@ sub getRecords { # not an index scan else { $record = $results[ $i - 1 ]->record($j)->raw(); - utf8::decode( $record ); # warn "RECORD $j:".$record; $results_hash->{'RECORDS'}[$j] = $record; @@ -502,7 +501,6 @@ sub getRecords { for ( my $j = 0 ; $j < $jmax ; $j++ ) { my $render_record = $results[ $i - 1 ]->record($j)->render(); - utf8::decode($render_record); my @used_datas = (); foreach my $tag ( @{ $facet->{tags} } ) { @@ -716,7 +714,6 @@ sub pazGetRecords { for (my $i = 0; $i < $count; $i++) { # FIXME -- may need to worry about diacritics here my $rec = $paz->record($recid, $i); - utf8::decode( $rec ); push @{ $result_group->{'RECORDS'} }, $rec; } @@ -1296,17 +1293,17 @@ sub buildQuery { if ( @limits ) { $q .= ' and '.join(' and ', @limits); } - return ( undef, $q, $q, "q=ccl=".uri_escape_utf8($q), $q, '', '', '', '', 'ccl' ); + return ( undef, $q, $q, "q=ccl=".uri_escape($q), $q, '', '', '', '', 'ccl' ); } if ( $query =~ /^cql=/ ) { - return ( undef, $', $', "q=cql=".uri_escape_utf8($'), $', '', '', '', '', 'cql' ); + return ( undef, $', $', "q=cql=".uri_escape($'), $', '', '', '', '', 'cql' ); } if ( $query =~ /^pqf=/ ) { if ($query_desc) { - $query_cgi = "q=".uri_escape_utf8($query_desc); + $query_cgi = "q=".uri_escape($query_desc); } else { $query_desc = $'; - $query_cgi = "q=pqf=".uri_escape_utf8($'); + $query_cgi = "q=pqf=".uri_escape($'); } return ( undef, $', $', $query_cgi, $query_desc, '', '', '', '', 'pqf' ); } @@ -1478,9 +1475,9 @@ sub buildQuery { $query .= " $operators[$i-1] "; $query .= " $index_plus " unless $indexes_set; $query .= " $operand"; - $query_cgi .= "&op=".uri_escape_utf8($operators[$i-1]); - $query_cgi .= "&idx=".uri_escape_utf8($index) if $index; - $query_cgi .= "&q=".uri_escape_utf8($operands[$i]) if $operands[$i]; + $query_cgi .= "&op=".uri_escape($operators[$i-1]); + $query_cgi .= "&idx=".uri_escape($index) if $index; + $query_cgi .= "&q=".uri_escape($operands[$i]) if $operands[$i]; $query_desc .= " $operators[$i-1] $index_plus $operands[$i]"; } @@ -1490,8 +1487,8 @@ sub buildQuery { $query .= " and "; $query .= "$index_plus " unless $indexes_set; $query .= "$operand"; - $query_cgi .= "&op=and&idx=".uri_escape_utf8($index) if $index; - $query_cgi .= "&q=".uri_escape_utf8($operands[$i]) if $operands[$i]; + $query_cgi .= "&op=and&idx=".uri_escape($index) if $index; + $query_cgi .= "&q=".uri_escape($operands[$i]) if $operands[$i]; $query_desc .= " and $index_plus $operands[$i]"; } } @@ -1503,8 +1500,8 @@ sub buildQuery { $query .= " $index_plus " unless $indexes_set; $query .= $operand; $query_desc .= " $index_plus $operands[$i]"; - $query_cgi .= "&idx=".uri_escape_utf8($index) if $index; - $query_cgi .= "&q=".uri_escape_utf8($operands[$i]) if $operands[$i]; + $query_cgi .= "&idx=".uri_escape($index) if $index; + $query_cgi .= "&q=".uri_escape($operands[$i]) if $operands[$i]; $previous_operand = 1; } } #/if $operands diff --git a/C4/Templates.pm b/C4/Templates.pm index 7f50143279..88b9b01687 100644 --- a/C4/Templates.pm +++ b/C4/Templates.pm @@ -67,7 +67,6 @@ sub new { COMPILE_DIR => C4::Context->config('template_cache_dir')?C4::Context->config('template_cache_dir'):'',, INCLUDE_PATH => \@includes, FILTERS => {}, - ENCODING => 'utf8', # templates don't have BOM, see Template::FAQ } ) or die Template->error(); my $self = { @@ -115,16 +114,57 @@ sub output { C4::Context->preference('opaclayoutstylesheet'); # add variables set via param to $vars for processing + # and clean any utf8 mess for my $k ( keys %{ $self->{VARS} } ) { $vars->{$k} = $self->{VARS}->{$k}; + if (ref($vars->{$k}) eq 'ARRAY'){ + utf8_arrayref($vars->{$k}); + } + elsif (ref($vars->{$k}) eq 'HASH'){ + utf8_hashref($vars->{$k}); + } + else { + utf8::encode($vars->{$k}) if utf8::is_utf8($vars->{$k}); + } } my $data; +# binmode( STDOUT, ":utf8" ); $template->process( $self->filename, $vars, \$data ) || die "Template process failed: ", $template->error(); return $data; } - +sub utf8_arrayref { + my $arrayref = shift; + foreach my $element (@$arrayref){ + if (ref($element) eq 'ARRAY'){ + utf8_arrayref($element); + next; + } + if (ref($element) eq 'HASH'){ + utf8_hashref($element); + next; + } + utf8::encode($element) if utf8::is_utf8($element); + } +} + +sub utf8_hashref { + my $hashref = shift; + for my $key (keys %{$hashref}){ + if (ref($hashref->{$key}) eq 'ARRAY'){ + utf8_arrayref($hashref->{$key}); + next; + } + if (ref($hashref->{$key}) eq 'HASH'){ + utf8_hashref($hashref->{$key}); + next; + } + utf8::encode($hashref->{$key}) if utf8::is_utf8($hashref->{$key}); + } +} + + # FIXME - this is a horrible hack to cache # the current known-good language, temporarily # put in place to resolve bug 4403. It is diff --git a/C4/XSLT.pm b/C4/XSLT.pm index 8f24df9e74..69516742cd 100644 --- a/C4/XSLT.pm +++ b/C4/XSLT.pm @@ -221,8 +221,7 @@ sub XSLTParse4Display { $stylesheet->{$xslfilename} = $xslt->parse_stylesheet($style_doc); } my $results = $stylesheet->{$xslfilename}->transform($source); - my $newxmlrecord = $stylesheet->{$xslfilename}->output_as_chars($results); - #no need to decode with UTF-8 in header of XSLT templates: BZ 6554 + my $newxmlrecord = $stylesheet->{$xslfilename}->output_string($results); return $newxmlrecord; } diff --git a/acqui/booksellers.pl b/acqui/booksellers.pl index 394ae92178..f6c7b2e928 100755 --- a/acqui/booksellers.pl +++ b/acqui/booksellers.pl @@ -74,7 +74,6 @@ my ( $template, $loggedinuser, $cookie ) = get_template_and_user( #parameters my $supplier = $query->param('supplier'); -utf8::decode($supplier); my $booksellerid = $query->param('booksellerid'); my @suppliers; diff --git a/acqui/histsearch.pl b/acqui/histsearch.pl index 0d4ea1a760..971fc18fa8 100755 --- a/acqui/histsearch.pl +++ b/acqui/histsearch.pl @@ -60,22 +60,13 @@ use C4::Debug; my $input = new CGI; my $title = $input->param( 'title'); -utf8::decode($title); my $author = $input->param('author'); -utf8::decode($author); my $isbn = $input->param('isbn'); -utf8::decode($isbn); my $name = $input->param( 'name' ); -utf8::decode($name); my $ean = $input->param('ean'); -utf8::decode($ean); my $basket = $input->param( 'basket' ); -utf8::decode($basket); my $basketgroupname = $input->param('basketgroupname'); -utf8::decode($basketgroupname); my $booksellerinvoicenumber = $input->param( 'booksellerinvoicenumber' ); -utf8::decode($booksellerinvoicenumber); - my $do_search = $input->param('do_search') || 0; my $from_placed_on = C4::Dates->new($input->param('from')); my $to_placed_on = C4::Dates->new($input->param('to')); diff --git a/admin/preferences.pl b/admin/preferences.pl index 2182591f29..db91e76e7d 100755 --- a/admin/preferences.pl +++ b/admin/preferences.pl @@ -21,7 +21,6 @@ use strict; use warnings; use CGI; - use C4::Auth; use C4::Context; use C4::Koha; @@ -35,7 +34,6 @@ use File::Spec; use IO::File; use YAML::Syck qw(); $YAML::Syck::ImplicitTyping = 1; -$YAML::Syck::ImplicitUnicode = 1; # force utf-8 for preference encoding our $lang; # use Smart::Comments; @@ -297,8 +295,7 @@ if ( $op eq 'save' ) { my @TABS; if ( $op eq 'search' ) { - my $searchfield = $input->param('searchfield'); - utf8::decode($searchfield); + my $searchfield = $input->param( 'searchfield' ); $searchfield =~ s/\p{IsC}//g; $searchfield =~ s/\s+/ /; diff --git a/admin/z3950servers.pl b/admin/z3950servers.pl index 897b55235c..92cc3e59f6 100755 --- a/admin/z3950servers.pl +++ b/admin/z3950servers.pl @@ -21,7 +21,7 @@ use strict; use warnings; -use CGI qw( -utf8 ); +use CGI; use C4::Context; use C4::Auth; use C4::Output; diff --git a/authorities/authorities-home.pl b/authorities/authorities-home.pl index 2fe4622d64..f67f5c9c40 100755 --- a/authorities/authorities-home.pl +++ b/authorities/authorities-home.pl @@ -125,7 +125,7 @@ if ( $op eq "do_search" ) { # next/previous would not work anymore # construction of the url of each page - my $value_url = uri_escape_utf8($value); + my $value_url = uri_escape($value); my $base_url = "authorities-home.pl?" ."marclist=$marclist" ."&and_or=$and_or" diff --git a/catalogue/search.pl b/catalogue/search.pl index 88e6825f18..8624b3b43a 100755 --- a/catalogue/search.pl +++ b/catalogue/search.pl @@ -154,7 +154,7 @@ use C4::Branch; # GetBranches my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold"); # create a new CGI object # FIXME: no_undef_params needs to be tested -use CGI qw( -no_undef_params -utf8 ); +use CGI qw('-no_undef_params'); my $cgi = new CGI; my ($template,$borrowernumber,$cookie); diff --git a/cataloguing/addbiblio.pl b/cataloguing/addbiblio.pl index e2ea1cdf89..3314f1a02a 100755 --- a/cataloguing/addbiblio.pl +++ b/cataloguing/addbiblio.pl @@ -880,7 +880,7 @@ if ( $op eq "addbiblio" ) { .'&frameworkcode='.$frameworkcode .'&circborrowernumber='.$fa_circborrowernumber .'&branch='.$fa_branch - .'&barcode='.uri_escape_utf8($fa_barcode) + .'&barcode='.uri_escape($fa_barcode) .'&stickyduedate='.$fa_stickyduedate .'&duedatespec='.$fa_duedatespec ); diff --git a/cataloguing/additem.pl b/cataloguing/additem.pl index 16fee3b055..7475103681 100755 --- a/cataloguing/additem.pl +++ b/cataloguing/additem.pl @@ -518,7 +518,7 @@ if ($op eq "additem") { print $input->redirect( '/cgi-bin/koha/circ/circulation.pl?' .'borrowernumber='.$fa_circborrowernumber - .'&barcode='.uri_escape_utf8($fa_barcode) + .'&barcode='.uri_escape($fa_barcode) .'&duedatespec='.$fa_duedatespec .'&stickyduedate=1' ); diff --git a/cataloguing/z3950_search.pl b/cataloguing/z3950_search.pl index 425a6c17f1..a80120990e 100755 --- a/cataloguing/z3950_search.pl +++ b/cataloguing/z3950_search.pl @@ -20,7 +20,7 @@ use strict; use warnings; -use CGI qw( -utf8 ); +use CGI; use C4::Auth; use C4::Output; diff --git a/circ/circulation.pl b/circ/circulation.pl index b664d4989a..ef0edf7cf8 100755 --- a/circ/circulation.pl +++ b/circ/circulation.pl @@ -24,7 +24,6 @@ use strict; use warnings; use CGI; - use C4::Output; use C4::Print; use C4::Auth qw/:DEFAULT get_session/; @@ -103,7 +102,6 @@ our %return_failed = (); for (@failedreturns) { $return_failed{$_} = 1; } my $findborrower = $query->param('findborrower') || q{}; -utf8::decode($findborrower); $findborrower =~ s|,| |g; my $borrowernumber = $query->param('borrowernumber'); diff --git a/installer/InstallAuth.pm b/installer/InstallAuth.pm index 1a8a4df89f..0015a14e2d 100644 --- a/installer/InstallAuth.pm +++ b/installer/InstallAuth.pm @@ -26,6 +26,7 @@ use Digest::MD5 qw(md5_base64); require Exporter; use C4::Context; +use C4::Output; use C4::Templates; use C4::Koha; use CGI::Session; @@ -393,9 +394,11 @@ sub checkauth { -HttpOnly => 1, -expires => '' ); - - require C4::Output; - C4::Output::output_html_with_http_headers($query,$cookie,$template->output); + print $query->header( + -type => 'text/html; charset=utf-8', + -cookie => $cookie + ), + $template->output; exit; } diff --git a/members/member.pl b/members/member.pl index 55f6a5bedd..d705810cfb 100755 --- a/members/member.pl +++ b/members/member.pl @@ -27,7 +27,7 @@ use strict; #use warnings; FIXME - Bug 2505 use C4::Auth; use C4::Output; -use CGI qw( -utf8 ); +use CGI; use C4::Members; use C4::Branch; use C4::Category; diff --git a/opac/opac-search.pl b/opac/opac-search.pl index 2502768863..363a11ea0e 100755 --- a/opac/opac-search.pl +++ b/opac/opac-search.pl @@ -60,7 +60,7 @@ use Business::ISBN; my $DisplayMultiPlaceHold = C4::Context->preference("DisplayMultiPlaceHold"); # create a new CGI object # FIXME: no_undef_params needs to be tested -use CGI qw( -no_undef_params -utf8 ); +use CGI qw('-no_undef_params'); my $cgi = new CGI; my $branch_group_limit = $cgi->param("branch_group_limit"); diff --git a/patroncards/members-search.pl b/patroncards/members-search.pl index 8ab249bc1e..ef7a7760f1 100755 --- a/patroncards/members-search.pl +++ b/patroncards/members-search.pl @@ -34,7 +34,6 @@ my $startfrom = $cgi->param('startfrom')||1; my $resultsperpage = $cgi->param('resultsperpage')||C4::Context->preference("PatronsPerPage")||20; my $category = $cgi->param('category') || undef; my $member = $cgi->param('member') || ''; -utf8::decode($member); my $orderby = $cgi->param('orderby') || undef; my @categories=C4::Category->all; diff --git a/reports/guided_reports.pl b/reports/guided_reports.pl index c1c5590694..46a65f1deb 100755 --- a/reports/guided_reports.pl +++ b/reports/guided_reports.pl @@ -736,7 +736,7 @@ elsif ($phase eq 'Run this report'){ my $totpages = int($total/$limit) + (($total % $limit) > 0 ? 1 : 0); my $url = "/cgi-bin/koha/reports/guided_reports.pl?reports=$report_id&phase=Run%20this%20report&limit=$limit"; if (@sql_params) { - $url = join('&sql_params=', $url, map { URI::Escape::uri_escape_utf8($_) } @sql_params); + $url = join('&sql_params=', $url, map { URI::Escape::uri_escape($_) } @sql_params); } $template->param( 'results' => \@rows, diff --git a/reserve/renewscript.pl b/reserve/renewscript.pl index 232db8bb89..dbdced9c80 100755 --- a/reserve/renewscript.pl +++ b/reserve/renewscript.pl @@ -127,7 +127,7 @@ foreach my $barcode (@barcodes) { # redirection to the referrer page # if ( $input->param('destination') eq "circ" ) { - $cardnumber = uri_escape_utf8($cardnumber); + $cardnumber = uri_escape($cardnumber); print $input->redirect( '/cgi-bin/koha/circ/circulation.pl?findborrower=' . $cardnumber . $failedrenews diff --git a/serials/routing-preview.pl b/serials/routing-preview.pl index 9869f1c925..437c85288b 100755 --- a/serials/routing-preview.pl +++ b/serials/routing-preview.pl @@ -127,7 +127,7 @@ $routingnotes =~ s/\n/\
/g; $template->param( title => $subs->{'bibliotitle'}, issue => $issue, - issue_escaped => URI::Escape::uri_escape_utf8($issue), + issue_escaped => URI::Escape::uri_escape($issue), subscriptionid => $subscriptionid, memberloop => $memberloop, routingnotes => $routingnotes, diff --git a/serials/routing.pl b/serials/routing.pl index 5dee5eafbb..a95631bdf6 100755 --- a/serials/routing.pl +++ b/serials/routing.pl @@ -62,7 +62,7 @@ if($op eq 'add'){ if($op eq 'save'){ my $sth = $dbh->prepare('UPDATE serial SET routingnotes = ? WHERE subscriptionid = ?'); $sth->execute($notes,$subscriptionid); - my $urldate = URI::Escape::uri_escape_utf8($date_selected); + my $urldate = URI::Escape::uri_escape($date_selected); print $query->redirect("routing-preview.pl?subscriptionid=$subscriptionid&issue=$urldate"); } diff --git a/serials/serials-search.pl b/serials/serials-search.pl index de26736e74..bc84ed68fe 100755 --- a/serials/serials-search.pl +++ b/serials/serials-search.pl @@ -38,13 +38,10 @@ use C4::Serials; my $query = new CGI; my $title = $query->param('title_filter') || ''; -utf8::decode($title); my $ISSN = $query->param('ISSN_filter') || ''; my $EAN = $query->param('EAN_filter') || ''; my $publisher = $query->param('publisher_filter') || ''; -utf8::decode($publisher); my $bookseller = $query->param('bookseller_filter') || ''; -utf8::decode($bookseller); my $biblionumber = $query->param('biblionumber') || ''; my $branch = $query->param('branch_filter') || ''; my $routing = $query->param('routing') || C4::Context->preference("RoutingSerials"); diff --git a/tags/review.pl b/tags/review.pl index 3f4cc50473..1affd539e7 100755 --- a/tags/review.pl +++ b/tags/review.pl @@ -23,7 +23,7 @@ use warnings; use strict; use Data::Dumper; use POSIX; -use CGI qw/-utf8/; +use CGI; use CGI::Cookie; # need to check cookies before having CGI parse the POST request use C4::Auth qw(:DEFAULT check_cookie_auth); diff --git a/tools/viewlog.pl b/tools/viewlog.pl index 533f00ede0..e48aaef4c5 100755 --- a/tools/viewlog.pl +++ b/tools/viewlog.pl @@ -21,7 +21,7 @@ use strict; #use warnings; FIXME - Bug 2505 use C4::Auth; -use CGI qw/-utf8/; +use CGI; use C4::Context; use C4::Koha; use C4::Dates;