From fafcbff015fe8e7abe31cf8a0668b17d1bc729bb Mon Sep 17 00:00:00 2001 From: Jonathan Druart Date: Thu, 11 Feb 2021 10:37:13 +0100 Subject: [PATCH] Bug 27673: Fix encoding issues MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit There is a difference between YAML::Load and YAML::XS::Load From YAML::XS pod: "YAML::XS only deals with streams of utf8 octets" Test plan: We are going to test 1 occurence and QA will confirm others don't contain typos. 0. Don't apply the patches 1. Create a new itemtype with code=❤️ 2. Create a new item using this itemtype (to biblionumber=1 will work) 3. Fill OpacHiddenItems with itype: [❤️] 4. Search for "street shuffle" or any terms that will return the biblio Notice that the item is there (there is an error in logs) 5. Apply the patches 6. Repeat 4 and confirm that the item is now hidden Signed-off-by: Kyle M Hall Signed-off-by: Joonas Kylmälä Signed-off-by: Jonathan Druart --- C4/Circulation.pm | 3 ++- C4/Context.pm | 2 +- C4/Items.pm | 3 ++- C4/Record.pm | 3 ++- C4/Ris.pm | 3 ++- Koha/Filter/MARC/EmbedItems.pm | 2 +- about.pl | 3 ++- acqui/addorderiso2709.pl | 5 +++-- circ/pendingreserves.pl | 3 ++- opac/opac-ISBDdetail.pl | 3 ++- opac/opac-search.pl | 3 ++- 11 files changed, 21 insertions(+), 12 deletions(-) diff --git a/C4/Circulation.pm b/C4/Circulation.pm index a563bc1871..a7302d862a 100644 --- a/C4/Circulation.pm +++ b/C4/Circulation.pm @@ -22,6 +22,7 @@ use Modern::Perl; use DateTime; use POSIX qw( floor ); use YAML::XS; +use Encode; use Koha::DateUtils; use C4::Context; @@ -2016,7 +2017,7 @@ sub AddReturn { if ($yaml) { $yaml = "$yaml\n\n"; # YAML is anal on ending \n. Surplus does not hurt my $rules; - eval { $rules = YAML::XS::Load($yaml); }; + eval { $rules = YAML::XS::Load(Encode::encode_utf8($yaml)); }; if ($@) { warn "Unable to parse UpdateNotForLoanStatusOnCheckin syspref : $@"; } diff --git a/C4/Context.pm b/C4/Context.pm index bf1df49336..f6406fd7ba 100644 --- a/C4/Context.pm +++ b/C4/Context.pm @@ -445,7 +445,7 @@ the value cannot be properly decoded as YAML. sub yaml_preference { my ( $self, $preference ) = @_; - my $yaml = eval { YAML::XS::Load( $self->preference( $preference ) ); }; + my $yaml = eval { YAML::XS::Load( Encode::encode_utf8( $self->preference( $preference ) ) ); }; if ($@) { warn "Unable to parse $preference syspref : $@"; return; diff --git a/C4/Items.pm b/C4/Items.pm index 508e638149..c5da60d40f 100644 --- a/C4/Items.pm +++ b/C4/Items.pm @@ -49,6 +49,7 @@ BEGIN { use Carp; use Try::Tiny; +use Encode; use C4::Context; use C4::Koha; use C4::Biblio; @@ -982,7 +983,7 @@ sub GetHiddenItemnumbers { $yaml = "$yaml\n\n"; # YAML is anal on ending \n. Surplus does not hurt my $hidingrules; eval { - $hidingrules = YAML::XS::Load($yaml); + $hidingrules = YAML::XS::Load(Encode::encode_utf8($yaml)); }; if ($@) { warn "Unable to parse OpacHiddenItems syspref : $@"; diff --git a/C4/Record.pm b/C4/Record.pm index 5616879cb6..0c3c670ef3 100644 --- a/C4/Record.pm +++ b/C4/Record.pm @@ -30,6 +30,7 @@ use C4::Biblio; #marc2bibtex use C4::Koha; #marc2csv use C4::XSLT (); use YAML::XS; #marcrecords2csv +use Encode; use Template; use Text::CSV::Encoded; #marc2csv use Koha::Items; @@ -800,7 +801,7 @@ sub marc2bibtex { my $additional_fields; if ($BibtexExportAdditionalFields) { $BibtexExportAdditionalFields = "$BibtexExportAdditionalFields\n\n"; - $additional_fields = eval { YAML::XS::Load($BibtexExportAdditionalFields); }; + $additional_fields = eval { YAML::XS::Load(Encode::encode_utf8($BibtexExportAdditionalFields)); }; if ($@) { warn "Unable to parse BibtexExportAdditionalFields : $@"; $additional_fields = undef; diff --git a/C4/Ris.pm b/C4/Ris.pm index 3b39e96707..867030c73c 100644 --- a/C4/Ris.pm +++ b/C4/Ris.pm @@ -64,6 +64,7 @@ use Modern::Perl; use List::MoreUtils qw/uniq/; use YAML::XS; +use Encode; use vars qw(@ISA @EXPORT); use Koha::SimpleMARC qw(read_field); @@ -120,7 +121,7 @@ sub marc2ris { my $ris_additional_fields; if ($RisExportAdditionalFields) { $RisExportAdditionalFields = "$RisExportAdditionalFields\n\n"; - $ris_additional_fields = eval { YAML::XS::Load($RisExportAdditionalFields); }; + $ris_additional_fields = eval { YAML::XS::Load(Encode::encode_utf8($RisExportAdditionalFields)); }; if ($@) { warn "Unable to parse RisExportAdditionalFields : $@"; $ris_additional_fields = undef; diff --git a/Koha/Filter/MARC/EmbedItems.pm b/Koha/Filter/MARC/EmbedItems.pm index c3623755ca..337a4dbbae 100644 --- a/Koha/Filter/MARC/EmbedItems.pm +++ b/Koha/Filter/MARC/EmbedItems.pm @@ -31,7 +31,7 @@ my $biblio = Koha::Biblios->find( my $opachiddenitems_rules; eval { my $yaml = C4::Context->preference('OpacHiddenItems') . "\n\n"; - $opachiddenitems_rules = YAML::XS::Load($yaml); + $opachiddenitems_rules = YAML::XS::Load(Encode::encode_utf8($yaml)); }; my @items = grep { !$_->hidden_in_opac({ rules => $opachiddenitems_rules }) @{$biblio->items}; diff --git a/about.pl b/about.pl index 3c9ceb908d..a8df6aa312 100755 --- a/about.pl +++ b/about.pl @@ -34,6 +34,7 @@ use Config; use Search::Elasticsearch; use Try::Tiny; use YAML::XS; +use Encode; use C4::Output; use C4::Auth; @@ -429,7 +430,7 @@ my @bad_yaml_prefs; foreach my $syspref (@yaml_prefs) { my $yaml = C4::Context->preference( $syspref ); if ( $yaml ) { - eval { YAML::XS::Load( "$yaml\n\n" ); }; + eval { YAML::XS::Load( Encode::encode_utf8("$yaml\n\n") ); }; if ($@) { push @bad_yaml_prefs, $syspref; } diff --git a/acqui/addorderiso2709.pl b/acqui/addorderiso2709.pl index 87973c1fca..202f74862e 100755 --- a/acqui/addorderiso2709.pl +++ b/acqui/addorderiso2709.pl @@ -26,6 +26,7 @@ use CGI qw ( -utf8 ); use Carp; use YAML::XS; use List::MoreUtils qw/uniq/; +use Encode; use C4::Context; use C4::Auth; @@ -678,7 +679,7 @@ sub get_infos_syspref { my $syspref = C4::Context->preference($syspref_name); $syspref = "$syspref\n\n"; # YAML is anal on ending \n. Surplus does not hurt my $yaml = eval { - YAML::XS::Load($syspref); + YAML::XS::Load(Encode::encode_utf8($syspref)); }; if ( $@ ) { warn "Unable to parse $syspref syspref : $@"; @@ -724,7 +725,7 @@ sub get_infos_syspref_on_item { my $syspref = C4::Context->preference($syspref_name); $syspref = "$syspref\n\n"; # YAML is anal on ending \n. Surplus does not hurt my $yaml = eval { - YAML::XS::Load($syspref); + YAML::XS::Load(Encode::encode_utf8($syspref)); }; if ( $@ ) { warn "Unable to parse $syspref syspref : $@"; diff --git a/circ/pendingreserves.pl b/circ/pendingreserves.pl index 6da5e3a35a..3ed65938b7 100755 --- a/circ/pendingreserves.pl +++ b/circ/pendingreserves.pl @@ -22,6 +22,7 @@ use Modern::Perl; use constant PULL_INTERVAL => 2; use List::MoreUtils qw( uniq ); use YAML::XS; +use Encode; use C4::Context; use C4::Output; @@ -110,7 +111,7 @@ if ( $op eq 'cancel_reserve' and $reserve_id ) { if ( my $yaml = C4::Context->preference('UpdateItemWhenLostFromHoldList') ) { $yaml = "$yaml\n\n"; # YAML is anal on ending \n. Surplus does not hurt my $assignments; - eval { $assignments = YAML::XS::Load($yaml); }; + eval { $assignments = YAML::XS::Load(Encode::encode_utf8($yaml)); }; if ($@) { warn "Unable to parse UpdateItemWhenLostFromHoldList syspref : $@" if $@; } diff --git a/opac/opac-ISBDdetail.pl b/opac/opac-ISBDdetail.pl index f8ff0d2267..6278177e10 100755 --- a/opac/opac-ISBDdetail.pl +++ b/opac/opac-ISBDdetail.pl @@ -41,6 +41,7 @@ the items attached to the biblio use Modern::Perl; use YAML::XS; +use Encode; use C4::Auth; use C4::Context; @@ -83,7 +84,7 @@ my $patron = Koha::Patrons->find($loggedinuser); my $opachiddenitems_rules; eval { my $yaml = C4::Context->preference('OpacHiddenItems') . "\n\n"; - $opachiddenitems_rules = YAML::XS::Load($yaml); + $opachiddenitems_rules = YAML::XS::Load(Encode::encode_utf8($yaml)); }; unless ( $patron and $patron->category->override_hidden_items ) { diff --git a/opac/opac-search.pl b/opac/opac-search.pl index c04960d207..dd0d4884dc 100755 --- a/opac/opac-search.pl +++ b/opac/opac-search.pl @@ -31,6 +31,7 @@ use C4::Context; use List::MoreUtils q/any/; use Try::Tiny; use YAML::XS; +use Encode; use Data::Dumper; # TODO remove @@ -246,7 +247,7 @@ my $yaml = C4::Context->preference('OpacHiddenItems'); if ( $yaml =~ /\S/ ) { $yaml = "$yaml\n\n"; # YAML expects trailing newline. Surplus does not hurt. eval { - $hidingrules = YAML::XS::Load($yaml); + $hidingrules = YAML::XS::Load(Encode::encode_utf8($yaml)); }; if ($@) { warn "Unable to parse OpacHiddenItems syspref : $@"; -- 2.39.5