Bug 27673: Fix encoding issues

There is a difference between YAML::Load and YAML::XS::Load
From YAML::XS pod:
"YAML::XS only deals with streams of utf8 octets"

Test plan:
We are going to test 1 occurence and QA will confirm others don't
contain typos.
0. Don't apply the patches
1. Create a new itemtype with code=❤️
2. Create a new item using this itemtype (to biblionumber=1 will work)
3. Fill OpacHiddenItems with
 itype: [❤️]
4. Search for "street shuffle" or any terms that will return the biblio
Notice that the item is there (there is an error in logs)
5. Apply the patches
6. Repeat 4 and confirm that the item is now hidden

Signed-off-by: Kyle M Hall <kyle@bywatersolutions.com>
Signed-off-by: Joonas Kylmälä <joonas.kylmala@helsinki.fi>

Signed-off-by: Jonathan Druart <jonathan.druart@bugs.koha-community.org>

C4/Circulation.pm

@@ -22,6 +22,7 @@ use Modern::Perl;
 use DateTime;
 use POSIX qw( floor );
 use YAML::XS;
+use Encode;
 
 use Koha::DateUtils;
 use C4::Context;
@@ -2016,7 +2017,7 @@ sub AddReturn {
     if ($yaml) {
         $yaml = "$yaml\n\n";  # YAML is anal on ending \n. Surplus does not hurt
         my $rules;
-        eval { $rules = YAML::XS::Load($yaml); };
+        eval { $rules = YAML::XS::Load(Encode::encode_utf8($yaml)); };
         if ($@) {
             warn "Unable to parse UpdateNotForLoanStatusOnCheckin syspref : $@";
         }

C4/Context.pm

@@ -445,7 +445,7 @@ the value cannot be properly decoded as YAML.
 sub yaml_preference {
     my ( $self, $preference ) = @_;
 
-    my $yaml = eval { YAML::XS::Load( $self->preference( $preference ) ); };
+    my $yaml = eval { YAML::XS::Load( Encode::encode_utf8( $self->preference( $preference ) ) ); };
     if ($@) {
         warn "Unable to parse $preference syspref : $@";
         return;

C4/Items.pm

@@ -49,6 +49,7 @@ BEGIN {
 
 use Carp;
 use Try::Tiny;
+use Encode;
 use C4::Context;
 use C4::Koha;
 use C4::Biblio;
@@ -982,7 +983,7 @@ sub GetHiddenItemnumbers {
     $yaml = "$yaml\n\n"; # YAML is anal on ending \n. Surplus does not hurt
     my $hidingrules;
     eval {
-        $hidingrules = YAML::XS::Load($yaml);
+        $hidingrules = YAML::XS::Load(Encode::encode_utf8($yaml));
     };
     if ($@) {
         warn "Unable to parse OpacHiddenItems syspref : $@";

C4/Record.pm

@@ -30,6 +30,7 @@ use C4::Biblio; #marc2bibtex
 use C4::Koha; #marc2csv
 use C4::XSLT ();
 use YAML::XS; #marcrecords2csv
+use Encode;
 use Template;
 use Text::CSV::Encoded; #marc2csv
 use Koha::Items;
@@ -800,7 +801,7 @@ sub marc2bibtex {
     my $additional_fields;
     if ($BibtexExportAdditionalFields) {
         $BibtexExportAdditionalFields = "$BibtexExportAdditionalFields\n\n";
-        $additional_fields = eval { YAML::XS::Load($BibtexExportAdditionalFields); };
+        $additional_fields = eval { YAML::XS::Load(Encode::encode_utf8($BibtexExportAdditionalFields)); };
         if ($@) {
             warn "Unable to parse BibtexExportAdditionalFields : $@";
             $additional_fields = undef;

C4/Ris.pm

@@ -64,6 +64,7 @@ use Modern::Perl;
 
 use List::MoreUtils qw/uniq/;
 use YAML::XS;
+use Encode;
 use vars qw(@ISA @EXPORT);
 
 use Koha::SimpleMARC qw(read_field);
@@ -120,7 +121,7 @@ sub marc2ris {
     my $ris_additional_fields;
     if ($RisExportAdditionalFields) {
         $RisExportAdditionalFields = "$RisExportAdditionalFields\n\n";
-        $ris_additional_fields = eval { YAML::XS::Load($RisExportAdditionalFields); };
+        $ris_additional_fields = eval { YAML::XS::Load(Encode::encode_utf8($RisExportAdditionalFields)); };
         if ($@) {
             warn "Unable to parse RisExportAdditionalFields : $@";
             $ris_additional_fields = undef;

Koha/Filter/MARC/EmbedItems.pm

@@ -31,7 +31,7 @@ my $biblio = Koha::Biblios->find(
 my $opachiddenitems_rules;
 eval {
     my $yaml = C4::Context->preference('OpacHiddenItems') . "\n\n";
-    $opachiddenitems_rules = YAML::XS::Load($yaml);
+    $opachiddenitems_rules = YAML::XS::Load(Encode::encode_utf8($yaml));
 };
 
 my @items  = grep { !$_->hidden_in_opac({ rules => $opachiddenitems_rules }) @{$biblio->items};

about.pl

@@ -34,6 +34,7 @@ use Config;
 use Search::Elasticsearch;
 use Try::Tiny;
 use YAML::XS;
+use Encode;
 
 use C4::Output;
 use C4::Auth;
@@ -429,7 +430,7 @@ my @bad_yaml_prefs;
 foreach my $syspref (@yaml_prefs) {
     my $yaml = C4::Context->preference( $syspref );
     if ( $yaml ) {
-        eval { YAML::XS::Load( "$yaml\n\n" ); };
+        eval { YAML::XS::Load( Encode::encode_utf8("$yaml\n\n") ); };
         if ($@) {
             push @bad_yaml_prefs, $syspref;
         }

acqui/addorderiso2709.pl

@@ -26,6 +26,7 @@ use CGI qw ( -utf8 );
 use Carp;
 use YAML::XS;
 use List::MoreUtils qw/uniq/;
+use Encode;
 
 use C4::Context;
 use C4::Auth;
@@ -678,7 +679,7 @@ sub get_infos_syspref {
     my $syspref = C4::Context->preference($syspref_name);
     $syspref = "$syspref\n\n"; # YAML is anal on ending \n. Surplus does not hurt
     my $yaml = eval {
-        YAML::XS::Load($syspref);
+        YAML::XS::Load(Encode::encode_utf8($syspref));
     };
     if ( $@ ) {
         warn "Unable to parse $syspref syspref : $@";
@@ -724,7 +725,7 @@ sub get_infos_syspref_on_item {
     my $syspref = C4::Context->preference($syspref_name);
     $syspref = "$syspref\n\n"; # YAML is anal on ending \n. Surplus does not hurt
     my $yaml = eval {
-        YAML::XS::Load($syspref);
+        YAML::XS::Load(Encode::encode_utf8($syspref));
     };
     if ( $@ ) {
         warn "Unable to parse $syspref syspref : $@";

circ/pendingreserves.pl

@@ -22,6 +22,7 @@ use Modern::Perl;
 use constant PULL_INTERVAL => 2;
 use List::MoreUtils qw( uniq );
 use YAML::XS;
+use Encode;
 
 use C4::Context;
 use C4::Output;
@@ -110,7 +111,7 @@ if ( $op eq 'cancel_reserve' and $reserve_id ) {
         if ( my $yaml = C4::Context->preference('UpdateItemWhenLostFromHoldList') ) {
             $yaml = "$yaml\n\n";  # YAML is anal on ending \n. Surplus does not hurt
             my $assignments;
-            eval { $assignments = YAML::XS::Load($yaml); };
+            eval { $assignments = YAML::XS::Load(Encode::encode_utf8($yaml)); };
             if ($@) {
                 warn "Unable to parse UpdateItemWhenLostFromHoldList syspref : $@" if $@;
             }

opac/opac-ISBDdetail.pl

@@ -41,6 +41,7 @@ the items attached to the biblio
 
 use Modern::Perl;
 use YAML::XS;
+use Encode;
 
 use C4::Auth;
 use C4::Context;
@@ -83,7 +84,7 @@ my $patron = Koha::Patrons->find($loggedinuser);
 my $opachiddenitems_rules;
 eval {
     my $yaml = C4::Context->preference('OpacHiddenItems') . "\n\n";
-    $opachiddenitems_rules = YAML::XS::Load($yaml);
+    $opachiddenitems_rules = YAML::XS::Load(Encode::encode_utf8($yaml));
 };
 
 unless ( $patron and $patron->category->override_hidden_items ) {

opac/opac-search.pl

@@ -31,6 +31,7 @@ use C4::Context;
 use List::MoreUtils q/any/;
 use Try::Tiny;
 use YAML::XS;
+use Encode;
 
 use Data::Dumper; # TODO remove
 
@@ -246,7 +247,7 @@ my $yaml = C4::Context->preference('OpacHiddenItems');
 if ( $yaml =~ /\S/ ) {
     $yaml = "$yaml\n\n"; # YAML expects trailing newline. Surplus does not hurt.
     eval {
-        $hidingrules = YAML::XS::Load($yaml);
+        $hidingrules = YAML::XS::Load(Encode::encode_utf8($yaml));
     };
     if ($@) {
         warn "Unable to parse OpacHiddenItems syspref : $@";