From fd829dbdcb9f5ea39146c88cd976137876e9a411 Mon Sep 17 00:00:00 2001 From: Jonathan Druart Date: Mon, 2 Aug 2021 11:06:57 +0200 Subject: [PATCH] Bug 25078: Keep atomic updates in "atomicupdate" dir The atomicupdate directory will contain all the atomic update files (old and new version). That will ease job for the RM and RMaints, no file (except skeletons) must be in this directory before push. This patch also fixes an inconsistency we had: the atomic update was run before the other db revs on the UI but after when the CLI script was used. Now we make sure that the CLI does not deal with the atomic update files when called from the installer (UI) Signed-off-by: Kyle M Hall Signed-off-by: Jonathan Druart --- C4/Installer.pm | 186 +++++++++++++----- installer/data/mysql/updatedatabase.pl | 29 ++- installer/install.pl | 11 +- .../prog/en/modules/installer/step3.tt | 55 +++++- 4 files changed, 204 insertions(+), 77 deletions(-) diff --git a/C4/Installer.pm b/C4/Installer.pm index 0fb9ba8f8e..e473171e0d 100644 --- a/C4/Installer.pm +++ b/C4/Installer.pm @@ -23,16 +23,18 @@ use Try::Tiny; use Encode qw( encode decode is_utf8 ); use DBIx::RunSQL; use YAML::XS; +use File::Slurp qw( read_file ); +use DBI; + use C4::Context; use Koha::Schema; -use DBI; use Koha; use vars qw(@ISA @EXPORT); BEGIN { require Exporter; @ISA = qw( Exporter ); - push @EXPORT, qw( primary_key_exists unique_key_exists foreign_key_exists index_exists column_exists TableExists marc_framework_sql_list TransformToNum CheckVersion NewVersion sanitize_zero_date update get_db_entries ); + push @EXPORT, qw( primary_key_exists unique_key_exists foreign_key_exists index_exists column_exists TableExists marc_framework_sql_list TransformToNum CheckVersion NewVersion SetVersion sanitize_zero_date update get_db_entries get_atomic_updates run_atomic_updates ); }; =head1 NAME @@ -718,44 +720,54 @@ sub get_db_entries { return \@need_update; } +sub run_db_rev { + my ($file) = @_; + + my $db_rev = do $file; + + my $error; + my $out = ''; + open my $outfh, '>', \$out; + try { + my $schema = Koha::Database->new->schema; + $schema->txn_do( + sub { + $db_rev->{up}->( { dbh => $schema->storage->dbh, out => $outfh } ); + } + ); + } + catch { + $error = $_; + }; + + close $outfh; + $out = decode( 'UTF-8', $out ); + + my $db_entry = { + filepath => $file, + bug_number => $db_rev->{bug_number}, + description => $db_rev->{description}, + exec_output => $out, + version => scalar version_from_file($file), + time => POSIX::strftime( "%H:%M:%S", localtime ), + error => $error + }; + $db_entry->{output} = generate_output_db_entry($db_entry, $out); + return $db_entry; +} + sub update { my ( $files, $params ) = @_; my $force = $params->{force} || 0; - my $schema = Koha::Database->new->schema; my ( @done, @errors ); for my $file ( @$files ) { - my $db_rev = do $file; - - my $error; - - my $out = ''; - open my $outfh, '>', \$out; - try { - $schema->txn_do( - sub { - $db_rev->{up}->({ dbh => $schema->storage->dbh, out => $outfh }); - } - ); - } catch { - $error = $_; - }; - - close $outfh; - $out = decode('UTF-8', $out); - - my $db_entry = { - bug_number => $db_rev->{bug_number}, - description => $db_rev->{description}, - version => version_from_file($file), - time => POSIX::strftime( "%H:%M:%S", localtime ), - }; - $db_entry->{output} = output_version( { %$db_entry, done => !$error, report => $out } ); + my $db_entry = run_db_rev($file); - if ( $error ) { - push @errors, { %$db_entry, error => $error }; + if ( $db_entry->{error} ) { + push @errors, $db_entry; $force ? next : last ; # We stop the update if an error occurred! } @@ -766,30 +778,39 @@ sub update { return { success => \@done, error => \@errors }; } -sub output_version { +sub generate_output_db_entry { my ( $db_entry ) = @_; my $description = $db_entry->{description}; - my $report = $db_entry->{report}; - my $DBversion = $db_entry->{version}; - my $bug_number = $db_entry->{bug_number}; - my $time = $db_entry->{time}; - my $done = defined $db_entry->{done} - ? $db_entry->{done} - ? " done" - : " failed" - : ""; # For old versions, we don't know if we succeed or failed + my $output = $db_entry->{output}; + my $DBversion = $db_entry->{version}; + my $bug_number = $db_entry->{bug_number}; + my $time = $db_entry->{time}; + my $exec_output = $db_entry->{exec_output}; + my $done = defined $db_entry->{done} + ? $db_entry->{done} + ? " done" + : " failed" + : ""; # For old versions, we don't know if we succeed or failed my @output; - if ($bug_number) { - push @output, sprintf('Upgrade to %s %s [%s]: Bug %5s - %s', $DBversion, $done, $time, $bug_number, $description); - } else { - push @output, sprintf('Upgrade to %s %s [%s]: %s', $DBversion, $done, $time, $description); + if ( $DBversion ) { + if ($bug_number) { + push @output, sprintf('Upgrade to %s %s [%s]: Bug %5s - %s', $DBversion, $done, $time, $bug_number, $description); + } else { + push @output, sprintf('Upgrade to %s %s [%s]: %s', $DBversion, $done, $time, $description); + } + } else { # Atomic update + if ($bug_number) { + push @output, sprintf('DEV atomic update %s %s [%s]: Bug %5s - %s', $db_entry->{filepath}, $done, $time, $bug_number, $description); + } else { # Old atomic update syntax + push @output, sprintf('DEV atomic update %s %s [%s]', $db_entry->{filepath}, $done, $time); + } } - if ($report) { - foreach my $line (split /\n/, $report) { + if ($exec_output) { + foreach my $line (split /\n/, $exec_output) { push @output, sprintf "\t\t\t\t\t\t - %s", $line; } } @@ -797,6 +818,75 @@ sub output_version { return \@output; } +sub get_atomic_updates { + my @atomic_upate_files; + # if there is anything in the atomicupdate, read and execute it. + my $update_dir = C4::Context->config('intranetdir') . '/installer/data/mysql/atomicupdate/'; + opendir( my $dirh, $update_dir ); + foreach my $file ( sort readdir $dirh ) { + next if $file !~ /\.(perl|pl)$/; #skip other files + next if $file eq 'skeleton.perl' || $file eq 'skeleton.pl'; # skip the skeleton files + + push @atomic_upate_files, $file; + } + return \@atomic_upate_files; +} + +sub run_atomic_updates { + my ( $files ) = @_; + + my $update_dir = C4::Context->config('intranetdir') . '/installer/data/mysql/atomicupdate/'; + my ( @done, @errors ); + for my $file ( @$files ) { + my $filepath = $update_dir . $file; + + my $atomic_update; + if ( $file =~ m{\.perl$} ) { + my $code = read_file( $filepath ); + my ( $out, $err ) = ('', ''); + { + open my $oldout, ">&STDOUT"; + close STDOUT; + open STDOUT,'>:encoding(utf8)', \$out; + my $DBversion = Koha::version; # We need $DBversion and $dbh for the eval + my $dbh = C4::Context->dbh; + eval $code; ## no critic (StringyEval) + $err = $@; + warn $err if $err; + close STDOUT; + open STDOUT, ">&", $oldout; + } + + $atomic_update = { + filepath => $filepath, + description => '', + version => undef, + time => POSIX::strftime( "%H:%M:%S", localtime ), + }; + + + $atomic_update->{output} = + $out + ? [ split "\n", $out ] + : generate_output_db_entry($atomic_update); # There wad an error, we didn't reach NewVersion) + + $atomic_update->{error} = $err if $err; + } elsif ( $file =~ m{\.pl$} ) { + $atomic_update = run_db_rev($filepath); + } else { + warn "Atomic update must be .perl or .pl ($file)"; + } + + if ( $atomic_update->{error} ) { + push @errors, $atomic_update; + } else { + push @done, $atomic_update; + } + } + + return { success => \@done, error => \@errors }; +} + =head2 DropAllForeignKeys($table) Drop all foreign keys of the table $table @@ -881,7 +971,7 @@ sub NewVersion { $description = $descriptions; } - my $output = output_version( { + my $output = generate_output_db_entry( { bug_number => $bug_number, description => $description, report => $report, diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl index e9f2a538c2..4f89a31f60 100755 --- a/installer/data/mysql/updatedatabase.pl +++ b/installer/data/mysql/updatedatabase.pl @@ -46,7 +46,6 @@ use MARC::Record; use MARC::File::XML ( BinaryEncoding => 'utf8' ); use File::Path qw[remove_tree]; # perl core module -use File::Slurp; # FIXME - The user might be installing a new database, so can't rely # on /etc/koha.conf anyway. @@ -24284,7 +24283,7 @@ if( CheckVersion( $DBversion ) ) { } unless ( $ENV{HTTP_HOST} ) { # Is that correct? - my $files = C4::Installer::get_db_entries; + my $files = get_db_entries; my $report = update( $files, { force => $force } ); for my $s ( @{ $report->{success} } ) { @@ -24294,24 +24293,18 @@ unless ( $ENV{HTTP_HOST} ) { # Is that correct? say Encode::encode_utf8(join "\n", @{$e->{output}}); say Encode::encode_utf8("ERROR - " . $e->{error}); } -} -# SEE bug 13068 -# if there is anything in the atomicupdate, read and execute it. -my $update_dir = C4::Context->config('intranetdir') . '/installer/data/mysql/atomicupdate/'; -opendir( my $dirh, $update_dir ); -foreach my $file ( sort readdir $dirh ) { - next if $file !~ /\.(sql|perl)$/; #skip other files - next if $file eq 'skeleton.perl'; # skip the skeleton file - print "DEV atomic update: $file\n"; - if ( $file =~ /\.sql$/ ) { - my $installer = C4::Installer->new(); - my $rv = $installer->load_sql( $update_dir . $file ) ? 0 : 1; - } elsif ( $file =~ /\.perl$/ ) { - my $code = read_file( $update_dir . $file ); - eval $code; ## no critic (StringyEval) - say "Atomic update generated errors: $@" if $@; + my $atomic_update_files = get_atomic_updates; + $report = run_atomic_updates($atomic_update_files); + for my $s ( @{ $report->{success} } ) { + say Encode::encode_utf8(join "\n", @{$s->{output}}); } + for my $e ( @{ $report->{error} } ) { + say Encode::encode_utf8(join "\n", @{$e->{output}}); + say Encode::encode_utf8("ERROR - " . $e->{error}); + } + + } exit; diff --git a/installer/install.pl b/installer/install.pl index 3d410f34cb..abb3897a94 100755 --- a/installer/install.pl +++ b/installer/install.pl @@ -431,8 +431,17 @@ elsif ( $step && $step == 3 ) { my $db_entries = get_db_entries(); my $report = update( $db_entries ); + my $atomic_update_files = get_atomic_updates; + my $atomic_update_report = run_atomic_updates( $atomic_update_files ); - $template->param( success => $report->{success}, error => $report->{error} ); + $template->param( + success => $report->{success}, + error => $report->{error}, + atomic_updates => { + success => $atomic_update_report->{success}, + error => $atomic_update_report->{error} + } + ); #warn "The following errors were returned while attempting to run the updatedatabase.pl script:\n"; #FIXME restore this $template->param( $op => 1 ); diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/installer/step3.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/installer/step3.tt index 2fca0c40dc..b2a6604f4d 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/installer/step3.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/installer/step3.tt @@ -263,7 +263,7 @@ [% END %] [%# Success for new versions %] - [% IF success %] + [% IF success.size %]
    [% FOR s IN success %] [% FOR o IN s.output %] @@ -276,21 +276,55 @@
[% END %] - [%# Errors for old versions %] - [% IF has_update_errors %] -

Update errors :

+ [% IF atomic_updates.success.size %] +

Atomic updates:

    - [% FOREACH update_error IN update_errors %] -
  • [% update_error.line | html %]
    • + [% FOR s IN atomic_updates.success %] + [% FOR o IN s.output %] +
  • [% o | html %]
    • + [% IF s.output.size > 1 %] + [% IF loop.first %]
      [% ELSIF loop.last %]
    [% END %] + [% END %] + [% END %] [% END %]
[% END %] - [%# Errors for new versions %] - [% IF error %] -

Update error :

+ [% IF has_update_errors OR error.size %] +

Update errors :

+ [%# Errors for old versions %] + [% IF has_update_errors %] +
    + [% FOREACH update_error IN update_errors %] +
  • [% update_error.line | html %]
    • + [% END %] +
+ [% END %] + + [%# Errors for new versions %] + [% IF error.size %] +
    + [% FOR e IN error %] + [% FOR o IN e.output %] +
  • + [% o | html %] +
    + ERROR: [% e.error | html %] + + [% IF e.output.size > 1 %] + [% IF loop.first %]
      [% ELSIF loop.last %]
    [% END %] + [% END %] +
    • + [% END %] + [% END %] +
+ [% END %] + [% END %] + + [% IF atomic_updates.error.size %] +

Atomic update error :

    - [% FOR e IN error %] + [% FOR e IN atomic_updates.error %] [% FOR o IN e.output %]
  • [% o | html %] @@ -305,6 +339,7 @@ [% END %]
[% END %] + [% UNLESS error OR has_update_errors %]

Everything went okay. Update done.

Continue to log in to Koha

-- 2.39.5