From 3d385d0e114cf7c3c35e1b6fcd8a51d1726a672d Mon Sep 17 00:00:00 2001
From: Nick Clemens <nick@bywatersolutions.com>
Date: Tue, 23 Jan 2024 18:47:13 +0000
Subject: [PATCH] Bug 29507: Speed up auto renew cronjob via parallel
 processing

The cron can take a very long time to run on systems with many issues.
For example, a partner with ~250k auto_renew issues is taking about 9 hours to run.

If we run that same number of issues in 5 parallel chunks
( splitting the number of issues as evenly as possible ), it could take under 2 hours.

Test Plan:
1) Generate a number of issues marked for auto_renew
2) Run the automatic_renewals.pl, use the `time` utility to track how much time it took to run
3) Set parallel_loops to 10 in auto_renew_cronjob section of config in koha-conf
4) Repeat step 2, note the improvement in speed
5) Experiment with other values

Signed-off-by: Matt Blenkinsop <matt.blenkinsop@ptfs-europe.com>
Signed-off-by: Martin Renvoize <martin.renvoize@ptfs-europe.com>
Signed-off-by: Katrin Fischer <katrin.fischer@bsz-bw.de>
---
 debian/templates/koha-conf-site.xml.in |  4 ++
 etc/koha-conf.xml                      |  4 ++
 misc/cronjobs/automatic_renewals.pl    | 60 ++++++++++++++++++++++----
 3 files changed, 60 insertions(+), 8 deletions(-)

diff --git a/debian/templates/koha-conf-site.xml.in b/debian/templates/koha-conf-site.xml.in
index 3cbb7fae31..1f034a26c5 100644
--- a/debian/templates/koha-conf-site.xml.in
+++ b/debian/templates/koha-conf-site.xml.in
@@ -491,5 +491,9 @@ __END_SRU_PUBLICSERVER__
 
  <mfa_range>1</mfa_range><!-- Number of 30 second iterations to allow for MFA code checking -->
 
+ <auto_renew_cronjob>
+   <parallel_loops_count>1</parallel_loops_count>
+ </auto_renew_cronjob>
+
 </config>
 </yazgfs>
diff --git a/etc/koha-conf.xml b/etc/koha-conf.xml
index d38a2f13f4..b162219f9c 100644
--- a/etc/koha-conf.xml
+++ b/etc/koha-conf.xml
@@ -302,5 +302,9 @@
 
  <mfa_range>1</mfa_range><!-- Number of 30 second iterations to allow for MFA code checking -->
 
+ <auto_renew_cronjob>
+   <parallel_loops_count>1</parallel_loops_count>
+ </auto_renew_cronjob>
+
 </config>
 </yazgfs>
diff --git a/misc/cronjobs/automatic_renewals.pl b/misc/cronjobs/automatic_renewals.pl
index 30d8e94693..bb0e053859 100755
--- a/misc/cronjobs/automatic_renewals.pl
+++ b/misc/cronjobs/automatic_renewals.pl
@@ -75,7 +75,8 @@ chosen 'Digests only' on the advance messages.
 =cut
 
 use Modern::Perl;
-use Pod::Usage qw( pod2usage );
+use Parallel::ForkManager;
+use Pod::Usage   qw( pod2usage );
 use Getopt::Long qw( GetOptions );
 
 use Koha::Script -cron;
@@ -134,7 +135,7 @@ $verbose = 1 unless $verbose or $confirm;
 print "Test run only\n" unless $confirm;
 
 print "getting auto renewals\n" if $verbose;
-my $auto_renews = Koha::Checkouts->search(
+my @auto_renews = Koha::Checkouts->search(
     {
         auto_renew                   => 1,
         'patron.autorenew_checkouts' => 1,
@@ -142,13 +143,58 @@ my $auto_renews = Koha::Checkouts->search(
     {
         join => ['patron','item']
     }
-);
-print "found " . $auto_renews->count . " auto renewals\n" if $verbose;
+)->as_list;
+print "found " . scalar @auto_renews . " auto renewals\n" if $verbose;
+
+my $cron_options = C4::Context->config('auto_renew_cronjob');
+my $loops = $cron_options ? $cron_options->{parallel_loops_count} // 1 : 1;
+
+# Split the list of issues into chunks to run in parallel
+my @chunks;
+if ( $loops > 1 ) {
+    my $i              = 0;
+    my $borrowernumber = 0;
+    while (@auto_renews) {
+        my $auto_renew = pop(@auto_renews);
+        if ( $borrowernumber != $auto_renew->borrowernumber ) {
+            $i++ if $borrowernumber;
+            $borrowernumber = $auto_renew->borrowernumber;
+        }
+        $i = 0 if $i >= $loops;
+        push( @{ $chunks[$i] }, $auto_renew );
+    }
+    my $pm = Parallel::ForkManager->new($loops);
+  DATA_LOOP:
+    foreach my $chunk (@chunks) {
+        my $pid = $pm->start and next DATA_LOOP;
+        _ProcessRenewals($chunk);
+        $pm->finish;
+    }
+    $pm->wait_all_children;
+}
+else {
+    _ProcessRenewals( \@auto_renews );
+}
+
+cronlogaction({ action => 'End', info => "COMPLETED" });
+
+
+=head1 METHODS
+
+=head2 _ProcessRenewals
+
+    Internal method to process the queue in chunks
+
+=cut
+
+sub _ProcessRenewals {
+    my $auto_renew_issues = shift;
 
 my $renew_digest = {};
 my %report;
 my @item_renewal_ids;
-while ( my $auto_renew = $auto_renews->next ) {
+
+    foreach my $auto_renew (@$auto_renew_issues) {
     print "examining item '" . $auto_renew->itemnumber . "' to auto renew\n" if $verbose;
 
     my ( $borrower_preferences, $wants_messages, $wants_digest ) = ( undef, 0, 0 );
@@ -296,9 +342,7 @@ if ( $send_notices && $confirm ) {
     }
 }
 
-cronlogaction({ action => 'End', info => "COMPLETED" });
-
-=head1 METHODS
+}
 
 =head2 send_digests
 
-- 
2.39.5