From 7c76176b85fb4f4a8ce03b41e42629821ae1d80f Mon Sep 17 00:00:00 2001
From: Kyle M Hall <kyle@bywatersolutions.com>
Date: Tue, 3 Jan 2023 15:35:42 +0000
Subject: [PATCH] Bug 32558: Add ability for background_jobs_worker.pl to
 process multiple jobs simultaneously up to a limit

Right now background_jobs_worker.pl only processes jobs in serial. It would make sense to handle jobs in parallel up to a user definable limit.

Test Plan:
1) Apply this patch
2) Stop background_jobs_worker.pl
3) Generate some background jobs by editing records, placing holds, etc
4) Watch processes in a new terminal: watch -n 0.1 'ps aux | grep background_jobs_worker.pl'
5) Run background_jobs_worker.pl with parameter -m 3 or some other
   number of max processes
6) Note the multiple forked processes in the ps output

Test notes - also tested the following on KTD:
1. Stop background_jobs_worker.pl
2. Edit /etc/koha/sites/kohadev/koha-conf.xml - set max_processes to 10
3. Generate some background jobs
4. Watch processes in a new terminal: watch -n 0.1 'ps aux | grep background_jobs_worker.pl'
5. Restart all
6. Confirm multiple forked processes in the ps output

Both methods work as expected and generate multiple forked processes
based on the value set for max processes.

Signed-off-by: emlam <emily.lamancusa@montgomerycountymd.gov>

Signed-off-by: Marcel de Rooy <m.de.rooy@rijksmuseum.nl>
Signed-off-by: Tomas Cohen Arazi <tomascohen@theke.io>
(cherry picked from commit 311d68815fa69d66d948c023651c07271a0248c9)
Signed-off-by: Jacob O'Mara <jacob.omara@ptfs-europe.com>
(cherry picked from commit 95a5d612f36cd3b06a184106a217fbc1e1716577)
Signed-off-by: Lucas Gass <lucas@bywatersolutions.com>
---
 debian/templates/koha-conf-site.xml.in |  5 +++++
 etc/koha-conf.xml                      |  5 +++++
 misc/background_jobs_worker.pl         | 22 +++++++++++++++++++++-
 3 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/debian/templates/koha-conf-site.xml.in b/debian/templates/koha-conf-site.xml.in
index bcf6bab0f6..dc9bcb6b66 100644
--- a/debian/templates/koha-conf-site.xml.in
+++ b/debian/templates/koha-conf-site.xml.in
@@ -456,6 +456,11 @@ __END_SRU_PUBLICSERVER__
    <vhost>__MESSAGE_BROKER_VHOST__</vhost>
  </message_broker>
 
+ <background_jobs_worker>
+     <!-- Max simultaneous processes per worker -->
+     <max_processes>1</max_processes>
+ </background_jobs_worker>
+
  <do_not_remove_cookie>__KEEP_COOKIE__</do_not_remove_cookie>
  <do_not_remove_cookie>catalogue_editor_\d+</do_not_remove_cookie>
  <!-- Uncomment lines like hereunder to not clear cookies at logout:
diff --git a/etc/koha-conf.xml b/etc/koha-conf.xml
index 793420437d..a52b9bdac4 100644
--- a/etc/koha-conf.xml
+++ b/etc/koha-conf.xml
@@ -273,6 +273,11 @@
    <vhost></vhost>
  </message_broker>
 
+ <background_jobs_worker>
+     <!-- Max simultaneous processes per worker -->
+     <max_processes>1</max_processes>
+ </background_jobs_worker>
+
  <do_not_remove_cookie>catalogue_editor_\d+</do_not_remove_cookie>
  <!-- Uncomment lines like hereunder to not clear cookies at logout:
       The cookie name is case sensitive.
diff --git a/misc/background_jobs_worker.pl b/misc/background_jobs_worker.pl
index b98d5f135f..add96711c2 100755
--- a/misc/background_jobs_worker.pl
+++ b/misc/background_jobs_worker.pl
@@ -21,7 +21,7 @@ background_jobs_worker.pl - Worker script that will process background jobs
 
 =head1 SYNOPSIS
 
-./background_jobs_worker.pl [--queue QUEUE]
+./background_jobs_worker.pl [--queue QUEUE] [-m|--max-processes MAX_PROCESSES]
 
 =head1 DESCRIPTION
 
@@ -30,6 +30,11 @@ or if a Stomp server is not active it will poll the database every 10s for new j
 
 You can specify some queues only (using --queue, which is repeatable) if you want to run several workers that will handle their own jobs.
 
+--m --max-processes specifies how many jobs to process simultaneously
+
+Max processes will be set from the command line option, the environment variable MAX_PROCESSES, or the koha-conf file, in that order of precedence.
+By default the script will only run one job at a time.
+
 =head1 OPTIONS
 
 =over
@@ -52,16 +57,25 @@ use JSON qw( decode_json );
 use Try::Tiny;
 use Pod::Usage;
 use Getopt::Long;
+use Parallel::ForkManager;
 
 use Koha::Logger;
 use Koha::BackgroundJobs;
+use C4::Context;
 
 my ( $help, @queues );
+
+my $max_processes = $ENV{MAX_PROCESSES};
+$max_processes ||= C4::Context->config('background_jobs_worker')->{max_processes} if C4::Context->config('background_jobs_worker');
+$max_processes ||= 1;
+
 GetOptions(
+    'm|max-processes=i' => \$max_processes,
     'h|help' => \$help,
     'queue=s' => \@queues,
 ) || pod2usage(1);
 
+
 pod2usage(0) if $help;
 
 unless (@queues) {
@@ -75,6 +89,8 @@ try {
     warn sprintf "Cannot connect to the message broker, the jobs will be processed anyway (%s)", $_;
 };
 
+my $pm = Parallel::ForkManager->new($max_processes);
+
 if ( $conn ) {
     # FIXME cf note in Koha::BackgroundJob about $namespace
     my $namespace = C4::Context->config('memcached_namespace');
@@ -115,7 +131,9 @@ while (1) {
             next;
         }
 
+        $pm->start and next;
         process_job( $job, $args );
+        $pm->finish;
 
     } else {
         my $jobs = Koha::BackgroundJobs->search({ status => 'new', queue => \@queues });
@@ -130,7 +148,9 @@ while (1) {
 
             next unless $args;
 
+            $pm->start and next;
             process_job( $job, { job_id => $job->id, %$args } );
+            $pm->finish;
 
         }
         sleep 10;
-- 
2.20.1