Allow timeout configuration per delayed job queue (#3945)

cloudfoundry · Sep 17, 2024 · 7f42106 · 7f42106
1 parent 3323919
commit 7f42106
Show file tree

Hide file tree

Showing 7 changed files with 35 additions and 4 deletions.
diff --git a/app/jobs/enqueuer.rb b/app/jobs/enqueuer.rb
@@ -52,6 +52,8 @@ def load_delayed_job_plugins
       end
 
       def job_timeout
+        return @timeout_calculator.calculate(@job.try(:job_name_in_configuration), @opts[:queue]) if @opts[:queue]
+
         @timeout_calculator.calculate(@job.try(:job_name_in_configuration))
       end
 

diff --git a/config/cloud_controller.yml b/config/cloud_controller.yml
@@ -33,6 +33,7 @@ system_hostnames:
 jobs:
   global:
     timeout_in_seconds: 14400
+  queues: {}
 
 app_usage_events:
   cutoff_age_in_days: 31

diff --git a/lib/cloud_controller/clock/job_timeout_calculator.rb b/lib/cloud_controller/clock/job_timeout_calculator.rb
@@ -1,21 +1,27 @@
 module VCAP::CloudController
   class JobTimeoutCalculator
     JOBS_WITH_TIMEOUTS = %i[app_usage_events_cleanup blobstore_delete diego_sync].freeze
+    QUEUES_WITH_TIMEOUT = %w[cc-generic].freeze
 
     def initialize(config)
       @config = config
     end
 
-    def calculate(job_name)
-      specified_timeout(job_name) || config.get(:jobs, :global, :timeout_in_seconds)
+    def calculate(job_name, queue_name=nil)
+      job_timeout(job_name) || (queue_name && queue_timeout(queue_name)) || config.get(:jobs, :global, :timeout_in_seconds)
     end
 
     private
 
-    def specified_timeout(job_name)
+    def job_timeout(job_name)
       JOBS_WITH_TIMEOUTS.include?(job_name) && config.get(:jobs, job_name.to_sym, :timeout_in_seconds)
     end
 
+    def queue_timeout(queue_name)
+      queue_symbol = queue_name == 'cc-generic' ? :cc_generic : queue_name.to_sym
+      QUEUES_WITH_TIMEOUT.include?(queue_name) && config.get(:jobs, :queues, queue_symbol, :timeout_in_seconds)
+    end
+
     attr_reader :config
   end
 end
diff --git a/lib/cloud_controller/config_schemas/base/api_schema.rb b/lib/cloud_controller/config_schemas/base/api_schema.rb
@@ -341,6 +341,9 @@ class ApiSchema < VCAP::Config
 
             jobs: {
               global: { timeout_in_seconds: Integer },
+              queues: {
+                optional(:cc_generic) => { timeout_in_seconds: Integer }
+              },
               optional(:enable_dynamic_job_priorities) => bool,
               optional(:app_usage_events_cleanup) => { timeout_in_seconds: Integer },
               optional(:blobstore_delete) => { timeout_in_seconds: Integer },

diff --git a/lib/cloud_controller/config_schemas/base/worker_schema.rb b/lib/cloud_controller/config_schemas/base/worker_schema.rb
@@ -167,6 +167,9 @@ class WorkerSchema < VCAP::Config
 
             jobs: {
               global: { timeout_in_seconds: Integer },
+              queues: {
+                optional(:cc_generic) => { timeout_in_seconds: Integer }
+              },
               optional(:enable_dynamic_job_priorities) => bool,
               optional(:app_usage_events_cleanup) => { timeout_in_seconds: Integer },
               optional(:blobstore_delete) => { timeout_in_seconds: Integer },

diff --git a/spec/unit/jobs/enqueuer_spec.rb b/spec/unit/jobs/enqueuer_spec.rb
@@ -13,6 +13,7 @@ module VCAP::CloudController::Jobs
           global: {
             timeout_in_seconds: global_timeout
           },
+          queues: {},
           **priorities
         }
       }
@@ -56,7 +57,7 @@ module VCAP::CloudController::Jobs
           original_enqueue.call(enqueued_job, opts)
         end
         Enqueuer.new(wrapped_job, opts).public_send(method_name)
-        expect(timeout_calculator).to have_received(:calculate).with(wrapped_job.job_name_in_configuration)
+        expect(timeout_calculator).to have_received(:calculate).with(wrapped_job.job_name_in_configuration, 'my-queue')
       end
 
       it 'uses the default priority' do

diff --git a/spec/unit/lib/cloud_controller/clock/job_timeout_calculator_spec.rb b/spec/unit/lib/cloud_controller/clock/job_timeout_calculator_spec.rb
@@ -9,6 +9,9 @@ module VCAP::CloudController
       Config.new({
                    jobs: {
                      global: { timeout_in_seconds: global_timeout },
+                     queues: {
+                       cc_generic: { timeout_in_seconds: 5.hours }
+                     },
                      app_usage_events_cleanup: { timeout_in_seconds: 2.hours },
                      blobstore_delete: { timeout_in_seconds: 3.hours },
                      diego_sync: { timeout_in_seconds: 4.hours }
@@ -35,5 +38,17 @@ module VCAP::CloudController
         expect(JobTimeoutCalculator.new(config).calculate(nil)).to eq(global_timeout)
       end
     end
+
+    context 'when a queue timeout is specified' do
+      it 'returns the queue timeout from the config' do
+        expect(JobTimeoutCalculator.new(config).calculate(:bommel, 'cc-generic')).to eq(5.hours)
+      end
+    end
+
+    context 'when the queue timeout is NOT specified in the config' do
+      it 'returns the global timeout' do
+        expect(JobTimeoutCalculator.new(config).calculate(:bommel, 'not-configured-queue')).to eq(global_timeout)
+      end
+    end
   end
 end