From db94443d16c9b5519c9c7d73bebd666e7ad08b46 Mon Sep 17 00:00:00 2001 From: Xavier Shay Date: Wed, 29 May 2019 15:43:55 +1000 Subject: [PATCH 1/5] Prevent jobs from being enqueued in the past. It's relatively easy for this to happen by accident (e.g. time drift on a rogue server), and it throws a spanner in the works if you're monitoring for oldest job in the queue. --- lib/que/job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/que/job.rb b/lib/que/job.rb index faa6315f..b1cf0895 100644 --- a/lib/que/job.rb +++ b/lib/que/job.rb @@ -17,7 +17,7 @@ class Job ( coalesce($1, 'default')::text, coalesce($2, 100)::smallint, - coalesce($3, now())::timestamptz, + greatest($3, now())::timestamptz, $4::text, coalesce($5, '[]')::jsonb, coalesce($6, '{}')::jsonb From df99cef42400ed81b4f397ccf38b2f19ef521a7e Mon Sep 17 00:00:00 2001 From: Ben Hoskings Date: Wed, 25 Mar 2020 11:21:45 +1100 Subject: [PATCH 2/5] Log worker configuration on startup. --- bin/command_line_interface.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/command_line_interface.rb b/bin/command_line_interface.rb index f4dd4df0..f97cc890 100644 --- a/bin/command_line_interface.rb +++ b/bin/command_line_interface.rb @@ -232,6 +232,7 @@ def parse( $stop_que_executable = false %w[INT TERM].each { |signal| trap(signal) { $stop_que_executable = true } } + output.puts "Que started with #{locker.workers.length} workers (priorities: #{locker.workers.map(&:priority)})" output.puts "Que waiting for jobs..." loop do From ae4f0fb83d63077a772cb4c1bb031ccb0ce15c93 Mon Sep 17 00:00:00 2001 From: Ben Hoskings Date: Sun, 12 Apr 2020 18:16:04 +1000 Subject: [PATCH 3/5] Fix CLI spec after df99cef. --- bin/command_line_interface.spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/command_line_interface.spec.rb b/bin/command_line_interface.spec.rb index 02d7d803..55711856 100644 --- a/bin/command_line_interface.spec.rb +++ b/bin/command_line_interface.spec.rb @@ -147,6 +147,7 @@ def write_file assert_equal( [ + "Que started with 6 workers (priorities: [10, 30, 50, nil, nil, nil])", "Que waiting for jobs...", "\nFinishing Que's current jobs before exiting...", "Que's jobs finished, exiting...", From d57305efb4747d4e42ae29289c172307598a460f Mon Sep 17 00:00:00 2001 From: Ben Hoskings Date: Tue, 14 Apr 2020 10:52:45 +1000 Subject: [PATCH 4/5] Improve QueJob#by_job_class performance. This method wasn't making use of the que_jobs.args index and was performing badly on large datasets. GIN indexes don't support the '=' operator, but support existence '?' and containment '@>' among others. To make use of the index the query has to check for containment of a subset document rather than match a value within the document directly. QueJob#by_job_args is already querying in this way; this change makes QueJob#by_job_class consistent with it. Context: https://www.postgresql.org/docs/9.6/datatype-json.html#JSON-INDEXING Given a jsonb column "data" indexed as follows: CREATE INDEX data_index ON table USING GIN (data); And with data in this format: { "name": "Melbourne", "type": "City" } This query will make use of the index: SELECT data FROM table WHERE data @> '{"name": "Melbourne"}'; But this one will not: SELECT data FROM table WHERE data->>'name' = "Melbourne"; --- lib/que/active_record/model.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/que/active_record/model.rb b/lib/que/active_record/model.rb index cab3119b..abb873be 100644 --- a/lib/que/active_record/model.rb +++ b/lib/que/active_record/model.rb @@ -25,9 +25,10 @@ class Model < ::ActiveRecord::Base class << self def by_job_class(job_class) job_class = job_class.name if job_class.is_a?(Class) + job_class_doc = "[{\"job_class\": \"#{job_class}\"}]" where( - "que_jobs.job_class = ? OR (que_jobs.job_class = 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper' AND que_jobs.args->0->>'job_class' = ?)", - job_class, job_class, + "que_jobs.job_class = ? OR (que_jobs.job_class = 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper' AND que_jobs.args @> ?)", + job_class, job_class_doc, ) end From a18102dab878abbc6397be716832c35060230a16 Mon Sep 17 00:00:00 2001 From: David Cristofaro Date: Thu, 4 Jun 2020 16:10:39 +1000 Subject: [PATCH 5/5] Port Rails 6 time fix In Rails 6 we get `no implicit conversion of Time into String` when enqueueing jobs. This was patched upstream, see [here](https://github.com/que-rb/que/issues/247). --- lib/que/connection.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/que/connection.rb b/lib/que/connection.rb index fbb8a1d1..30adf777 100644 --- a/lib/que/connection.rb +++ b/lib/que/connection.rb @@ -152,7 +152,13 @@ def convert_params(params) }, # Timestamp with time zone - 1184 => Time.method(:parse), + 1184 => -> (value) { + case value + when Time then value + when String then Time.parse(value) + else raise "Unexpected time class: #{value.class} (#{value.inspect})" + end + } } # JSON, JSONB