MattT/APPEALS-51115: Initialize FIFO SQS Queues Locally (#22182)

* Set local env to utilize FIFO queues where appropriate

FIFO queues are configured with the same attributes as our queues in higher environments.

* Resolve failing test

* Remove defunct test

* Update make commands

---------

Co-authored-by: Matthew Thornton <ThorntonMatthew@users.noreply.github.com>

Makefile.example

@@ -287,7 +287,7 @@ one-test:  ## run the rspec test passed in
 	bundle exec rspec $(RUN_ARGS)
 
 run-all-queues: ## start shoryuken with all queues
-	bundle exec shoryuken -q caseflow_development_send_notifications caseflow_development_high_priority caseflow_development_low_priority  -R
+	bundle exec shoryuken -q caseflow_development_send_notifications.fifo caseflow_development_high_priority caseflow_development_low_priority  -R
 
 run-low-priority: ## start shoryuken with just the low priority queue
 	bundle exec shoryuken -q caseflow_development_low_priority  -R
@@ -296,7 +296,7 @@ run-high-priority: ## start shoryuken with just the high priority queue
 	bundle exec shoryuken -q caseflow_development_high_priority  -R
 
 run-send-notifications: ## start shoryuken with just the send_notification queue
-	bundle exec shoryuken -q caseflow_development_send_notifications  -R
+	bundle exec shoryuken -q caseflow_development_send_notifications.fifo  -R
 
 jest: ## Run jest tests
 	cd client && yarn jest

app/jobs/send_notification_job.rb

@@ -43,7 +43,7 @@ class SendNotificationJobError < StandardError; end
 
   class << self
     def queue_name_suffix
-      ApplicationController.dependencies_faked? ? :send_notifications : :"send_notifications.fifo"
+      :"send_notifications.fifo"
     end
   end
 

config/environments/development.rb

@@ -98,10 +98,10 @@
   # Set to true to get the documents from efolder running locally on port 4000.
   config.use_efolder_locally = false
 
-  # set to true to create queues and override the sqs endpiont
+  # set to true to create queues and override the sqs endpoint
   config.sqs_create_queues = true
 
-  config.sqs_endpoint = ENV.has_key?('DOCKERIZED') ? 'http://localstack:4576' : 'http://localhost:4576'
+  config.sqs_endpoint = ENV.has_key?('DOCKERIZED') ? 'http://localstack:4566' : 'http://localhost:4566'
 
   # since we mock aws using localstack, provide dummy creds to the aws gem
   ENV["AWS_ACCESS_KEY_ID"] ||= "dummykeyid"

config/initializers/message_queues.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# Initializes SQS message queues not for intended for use with
+#  asynchronous jobs.
+#
+# This will primarily be utilized in our development and demo environments.
+
+QUEUE_PREFIX = "caseflow_#{ENV['DEPLOY_ENV']}_"
+
+MESSAGE_QUEUES = [
+  {
+    name: "receive_notifications.fifo",
+    attributes: {
+      "FifoQueue" => "true",
+      "FifoThroughputLimit" => "perQueue"
+    }
+  }
+].freeze
+
+if Rails.application.config.sqs_create_queues
+  sqs_client = Aws::SQS::Client.new
+  sqs_client.config[:endpoint] = URI(Rails.application.config.sqs_endpoint)
+
+  MESSAGE_QUEUES.each do |queue_info|
+    sqs_client.create_queue({
+      queue_name: "#{QUEUE_PREFIX}#{queue_info[:name]}".to_sym,
+      attributes: queue_info[:attributes]
+    })
+  end
+end

config/initializers/shoryuken.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "#{Rails.root}/app/jobs/middleware/job_monitoring_middleware"
 require "#{Rails.root}/app/jobs/middleware/job_request_store_middleware"
 require "#{Rails.root}/app/jobs/middleware/job_sentry_scope_middleware"
@@ -8,16 +10,23 @@
   .shoryuken_options(retry_intervals: [3.seconds, 30.seconds, 5.minutes, 30.minutes, 2.hours, 5.hours])
 
 if Rails.application.config.sqs_endpoint
-  # override the sqs_endpoint
   Shoryuken::Client.sqs.config[:endpoint] = URI(Rails.application.config.sqs_endpoint)
 end
 
 if Rails.application.config.sqs_create_queues
   # create the development queues
-  Shoryuken::Client.sqs.create_queue({ queue_name: ActiveJob::Base.queue_name_prefix + '_low_priority' })
-  Shoryuken::Client.sqs.create_queue({ queue_name: ActiveJob::Base.queue_name_prefix + '_high_priority' })
-  Shoryuken::Client.sqs.create_queue({ queue_name: ActiveJob::Base.queue_name_prefix + '_send_notifications' })
-  Shoryuken::Client.sqs.create_queue({ queue_name: ActiveJob::Base.queue_name_prefix + '_receive_notifications' })
+  Shoryuken::Client.sqs.create_queue({ queue_name: ActiveJob::Base.queue_name_prefix + "_low_priority" })
+  Shoryuken::Client.sqs.create_queue({ queue_name: ActiveJob::Base.queue_name_prefix + "_high_priority" })
+  Shoryuken::Client.sqs.create_queue({
+                                       queue_name: (
+                                        ActiveJob::Base.queue_name_prefix + "_send_notifications.fifo"
+                                      ).to_sym,
+                                       attributes: {
+                                         "FifoQueue" => "true",
+                                         "FifoThroughputLimit" => "perQueue",
+                                         "ContentBasedDeduplication" => "false"
+                                       }
+                                     })
 end
 
 Shoryuken.configure_server do |config|

docker-compose-m1.yml

@@ -19,10 +19,9 @@ services:
   appeals-localstack-aws:
     platform: linux/amd64
     container_name: localstack
-    image: localstack/localstack:0.11.4
+    image: localstack/localstack:0.14.5
     ports:
-      - "4567-4583:4567-4583"
-      - "8082:${PORT_WEB_UI-8080}"
+      - "4566:4566"
     environment:
       - SERVICES=sqs
     volumes:

docker-compose.yml

@@ -24,10 +24,9 @@ services:
 
   appeals-localstack-aws:
     container_name: localstack
-    image: localstack/localstack:0.11.4
+    image: localstack/localstack:0.14.5
     ports:
-      - "4567-4583:4567-4583"
-      - "8082:${PORT_WEB_UI-8080}"
+      - "4566:4566"
     environment:
       - SERVICES=sqs
     volumes:

spec/jobs/send_notification_job_spec.rb

@@ -178,21 +178,11 @@
   context "#queue_name_suffix" do
     subject { described_class.queue_name_suffix }
 
-    it "returns non-FIFO name in development environment" do
-      is_expected.to eq :send_notifications
-    end
-
-    it "returns FIFO name in non-development environment" do
-      allow(ApplicationController).to receive(:dependencies_faked?).and_return(false)
-
+    it "returns FIFO name" do
       is_expected.to eq :"send_notifications.fifo"
     end
   end
 
-  it "it is the correct queue" do
-    expect(SendNotificationJob.new.queue_name).to eq(queue_name)
-  end
-
   context ".perform" do
     subject(:job) { SendNotificationJob.perform_later(good_message.to_json) }