refine retries

app/controllers/application_controller.rb

@@ -7,6 +7,8 @@ def close
   end
 end
 
+class RetryException < RuntimeError; end
+
 class ApplicationController < ActionController::Base
   include DuaMixin
   include Encoder

app/controllers/file_controller.rb

@@ -56,13 +56,11 @@ def check_download
       obj = @file.inv_version.inv_object
       return if current_user_can_download?(obj)
     rescue StandardError => e
-      # :nocov:
       retries += 1
-      raise(e) if retries > RETRY_LIMIT
+      raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
       sleep 1
       retry
-      # :nocov:
     end
 
     flash[:error] = 'You do not have download permissions.'
@@ -86,15 +84,13 @@ def storage_key_do
     retries = 0
     begin
       do_storage_key_do
-    # :nocov:
     rescue StandardError => e
       retries += 1
-      raise(e) if retries > RETRY_LIMIT
+      raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
       sleep 1
       retry
     end
-    # :nocov:
   end
 
   # Perform database lookup for storge node and key
@@ -225,13 +221,11 @@ def load_file
         .first
       raise ActiveRecord::RecordNotFound if @file.nil?
     rescue StandardError => e
-      # :nocov:
       retries += 1
-      raise(e) if retries > RETRY_LIMIT
+      raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
       sleep 1
       retry
-      # :nocov:
     end
   end
   # rubocop:enable Metrics/AbcSize

app/controllers/object_controller.rb

@@ -36,13 +36,11 @@ def load_object
     begin
       @object = InvObject.where('ark = ?', params_u(:object)).includes(:inv_collections, inv_versions: [:inv_files]).first
     rescue StandardError => e
-      # :nocov:
       retries += 1
-      raise(e) if retries > RETRY_LIMIT
+      raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
       sleep 1
       retry
-      # :nocov:
     end
 
     raise ActiveRecord::RecordNotFound if @object.nil?
@@ -105,15 +103,13 @@ def recent
     retries = 0
     begin
       do_recent
-    # :nocov:
     rescue StandardError => e
       retries += 1
-      raise(e) if retries > RETRY_LIMIT
+      raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
       sleep 1
       retry
     end
-    # :nocov:
   end
 
   def do_recent

app/models/inv_object.rb

@@ -57,20 +57,18 @@ def bytestream_uri3
     URI.parse("#{APP_CONFIG['uri_3']}#{node_number}/#{to_param}")
   end
 
-  # :nocov:
   def dua_exists?
     retries = 0
     begin
       !inv_duas.blank?
     rescue StandardError => e
       retries += 1
-      raise(e) if retries > RETRY_LIMIT
+      raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
       sleep 1
       retry
     end
   end
-  # :nocov:
 
   # :nocov:
   def dua_uri
@@ -98,7 +96,7 @@ def current_version
     # :nocov:
     rescue StandardError => e
       retries += 1
-      raise(e) if retries > RETRY_LIMIT
+      raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
       sleep 1
       retry
@@ -113,7 +111,7 @@ def inv_collection
     # :nocov:
     rescue StandardError => e
       retries += 1
-      raise(e) if retries > RETRY_LIMIT
+      raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
       sleep 1
       retry
@@ -136,7 +134,7 @@ def all_local_ids
     # :nocov:
     rescue StandardError => e
       retries += 1
-      raise(e) if retries > RETRY_LIMIT
+      raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
       sleep 1
       retry
@@ -228,7 +226,7 @@ def object_info_add_versions(json, maxfile)
       # :nocov:
       rescue StandardError => e
         retries += 1
-        raise(e) if retries > RETRY_LIMIT
+        raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
         sleep 1
         retry

app/models/inv_version.rb

@@ -28,7 +28,7 @@ def total_size
     # :nocov:
     rescue StandardError => e
       retries += 1
-      raise(e) if retries > RETRY_LIMIT
+      raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
       sleep 1
       retry
@@ -43,7 +43,7 @@ def system_files
     # :nocov:
     rescue StandardError => e
       retries += 1
-      raise(e) if retries > RETRY_LIMIT
+      raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
       sleep 1
       retry
@@ -58,7 +58,7 @@ def producer_files
     # :nocov:
     rescue StandardError => e
       retries += 1
-      raise(e) if retries > RETRY_LIMIT
+      raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
       sleep 1
       retry
@@ -73,7 +73,7 @@ def metadata(element)
     # :nocov:
     rescue StandardError => e
       retries += 1
-      raise(e) if retries > RETRY_LIMIT
+      raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
       sleep 1
       retry
@@ -105,15 +105,13 @@ def total_actual_size
     retries = 0
     begin
       inv_files.sum('full_size')
-    # :nocov:
     rescue StandardError => e
       retries += 1
-      raise(e) if retries > RETRY_LIMIT
+      raise(RetryException.new(e)) if retries > RETRY_LIMIT
 
       sleep 1
       retry
     end
-    # :nocov:
   end
 
   def exceeds_sync_size?

spec/controllers/file_controller_spec.rb

@@ -155,6 +155,62 @@ def my_presign_wrapper
       end
     end
 
+    it 'redirects to presign url for the file - retry failure on pathname match' do
+      mock_permissions_all(user_id, collection_id)
+
+      request.session.merge!({ uid: user_id })
+      allow(InvFile)
+        .to receive(:joins)
+        .with(any_args)
+        .and_raise(Mysql2::Error::ConnectionError.new('Simulate Failure'))
+
+      expect{
+        get(:presign, params: params)
+      }.to raise_error(RetryException)
+    end
+
+    it 'redirects to presign url for the file - retry failure on object retreival' do
+      mock_permissions_all(user_id, collection_id)
+
+      request.session.merge!({ uid: user_id })
+      allow_any_instance_of(InvFile)
+        .to receive(:inv_version)
+        .with(any_args)
+        .and_raise(Mysql2::Error::ConnectionError.new('Simulate Failure'))
+
+      expect{
+        get(:presign, params: params)
+      }.to raise_error(RetryException)
+    end
+
+    it 'redirects to presign url for the file - retry version check' do
+      mock_permissions_all(user_id, collection_id)
+
+      request.session.merge!({ uid: user_id })
+      allow_any_instance_of(Mysql2::Client)
+        .to receive(:prepare)
+        .with(any_args)
+        .and_raise(Mysql2::Error::ConnectionError.new('Simulate Failure'))
+
+      expect{
+        get(:presign, params: params)
+      }.to raise_error(RetryException)
+    end
+
+    it 'redirects to presign url for the file - retry dua check' do
+      mock_permissions_all(user_id, collection_id)
+
+      request.session.merge!({ uid: user_id })
+      allow_any_instance_of(InvObject)
+        .to receive(:inv_duas)
+        .with(any_args)
+        .and_raise(Mysql2::Error::ConnectionError.new('Simulate Failure'))
+
+      expect{
+        get(:presign, params: params)
+      }.to raise_error(RetryException)
+    end
+
     it 'test ark encoding recovery' do
       mock_permissions_all(user_id, collection_id)
 

spec/controllers/object_controller_spec.rb

@@ -221,6 +221,27 @@
         get(:index, params: { object: object_ark })
         expect(response.status).to eq(401)
       end
+
+      it 'successful object load' do
+        mock_permissions_all(user_id, collection_id)
+        request.session.merge!({ uid: user_id })
+        get(:index, params: { object: object_ark })
+        expect(response.status).to eq(200)
+      end
+
+      it 'successful object load - retry failure on load_object' do
+        mock_permissions_all(user_id, collection_id)
+        request.session.merge!({ uid: user_id })
+        allow(InvObject)
+          .to receive(:where)
+          .with(any_args)
+          .and_raise(Mysql2::Error::ConnectionError.new('Simulate Failure'))
+
+        expect{
+          get(:index, params: { object: object_ark })
+        }.to raise_error(RetryException)
+      end
+
     end
 
     describe ':object_info' do
@@ -744,6 +765,18 @@
         end
       end
 
+      it 'gets the list of objects - retry failure on query' do
+        mock_permissions_all(user_id, collection_id)
+        allow(InvCollection)
+          .to receive(:where)
+          .with(any_args)
+          .and_raise(Mysql2::Error::ConnectionError.new('Simulate Failure'))
+
+        expect{
+          get(:recent, params: { collection: collection.ark })
+        }.to raise_error(RetryException)
+      end
+
       it 'gets the list of objects - no collection permissions' do
         request.accept = 'application/atom+xml'
         get(:recent, params: { collection: collection.ark })

spec/controllers/version_controller_spec.rb

@@ -50,6 +50,20 @@
       expect(response.status).to eq(401)
     end
 
+    it 'total actual size - retry error' do
+      mock_permissions_all(user_id, collection_id)
+
+      request.session.merge!({ uid: user_id })
+      allow_any_instance_of(ActiveRecord::Associations::CollectionProxy)
+        .to receive(:sum)
+        .with(any_args)
+        .and_raise(Mysql2::Error::ConnectionError.new('Simulate Failure'))
+
+      expect{
+        get(:download, params: params)
+      }.to raise_error(RetryException)
+    end
+
     it 'prevents download when download size exceeded' do
       mock_permissions_all(user_id, collection_id)