Merge pull request #266 from Shopify/cbruckmayer/only-process-retry-q…

…ueue-on-manual-retries

Only process retry queue on manual retries

ruby/lib/minitest/queue/runner.rb

@@ -49,7 +49,8 @@ def retry_command
 
       def run_command
         require_worker_id!
-        if queue.retrying? || retry?
+        # if it's an automatic job retry we should process the main queue
+        if manual_retry?
           if queue.expired?
             abort! "The test run is too old and can't be retried"
           end
@@ -671,6 +672,12 @@ def abort!(message, exit_status=1)
         exit! exit_status # exit! is required to avoid minitest at_exit callback
       end
 
+      def manual_retry?
+        # this env variable only exists on Buildkite so we should default to manual for backward compatibility
+        (retry? || queue.retrying?) &&
+          ENV.fetch("BUILDKITE_RETRY_TYPE", "manual") == "manual"
+      end
+
       def retry?
         ENV["BUILDKITE_RETRY_COUNT"].to_i > 0 ||
           ENV["SEMAPHORE_PIPELINE_RERUN"] == "true"

ruby/test/integration/minitest_redis_test.rb

@@ -85,7 +85,6 @@ def test_lost_test_with_heartbeat_monitor
             )
         end
 
-
         assert_empty err
         result = normalize(out.lines[1].strip)
         assert_equal "Ran 1 tests, 0 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs (aggregated)", result
@@ -349,6 +348,47 @@ def test_retry_success
       assert_equal 'All tests were ran already', output
     end
 
+    def test_automatic_retry
+      out, err = capture_subprocess_io do
+        system(
+          @exe, 'run',
+          '--queue', @redis_url,
+          '--seed', 'foobar',
+          '--build', '1',
+          '--worker', '1',
+          '--timeout', '1',
+          '--max-requeues', '1',
+          '--requeue-tolerance', '1',
+          '-Itest',
+          'test/failing_test.rb',
+          chdir: 'test/fixtures/',
+        )
+      end
+      assert_empty err
+      output = normalize(out.lines.last.strip)
+      assert_equal 'Ran 200 tests, 200 assertions, 100 failures, 0 errors, 0 skips, 100 requeues in X.XXs', output
+
+      out, err = capture_subprocess_io do
+        system(
+          { "BUILDKITE_RETRY_TYPE" => "automatic" },
+          @exe, 'run',
+          '--queue', @redis_url,
+          '--seed', 'foobar',
+          '--build', '1',
+          '--worker', '1',
+          '--timeout', '1',
+          '--max-requeues', '1',
+          '--requeue-tolerance', '1',
+          '-Itest',
+          'test/failing_test.rb',
+          chdir: 'test/fixtures/',
+        )
+      end
+      assert_empty err
+      output = normalize(out.lines.last.strip)
+      assert_equal 'All tests were ran already', output
+    end
+
     def test_retry_fails_when_test_run_is_expired
       out, err = capture_subprocess_io do
         system(