Test: Add fail-fast mechanism

NEWS.md

@@ -99,6 +99,11 @@ Standard library changes
 
 #### SparseArrays
 
+#### Test
+* New fail-fast mode for testsets that will terminate the test run early if a failure or error.
+  Set either via the `@testset` kwarg `failfast=true` or by setting env var `JULIA_TEST_FAILFAST`
+  to `"true"` i.e. in CI runs to request the job failure be posted eagerly when issues occur ([#xxx])
+
 #### Dates
 
 #### Downloads

stdlib/Test/src/Test.jl

@@ -41,6 +41,8 @@ const DISPLAY_FAILED = (
     :contains
 )
 
+const FAIL_FAST = Ref{Bool}(false)
+
 #-----------------------------------------------------------------------
 
 # Backtrace utility functions
@@ -963,8 +965,12 @@ mutable struct DefaultTestSet <: AbstractTestSet
     showtiming::Bool
     time_start::Float64
     time_end::Union{Float64,Nothing}
+    failfast::Bool
 end
-DefaultTestSet(desc::AbstractString; verbose::Bool = false, showtiming::Bool = true) = DefaultTestSet(String(desc)::String, [], 0, false, verbose, showtiming, time(), nothing)
+DefaultTestSet(desc::AbstractString; verbose::Bool = false, showtiming::Bool = true, failfast::Bool = false) =
+    DefaultTestSet(String(desc)::String, [], 0, false, verbose, showtiming, time(), nothing, failfast)
+
+struct FailFastError <: Exception end
 
 # For a broken result, simply store the result
 record(ts::DefaultTestSet, t::Broken) = (push!(ts.results, t); t)
@@ -986,6 +992,7 @@ function record(ts::DefaultTestSet, t::Union{Fail, Error})
         end
     end
     push!(ts.results, t)
+    (FAIL_FAST[] || ts.failfast) && throw(FailFastError())
     return t
 end
 
@@ -1128,6 +1135,14 @@ function filter_errors(ts::DefaultTestSet)
     efs
 end
 
+function any_fail_or_error(ts::DefaultTestSet)
+    for t in ts.results
+        isa(t, Fail) && return true
+        isa(t, Error) && return true
+    end
+    false
+end
+
 # Recursive function that counts the number of test results of each
 # type directly in the testset, and totals across the child testsets
 function get_test_counts(ts::DefaultTestSet)
@@ -1262,11 +1277,17 @@ along with a summary of the test results.
 Any custom testset type (subtype of `AbstractTestSet`) can be given and it will
 also be used for any nested `@testset` invocations. The given options are only
 applied to the test set where they are given. The default test set type
-accepts two boolean options:
+accepts three boolean options:
 - `verbose`: if `true`, the result summary of the nested testsets is shown even
 when they all pass (the default is `false`).
 - `showtiming`: if `true`, the duration of each displayed testset is shown
 (the default is `true`).
+- `failfast`: if `true`, any test failure or error will cause the testset to return
+immediately (the default is `false`). This can also be set globally via the env var
+`JULIA_TEST_FAILFAST`.
+
+!!! compat "Julia 1.9"
+    `failfast` requires at least Julia 1.9.
 
 The description string accepts interpolation from the loop indices.
 If no description is provided, one is constructed based on the variables.
@@ -1310,6 +1331,8 @@ macro testset(args...)
         error("Expected function call, begin/end block or for loop as argument to @testset")
     end
 
+    FAIL_FAST[] = something(tryparse(Bool, get(ENV, "JULIA_TEST_FAILFAST", "false")), false)
+
     if tests.head === :for
         return testset_forloop(args, tests, __source__)
     else
@@ -1364,7 +1387,11 @@ function testset_beginend_call(args, tests, source)
             # something in the test block threw an error. Count that as an
             # error in this test set
             trigger_test_failure_break(err)
-            record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source))))
+            if err isa FailFastError
+                failfast_print()
+            else
+                record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source))))
+            end
         finally
             copy!(RNG, oldrng)
             Random.set_global_seed!(oldseed)
@@ -1380,6 +1407,10 @@ function testset_beginend_call(args, tests, source)
     return ex
 end
 
+function failfast_print()
+    printstyled("\nFail-fast enabled:"; color = Base.error_color(), bold=true)
+    printstyled(" Fail or Error occured\n\n"; color = Base.error_color())
+end
 
 """
 Generate the code for a `@testset` with a `for` loop argument
@@ -1440,7 +1471,9 @@ function testset_forloop(args, testloop, source)
             # Something in the test block threw an error. Count that as an
             # error in this test set
             trigger_test_failure_break(err)
-            record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source))))
+            if !isa(err, FailFastError)
+                record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source))))
+            end
         end
     end
     quote

stdlib/Test/test/runtests.jl

@@ -1160,6 +1160,85 @@ end
     end
 end
 
+@testset "failfast option" begin
+    @testset "non failfast (default)" begin
+        expected = r"""
+        Test Summary: | Pass  Fail  Error  Total  Time
+        Foo           |    1     2      1      4  \s*\d*.\ds
+          Bar         |    1     1             2  \s*\d*.\ds
+        """
+
+        mktemp() do f, _
+            write(f,
+            """
+            using Test
+
+            @testset "Foo" begin
+                @test false
+                @test error()
+                @testset "Bar" begin
+                    @test false
+                    @test true
+                end
+            end
+            """)
+            cmd    = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
+            result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
+            @test occursin(expected, result)
+        end
+    end
+    @testset "failfast" begin
+        expected = r"""
+        Test Summary: | Fail  Total  Time
+        Foo           |    1      1  \s*\d*.\ds
+        """
+
+        mktemp() do f, _
+            write(f,
+            """
+            using Test
+
+            @testset "Foo" failfast=true begin
+                @test false
+                @test error()
+                @testset "Bar" begin
+                    @test false
+                    @test true
+                end
+            end
+            """)
+            cmd    = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
+            result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
+            @test occursin(expected, result)
+        end
+    end
+    @testset "failfast via env var" begin
+        expected = r"""
+        Test Summary: | Fail  Total  Time
+        Foo           |    1      1  \s*\d*.\ds
+        """
+
+        mktemp() do f, _
+            write(f,
+            """
+            using Test
+            ENV["JULIA_TEST_FAILFAST"] = true
+            @testset "Foo" begin
+                @test false
+                @test error()
+                @testset "Bar" begin
+                    @test false
+                    @test true
+                end
+            end
+            """)
+            cmd    = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
+            result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
+            @test occursin(expected, result)
+        end
+    end
+end
+
 # Non-booleans in @test (#35888)
 struct T35888 end
 Base.isequal(::T35888, ::T35888) = T35888()