From 88def1afe16acdfe41b15dc956742359d837ce04 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 20 May 2022 19:30:58 -0400 Subject: [PATCH] Test: Add fail-fast mechanism (#45317) --- NEWS.md | 5 ++ stdlib/Test/src/Test.jl | 43 ++++++++++++-- stdlib/Test/test/runtests.jl | 105 +++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index 9c2468e229861..63027b9aabf7c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -104,6 +104,11 @@ Standard library changes #### SparseArrays +#### Test +* New fail-fast mode for testsets that will terminate the test run early if a failure or error occurs. + Set either via the `@testset` kwarg `failfast=true` or by setting env var `JULIA_TEST_FAILFAST` + to `"true"` i.e. in CI runs to request the job failure be posted eagerly when issues occur ([#45317]) + #### Dates #### Downloads diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index c4c2359253943..11ec4f29961f6 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -41,6 +41,8 @@ const DISPLAY_FAILED = ( :contains ) +const FAIL_FAST = Ref{Bool}(false) + #----------------------------------------------------------------------- # Backtrace utility functions @@ -963,8 +965,22 @@ mutable struct DefaultTestSet <: AbstractTestSet showtiming::Bool time_start::Float64 time_end::Union{Float64,Nothing} + failfast::Bool end -DefaultTestSet(desc::AbstractString; verbose::Bool = false, showtiming::Bool = true) = DefaultTestSet(String(desc)::String, [], 0, false, verbose, showtiming, time(), nothing) +function DefaultTestSet(desc::AbstractString; verbose::Bool = false, showtiming::Bool = true, failfast::Union{Nothing,Bool} = nothing) + if isnothing(failfast) + # pass failfast state into child testsets + parent_ts = get_testset() + if parent_ts isa DefaultTestSet + failfast = parent_ts.failfast + else + failfast = false + end + end + return DefaultTestSet(String(desc)::String, [], 0, false, verbose, showtiming, time(), nothing, failfast) +end + +struct FailFastError <: Exception end # For a broken result, simply store the result record(ts::DefaultTestSet, t::Broken) = (push!(ts.results, t); t) @@ -986,6 +1002,7 @@ function record(ts::DefaultTestSet, t::Union{Fail, Error}) end end push!(ts.results, t) + (FAIL_FAST[] || ts.failfast) && throw(FailFastError()) return t end @@ -1262,11 +1279,17 @@ along with a summary of the test results. Any custom testset type (subtype of `AbstractTestSet`) can be given and it will also be used for any nested `@testset` invocations. The given options are only applied to the test set where they are given. The default test set type -accepts two boolean options: +accepts three boolean options: - `verbose`: if `true`, the result summary of the nested testsets is shown even when they all pass (the default is `false`). - `showtiming`: if `true`, the duration of each displayed testset is shown (the default is `true`). +- `failfast`: if `true`, any test failure or error will cause the testset and any +child testsets to return immediately (the default is `false`). This can also be set +globally via the env var `JULIA_TEST_FAILFAST`. + +!!! compat "Julia 1.9" + `failfast` requires at least Julia 1.9. The description string accepts interpolation from the loop indices. If no description is provided, one is constructed based on the variables. @@ -1310,6 +1333,8 @@ macro testset(args...) error("Expected function call, begin/end block or for loop as argument to @testset") end + FAIL_FAST[] = something(tryparse(Bool, get(ENV, "JULIA_TEST_FAILFAST", "false")), false) + if tests.head === :for return testset_forloop(args, tests, __source__) else @@ -1364,7 +1389,11 @@ function testset_beginend_call(args, tests, source) # something in the test block threw an error. Count that as an # error in this test set trigger_test_failure_break(err) - record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)))) + if err isa FailFastError + get_testset_depth() > 1 ? rethrow() : failfast_print() + else + record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)))) + end finally copy!(RNG, oldrng) Random.set_global_seed!(oldseed) @@ -1380,6 +1409,10 @@ function testset_beginend_call(args, tests, source) return ex end +function failfast_print() + printstyled("\nFail-fast enabled:"; color = Base.error_color(), bold=true) + printstyled(" Fail or Error occured\n\n"; color = Base.error_color()) +end """ Generate the code for a `@testset` with a `for` loop argument @@ -1443,7 +1476,9 @@ function testset_forloop(args, testloop, source) # Something in the test block threw an error. Count that as an # error in this test set trigger_test_failure_break(err) - record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)))) + if !isa(err, FailFastError) + record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)))) + end end end quote diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 579b81cd5ace9..38a4fb0031dd7 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -1160,6 +1160,111 @@ end end end +@testset "failfast option" begin + @testset "non failfast (default)" begin + expected = r""" + Test Summary: | Pass Fail Error Total Time + Foo | 1 2 1 4 \s*\d*.\ds + Bar | 1 1 2 \s*\d*.\ds + """ + + mktemp() do f, _ + write(f, + """ + using Test + + @testset "Foo" begin + @test false + @test error() + @testset "Bar" begin + @test false + @test true + end + end + """) + cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f` + result = read(pipeline(ignorestatus(cmd), stderr=devnull), String) + @test occursin(expected, result) + end + end + @testset "failfast" begin + expected = r""" + Test Summary: | Fail Total Time + Foo | 1 1 \s*\d*.\ds + """ + + mktemp() do f, _ + write(f, + """ + using Test + + @testset "Foo" failfast=true begin + @test false + @test error() + @testset "Bar" begin + @test false + @test true + end + end + """) + cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f` + result = read(pipeline(ignorestatus(cmd), stderr=devnull), String) + @test occursin(expected, result) + end + end + @testset "failfast passes to child testsets" begin + expected = r""" + Test Summary: | Fail Total Time + PackageName | 1 1 \s*\d*.\ds + 1 | 1 1 \s*\d*.\ds + """ + + mktemp() do f, _ + write(f, + """ + using Test + + @testset "Foo" failfast=true begin + @testset "1" begin + @test false + end + @testset "2" begin + @test true + end + end + """) + cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f` + result = read(pipeline(ignorestatus(cmd), stderr=devnull), String) + @test occursin(expected, result) + end + end + @testset "failfast via env var" begin + expected = r""" + Test Summary: | Fail Total Time + Foo | 1 1 \s*\d*.\ds + """ + + mktemp() do f, _ + write(f, + """ + using Test + ENV["JULIA_TEST_FAILFAST"] = true + @testset "Foo" begin + @test false + @test error() + @testset "Bar" begin + @test false + @test true + end + end + """) + cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f` + result = read(pipeline(ignorestatus(cmd), stderr=devnull), String) + @test occursin(expected, result) + end + end +end + # Non-booleans in @test (#35888) struct T35888 end Base.isequal(::T35888, ::T35888) = T35888()