Skip to content

Commit

Permalink
Test: Add fail-fast mechanism
Browse files Browse the repository at this point in the history
  • Loading branch information
@IanButterworth
IanButterworth committed May 16, 2022
1 parent f9aa28f commit 226566f
Show file tree
Hide file tree
Showing 3 changed files with 149 additions and 4 deletions.
5 changes: 5 additions & 0 deletions NEWS.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,11 @@ Standard library changes

#### SparseArrays

#### Test
* New fail-fast mode for testsets that will terminate the test run early if a failure or error occurs.
Set either via the `@testset` kwarg `failfast=true` or by setting env var `JULIA_TEST_FAILFAST`
to `"true"` i.e. in CI runs to request the job failure be posted eagerly when issues occur ([#xxx])

#### Dates

#### Downloads
Expand Down
43 changes: 39 additions & 4 deletions stdlib/Test/src/Test.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ const DISPLAY_FAILED = (
:contains
)

const FAIL_FAST = Ref{Bool}(false)

#-----------------------------------------------------------------------

# Backtrace utility functions
Expand Down Expand Up @@ -963,8 +965,22 @@ mutable struct DefaultTestSet <: AbstractTestSet
showtiming::Bool
time_start::Float64
time_end::Union{Float64,Nothing}
failfast::Bool
end
DefaultTestSet(desc::AbstractString; verbose::Bool = false, showtiming::Bool = true) = DefaultTestSet(String(desc)::String, [], 0, false, verbose, showtiming, time(), nothing)
function DefaultTestSet(desc::AbstractString; verbose::Bool = false, showtiming::Bool = true, failfast::Union{Nothing,Bool} = nothing)
if isnothing(failfast)
# pass failfast state into child testsets
parent_ts = get_testset()
if parent_ts isa DefaultTestSet
failfast = something(parent_ts.failfast, false)
else
failfast = false
end
end
return DefaultTestSet(String(desc)::String, [], 0, false, verbose, showtiming, time(), nothing, failfast)
end

struct FailFastError <: Exception end

# For a broken result, simply store the result
record(ts::DefaultTestSet, t::Broken) = (push!(ts.results, t); t)
Expand All @@ -986,6 +1002,7 @@ function record(ts::DefaultTestSet, t::Union{Fail, Error})
end
end
push!(ts.results, t)
(FAIL_FAST[] || ts.failfast) && throw(FailFastError())
return t
end

Expand Down Expand Up @@ -1262,11 +1279,17 @@ along with a summary of the test results.
Any custom testset type (subtype of `AbstractTestSet`) can be given and it will
also be used for any nested `@testset` invocations. The given options are only
applied to the test set where they are given. The default test set type
accepts two boolean options:
accepts three boolean options:
- `verbose`: if `true`, the result summary of the nested testsets is shown even
when they all pass (the default is `false`).
- `showtiming`: if `true`, the duration of each displayed testset is shown
(the default is `true`).
- `failfast`: if `true`, any test failure or error will cause the testset and any
child testsets to return immediately (the default is `false`). This can also be set
globally via the env var `JULIA_TEST_FAILFAST`.
!!! compat "Julia 1.9"
`failfast` requires at least Julia 1.9.
The description string accepts interpolation from the loop indices.
If no description is provided, one is constructed based on the variables.
Expand Down Expand Up @@ -1310,6 +1333,8 @@ macro testset(args...)
error("Expected function call, begin/end block or for loop as argument to @testset")
end

FAIL_FAST[] = something(tryparse(Bool, get(ENV, "JULIA_TEST_FAILFAST", "false")), false)

if tests.head === :for
return testset_forloop(args, tests, __source__)
else
Expand Down Expand Up @@ -1364,7 +1389,11 @@ function testset_beginend_call(args, tests, source)
# something in the test block threw an error. Count that as an
# error in this test set
trigger_test_failure_break(err)
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source))))
if err isa FailFastError
get_testset_depth() > 1 ? rethrow() : failfast_print()
else
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source))))
end
finally
copy!(RNG, oldrng)
Random.set_global_seed!(oldseed)
Expand All @@ -1380,6 +1409,10 @@ function testset_beginend_call(args, tests, source)
return ex
end

function failfast_print()
printstyled("\nFail-fast enabled:"; color = Base.error_color(), bold=true)
printstyled(" Fail or Error occured\n\n"; color = Base.error_color())
end

"""
Generate the code for a `@testset` with a `for` loop argument
Expand Down Expand Up @@ -1440,7 +1473,9 @@ function testset_forloop(args, testloop, source)
# Something in the test block threw an error. Count that as an
# error in this test set
trigger_test_failure_break(err)
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source))))
if !isa(err, FailFastError)
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source))))
end
end
end
quote
Expand Down
105 changes: 105 additions & 0 deletions stdlib/Test/test/runtests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1160,6 +1160,111 @@ end
end
end

@testset "failfast option" begin
@testset "non failfast (default)" begin
expected = r"""
Test Summary: | Pass Fail Error Total Time
Foo | 1 2 1 4 \s*\d*.\ds
Bar | 1 1 2 \s*\d*.\ds
"""

mktemp() do f, _
write(f,
"""
using Test
@testset "Foo" begin
@test false
@test error()
@testset "Bar" begin
@test false
@test true
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
end
@testset "failfast" begin
expected = r"""
Test Summary: | Fail Total Time
Foo | 1 1 \s*\d*.\ds
"""

mktemp() do f, _
write(f,
"""
using Test
@testset "Foo" failfast=true begin
@test false
@test error()
@testset "Bar" begin
@test false
@test true
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
end
@testset "failfast passes to child testsets" begin
expected = r"""
Test Summary: | Fail Total Time
PackageName | 1 1 \s*\d*.\ds
1 | 1 1 \s*\d*.\ds
"""

mktemp() do f, _
write(f,
"""
using Test
@testset "Foo" failfast=true begin
@testset "1" begin
@test false
end
@testset "2" begin
@test true
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
end
@testset "failfast via env var" begin
expected = r"""
Test Summary: | Fail Total Time
Foo | 1 1 \s*\d*.\ds
"""

mktemp() do f, _
write(f,
"""
using Test
ENV["JULIA_TEST_FAILFAST"] = true
@testset "Foo" begin
@test false
@test error()
@testset "Bar" begin
@test false
@test true
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
end
end

# Non-booleans in @test (#35888)
struct T35888 end
Base.isequal(::T35888, ::T35888) = T35888()
Expand Down

0 comments on commit 226566f

Please sign in to comment.