Skip to content

use ScopedValues for TestSets #53462

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Aug 26, 2025
Merged

use ScopedValues for TestSets #53462

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
564e1d5
Rebase #53462
simonbyrne Aug 23, 2025
b0b4cf9
FAIL_FAST to OncePerProcess
Keno Aug 23, 2025
e3c4c64
rm unrelated ScopedValue changes
Keno Aug 23, 2025
e6b0f73
Add NEWS
Keno Aug 23, 2025
888b9aa
Also move over TESTSET_PRINT_ENABLE
Keno Aug 24, 2025
e503185
Turns out the scopedvalues test changes are required
Keno Aug 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions NEWS.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ Standard library changes
* Test failures when using the `@test` macro now show evaluated arguments for all function calls ([#57825], [#57839]).
* Transparent test sets (`@testset let`) now show context when tests error ([#58727]).
* `@test_throws` now supports a three-argument form `@test_throws ExceptionType pattern expr` to test both exception type and message pattern in one call ([#59117]).
* The testset stack was changed to use `ScopedValue` rather than task local storage ([#53462]).

#### InteractiveUtils

Expand Down
134 changes: 42 additions & 92 deletions stdlib/Test/src/Test.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,16 @@ using Random: AbstractRNG, default_rng
using InteractiveUtils: gen_call_with_extracted_types
using Base: typesplit, remove_linenums!
using Serialization: Serialization

const FAIL_FAST = Ref{Bool}(false)
using Base.ScopedValues: ScopedValue, @with

const record_passes = OncePerProcess{Bool}() do
return Base.get_bool_env("JULIA_TEST_RECORD_PASSES", false)
end

const global_fail_fast = OncePerProcess{Bool}() do
return Base.get_bool_env("JULIA_TEST_FAILFAST", false)
end

#-----------------------------------------------------------------------

# Backtrace utility functions
Expand Down Expand Up @@ -1109,7 +1112,7 @@ if get_testset_depth() != 0
end
```
"""
function finish end
finish(ts::AbstractTestSet) = ts

"""
TestSetException
Expand Down Expand Up @@ -1144,7 +1147,6 @@ end
A simple fallback test set that throws immediately on a failure.
"""
struct FallbackTestSet <: AbstractTestSet end
const fallback_testset = FallbackTestSet()

struct FallbackTestSetException <: Exception
msg::String
Expand All @@ -1161,8 +1163,6 @@ function record(ts::FallbackTestSet, t::Union{Fail, Error})
println(t)
throw(FallbackTestSetException("There was an error during testing"))
end
# We don't need to do anything as we don't record anything
finish(ts::FallbackTestSet) = ts

#-----------------------------------------------------------------------

Expand Down Expand Up @@ -1237,7 +1237,7 @@ function DefaultTestSet(desc::AbstractString; verbose::Bool = false, showtiming:
if parent_ts isa DefaultTestSet
failfast = parent_ts.failfast
else
failfast = false
failfast = global_fail_fast()
end
end
return DefaultTestSet(String(desc)::String,
Expand Down Expand Up @@ -1281,7 +1281,7 @@ function record(ts::DefaultTestSet, t::Union{Fail, Error}; print_result::Bool=TE
end
end
@lock ts.results_lock push!(ts.results, t)
(FAIL_FAST[] || ts.failfast) && throw(FailFastError())
ts.failfast && throw(FailFastError())
return t
end

Expand Down Expand Up @@ -1408,9 +1408,6 @@ function print_test_results(io::IO, ts::AbstractTestSet, depth_pad=0)
end
end


const TESTSET_PRINT_ENABLE = Ref(true)

# Called at the end of a @testset, behaviour depends on whether
# this is a child of another testset, or the "root" testset
function finish(ts::DefaultTestSet; print_results::Bool=TESTSET_PRINT_ENABLE[])
Expand Down Expand Up @@ -1836,8 +1833,6 @@ macro testset(args...)
error("Expected function call, begin/end block or for loop as argument to @testset")
end

FAIL_FAST[] = Base.get_bool_env("JULIA_TEST_FAILFAST", false)

if tests.head === :for
return testset_forloop(args, tests, __source__)
elseif tests.head === :let
Expand Down Expand Up @@ -1881,21 +1876,11 @@ function testset_context(args, ex, source)
else
error("Malformed `let` expression is given")
end
reverse!(contexts)

test_ex = ex.args[2]

ex.args[2] = quote
$(map(contexts) do context
:($push_testset($(ContextTestSet)($(QuoteNode(context)), $context; $options...)))
end...)
try
$(test_ex)
finally
$(map(_->:($pop_testset()), contexts)...)
end
for context in contexts
test_ex = :($Test.@with_testset($ContextTestSet($(QuoteNode(context)), $context; $options...), $test_ex))
end

ex.args[2] = test_ex
return esc(ex)
end

Expand Down Expand Up @@ -1946,7 +1931,7 @@ function testset_beginend_call(args, tests, source)
else
$(testsettype)($desc; $options...)
end
push_testset(ts)

# we reproduce the logic of guardseed, but this function
# cannot be used as it changes slightly the semantic of @testset,
# by wrapping the body in a function
Expand All @@ -1955,26 +1940,27 @@ function testset_beginend_call(args, tests, source)
local ts_rng = get_rng(ts)
local tls_seed = isnothing(ts_rng) ? set_rng!(ts, tls_seed_orig) : ts_rng
try
# default RNG is reset to its state from last `seed!()` to ease reproduce a failed test
copy!(Random.default_rng(), tls_seed)
copy!(Random.get_tls_seed(), Random.default_rng())
let
$(esc(tests))
@with_testset ts begin
# default RNG is reset to its state from last `seed!()` to ease reproduce a failed test
copy!(Random.default_rng(), tls_seed)
copy!(Random.get_tls_seed(), Random.default_rng())
let
$(esc(tests))
end
end
catch err
err isa InterruptException && rethrow()
# something in the test block threw an error. Count that as an
# error in this test set
trigger_test_failure_break(err)
if is_failfast_error(err)
get_testset_depth() > 1 ? rethrow() : failfast_print()
get_testset_depth() > 0 ? rethrow() : failfast_print()
Comment on lines -1970 to +1957
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can't immediately tell the motivation for this change, could you please elaborate?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Previously the check used to be inside the scope, now it is outside

else
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)), nothing))
end
finally
copy!(default_rng(), default_rng_orig)
copy!(Random.get_tls_seed(), tls_seed_orig)
pop_testset()
ret = finish(ts)
end
ret
Expand Down Expand Up @@ -2031,59 +2017,41 @@ function testset_forloop(args, testloop, source)
tests = testloop.args[2]
blk = quote
_check_testset($testsettype, $(QuoteNode(testsettype.args[1])))
# Trick to handle `break` and `continue` in the test code before
# they can be handled properly by `finally` lowering.
if !first_iteration
pop_testset()
finish_errored = true
push!(arr, finish(ts))
finish_errored = false
copy!(default_rng(), tls_seed)
end
ts = if ($testsettype === $DefaultTestSet) && $(isa(source, LineNumberNode))
$(testsettype)($desc; source=$(QuoteNode(source.file)), $options..., rng=tls_seed)
else
$(testsettype)($desc; $options...)
end
push_testset(ts)
first_iteration = false
try
$(esc(tests))
@with_testset ts begin
# default RNG is reset to its state from last `seed!()` to ease reproduce a failed test
copy!(Random.default_rng(), tls_seed)
$(esc(tests))
end
catch err
err isa InterruptException && rethrow()
# Something in the test block threw an error. Count that as an
# error in this test set
trigger_test_failure_break(err)
if is_failfast_error(err)
get_testset_depth() > 1 ? rethrow() : failfast_print()
get_testset_depth() > 0 ? rethrow() : failfast_print()
else
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)), nothing))
end
finally
copy!(default_rng(), default_rng_orig)
copy!(Random.get_tls_seed(), tls_seed_orig)
push!(arr, finish(ts))
end
end
quote
local arr = Vector{Any}()
local first_iteration = true
local ts
local rng_option = get($(options), :rng, nothing)
local finish_errored = false
local default_rng_orig = copy(default_rng())
local tls_seed_orig = copy(Random.get_tls_seed())
local tls_seed = isnothing(rng_option) ? copy(Random.get_tls_seed()) : rng_option
copy!(Random.default_rng(), tls_seed)
try
let
$(Expr(:for, Expr(:block, [esc(v) for v in loopvars]...), blk))
end
finally
# Handle `return` in test body
if !first_iteration && !finish_errored
pop_testset()
@assert @isdefined(ts) "Assertion to tell the compiler about the definedness of this variable"
push!(arr, finish(ts))
end
copy!(default_rng(), default_rng_orig)
copy!(Random.get_tls_seed(), tls_seed_orig)
let
$(Expr(:for, Expr(:block, [esc(v) for v in loopvars]...), blk))
end
arr
end
Expand Down Expand Up @@ -2132,39 +2100,22 @@ end
#-----------------------------------------------------------------------
# Various helper methods for test sets

const CURRENT_TESTSET = ScopedValue{AbstractTestSet}(FallbackTestSet())
const TESTSET_DEPTH = ScopedValue{Int}(0)
const TESTSET_PRINT_ENABLE = ScopedValue{Bool}(true)

macro with_testset(ts, expr)
:(@with(CURRENT_TESTSET => $(esc(ts)), TESTSET_DEPTH => get_testset_depth() + 1, $(esc(expr))))
end

"""
get_testset()

Retrieve the active test set from the task's local storage. If no
test set is active, use the fallback default test set.
"""
function get_testset()
testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[])
return isempty(testsets) ? fallback_testset : testsets[end]
end

"""
push_testset(ts::AbstractTestSet)

Adds the test set to the `task_local_storage`.
"""
function push_testset(ts::AbstractTestSet)
testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[])
push!(testsets, ts)
setindex!(task_local_storage(), testsets, :__BASETESTNEXT__)
end
Comment on lines -2146 to -2155
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Apparently some (testing) packages were using this: https://juliahub.com/ui/Search?type=symbols&q=push_testset&u=use. Presumably it's ok to ask them to adapt their use of internal functions? These aren't documented.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, unfortunately I don't see how we could keep that working

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How would one update a codebase that is using push_testset? is it as simple as changing from

push_testset(ts)
try
   ...
finally
  pop_testset()
end

to

@with_testset ts begin
  ...
end

?

(p.s. Very pleased to see this improvement!)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, that should work I think.


"""
pop_testset()

Pops the last test set added to the `task_local_storage`. If there are no
active test sets, returns the fallback default test set.
"""
function pop_testset()
testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[])
ret = isempty(testsets) ? fallback_testset : pop!(testsets)
setindex!(task_local_storage(), testsets, :__BASETESTNEXT__)
return ret
something(Base.ScopedValues.get(CURRENT_TESTSET))
end

"""
Expand All @@ -2173,8 +2124,7 @@ end
Return the number of active test sets, not including the default test set
"""
function get_testset_depth()
testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[])
return length(testsets)
something(Base.ScopedValues.get(TESTSET_DEPTH))
end

_args_and_call((args..., f)...; kwargs...) = (args, kwargs, f(args...; kwargs...))
Expand Down
4 changes: 2 additions & 2 deletions stdlib/Test/test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1541,7 +1541,7 @@ end
write(f,
"""
using Test
ENV["JULIA_TEST_FAILFAST"] = true

@testset "Foo" begin
@test false
@test error()
Expand All @@ -1551,7 +1551,7 @@ end
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
cmd = addenv(`$(Base.julia_cmd()) --startup-file=no --color=no $f`, "JULIA_TEST_FAILFAST"=>"true")
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin(expected, result)
end
Expand Down
Loading