From f3f7f1867450fa4d4c785d7aa69fd984f3fd9f0f Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Thu, 17 Aug 2023 12:42:59 -0400 Subject: [PATCH 01/23] Add ScopedVariables ScopedVariables are containers whose observed value depends the current dynamic scope. This implementation is inspired by https://openjdk.org/jeps/446 A scope is introduced with the `scoped` function that takes a lambda to execute within the new scope. The value of a `ScopedVariable` is constant within that scope and can only be set upon introduction of a new scope. Scopes are propagated across tasks boundaries. In contrast to #35833 the storage of the per-scope data is assoicated with the ScopedVariables object and does not require copies upon scope entry. This also means that libraries can use scoped variables without paying for scoped variables introduces in other libraries. Finding the current value of a ScopedVariable, involves walking the scope chain upwards and checking if the scoped variable has a value for the current or one of its parent scopes. This means the cost of a lookup scales with the depth of the dynamic scoping. This could be amortized by using a task-local cache. --- base/Base.jl | 12 ++-- base/boot.jl | 2 +- base/exports.jl | 4 ++ base/logging.jl | 16 ++--- base/scopedvariables.jl | 128 ++++++++++++++++++++++++++++++++++++++++ src/jltypes.c | 2 +- src/julia.h | 2 +- src/task.c | 6 +- test/choosetests.jl | 1 + test/scopedvariables.jl | 48 +++++++++++++++ 10 files changed, 199 insertions(+), 22 deletions(-) create mode 100644 base/scopedvariables.jl create mode 100644 test/scopedvariables.jl diff --git a/base/Base.jl b/base/Base.jl index 0673a1081ae69..5577769406537 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -331,10 +331,6 @@ using .Libc: getpid, gethostname, time, memcpy, memset, memmove, memcmp const libblas_name = "libblastrampoline" * (Sys.iswindows() ? "-5" : "") const liblapack_name = libblas_name -# Logging -include("logging.jl") -using .CoreLogging - # Concurrency (part 2) # Note that `atomics.jl` here should be deprecated Core.eval(Threads, :(include("atomics.jl"))) @@ -344,6 +340,14 @@ include("task.jl") include("threads_overloads.jl") include("weakkeydict.jl") +# ScopedVariables +include("scopedvariables.jl") +using .ScopedVariables + +# Logging +include("logging.jl") +using .CoreLogging + include("env.jl") # functions defined in Random diff --git a/base/boot.jl b/base/boot.jl index e24a6f4ffc0e0..a7630fe1cbb60 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -163,7 +163,7 @@ # result::Any # exception::Any # backtrace::Any -# logstate::Any +# scope::Any # code::Any #end diff --git a/base/exports.jl b/base/exports.jl index 0959fa1c391e2..8d68fd3f17b22 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -648,6 +648,10 @@ export sprint, summary, +# ScopedVariable + scoped, + ScopedVariable, + # logging @debug, @info, diff --git a/base/logging.jl b/base/logging.jl index c42af08d8f4ae..d7034cb8c8cae 100644 --- a/base/logging.jl +++ b/base/logging.jl @@ -492,8 +492,10 @@ end LogState(logger) = LogState(LogLevel(_invoked_min_enabled_level(logger)), logger) +const CURRENT_LOGSTATE = ScopedVariable{Union{Nothing, LogState}}(nothing) + function current_logstate() - logstate = current_task().logstate + logstate = CURRENT_LOGSTATE[] return (logstate !== nothing ? logstate : _global_logstate)::LogState end @@ -506,17 +508,7 @@ end return nothing end -function with_logstate(f::Function, logstate) - @nospecialize - t = current_task() - old = t.logstate - try - t.logstate = logstate - f() - finally - t.logstate = old - end -end +with_logstate(f::Function, logstate) = scoped(f, CURRENT_LOGSTATE => logstate) #------------------------------------------------------------------------------- # Control of the current logger and early log filtering diff --git a/base/scopedvariables.jl b/base/scopedvariables.jl new file mode 100644 index 0000000000000..e56b99093179a --- /dev/null +++ b/base/scopedvariables.jl @@ -0,0 +1,128 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module ScopedVariables + +export ScopedVariable, scoped + +mutable struct Scope + const parent::Union{Nothing, Scope} +end + +current_scope() = current_task().scope::Union{Nothing, Scope} + +""" + ScopedVariable(x) + +Create a container that propagates values across scopes. +Use [`scoped`](@ref) to create and enter a new scope. + +Values can only be set when entering a new scope, +and the value referred to will be constant during the +execution of a scope. + +Dynamic scopes are propagated across tasks. + +# Examples +```jldoctest +julia> const svar = ScopedVariable(1); + +julia> svar[] +1 + +julia> scoped(svar => 2) do + svar[] + end +2 +``` + +!!! compat "Julia 1.11" + This method requires at least Julia 1.11. In Julia 1.7+ this + is available from the package ScopedVariables.jl. +""" +mutable struct ScopedVariable{T} + const values::WeakKeyDict{Scope, T} + const initial_value::T + ScopedVariable{T}(initial_value) where {T} = new{T}(WeakKeyDict{Scope, T}(), initial_value) +end +ScopedVariable(initial_value::T) where {T} = ScopedVariable{T}(initial_value) + +Base.eltype(::Type{ScopedVariable{T}}) where {T} = T + +function Base.getindex(var::ScopedVariable{T})::T where T + scope = current_scope() + if scope === nothing + return var.initial_value + end + @lock var.values begin + while scope !== nothing + if haskey(var.values.ht, scope) + return var.values.ht[scope] + end + scope = scope.parent + end + end + return var.initial_value +end + +function Base.show(io::IO, var::ScopedVariable) + print(io, ScopedVariable) + print(io, '{', eltype(var), '}') + print(io, '(') + show(io, var[]) + print(io, ')') +end + +function __set_var!(scope::Scope, var::ScopedVariable{T}, val::T) where T + # PRIVATE API! Wrong usage will break invariants of ScopedVariable. + if scope === nothing + error("ScopedVariable: Currently not in scope.") + end + @lock var.values begin + if haskey(var.values.ht, scope) + error("ScopedVariable: Variable is already set for this scope.") + end + var.values[scope] = val + end +end + +""" + scoped(f, var::ScopedVariable{T} => val::T) + +Execute `f` in a new scope with `var` set to `val`. +""" +function scoped(f, pair::Pair{<:ScopedVariable{T}, T}) where T + @nospecialize + ct = Base.current_task() + current_scope = ct.scope::Union{Nothing, Scope} + try + scope = Scope(current_scope) + __set_var!(scope, pair...) + ct.scope = scope + return f() + finally + ct.scope = current_scope + end +end + +""" + scoped(f, vars...::ScopedVariable{T} => val::T) + +Execute `f` in a new scope with each scoped variable set to the provided `val`. +""" +function scoped(f, pairs::Pair{<:ScopedVariable}...) + @nospecialize + ct = Base.current_task() + current_scope = ct.scope::Union{Nothing, Scope} + try + scope = Scope(current_scope) + for (var, val) in pairs + __set_var!(scope, var, val) + end + ct.scope = scope + return f() + finally + ct.scope = current_scope + end +end + +end # module ScopedVariables diff --git a/src/jltypes.c b/src/jltypes.c index f3273ae936db3..de28631ef95be 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3232,7 +3232,7 @@ void jl_init_types(void) JL_GC_DISABLED "storage", "donenotify", "result", - "logstate", + "scope", "code", "rngState0", "rngState1", diff --git a/src/julia.h b/src/julia.h index a96b4a1f5e562..3d94c44c4f2ad 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2013,7 +2013,7 @@ typedef struct _jl_task_t { jl_value_t *tls; jl_value_t *donenotify; jl_value_t *result; - jl_value_t *logstate; + jl_value_t *scope; jl_function_t *start; // 4 byte padding on 32-bit systems // uint32_t padding0; diff --git a/src/task.c b/src/task.c index 1dab8688cb079..2e95a6f1770c4 100644 --- a/src/task.c +++ b/src/task.c @@ -1068,8 +1068,8 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, jl_value_t *completion t->result = jl_nothing; t->donenotify = completion_future; jl_atomic_store_relaxed(&t->_isexception, 0); - // Inherit logger state from parent task - t->logstate = ct->logstate; + // Inherit scope from parent task + t->scope = ct->scope; // Fork task-local random state from parent jl_rng_split(t->rngState, ct->rngState); // there is no active exception handler available on this stack yet @@ -1670,7 +1670,7 @@ jl_task_t *jl_init_root_task(jl_ptls_t ptls, void *stack_lo, void *stack_hi) ct->result = jl_nothing; ct->donenotify = jl_nothing; jl_atomic_store_relaxed(&ct->_isexception, 0); - ct->logstate = jl_nothing; + ct->scope = jl_nothing; ct->eh = NULL; ct->gcstack = NULL; ct->excstack = NULL; diff --git a/test/choosetests.jl b/test/choosetests.jl index c38817bb4eeb9..0f294497177da 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -29,6 +29,7 @@ const TESTNAMES = [ "channels", "iostream", "secretbuffer", "specificity", "reinterpretarray", "syntax", "corelogging", "missing", "asyncmap", "smallarrayshrink", "opaque_closure", "filesystem", "download", + "scopedvariables", ] const INTERNET_REQUIRED_LIST = [ diff --git a/test/scopedvariables.jl b/test/scopedvariables.jl new file mode 100644 index 0000000000000..0642e0584b50d --- /dev/null +++ b/test/scopedvariables.jl @@ -0,0 +1,48 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +const svar1 = ScopedVariable(1) + +@testset "errors" begin + var = ScopedVariable(1) + @test_throws MethodError var[] = 2 + scoped() do + @test_throws MethodError var[] = 2 + end +end + +const svar = ScopedVariable(1) +@testset "inheritance" begin + @test svar[] == 1 + scoped() do + @test svar[] == 1 + scoped() do + @test svar[] == 1 + end + scoped(svar => 2) do + @test svar[] == 2 + end + @test svar[] == 1 + end + @test svar[] == 1 +end + +const svar_float = ScopedVariable(1.0) + +@testset "multiple scoped variables" begin + scoped(svar => 2, svar_float => 2.0) do + @test svar[] == 2 + @test svar_float[] == 2.0 + end +end + +import Base.Threads: @spawn +@testset "tasks" begin + @test fetch(@spawn begin + svar[] + end) == 1 + scoped(svar => 2) do + @test fetch(@spawn begin + svar[] + end) == 2 + end +end From 87007ed5c24c7bd822774567e3162281ecd3ac40 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Thu, 17 Aug 2023 13:36:43 -0400 Subject: [PATCH 02/23] Update base/scopedvariables.jl --- base/scopedvariables.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/base/scopedvariables.jl b/base/scopedvariables.jl index e56b99093179a..ff0ee1a53c654 100644 --- a/base/scopedvariables.jl +++ b/base/scopedvariables.jl @@ -74,9 +74,6 @@ end function __set_var!(scope::Scope, var::ScopedVariable{T}, val::T) where T # PRIVATE API! Wrong usage will break invariants of ScopedVariable. - if scope === nothing - error("ScopedVariable: Currently not in scope.") - end @lock var.values begin if haskey(var.values.ht, scope) error("ScopedVariable: Variable is already set for this scope.") From a8445f6f1fd82869bd3955f5a488bc4fd6b28df0 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Thu, 17 Aug 2023 16:54:39 -0400 Subject: [PATCH 03/23] store state in Scope --- base/scopedvariables.jl | 83 ++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/base/scopedvariables.jl b/base/scopedvariables.jl index ff0ee1a53c654..55a86f60b1ff8 100644 --- a/base/scopedvariables.jl +++ b/base/scopedvariables.jl @@ -4,12 +4,6 @@ module ScopedVariables export ScopedVariable, scoped -mutable struct Scope - const parent::Union{Nothing, Scope} -end - -current_scope() = current_task().scope::Union{Nothing, Scope} - """ ScopedVariable(x) @@ -40,28 +34,55 @@ julia> scoped(svar => 2) do is available from the package ScopedVariables.jl. """ mutable struct ScopedVariable{T} - const values::WeakKeyDict{Scope, T} const initial_value::T - ScopedVariable{T}(initial_value) where {T} = new{T}(WeakKeyDict{Scope, T}(), initial_value) + ScopedVariable{T}(initial_value) where {T} = new{T}(initial_value) end ScopedVariable(initial_value::T) where {T} = ScopedVariable{T}(initial_value) Base.eltype(::Type{ScopedVariable{T}}) where {T} = T +mutable struct Scope + const parent::Union{Nothing, Scope} + # XXX: Probably want this to be an upgradeable RWLock + const lock::Base.Threads.SpinLock + # IdDict trades off some potential space savings for performance. + # IdDict ~60ns; WeakKeyDict ~100ns + # Space savings would come from ScopedVariables being GC'd. + # Now we hold onto them until Scope get's GC'd + const values::IdDict{<:ScopedVariable, Any} + Scope(parent) = new(parent, Base.Threads.SpinLock(), IdDict{ScopedVariable, Any}()) +end + +current_scope() = current_task().scope::Union{Nothing, Scope} + +function Base.show(io::IO, ::Scope) + print(io, Scope) +end + +Base.lock(scope::Scope) = lock(scope.lock) +Base.unlock(scope::Scope) = unlock(scope.lock) + function Base.getindex(var::ScopedVariable{T})::T where T - scope = current_scope() - if scope === nothing - return var.initial_value - end - @lock var.values begin - while scope !== nothing - if haskey(var.values.ht, scope) - return var.values.ht[scope] + cs = scope = current_scope() + val = var.initial_value + while scope !== nothing + @lock scope begin + if haskey(scope.values, var) + val = scope.values[var] + break end - scope = scope.parent + end + scope = scope.parent + end + if scope !== cs + # found the value in an upper scope, copy it down to cache. + # this is beneficial since in contrast to storing the values in the ScopedVariable + # we now need to acquire n-Locks for an n-depth scope. + @lock cs begin + cs.values[var] = val end end - return var.initial_value + return val end function Base.show(io::IO, var::ScopedVariable) @@ -74,12 +95,8 @@ end function __set_var!(scope::Scope, var::ScopedVariable{T}, val::T) where T # PRIVATE API! Wrong usage will break invariants of ScopedVariable. - @lock var.values begin - if haskey(var.values.ht, scope) - error("ScopedVariable: Variable is already set for this scope.") - end - var.values[scope] = val - end + @assert !haskey(scope.values, var) + scope.values[var] = val end """ @@ -91,10 +108,13 @@ function scoped(f, pair::Pair{<:ScopedVariable{T}, T}) where T @nospecialize ct = Base.current_task() current_scope = ct.scope::Union{Nothing, Scope} - try - scope = Scope(current_scope) + scope = Scope(current_scope) + @lock scope begin __set_var!(scope, pair...) - ct.scope = scope + end + + ct.scope = scope + try return f() finally ct.scope = current_scope @@ -110,12 +130,15 @@ function scoped(f, pairs::Pair{<:ScopedVariable}...) @nospecialize ct = Base.current_task() current_scope = ct.scope::Union{Nothing, Scope} - try - scope = Scope(current_scope) + scope = Scope(current_scope) + @lock scope begin for (var, val) in pairs __set_var!(scope, var, val) end - ct.scope = scope + end + + ct.scope = scope + try return f() finally ct.scope = current_scope From 54583a6178c1f359d41871b7b82b4a484ac6bbb4 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Thu, 17 Aug 2023 22:00:23 -0400 Subject: [PATCH 04/23] rename to ScopedValue --- base/Base.jl | 6 +-- base/exports.jl | 4 +- base/logging.jl | 2 +- base/{scopedvariables.jl => scopedvalue.jl} | 48 ++++++++++----------- test/choosetests.jl | 2 +- test/{scopedvariables.jl => scopedvalue.jl} | 10 ++--- 6 files changed, 36 insertions(+), 36 deletions(-) rename base/{scopedvariables.jl => scopedvalue.jl} (71%) rename test/{scopedvariables.jl => scopedvalue.jl} (82%) diff --git a/base/Base.jl b/base/Base.jl index 5577769406537..62b6d21589ed7 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -340,9 +340,9 @@ include("task.jl") include("threads_overloads.jl") include("weakkeydict.jl") -# ScopedVariables -include("scopedvariables.jl") -using .ScopedVariables +# ScopedValues +include("scopedvalues.jl") +using .ScopedValues # Logging include("logging.jl") diff --git a/base/exports.jl b/base/exports.jl index 8d68fd3f17b22..0a35323022e1d 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -648,9 +648,9 @@ export sprint, summary, -# ScopedVariable +# ScopedValue scoped, - ScopedVariable, + ScopedValue, # logging @debug, diff --git a/base/logging.jl b/base/logging.jl index d7034cb8c8cae..808e3592c9fe1 100644 --- a/base/logging.jl +++ b/base/logging.jl @@ -492,7 +492,7 @@ end LogState(logger) = LogState(LogLevel(_invoked_min_enabled_level(logger)), logger) -const CURRENT_LOGSTATE = ScopedVariable{Union{Nothing, LogState}}(nothing) +const CURRENT_LOGSTATE = ScopedValue{Union{Nothing, LogState}}(nothing) function current_logstate() logstate = CURRENT_LOGSTATE[] diff --git a/base/scopedvariables.jl b/base/scopedvalue.jl similarity index 71% rename from base/scopedvariables.jl rename to base/scopedvalue.jl index 55a86f60b1ff8..bb188c16ae76e 100644 --- a/base/scopedvariables.jl +++ b/base/scopedvalue.jl @@ -1,11 +1,11 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -module ScopedVariables +module ScopedValues -export ScopedVariable, scoped +export ScopedValue, scoped """ - ScopedVariable(x) + ScopedValue(x) Create a container that propagates values across scopes. Use [`scoped`](@ref) to create and enter a new scope. @@ -18,7 +18,7 @@ Dynamic scopes are propagated across tasks. # Examples ```jldoctest -julia> const svar = ScopedVariable(1); +julia> const svar = ScopedValue(1); julia> svar[] 1 @@ -31,15 +31,15 @@ julia> scoped(svar => 2) do !!! compat "Julia 1.11" This method requires at least Julia 1.11. In Julia 1.7+ this - is available from the package ScopedVariables.jl. + is available from the package ScopedValues.jl. """ -mutable struct ScopedVariable{T} +mutable struct ScopedValue{T} const initial_value::T - ScopedVariable{T}(initial_value) where {T} = new{T}(initial_value) + ScopedValue{T}(initial_value) where {T} = new{T}(initial_value) end -ScopedVariable(initial_value::T) where {T} = ScopedVariable{T}(initial_value) +ScopedValue(initial_value::T) where {T} = ScopedValue{T}(initial_value) -Base.eltype(::Type{ScopedVariable{T}}) where {T} = T +Base.eltype(::Type{ScopedValue{T}}) where {T} = T mutable struct Scope const parent::Union{Nothing, Scope} @@ -47,10 +47,10 @@ mutable struct Scope const lock::Base.Threads.SpinLock # IdDict trades off some potential space savings for performance. # IdDict ~60ns; WeakKeyDict ~100ns - # Space savings would come from ScopedVariables being GC'd. + # Space savings would come from ScopedValues being GC'd. # Now we hold onto them until Scope get's GC'd - const values::IdDict{<:ScopedVariable, Any} - Scope(parent) = new(parent, Base.Threads.SpinLock(), IdDict{ScopedVariable, Any}()) + const values::IdDict{<:ScopedValue, Any} + Scope(parent) = new(parent, Base.Threads.SpinLock(), IdDict{ScopedValue, Any}()) end current_scope() = current_task().scope::Union{Nothing, Scope} @@ -62,7 +62,7 @@ end Base.lock(scope::Scope) = lock(scope.lock) Base.unlock(scope::Scope) = unlock(scope.lock) -function Base.getindex(var::ScopedVariable{T})::T where T +function Base.getindex(var::ScopedValue{T})::T where T cs = scope = current_scope() val = var.initial_value while scope !== nothing @@ -76,7 +76,7 @@ function Base.getindex(var::ScopedVariable{T})::T where T end if scope !== cs # found the value in an upper scope, copy it down to cache. - # this is beneficial since in contrast to storing the values in the ScopedVariable + # this is beneficial since in contrast to storing the values in the ScopedValue # we now need to acquire n-Locks for an n-depth scope. @lock cs begin cs.values[var] = val @@ -85,26 +85,26 @@ function Base.getindex(var::ScopedVariable{T})::T where T return val end -function Base.show(io::IO, var::ScopedVariable) - print(io, ScopedVariable) +function Base.show(io::IO, var::ScopedValue) + print(io, ScopedValue) print(io, '{', eltype(var), '}') print(io, '(') show(io, var[]) print(io, ')') end -function __set_var!(scope::Scope, var::ScopedVariable{T}, val::T) where T - # PRIVATE API! Wrong usage will break invariants of ScopedVariable. +function __set_var!(scope::Scope, var::ScopedValue{T}, val::T) where T + # PRIVATE API! Wrong usage will break invariants of ScopedValue. @assert !haskey(scope.values, var) scope.values[var] = val end """ - scoped(f, var::ScopedVariable{T} => val::T) + scoped(f, var::ScopedValue{T} => val::T) Execute `f` in a new scope with `var` set to `val`. """ -function scoped(f, pair::Pair{<:ScopedVariable{T}, T}) where T +function scoped(f, pair::Pair{<:ScopedValue{T}, T}) where T @nospecialize ct = Base.current_task() current_scope = ct.scope::Union{Nothing, Scope} @@ -122,11 +122,11 @@ function scoped(f, pair::Pair{<:ScopedVariable{T}, T}) where T end """ - scoped(f, vars...::ScopedVariable{T} => val::T) + scoped(f, vars...::ScopedValue{T} => val::T) -Execute `f` in a new scope with each scoped variable set to the provided `val`. +Execute `f` in a new scope with each scoped value set to the provided `val`. """ -function scoped(f, pairs::Pair{<:ScopedVariable}...) +function scoped(f, pairs::Pair{<:ScopedValue}...) @nospecialize ct = Base.current_task() current_scope = ct.scope::Union{Nothing, Scope} @@ -145,4 +145,4 @@ function scoped(f, pairs::Pair{<:ScopedVariable}...) end end -end # module ScopedVariables +end # module ScopedValues diff --git a/test/choosetests.jl b/test/choosetests.jl index 0f294497177da..4d493a16d0484 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -29,7 +29,7 @@ const TESTNAMES = [ "channels", "iostream", "secretbuffer", "specificity", "reinterpretarray", "syntax", "corelogging", "missing", "asyncmap", "smallarrayshrink", "opaque_closure", "filesystem", "download", - "scopedvariables", + "scopedvalues", ] const INTERNET_REQUIRED_LIST = [ diff --git a/test/scopedvariables.jl b/test/scopedvalue.jl similarity index 82% rename from test/scopedvariables.jl rename to test/scopedvalue.jl index 0642e0584b50d..bbb07286d30dc 100644 --- a/test/scopedvariables.jl +++ b/test/scopedvalue.jl @@ -1,16 +1,16 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -const svar1 = ScopedVariable(1) +const svar1 = ScopedValue(1) @testset "errors" begin - var = ScopedVariable(1) + var = ScopedValue(1) @test_throws MethodError var[] = 2 scoped() do @test_throws MethodError var[] = 2 end end -const svar = ScopedVariable(1) +const svar = ScopedValue(1) @testset "inheritance" begin @test svar[] == 1 scoped() do @@ -26,9 +26,9 @@ const svar = ScopedVariable(1) @test svar[] == 1 end -const svar_float = ScopedVariable(1.0) +const svar_float = ScopedValue(1.0) -@testset "multiple scoped variables" begin +@testset "multiple scoped values" begin scoped(svar => 2, svar_float => 2.0) do @test svar[] == 2 @test svar_float[] == 2.0 From 063140be9f14c850f65b523dee0aec39d6d3ca32 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Thu, 17 Aug 2023 22:48:44 -0400 Subject: [PATCH 05/23] add docs and optimize implementation --- base/scopedvalue.jl | 148 ----------------------- base/scopedvalues.jl | 148 +++++++++++++++++++++++ doc/make.jl | 1 + doc/src/base/scopedvalues.md | 86 +++++++++++++ test/{scopedvalue.jl => scopedvalues.jl} | 0 5 files changed, 235 insertions(+), 148 deletions(-) delete mode 100644 base/scopedvalue.jl create mode 100644 base/scopedvalues.jl create mode 100644 doc/src/base/scopedvalues.md rename test/{scopedvalue.jl => scopedvalues.jl} (100%) diff --git a/base/scopedvalue.jl b/base/scopedvalue.jl deleted file mode 100644 index bb188c16ae76e..0000000000000 --- a/base/scopedvalue.jl +++ /dev/null @@ -1,148 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -module ScopedValues - -export ScopedValue, scoped - -""" - ScopedValue(x) - -Create a container that propagates values across scopes. -Use [`scoped`](@ref) to create and enter a new scope. - -Values can only be set when entering a new scope, -and the value referred to will be constant during the -execution of a scope. - -Dynamic scopes are propagated across tasks. - -# Examples -```jldoctest -julia> const svar = ScopedValue(1); - -julia> svar[] -1 - -julia> scoped(svar => 2) do - svar[] - end -2 -``` - -!!! compat "Julia 1.11" - This method requires at least Julia 1.11. In Julia 1.7+ this - is available from the package ScopedValues.jl. -""" -mutable struct ScopedValue{T} - const initial_value::T - ScopedValue{T}(initial_value) where {T} = new{T}(initial_value) -end -ScopedValue(initial_value::T) where {T} = ScopedValue{T}(initial_value) - -Base.eltype(::Type{ScopedValue{T}}) where {T} = T - -mutable struct Scope - const parent::Union{Nothing, Scope} - # XXX: Probably want this to be an upgradeable RWLock - const lock::Base.Threads.SpinLock - # IdDict trades off some potential space savings for performance. - # IdDict ~60ns; WeakKeyDict ~100ns - # Space savings would come from ScopedValues being GC'd. - # Now we hold onto them until Scope get's GC'd - const values::IdDict{<:ScopedValue, Any} - Scope(parent) = new(parent, Base.Threads.SpinLock(), IdDict{ScopedValue, Any}()) -end - -current_scope() = current_task().scope::Union{Nothing, Scope} - -function Base.show(io::IO, ::Scope) - print(io, Scope) -end - -Base.lock(scope::Scope) = lock(scope.lock) -Base.unlock(scope::Scope) = unlock(scope.lock) - -function Base.getindex(var::ScopedValue{T})::T where T - cs = scope = current_scope() - val = var.initial_value - while scope !== nothing - @lock scope begin - if haskey(scope.values, var) - val = scope.values[var] - break - end - end - scope = scope.parent - end - if scope !== cs - # found the value in an upper scope, copy it down to cache. - # this is beneficial since in contrast to storing the values in the ScopedValue - # we now need to acquire n-Locks for an n-depth scope. - @lock cs begin - cs.values[var] = val - end - end - return val -end - -function Base.show(io::IO, var::ScopedValue) - print(io, ScopedValue) - print(io, '{', eltype(var), '}') - print(io, '(') - show(io, var[]) - print(io, ')') -end - -function __set_var!(scope::Scope, var::ScopedValue{T}, val::T) where T - # PRIVATE API! Wrong usage will break invariants of ScopedValue. - @assert !haskey(scope.values, var) - scope.values[var] = val -end - -""" - scoped(f, var::ScopedValue{T} => val::T) - -Execute `f` in a new scope with `var` set to `val`. -""" -function scoped(f, pair::Pair{<:ScopedValue{T}, T}) where T - @nospecialize - ct = Base.current_task() - current_scope = ct.scope::Union{Nothing, Scope} - scope = Scope(current_scope) - @lock scope begin - __set_var!(scope, pair...) - end - - ct.scope = scope - try - return f() - finally - ct.scope = current_scope - end -end - -""" - scoped(f, vars...::ScopedValue{T} => val::T) - -Execute `f` in a new scope with each scoped value set to the provided `val`. -""" -function scoped(f, pairs::Pair{<:ScopedValue}...) - @nospecialize - ct = Base.current_task() - current_scope = ct.scope::Union{Nothing, Scope} - scope = Scope(current_scope) - @lock scope begin - for (var, val) in pairs - __set_var!(scope, var, val) - end - end - - ct.scope = scope - try - return f() - finally - ct.scope = current_scope - end -end - -end # module ScopedValues diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl new file mode 100644 index 0000000000000..7f6555fec1a7f --- /dev/null +++ b/base/scopedvalues.jl @@ -0,0 +1,148 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +module ScopedValues + +export ScopedValue, scoped + +""" + ScopedValue(x) + +Create a container that propagates values across dynamic scopes. +Use [`scoped`](@ref) to create and enter a new dynamic scope. + +Values can only be set when entering a new dynamic scope, +and the value referred to will be constant during the +execution of a dynamic scope. + +Dynamic scopes are propagated across tasks. + +# Examples +```jldoctest +julia> const svar = ScopedValue(1); + +julia> svar[] +1 + +julia> scoped(svar => 2) do + svar[] + end +2 +``` + +!!! compat "Julia 1.11" + Scoped values were introduced in Julia 1.11. In Julia 1.7+ a compatible + implementation is available from the package ScopedValues.jl. +""" +mutable struct ScopedValue{T} + const initial_value::T + ScopedValue{T}(initial_value) where {T} = new{T}(initial_value) +end +ScopedValue(initial_value::T) where {T} = ScopedValue{T}(initial_value) + +Base.eltype(::Type{ScopedValue{T}}) where {T} = T + +import Base: ImmutableDict + +## +# Notes on the implementation. +# We want lookup to be unreasonable fast. +# - IDDict/Dict are ~10ns +# - ImmutableDict is faster up to about ~15 entries +# - ScopedValue are meant to be constant, Immutabilty +# is thus a boon +# - If we were to use IDDict/Dict we would need to split +# the cache portion and the value portion of the hash-table, +# the value portion is read-only/write-once, but the cache version +# would need a lock which makes ImmutableDict incredibly attractive. +# We could also use task-local-storage, but that added about 12ns. +# - Values are GC'd when scopes become unreachable, one could use +# a WeakKeyDict to also ensure that values get GC'd when ScopedValues +# become unreachable. + +mutable struct Scope + const parent::Union{Nothing, Scope} + @atomic values::ImmutableDict{ScopedValue, Any} +end +Scope(parent) = Scope(parent, ImmutableDict{ScopedValue, Any}()) +current_scope() = current_task().scope::Union{Nothing, Scope} + +function Base.show(io::IO, ::Scope) + print(io, Scope) +end + +# VC: I find it rather useful to have one function to use for both +# haskey and get. +@inline function get(dict::ImmutableDict, key, ::Type{T}) where T + while isdefined(dict, :parent) + isequal(dict.key, key) && return Some(dict.value::T) + dict = dict.parent + end + return nothing +end + +function Base.getindex(var::ScopedValue{T})::T where T + scope = current_scope() + if scope === nothing + return var.initial_value + end + cs = scope + + val = var.initial_value + while scope !== nothing + values = @atomic :acquire scope.values + _val = get(values, var, T) + if _val !== nothing + val = something(_val) + break + end + scope = scope.parent + end + + if cs != scope + # found the value in an upper scope, copy it down to the cache. + # We are using the same dict for both cache and values. + # One can split these and potentially use `ImmutableDict` only for values + # and a Dict with SpinLock for the cache. + success = false + old = @atomic :acquire cs.values + while !success + new = ImmutableDict(old, var => val) + old, success = @atomicreplace :acquire_release :acquire cs.values old => new + end + end + + return val +end + +function Base.show(io::IO, var::ScopedValue) + print(io, ScopedValue) + print(io, '{', eltype(var), '}') + print(io, '(') + show(io, var[]) + print(io, ')') +end + +""" + scoped(f, (var::ScopedValue{T} => val::T)...) + +Execute `f` in a new scope with `var` set to `val`. +""" +function scoped(f, pair::Pair{<:ScopedValue}, rest::Pair{<:ScopedValue}...) + @nospecialize + values = ImmutableDict{ScopedValue, Any}(pair...) + for pair in rest + values = ImmutableDict{ScopedValue, Any}(values, pair...) + end + ct = Base.current_task() + current_scope = ct.scope::Union{Nothing, Scope} + ct.scope = Scope(current_scope, values) + try + return f() + finally + ct.scope = current_scope + end +end + +scoped(@nospecialize(f)) = f() + +end # module ScopedValues diff --git a/doc/make.jl b/doc/make.jl index 087b033fcf79c..0ae74a55aceee 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -112,6 +112,7 @@ BaseDocs = [ "base/arrays.md", "base/parallel.md", "base/multi-threading.md", + "base/scopedvalues.md", "base/constants.md", "base/file.md", "base/io-network.md", diff --git a/doc/src/base/scopedvalues.md b/doc/src/base/scopedvalues.md new file mode 100644 index 0000000000000..9d962f81d51da --- /dev/null +++ b/doc/src/base/scopedvalues.md @@ -0,0 +1,86 @@ +# Scoped Values + +Scoped values provide an implementation of dynamical scoping in Julia. +In particular dynamical scopes are propagated through [Task](@ref)s. + +!!! compat "Julia 1.11" + Scoped values were introduced in Julia 1.11. In Julia 1.7+ a compatible + implementation is available from the package ScopedValues.jl. + +In it's simplest form you can create a [`ScopedValue`](@ref) with a +default value and then use [`scoped`](@ref) to enter a new dynamical +scope. The new scope will inherit all values from the parent scope +(and recursivly from all outer scopes) with the the provided scoped +value taking priority over previous definitions. + +```julia +const scoped_val = ScopedValue(1) +const scoped_val2 = ScopedValue(2) + +# Enter a new dynamic scope and set value +scoped(scoped_val => 2) do + @show scoped_val[] # 2 + @show scoped_val2[] # 1 + scoped(scoped_val => 3, scoped_val2 => 5) do + @show scoped_val[] # 3 + @show scoped_val2[] # 5 + end +end +``` + +Scoped values are constant throughout a scope, but you can store mutable +state in a scoped value. Just keep in mind that the usual caveats +for global variables apply in the context of concurrent programming. + +```julia +const scoped_dict = ScopedValue(Dict()) + +# Important we are using `merge` to "unshare" the mutable values +# across the different views of the same scoped value. +scoped(svar_dict => merge(svar_dict, Dict(:a => 10))) do + @show svar_dict[][:a] +end +``` + +Care is also required when storing references to mutable state in scoped +values. You might want to explicitly unshare when entering a new dynamic scope. + + +In the example below we use a scoped value to implement a permission check in +a web-application. After determining the permissions of the request. +A new dynamic scope is entered and the scoped value `LEVEL` is set. +Other parts of the application can now query the scoped value and will receive +the appropriate value. Other alternatives like task-local storage and global variables, +are not well suited for this kind of propagation and our only alternative would have +been to thread a value through the entire call-chain. + +```julia +const LEVEL = ScopedValue(:GUEST) + +function serve(request, response) + level = isAdmin(request) ? :ADMIN : :GUEST + scoped(LEVEL => level) do + Threads.@spawn handle(request, respone) + end +end + +function open(connection::Database) + level = LEVEL[] + if level !== :ADMIN + error("Access disallowed") + end + # ... open connection +end + +function handle(request, response) + open(Database(#=...=#)) + # ... +end +``` + +## API docs + +```@docs +Base.ScopedValues.ScopedValue +Base.ScopedValues.scoped +``` diff --git a/test/scopedvalue.jl b/test/scopedvalues.jl similarity index 100% rename from test/scopedvalue.jl rename to test/scopedvalues.jl From 2e3b0579aa2b983ec848898fed11c86355985963 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 18 Aug 2023 07:57:46 -0400 Subject: [PATCH 06/23] Apply suggestions from code review Co-authored-by: Rafael Fourquet --- doc/src/base/scopedvalues.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/base/scopedvalues.md b/doc/src/base/scopedvalues.md index 9d962f81d51da..34a5add54e71d 100644 --- a/doc/src/base/scopedvalues.md +++ b/doc/src/base/scopedvalues.md @@ -7,7 +7,7 @@ In particular dynamical scopes are propagated through [Task](@ref)s. Scoped values were introduced in Julia 1.11. In Julia 1.7+ a compatible implementation is available from the package ScopedValues.jl. -In it's simplest form you can create a [`ScopedValue`](@ref) with a +In its simplest form you can create a [`ScopedValue`](@ref) with a default value and then use [`scoped`](@ref) to enter a new dynamical scope. The new scope will inherit all values from the parent scope (and recursivly from all outer scopes) with the the provided scoped @@ -20,7 +20,7 @@ const scoped_val2 = ScopedValue(2) # Enter a new dynamic scope and set value scoped(scoped_val => 2) do @show scoped_val[] # 2 - @show scoped_val2[] # 1 + @show scoped_val2[] # 2 scoped(scoped_val => 3, scoped_val2 => 5) do @show scoped_val[] # 3 @show scoped_val2[] # 5 From 179a9c6e12b963648f93349d37d3bc5e140f0784 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 18 Aug 2023 10:20:42 -0400 Subject: [PATCH 07/23] Elegance through simplicity --- base/scopedvalues.jl | 62 ++++++++++++-------------------------------- test/scopedvalues.jl | 7 +++++ 2 files changed, 24 insertions(+), 45 deletions(-) diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index 7f6555fec1a7f..fb28652937e1e 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -58,61 +58,33 @@ import Base: ImmutableDict # - Values are GC'd when scopes become unreachable, one could use # a WeakKeyDict to also ensure that values get GC'd when ScopedValues # become unreachable. +# - Scopes are an inline implementation of an ImmutableDict, if we wanted +# be really fance we could use a CTrie or HAMT. -mutable struct Scope +mutable struct Scope{T} const parent::Union{Nothing, Scope} - @atomic values::ImmutableDict{ScopedValue, Any} + const key::ScopedValue{T} + const value::T end -Scope(parent) = Scope(parent, ImmutableDict{ScopedValue, Any}()) +Scope(parent, key::ScopedValue{T}, value) where T = + Scope(parent, key, convert(T, value)) + current_scope() = current_task().scope::Union{Nothing, Scope} function Base.show(io::IO, ::Scope) print(io, Scope) end -# VC: I find it rather useful to have one function to use for both -# haskey and get. -@inline function get(dict::ImmutableDict, key, ::Type{T}) where T - while isdefined(dict, :parent) - isequal(dict.key, key) && return Some(dict.value::T) - dict = dict.parent - end - return nothing -end - function Base.getindex(var::ScopedValue{T})::T where T scope = current_scope() - if scope === nothing - return var.initial_value - end - cs = scope - - val = var.initial_value while scope !== nothing - values = @atomic :acquire scope.values - _val = get(values, var, T) - if _val !== nothing - val = something(_val) - break + if scope.key === var + return scope.value::T end scope = scope.parent end - - if cs != scope - # found the value in an upper scope, copy it down to the cache. - # We are using the same dict for both cache and values. - # One can split these and potentially use `ImmutableDict` only for values - # and a Dict with SpinLock for the cache. - success = false - old = @atomic :acquire cs.values - while !success - new = ImmutableDict(old, var => val) - old, success = @atomicreplace :acquire_release :acquire cs.values old => new - end - end - - return val -end + return var.initial_value +en function Base.show(io::IO, var::ScopedValue) print(io, ScopedValue) @@ -129,13 +101,13 @@ Execute `f` in a new scope with `var` set to `val`. """ function scoped(f, pair::Pair{<:ScopedValue}, rest::Pair{<:ScopedValue}...) @nospecialize - values = ImmutableDict{ScopedValue, Any}(pair...) - for pair in rest - values = ImmutableDict{ScopedValue, Any}(values, pair...) - end ct = Base.current_task() current_scope = ct.scope::Union{Nothing, Scope} - ct.scope = Scope(current_scope, values) + scope = Scope(current_scope, pair...) + for pair in rest + scope = Scope(scope, pair...) + end + ct.scope = scope try return f() finally diff --git a/test/scopedvalues.jl b/test/scopedvalues.jl index bbb07286d30dc..dde9300173e37 100644 --- a/test/scopedvalues.jl +++ b/test/scopedvalues.jl @@ -35,6 +35,13 @@ const svar_float = ScopedValue(1.0) end end +emptyf() = nothing + +@testset "conversion" begin + scoped(emptyf, gvar_float=>2) + @test_throws MethodError scoped(emptyf, gvar_float=>"hello") +end + import Base.Threads: @spawn @testset "tasks" begin @test fetch(@spawn begin From 933b90c2703273df87c10834a774fdcf689bb386 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 18 Aug 2023 14:19:28 -0400 Subject: [PATCH 08/23] rework some docs --- doc/src/base/scopedvalues.md | 63 ++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/doc/src/base/scopedvalues.md b/doc/src/base/scopedvalues.md index 34a5add54e71d..f76e2276c3bbb 100644 --- a/doc/src/base/scopedvalues.md +++ b/doc/src/base/scopedvalues.md @@ -10,48 +10,76 @@ In particular dynamical scopes are propagated through [Task](@ref)s. In its simplest form you can create a [`ScopedValue`](@ref) with a default value and then use [`scoped`](@ref) to enter a new dynamical scope. The new scope will inherit all values from the parent scope -(and recursivly from all outer scopes) with the the provided scoped +(and recursivly from all outer scopes) with the provided scoped value taking priority over previous definitions. ```julia const scoped_val = ScopedValue(1) -const scoped_val2 = ScopedValue(2) +const scoped_val2 = ScopedValue(0) # Enter a new dynamic scope and set value +@show scoped_val[] # 1 +@show scoped_val2[] # 0 scoped(scoped_val => 2) do @show scoped_val[] # 2 - @show scoped_val2[] # 2 + @show scoped_val2[] # 0 scoped(scoped_val => 3, scoped_val2 => 5) do @show scoped_val[] # 3 @show scoped_val2[] # 5 end + @show scoped_val[] # 2 + @show scoped_val2[] # 0 end +@show scoped_val[] # 1 +@show scoped_val2[] # 0 ``` Scoped values are constant throughout a scope, but you can store mutable state in a scoped value. Just keep in mind that the usual caveats for global variables apply in the context of concurrent programming. -```julia -const scoped_dict = ScopedValue(Dict()) +Care is also required when storing references to mutable state in scoped +values. You might want to explicitly unshare when entering a new dynamic scope. -# Important we are using `merge` to "unshare" the mutable values -# across the different views of the same scoped value. -scoped(svar_dict => merge(svar_dict, Dict(:a => 10))) do - @show svar_dict[][:a] +```julia +import Threads: @spawn +const sval_dict = ScopedValue(Dict()) + +# Example of using a mutable value wrongly +@sync begin + # `Dict` is not thread-safe the usage below is invalid + @spawn (sval_dict[][:a] = 3) + @spawn (sval_dict[][:b] = 3) end -``` -Care is also required when storing references to mutable state in scoped -values. You might want to explicitly unshare when entering a new dynamic scope. +@sync begin + # If we instead pass a unique dictionary to each + # task we can access the dictonaries race free. + scoped(sval_dict => Dict()) + @spawn (sval_dict[][:a] = 3) + end + scoped(sval_dict => Dict()) + @spawn (sval_dict[][:b] = 3) + end +end +# If you want to add new values to the dict, instead of replacing +# it, unshare the values explicitly. In this example we use `merge` +# to unshare the state of the dictonary in parent scope. +@sync begin + scoped(sval_dict => merge(sval_dict, Dict(:a => 10))) do + @spawn @show sval_dict[][:a] + end + @spawn sval_dict[][:a] = 3 # Not a race since they are unshared. +end +``` In the example below we use a scoped value to implement a permission check in -a web-application. After determining the permissions of the request. -A new dynamic scope is entered and the scoped value `LEVEL` is set. -Other parts of the application can now query the scoped value and will receive -the appropriate value. Other alternatives like task-local storage and global variables, -are not well suited for this kind of propagation and our only alternative would have +a web-application. After determining the permissions of the request, +a new dynamic scope is entered and the scoped value `LEVEL` is set. +Other parts of the application can query the scoped value and will receive +the appropriate value. Other alternatives like task-local storage and global variables +are not well suited for this kind of propagation; our only alternative would have been to thread a value through the entire call-chain. ```julia @@ -73,6 +101,7 @@ function open(connection::Database) end function handle(request, response) + # ... open(Database(#=...=#)) # ... end From 29e36216ef949089fdbfbcc3b34e2d7e91480646 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 18 Aug 2023 14:20:08 -0400 Subject: [PATCH 09/23] fixup! Elegance through simplicity --- base/scopedvalues.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index fb28652937e1e..86e29dfe0bd9e 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -84,7 +84,7 @@ function Base.getindex(var::ScopedValue{T})::T where T scope = scope.parent end return var.initial_value -en +end function Base.show(io::IO, var::ScopedValue) print(io, ScopedValue) From 4e5b7d8a28eef3c1ec91fbffa40384fa3c56f6af Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sat, 19 Aug 2023 12:31:48 -0400 Subject: [PATCH 10/23] fixup! fixup! Elegance through simplicity --- test/scopedvalues.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/scopedvalues.jl b/test/scopedvalues.jl index dde9300173e37..85a418836aa72 100644 --- a/test/scopedvalues.jl +++ b/test/scopedvalues.jl @@ -38,8 +38,8 @@ end emptyf() = nothing @testset "conversion" begin - scoped(emptyf, gvar_float=>2) - @test_throws MethodError scoped(emptyf, gvar_float=>"hello") + scoped(emptyf, svar_float=>2) + @test_throws MethodError scoped(emptyf, svar_float=>"hello") end import Base.Threads: @spawn From bae6d5eadba4086b69bb7ce8e03c5026990bca51 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sat, 19 Aug 2023 17:41:31 -0400 Subject: [PATCH 11/23] fixup! rework some docs --- doc/src/base/scopedvalues.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/base/scopedvalues.md b/doc/src/base/scopedvalues.md index f76e2276c3bbb..f761ae8f8a313 100644 --- a/doc/src/base/scopedvalues.md +++ b/doc/src/base/scopedvalues.md @@ -1,7 +1,7 @@ # Scoped Values Scoped values provide an implementation of dynamical scoping in Julia. -In particular dynamical scopes are propagated through [Task](@ref)s. +In particular dynamical scopes are propagated through [`Task`](@ref)s. !!! compat "Julia 1.11" Scoped values were introduced in Julia 1.11. In Julia 1.7+ a compatible From 60020f7d79a62d7e2ca3f278a13aa322ebbffc60 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 22 Aug 2023 11:18:10 -0400 Subject: [PATCH 12/23] Apply suggestions from code review Co-authored-by: Nick Robinson --- base/scopedvalues.jl | 2 +- doc/src/base/scopedvalues.md | 8 ++++---- test/scopedvalues.jl | 6 ++++++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index 86e29dfe0bd9e..affccecfe3374 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -90,7 +90,7 @@ function Base.show(io::IO, var::ScopedValue) print(io, ScopedValue) print(io, '{', eltype(var), '}') print(io, '(') - show(io, var[]) + show(IOContext(io, :typeinfo => eltype(var)), var[]) print(io, ')') end diff --git a/doc/src/base/scopedvalues.md b/doc/src/base/scopedvalues.md index f761ae8f8a313..bcf5c59f4f164 100644 --- a/doc/src/base/scopedvalues.md +++ b/doc/src/base/scopedvalues.md @@ -1,14 +1,14 @@ # Scoped Values -Scoped values provide an implementation of dynamical scoping in Julia. -In particular dynamical scopes are propagated through [`Task`](@ref)s. +Scoped values provide an implementation of dynamic scoping in Julia. +In particular dynamic scopes are propagated through [`Task`](@ref)s. !!! compat "Julia 1.11" Scoped values were introduced in Julia 1.11. In Julia 1.7+ a compatible implementation is available from the package ScopedValues.jl. In its simplest form you can create a [`ScopedValue`](@ref) with a -default value and then use [`scoped`](@ref) to enter a new dynamical +default value and then use [`scoped`](@ref) to enter a new dynamic scope. The new scope will inherit all values from the parent scope (and recursivly from all outer scopes) with the provided scoped value taking priority over previous definitions. @@ -42,7 +42,7 @@ Care is also required when storing references to mutable state in scoped values. You might want to explicitly unshare when entering a new dynamic scope. ```julia -import Threads: @spawn +import Base.Threads: @spawn const sval_dict = ScopedValue(Dict()) # Example of using a mutable value wrongly diff --git a/test/scopedvalues.jl b/test/scopedvalues.jl index 85a418836aa72..e52520d5c7a1d 100644 --- a/test/scopedvalues.jl +++ b/test/scopedvalues.jl @@ -3,11 +3,14 @@ const svar1 = ScopedValue(1) @testset "errors" begin + @test ScopedValue{Float64}(1) == 1.0 var = ScopedValue(1) @test_throws MethodError var[] = 2 scoped() do @test_throws MethodError var[] = 2 end + @test_throws MethodError ScopedValue{Int}() + @test_throws MethodError ScopedValue() end const svar = ScopedValue(1) @@ -33,6 +36,9 @@ const svar_float = ScopedValue(1.0) @test svar[] == 2 @test svar_float[] == 2.0 end + scoped(svar => 2, svar => 3) do + @test svar[] == 3 + end end emptyf() = nothing From c89c75cb2ae2bde51625809231e05a22e35361b1 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 22 Aug 2023 12:13:21 -0400 Subject: [PATCH 13/23] More review fixes --- base/scopedvalues.jl | 4 ---- test/scopedvalues.jl | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index affccecfe3374..8e191d8fec8cd 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -35,14 +35,10 @@ julia> scoped(svar => 2) do """ mutable struct ScopedValue{T} const initial_value::T - ScopedValue{T}(initial_value) where {T} = new{T}(initial_value) end -ScopedValue(initial_value::T) where {T} = ScopedValue{T}(initial_value) Base.eltype(::Type{ScopedValue{T}}) where {T} = T -import Base: ImmutableDict - ## # Notes on the implementation. # We want lookup to be unreasonable fast. diff --git a/test/scopedvalues.jl b/test/scopedvalues.jl index e52520d5c7a1d..979b61db5f868 100644 --- a/test/scopedvalues.jl +++ b/test/scopedvalues.jl @@ -3,7 +3,7 @@ const svar1 = ScopedValue(1) @testset "errors" begin - @test ScopedValue{Float64}(1) == 1.0 + @test ScopedValue{Float64}(1)[] == 1.0 var = ScopedValue(1) @test_throws MethodError var[] = 2 scoped() do From ed531c986efe883a5b69e2a29d570c604c49823b Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 22 Aug 2023 12:28:23 -0400 Subject: [PATCH 14/23] docstring on current_scope --- base/scopedvalues.jl | 24 +++++++++++++++++++-- test/scopedvalues.jl | 51 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 71 insertions(+), 4 deletions(-) diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index 8e191d8fec8cd..a132dbb7806f0 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -65,10 +65,30 @@ end Scope(parent, key::ScopedValue{T}, value) where T = Scope(parent, key, convert(T, value)) +""" + current_scope()::Union{Nothing, Scope} + +Return the current dynamic scope. +""" current_scope() = current_task().scope::Union{Nothing, Scope} -function Base.show(io::IO, ::Scope) - print(io, Scope) +function Base.show(io::IO, scope::Scope) + print(io, Scope, "(") + seen = Set{ScopedValue}() + while scope !== nothing + if scope.key ∉ seen + if !isempty(seen) + print(io, ", ") + end + print(io, typeof(scope.key), "@") + show(io, Base.objectid(scope.key)) + print(io, " => ") + show(IOContext(io, :typeinfo => eltype(scope.key)), scope.value) + push!(seen, scope.key) + end + scope = scope.parent + end + print(io, ")") end function Base.getindex(var::ScopedValue{T})::T where T diff --git a/test/scopedvalues.jl b/test/scopedvalues.jl index 979b61db5f868..a33214ce93e71 100644 --- a/test/scopedvalues.jl +++ b/test/scopedvalues.jl @@ -1,9 +1,8 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -const svar1 = ScopedValue(1) - @testset "errors" begin @test ScopedValue{Float64}(1)[] == 1.0 + @test_throws InexactError ScopedValue{Int}(1.5) var = ScopedValue(1) @test_throws MethodError var[] = 2 scoped() do @@ -59,3 +58,51 @@ import Base.Threads: @spawn end) == 2 end end + +@testset "show" begin + @test sprint(show, svar) == "ScopedValue{$Int}(1)" + @test sprint(show, ScopedValues.current_scope()) == "nothing" + scoped(svar => 2.0) do + @test sprint(show, svar) == "ScopedValue{$Int}(2)" + objid = sprint(show, Base.objectid(svar)) + @test sprint(show, ScopedValues.current_scope()) == "ScopedValues.Scope(ScopedValue{$Int}@$objid => 2)" + end +end + +const depth = ScopedValue(0) +function nth_scoped(f, n) + if n <= 0 + f() + else + scoped(depth => n) do + nth_scoped(f, n-1) + end + end +end + + +@testset "nested scoped" begin + @testset for depth in 1:16 + nth_scoped(depth) do + @test svar_float[] == 1.0 + end + scoped(svar_float=>2.0) do + nth_scoped(depth) do + @test svar_float[] == 2.0 + end + end + nth_scoped(depth) do + scoped(svar_float=>2.0) do + @test svar_float[] == 2.0 + end + end + end + scoped(svar_float=>2.0) do + nth_scoped(15) do + @test svar_float[] == 2.0 + scoped(svar_float => 3.0) do + @test svar_float[] == 3.0 + end + end + end +end From 9640b6ed6365b80febb86335301cc84d65481e21 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 22 Aug 2023 12:39:10 -0400 Subject: [PATCH 15/23] improve docs --- doc/src/base/scopedvalues.md | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/doc/src/base/scopedvalues.md b/doc/src/base/scopedvalues.md index bcf5c59f4f164..5380d81ceb8f5 100644 --- a/doc/src/base/scopedvalues.md +++ b/doc/src/base/scopedvalues.md @@ -1,10 +1,12 @@ # Scoped Values Scoped values provide an implementation of dynamic scoping in Julia. -In particular dynamic scopes are propagated through [`Task`](@ref)s. +Dynamic scope means that the state of the value is dependent on the execution path +of the program. This means that for a scoped value you may observe +multiple different values at the same time. !!! compat "Julia 1.11" - Scoped values were introduced in Julia 1.11. In Julia 1.7+ a compatible + Scoped values were introduced in Julia 1.11. In Julia 1.8+ a compatible implementation is available from the package ScopedValues.jl. In its simplest form you can create a [`ScopedValue`](@ref) with a @@ -34,6 +36,27 @@ end @show scoped_val2[] # 0 ``` +!!! note + Dynamic scopes are propagated through [`Task`](@ref)s. + +In the example below we open a new dynamic scope before launching a task. +The parent task and the two child tasks observe independent values of the +same scoped value at the same time. + +```julia +import Base.Threads: @spawn +const scoped_val = ScopedValue(1) +@sync begin + scoped(scoped_val => 2) + @spawn @show scoped_val[] # 2 + end + scoped(scoped_val => 3) + @spawn @show scoped_val[] # 3 + end + @show scoped_val[] # 1 +end +``` + Scoped values are constant throughout a scope, but you can store mutable state in a scoped value. Just keep in mind that the usual caveats for global variables apply in the context of concurrent programming. From 302b1b5e8e324769db0cde03ac0c3ba8a9db6639 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 22 Aug 2023 12:49:42 -0400 Subject: [PATCH 16/23] add news --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 16afb8c168443..a0a3d84744880 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,7 @@ Julia v1.11 Release Notes New language features --------------------- +* `ScopedValue` implement dynamic scope with inheritance across tasks ([#50958]). Language changes ---------------- From 7b707fa482f04c34c47cdaf7077591a967fb7bee Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 22 Aug 2023 19:20:13 -0400 Subject: [PATCH 17/23] address review comments --- base/scopedvalues.jl | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index a132dbb7806f0..2023fdd94499a 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -41,7 +41,7 @@ Base.eltype(::Type{ScopedValue{T}}) where {T} = T ## # Notes on the implementation. -# We want lookup to be unreasonable fast. +# We want lookup to be unreasonably fast. # - IDDict/Dict are ~10ns # - ImmutableDict is faster up to about ~15 entries # - ScopedValue are meant to be constant, Immutabilty @@ -55,16 +55,25 @@ Base.eltype(::Type{ScopedValue{T}}) where {T} = T # a WeakKeyDict to also ensure that values get GC'd when ScopedValues # become unreachable. # - Scopes are an inline implementation of an ImmutableDict, if we wanted -# be really fance we could use a CTrie or HAMT. +# be really fancy we could use a CTrie or HAMT. -mutable struct Scope{T} +mutable struct Scope const parent::Union{Nothing, Scope} - const key::ScopedValue{T} - const value::T + const key::ScopedValue + const value::Any + Scope(parent, key::ScopedValue{T}, value::T) where T = new(parent, key, value) end Scope(parent, key::ScopedValue{T}, value) where T = Scope(parent, key, convert(T, value)) +Scope(parent, pair::Pair{<:ScopedValue}, rest::Pair{<:ScopedValue}...) + scope = Scope(parent, pair...) + for pair in rest + scope = Scope(scope, pair...) + end + return scope +end + """ current_scope()::Union{Nothing, Scope} @@ -119,10 +128,7 @@ function scoped(f, pair::Pair{<:ScopedValue}, rest::Pair{<:ScopedValue}...) @nospecialize ct = Base.current_task() current_scope = ct.scope::Union{Nothing, Scope} - scope = Scope(current_scope, pair...) - for pair in rest - scope = Scope(scope, pair...) - end + scope = Scope(current_scope, pair, rest...) ct.scope = scope try return f() From 9fefaf19e0689ff50bc304530b858e0c88fdf454 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 22 Aug 2023 19:20:24 -0400 Subject: [PATCH 18/23] add macro --- base/scopedvalues.jl | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index 2023fdd94499a..9f802f106e1c9 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -2,7 +2,7 @@ module ScopedValues -export ScopedValue, scoped +export ScopedValue, scoped, @scoped """ ScopedValue(x) @@ -139,4 +139,28 @@ end scoped(@nospecialize(f)) = f() +macro scoped(exprs...) + ex = last(exprs) + if length(exprs) > 1 + exprs = exprs[1:end-1] + else + exprs = () + end + for expr in exprs + if expr.head !== :call || first(expr.args) !== :(=>) + error("@scoped expects arguments of the form `A => 2` got $expr") + end + end + exprs = map(esc, exprs) + ct = gensym(:ct) + current_scope = gensym(:current_scope) + body = Expr(:tryfinally, esc(ex), :($(ct).scope = $cs)) + quote + $(ct) = $(Base.current_task)() + $(current_scope) = $(ct).scope::$(Union{Nothing, Scope}) + $(ct).scope = $(Scope)($(current_scope), $(exprs...)) + $body + end +end + end # module ScopedValues From 1a632a759bc9709d0fd466d7ec8ec62bc6680201 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 22 Aug 2023 19:49:05 -0400 Subject: [PATCH 19/23] docs and more --- base/exports.jl | 1 + base/logging.jl | 2 +- base/scopedvalues.jl | 14 ++++++-- doc/src/base/scopedvalues.md | 64 ++++++++++++++++++++++++++++-------- test/scopedvalues.jl | 7 ++++ 5 files changed, 71 insertions(+), 17 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index 0a35323022e1d..abf140d0ad81f 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -650,6 +650,7 @@ export # ScopedValue scoped, + @scoped, ScopedValue, # logging diff --git a/base/logging.jl b/base/logging.jl index 808e3592c9fe1..8792074fe8b01 100644 --- a/base/logging.jl +++ b/base/logging.jl @@ -508,7 +508,7 @@ end return nothing end -with_logstate(f::Function, logstate) = scoped(f, CURRENT_LOGSTATE => logstate) +with_logstate(f::Function, logstate) = @scoped(CURRENT_LOGSTATE => logstate, f()) #------------------------------------------------------------------------------- # Control of the current logger and early log filtering diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index 9f802f106e1c9..9fbb37425dd47 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -27,10 +27,13 @@ julia> scoped(svar => 2) do svar[] end 2 + +julia> svar[] +1 ``` !!! compat "Julia 1.11" - Scoped values were introduced in Julia 1.11. In Julia 1.7+ a compatible + Scoped values were introduced in Julia 1.11. In Julia 1.8+ a compatible implementation is available from the package ScopedValues.jl. """ mutable struct ScopedValue{T} @@ -66,7 +69,7 @@ end Scope(parent, key::ScopedValue{T}, value) where T = Scope(parent, key, convert(T, value)) -Scope(parent, pair::Pair{<:ScopedValue}, rest::Pair{<:ScopedValue}...) +function Scope(parent, pair::Pair{<:ScopedValue}, rest::Pair{<:ScopedValue}...) scope = Scope(parent, pair...) for pair in rest scope = Scope(scope, pair...) @@ -139,6 +142,13 @@ end scoped(@nospecialize(f)) = f() +""" + @scoped vars... expr + +Macro version of `scoped(f, vars...)` but with `expr` instead of `f` function. +This is similar to using [`scoped`](@ref) with a `do` block, but avoids creating +a closure. +""" macro scoped(exprs...) ex = last(exprs) if length(exprs) > 1 diff --git a/doc/src/base/scopedvalues.md b/doc/src/base/scopedvalues.md index 5380d81ceb8f5..78ad91d63a538 100644 --- a/doc/src/base/scopedvalues.md +++ b/doc/src/base/scopedvalues.md @@ -10,9 +10,10 @@ multiple different values at the same time. implementation is available from the package ScopedValues.jl. In its simplest form you can create a [`ScopedValue`](@ref) with a -default value and then use [`scoped`](@ref) to enter a new dynamic -scope. The new scope will inherit all values from the parent scope -(and recursivly from all outer scopes) with the provided scoped +default value and then use [`scoped`](@ref) or [`@scoped`](@ref) to +enter a new dynamic scope. +The new scope will inherit all values from the parent scope +(and recursively from all outer scopes) with the provided scoped value taking priority over previous definitions. ```julia @@ -36,6 +37,14 @@ end @show scoped_val2[] # 0 ``` +Since `scoped` requires a closure or a function and creates another call-frame, +it can sometimes be beneficial to use the macro form. + +```julia +const STATE = ScopedValue{Union{Nothing, State}}() +with_state(f, state::State) = @scoped(STATE => state, f()) +``` + !!! note Dynamic scopes are propagated through [`Task`](@ref)s. @@ -62,7 +71,8 @@ state in a scoped value. Just keep in mind that the usual caveats for global variables apply in the context of concurrent programming. Care is also required when storing references to mutable state in scoped -values. You might want to explicitly unshare when entering a new dynamic scope. +values. You might want to explicitly [unshare mutable state](@ref unshare_mutable_state) +when entering a new dynamic scope. ```julia import Base.Threads: @spawn @@ -85,18 +95,10 @@ end @spawn (sval_dict[][:b] = 3) end end - -# If you want to add new values to the dict, instead of replacing -# it, unshare the values explicitly. In this example we use `merge` -# to unshare the state of the dictonary in parent scope. -@sync begin - scoped(sval_dict => merge(sval_dict, Dict(:a => 10))) do - @spawn @show sval_dict[][:a] - end - @spawn sval_dict[][:a] = 3 # Not a race since they are unshared. -end ``` +## Example + In the example below we use a scoped value to implement a permission check in a web-application. After determining the permissions of the request, a new dynamic scope is entered and the scoped value `LEVEL` is set. @@ -130,9 +132,43 @@ function handle(request, response) end ``` +## Idioms +### [Unshare mutable state]((@id unshare_mutable_state)) + +```julia +import Base.Threads: @spawn +const sval_dict = ScopedValue(Dict()) + +# If you want to add new values to the dict, instead of replacing +# it, unshare the values explicitly. In this example we use `merge` +# to unshare the state of the dictonary in parent scope. +@sync begin + scoped(sval_dict => merge(sval_dict, Dict(:a => 10))) do + @spawn @show sval_dict[][:a] + end + @spawn sval_dict[][:a] = 3 # Not a race since they are unshared. +end +``` + +### Local caching + +Since lookup of a scoped variable is linear in scope depth, it can be beneficial +for a library at an API boundary to cache the state of the scoped value. + +```julia +const DEVICE = ScopedValue(:CPU) + +function solve_problem(args...) + # Cache current device + @scoped DEVICE => DEVICE[] begin + # call functions that use `DEVICE[]` repeatedly. + end +``` + ## API docs ```@docs Base.ScopedValues.ScopedValue Base.ScopedValues.scoped +Base.ScopedValues.@scoped ``` diff --git a/test/scopedvalues.jl b/test/scopedvalues.jl index a33214ce93e71..d98fbcda9ceff 100644 --- a/test/scopedvalues.jl +++ b/test/scopedvalues.jl @@ -106,3 +106,10 @@ end end end end + +@testset "macro" begin + @scoped svar=>2 svar_float=>2.0 begin + @test svar[] == 2 + @test svar_float[] == 2.0 + end +end From 7b3811ab2b5af7d0ab85103b0ad05091fd07b076 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 22 Aug 2023 19:50:08 -0400 Subject: [PATCH 20/23] fix macro --- base/scopedvalues.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index 9fbb37425dd47..abd1870ff69a8 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -164,7 +164,7 @@ macro scoped(exprs...) exprs = map(esc, exprs) ct = gensym(:ct) current_scope = gensym(:current_scope) - body = Expr(:tryfinally, esc(ex), :($(ct).scope = $cs)) + body = Expr(:tryfinally, esc(ex), :($(ct).scope = $(current_scope))) quote $(ct) = $(Base.current_task)() $(current_scope) = $(ct).scope::$(Union{Nothing, Scope}) From 2ed9bb9f91c189bebb4a93301f90f9ebc84c0ff0 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 22 Aug 2023 19:53:57 -0400 Subject: [PATCH 21/23] simplify Scope(parent, pairs...) --- base/scopedvalues.jl | 3 +-- test/scopedvalues.jl | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index abd1870ff69a8..aa9bac878e49a 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -69,8 +69,7 @@ end Scope(parent, key::ScopedValue{T}, value) where T = Scope(parent, key, convert(T, value)) -function Scope(parent, pair::Pair{<:ScopedValue}, rest::Pair{<:ScopedValue}...) - scope = Scope(parent, pair...) +function Scope(scope, pairs::Pair{<:ScopedValue}...) for pair in rest scope = Scope(scope, pair...) end diff --git a/test/scopedvalues.jl b/test/scopedvalues.jl index d98fbcda9ceff..4313612218215 100644 --- a/test/scopedvalues.jl +++ b/test/scopedvalues.jl @@ -112,4 +112,9 @@ end @test svar[] == 2 @test svar_float[] == 2.0 end + # Doesn't do much... + @scoped begin + @test svar[] == 1 + @test svar_float[] == 1.0 + end end From e10368d80e3bdffbd97ef5d42e27bc00c32d36ac Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 22 Aug 2023 20:43:59 -0400 Subject: [PATCH 22/23] fixes --- base/scopedvalues.jl | 2 +- test/scopedvalues.jl | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/base/scopedvalues.jl b/base/scopedvalues.jl index aa9bac878e49a..755f3a03446ed 100644 --- a/base/scopedvalues.jl +++ b/base/scopedvalues.jl @@ -70,7 +70,7 @@ Scope(parent, key::ScopedValue{T}, value) where T = Scope(parent, key, convert(T, value)) function Scope(scope, pairs::Pair{<:ScopedValue}...) - for pair in rest + for pair in pairs scope = Scope(scope, pair...) end return scope diff --git a/test/scopedvalues.jl b/test/scopedvalues.jl index 4313612218215..367249cf74dbd 100644 --- a/test/scopedvalues.jl +++ b/test/scopedvalues.jl @@ -1,4 +1,5 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +import Base: ScopedValues @testset "errors" begin @test ScopedValue{Float64}(1)[] == 1.0 @@ -65,7 +66,7 @@ end scoped(svar => 2.0) do @test sprint(show, svar) == "ScopedValue{$Int}(2)" objid = sprint(show, Base.objectid(svar)) - @test sprint(show, ScopedValues.current_scope()) == "ScopedValues.Scope(ScopedValue{$Int}@$objid => 2)" + @test sprint(show, ScopedValues.current_scope()) == "Base.ScopedValues.Scope(ScopedValue{$Int}@$objid => 2)" end end From b344505524b2f9be79ce5f376aa330a089e7f7cc Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 22 Aug 2023 21:30:42 -0400 Subject: [PATCH 23/23] Implement Test/Testset using ScopedValue --- stdlib/Test/src/Test.jl | 141 ++++++++++++++-------------------------- test/runtests.jl | 27 ++++---- 2 files changed, 62 insertions(+), 106 deletions(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 622c696b383a0..29b9f4576d4c8 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -982,7 +982,11 @@ end A simple fallback test set that throws immediately on a failure. """ struct FallbackTestSet <: AbstractTestSet end -fallback_testset = FallbackTestSet() + +const CURRENT_TESTSET = ScopedValue{AbstractTestSet}(FallbackTestSet()) +const CURRENT_DEPTH = ScopedValue(0) +get_testset() = CURRENT_TESTSET[] +get_testset_depth() = CURRENT_DEPTH[] struct FallbackTestSetException <: Exception msg::String @@ -1520,14 +1524,13 @@ function testset_context(args, ex, source) test_ex = ex.args[2] ex.args[2] = quote + testset = $(CURRENT_TESTSET)[] $(map(contexts) do context - :($push_testset($(ContextTestSet)($(QuoteNode(context)), $context; $options...))) + :(testset = $(ContextTestSet)(testset, $(QuoteNode(context)), $context; $options...)) end...) - try - $(test_ex) - finally - $(map(_->:($pop_testset()), contexts)...) - end + @scoped($(CURRENT_TESTSET) => testset, + $(CURRENT_DEPTH) => $CURRENT_DEPTH[]+1, + $(test_ex)) end return esc(ex) @@ -1563,34 +1566,34 @@ function testset_beginend_call(args, tests, source) else $(testsettype)($desc; $options...) end - push_testset(ts) - # we reproduce the logic of guardseed, but this function - # cannot be used as it changes slightly the semantic of @testset, - # by wrapping the body in a function - local RNG = default_rng() - local oldrng = copy(RNG) - local oldseed = Random.GLOBAL_SEED - try - # RNG is re-seeded with its own seed to ease reproduce a failed test - Random.seed!(Random.GLOBAL_SEED) - let - $(esc(tests)) - end - catch err - err isa InterruptException && rethrow() - # something in the test block threw an error. Count that as an - # error in this test set - trigger_test_failure_break(err) - if err isa FailFastError - get_testset_depth() > 1 ? rethrow() : failfast_print() - else - record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)))) + @scoped $(CURRENT_TESTSET) => ts $(CURRENT_DEPTH) => $(CURRENT_DEPTH)[] + 1 begin + # we reproduce the logic of guardseed, but this function + # cannot be used as it changes slightly the semantic of @testset, + # by wrapping the body in a function + local RNG = default_rng() + local oldrng = copy(RNG) + local oldseed = Random.GLOBAL_SEED + try + # RNG is re-seeded with its own seed to ease reproduce a failed test + Random.seed!(Random.GLOBAL_SEED) + let + $(esc(tests)) + end + catch err + err isa InterruptException && rethrow() + # something in the test block threw an error. Count that as an + # error in this test set + trigger_test_failure_break(err) + if err isa FailFastError + get_testset_depth() > 1 ? rethrow() : failfast_print() + else + record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)))) + end + finally + copy!(RNG, oldrng) + Random.set_global_seed!(oldseed) + ret = finish(ts) end - finally - copy!(RNG, oldrng) - Random.set_global_seed!(oldseed) - pop_testset() - ret = finish(ts) end ret end @@ -1649,7 +1652,6 @@ function testset_forloop(args, testloop, source) # Trick to handle `break` and `continue` in the test code before # they can be handled properly by `finally` lowering. if !first_iteration - pop_testset() finish_errored = true push!(arr, finish(ts)) finish_errored = false @@ -1663,17 +1665,18 @@ function testset_forloop(args, testloop, source) else $(testsettype)($desc; $options...) end - push_testset(ts) - first_iteration = false - try - $(esc(tests)) - catch err - err isa InterruptException && rethrow() - # Something in the test block threw an error. Count that as an - # error in this test set - trigger_test_failure_break(err) - if !isa(err, FailFastError) - record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)))) + @scoped $(CURRENT_TESTSET) => ts $(CURRENT_DEPTH) => $(CURRENT_DEPTH)[] + 1 begin + first_iteration = false + try + $(esc(tests)) + catch err + err isa InterruptException && rethrow() + # Something in the test block threw an error. Count that as an + # error in this test set + trigger_test_failure_break(err) + if !isa(err, FailFastError) + record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)))) + end end end end @@ -1694,7 +1697,6 @@ function testset_forloop(args, testloop, source) finally # Handle `return` in test body if !first_iteration && !finish_errored - pop_testset() push!(arr, finish(ts)) end copy!(RNG, oldrng) @@ -1736,51 +1738,6 @@ end #----------------------------------------------------------------------- # Various helper methods for test sets -""" - get_testset() - -Retrieve the active test set from the task's local storage. If no -test set is active, use the fallback default test set. -""" -function get_testset() - testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) - return isempty(testsets) ? fallback_testset : testsets[end] -end - -""" - push_testset(ts::AbstractTestSet) - -Adds the test set to the `task_local_storage`. -""" -function push_testset(ts::AbstractTestSet) - testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) - push!(testsets, ts) - setindex!(task_local_storage(), testsets, :__BASETESTNEXT__) -end - -""" - pop_testset() - -Pops the last test set added to the `task_local_storage`. If there are no -active test sets, returns the fallback default test set. -""" -function pop_testset() - testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) - ret = isempty(testsets) ? fallback_testset : pop!(testsets) - setindex!(task_local_storage(), testsets, :__BASETESTNEXT__) - return ret -end - -""" - get_testset_depth() - -Return the number of active test sets, not including the default test set -""" -function get_testset_depth() - testsets = get(task_local_storage(), :__BASETESTNEXT__, AbstractTestSet[]) - return length(testsets) -end - _args_and_call(args...; kwargs...) = (args[1:end-1], kwargs, args[end](args[1:end-1]...; kwargs...)) _materialize_broadcasted(f, args...) = Broadcast.materialize(Broadcast.broadcasted(f, args...)) diff --git a/test/runtests.jl b/test/runtests.jl index 1264acae985b0..e5c9c64913e55 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -373,15 +373,14 @@ cd(@__DIR__) do Test.TESTSET_PRINT_ENABLE[] = false o_ts = Test.DefaultTestSet("Overall") o_ts.time_end = o_ts.time_start + o_ts_duration # manually populate the timing - Test.push_testset(o_ts) completed_tests = Set{String}() - for (testname, (resp,), duration) in results + @scoped Test.CURRENT_TESTSET => o_ts for (testname, (resp,), duration) in results push!(completed_tests, testname) if isa(resp, Test.DefaultTestSet) resp.time_end = resp.time_start + duration - Test.push_testset(resp) - Test.record(o_ts, resp) - Test.pop_testset() + @scoped Test.CURRENT_TESTSET => resp begin + Test.record(o_ts, resp) + end elseif isa(resp, Test.TestSetException) fake = Test.DefaultTestSet(testname) fake.time_end = fake.time_start + duration @@ -394,9 +393,9 @@ cd(@__DIR__) do for t in resp.errors_and_fails Test.record(fake, t) end - Test.push_testset(fake) - Test.record(o_ts, fake) - Test.pop_testset() + @scoped Test.CURRENT_TESTSET => fake begin + Test.record(o_ts, fake) + end else if !isa(resp, Exception) resp = ErrorException(string("Unknown result type : ", typeof(resp))) @@ -408,18 +407,18 @@ cd(@__DIR__) do fake = Test.DefaultTestSet(testname) fake.time_end = fake.time_start + duration Test.record(fake, Test.Error(:nontest_error, testname, nothing, Any[(resp, [])], LineNumberNode(1))) - Test.push_testset(fake) - Test.record(o_ts, fake) - Test.pop_testset() + @scoped Test.CURRENT_TESTSET => fake begin + Test.record(o_ts, fake) + end end end for test in all_tests (test in completed_tests) && continue fake = Test.DefaultTestSet(test) Test.record(fake, Test.Error(:test_interrupted, test, nothing, [("skipped", [])], LineNumberNode(1))) - Test.push_testset(fake) - Test.record(o_ts, fake) - Test.pop_testset() + @scoped Test.CURRENT_TESTSET => fake begin + Test.record(o_ts, fake) + end end Test.TESTSET_PRINT_ENABLE[] = true println()