From c38e89295aa19aa2b1c3ea897bce2fe9aaaf10fa Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Tue, 22 Feb 2022 14:19:29 -0500 Subject: [PATCH 01/68] Distributed test suite: wrap another non-thread-safe test in `poll_while` (#44280) (cherry picked from commit e4920250cea633e5460384b7529c7554d96b73a5) --- stdlib/Distributed/test/distributed_exec.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Distributed/test/distributed_exec.jl b/stdlib/Distributed/test/distributed_exec.jl index 69d9fb47eccc5..d2d46aebf0f1b 100644 --- a/stdlib/Distributed/test/distributed_exec.jl +++ b/stdlib/Distributed/test/distributed_exec.jl @@ -298,7 +298,7 @@ let wid1 = workers()[1], end remotecall_fetch(r -> (finalize(take!(r)); yield(); nothing), wid2, fstore) # finalize remotely sleep(0.5) # to ensure that wid2 messages have been executed on wid1 - @test remotecall_fetch(k -> haskey(Distributed.PGRP.refs, k), wid1, rrid) == false + @test poll_while(() -> remotecall_fetch(k -> haskey(Distributed.PGRP.refs, k), wid1, rrid)) end # Tests for issue #23109 - should not hang. From 9daeaddae0d6459bb43f16de0e3d4077c7a52632 Mon Sep 17 00:00:00 2001 From: Ian Atol Date: Mon, 28 Mar 2022 12:34:45 -0700 Subject: [PATCH 02/68] Pass around effects during cacheing (#44777) (cherry picked from commit d7782de62f974202f6be63f888bc173b8af7dddc) --- base/compiler/typeinfer.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index c20fcd14be0ef..c72d03a226472 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -356,7 +356,8 @@ function maybe_compress_codeinfo(interp::AbstractInterpreter, linfo::MethodInsta end function transform_result_for_cache(interp::AbstractInterpreter, linfo::MethodInstance, - valid_worlds::WorldRange, @nospecialize(inferred_result)) + valid_worlds::WorldRange, @nospecialize(inferred_result), + ipo_effects::Effects) # If we decided not to optimize, drop the OptimizationState now. # External interpreters can override as necessary to cache additional information if inferred_result isa OptimizationState @@ -391,7 +392,7 @@ function cache_result!(interp::AbstractInterpreter, result::InferenceResult) # TODO: also don't store inferred code if we've previously decided to interpret this function if !already_inferred - inferred_result = transform_result_for_cache(interp, linfo, valid_worlds, result.src) + inferred_result = transform_result_for_cache(interp, linfo, valid_worlds, result.src, result.ipo_effects) code_cache(interp)[linfo] = CodeInstance(result, inferred_result, valid_worlds) if track_newly_inferred[] m = linfo.def From dc89240c653db9303f58b993858b9317812a5c38 Mon Sep 17 00:00:00 2001 From: Simeon Schaub Date: Mon, 28 Mar 2022 15:35:10 -0400 Subject: [PATCH 03/68] errors: fix handling of `.op` in lowering (#44770) Discovered while running the syntax tests with lines 25-28 of `jlfrontend.scm` commented out. Turned out we didn't handle `.op` correctly in neither `check-dotop` nor in `deparse`. This meant we just got `error: malformed expression` instead of an actually useful error message. (cherry picked from commit 9112135140efe9ebd39727054e904bd881d46289) --- src/ast.scm | 18 ++++++++++-------- test/syntax.jl | 7 ++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/ast.scm b/src/ast.scm index 688b4e852e7c4..04438e50692f9 100644 --- a/src/ast.scm +++ b/src/ast.scm @@ -79,13 +79,15 @@ ((char? e) (string "'" e "'")) ((atom? e) (string e)) ((eq? (car e) '|.|) - (string (deparse (cadr e)) '|.| - (cond ((and (pair? (caddr e)) (memq (caaddr e) '(quote inert))) - (deparse-colon-dot (cadr (caddr e)))) - ((and (pair? (caddr e)) (eq? (caaddr e) 'copyast)) - (deparse-colon-dot (cadr (cadr (caddr e))))) - (else - (string #\( (deparse (caddr e)) #\)))))) + (if (length= e 2) + (string "(." (deparse (cadr e)) ")") + (string (deparse (cadr e)) '|.| + (cond ((and (pair? (caddr e)) (memq (caaddr e) '(quote inert))) + (deparse-colon-dot (cadr (caddr e)))) + ((and (pair? (caddr e)) (eq? (caaddr e) 'copyast)) + (deparse-colon-dot (cadr (cadr (caddr e))))) + (else + (string #\( (deparse (caddr e)) #\))))))) ((memq (car e) '(... |'|)) (string (deparse (cadr e)) (car e))) ((or (syntactic-op? (car e)) (eq? (car e) '|<:|) (eq? (car e) '|>:|) (eq? (car e) '-->)) @@ -445,7 +447,7 @@ (if (dotop-named? e) (error (string "invalid function name \"" (deparse e) "\"")) (if (pair? e) - (if (eq? (car e) '|.|) + (if (and (eq? (car e) '|.|) (length= e 3)) (check-dotop (caddr e)) (if (quoted? e) (check-dotop (cadr e)))))) diff --git a/test/syntax.jl b/test/syntax.jl index 805d006f971ba..8320060678d16 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -1909,7 +1909,12 @@ f31404(a, b; kws...) = (a, b, values(kws)) # issue #28992 macro id28992(x) x end @test @id28992(1 .+ 2) == 3 -@test Meta.isexpr(Meta.lower(@__MODULE__, :(@id28992((.+)(a,b) = 0))), :error) +@test Meta.@lower(.+(a,b) = 0) == Expr(:error, "invalid function name \".+\"") +@test Meta.@lower((.+)(a,b) = 0) == Expr(:error, "invalid function name \"(.+)\"") +let m = @__MODULE__ + @test Meta.lower(m, :($m.@id28992(.+(a,b) = 0))) == Expr(:error, "invalid function name \"$(nameof(m)).:.+\"") + @test Meta.lower(m, :($m.@id28992((.+)(a,b) = 0))) == Expr(:error, "invalid function name \"(.$(nameof(m)).+)\"") +end @test @id28992([1] .< [2] .< [3]) == [true] @test @id28992(2 ^ -2) == 0.25 @test @id28992(2 .^ -2) == 0.25 From 807b74cab20c5e34167a379ab2c6f65bc9a30763 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Tue, 29 Mar 2022 15:53:13 +0900 Subject: [PATCH 04/68] fix #44732, propagate effects of a thrown concrete call correctly (#44762) (cherry picked from commit f86b4ef88f487337d61eb373f3ec8ddbde76c78c) --- base/compiler/abstractinterpretation.jl | 33 +++++++++++-------------- base/compiler/ssair/inlining.jl | 6 ++--- base/compiler/stmtinfo.jl | 5 ++-- test/compiler/inline.jl | 22 +++++++++++++++++ 4 files changed, 43 insertions(+), 23 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 3cc709bc35849..e7a38b258e9bb 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -128,9 +128,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), end tristate_merge!(sv, effects) push!(const_results, const_result) - if const_result !== nothing - any_const_result = true - end + any_const_result |= const_result !== nothing this_rt = tmerge(this_rt, rt) if bail_out_call(interp, this_rt, sv) break @@ -179,9 +177,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), end tristate_merge!(sv, effects) push!(const_results, const_result) - if const_result !== nothing - any_const_result = true - end + any_const_result |= const_result !== nothing end @assert !(this_conditional isa Conditional) "invalid lattice element returned from inter-procedural context" seen += 1 @@ -694,12 +690,12 @@ function pure_eval_call(interp::AbstractInterpreter, end function _pure_eval_call(@nospecialize(f), arginfo::ArgInfo) args = collect_const_args(arginfo) - try - value = Core._apply_pure(f, args) - return Const(value) + value = try + Core._apply_pure(f, args) catch return nothing end + return Const(value) end function concrete_eval_eligible(interp::AbstractInterpreter, @@ -733,17 +729,18 @@ function concrete_eval_call(interp::AbstractInterpreter, @nospecialize(f), result::MethodCallResult, arginfo::ArgInfo, sv::InferenceState) concrete_eval_eligible(interp, f, result, arginfo, sv) || return nothing args = collect_const_args(arginfo) - try - value = Core._call_in_world_total(get_world_counter(interp), f, args...) - if is_inlineable_constant(value) || call_result_unused(sv) - # If the constant is not inlineable, still do the const-prop, since the - # code that led to the creation of the Const may be inlineable in the same - # circumstance and may be optimizable. - return ConstCallResults(Const(value), ConstResult(result.edge, value), EFFECTS_TOTAL) - end + world = get_world_counter(interp) + value = try + Core._call_in_world_total(world, f, args...) catch # The evaulation threw. By :consistent-cy, we're guaranteed this would have happened at runtime - return ConstCallResults(Union{}, ConstResult(result.edge), result.edge_effects) + return ConstCallResults(Union{}, ConstResult(result.edge, result.edge_effects), result.edge_effects) + end + if is_inlineable_constant(value) || call_result_unused(sv) + # If the constant is not inlineable, still do the const-prop, since the + # code that led to the creation of the Const may be inlineable in the same + # circumstance and may be optimizable. + return ConstCallResults(Const(value), ConstResult(result.edge, EFFECTS_TOTAL, value), EFFECTS_TOTAL) end return nothing end diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index a5b68d6d5198d..5e3ad4febe60f 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -1310,10 +1310,10 @@ end function const_result_item(result::ConstResult, state::InliningState) if !isdefined(result, :result) || !is_inlineable_constant(result.result) - return compileable_specialization(state.et, result.mi, EFFECTS_TOTAL) - else - return ConstantCase(quoted(result.result)) + return compileable_specialization(state.et, result.mi, result.effects) end + @assert result.effects === EFFECTS_TOTAL + return ConstantCase(quoted(result.result)) end function handle_cases!(ir::IRCode, idx::Int, stmt::Expr, @nospecialize(atype), diff --git a/base/compiler/stmtinfo.jl b/base/compiler/stmtinfo.jl index e3f69b2c43e54..3eeff0c2c86a8 100644 --- a/base/compiler/stmtinfo.jl +++ b/base/compiler/stmtinfo.jl @@ -49,9 +49,10 @@ end struct ConstResult mi::MethodInstance + effects::Effects result - ConstResult(mi::MethodInstance) = new(mi) - ConstResult(mi::MethodInstance, @nospecialize val) = new(mi, val) + ConstResult(mi::MethodInstance, effects::Effects) = new(mi, effects) + ConstResult(mi::MethodInstance, effects::Effects, @nospecialize val) = new(mi, effects, val) end """ diff --git a/test/compiler/inline.jl b/test/compiler/inline.jl index 1af52422b2f71..59604bd317f2e 100644 --- a/test/compiler/inline.jl +++ b/test/compiler/inline.jl @@ -1095,6 +1095,28 @@ Base.@assume_effects :consistent :effect_free :terminates_globally consteval( consteval(getindex, ___CONST_DICT___, :a) end +# https://github.com/JuliaLang/julia/issues/44732 +struct Component44732 + v +end +struct Container44732 + x::Union{Nothing,Component44732} +end + +# NOTE make sure to prevent inference bail out +validate44732(::Component44732) = nothing +validate44732(::Any) = error("don't erase this error!") + +function issue44732(c::Container44732) + validate44732(c.x) + return nothing +end + +let src = code_typed1(issue44732, (Container44732,)) + @test any(isinvoke(:validate44732), src.code) +end +@test_throws ErrorException("don't erase this error!") issue44732(Container44732(nothing)) + global x44200::Int = 0 function f44200() global x = 0 From f560674e7849419c77d88b33d0fa997d6d051b6c Mon Sep 17 00:00:00 2001 From: Jerry Ling Date: Tue, 29 Mar 2022 10:01:32 -0400 Subject: [PATCH 05/68] fix `===` when encountering null pointer (#44749) (cherry picked from commit 1a7355b3866fecfebdcb1923cfd691b48fe5a762) --- src/builtins.c | 21 +++++++++++++-------- test/core.jl | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/builtins.c b/src/builtins.c index 11da13285e9ca..64f0755182816 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -94,20 +94,25 @@ static int NOINLINE compare_fields(const jl_value_t *a, const jl_value_t *b, jl_ else { jl_datatype_t *ft = (jl_datatype_t*)jl_field_type_concrete(dt, f); if (jl_is_uniontype(ft)) { - uint8_t asel = ((uint8_t*)ao)[jl_field_size(dt, f) - 1]; - uint8_t bsel = ((uint8_t*)bo)[jl_field_size(dt, f) - 1]; + size_t idx = jl_field_size(dt, f) - 1; + uint8_t asel = ((uint8_t*)ao)[idx]; + uint8_t bsel = ((uint8_t*)bo)[idx]; if (asel != bsel) return 0; ft = (jl_datatype_t*)jl_nth_union_component((jl_value_t*)ft, asel); } else if (ft->layout->first_ptr >= 0) { - // If the field is a inline immutable that can be can be undef - // we need to check to check for undef first since undef struct + // If the field is a inline immutable that can be undef + // we need to check for undef first since undef struct // may have fields that are different but should still be treated as equal. - jl_value_t *ptra = ((jl_value_t**)ao)[ft->layout->first_ptr]; - jl_value_t *ptrb = ((jl_value_t**)bo)[ft->layout->first_ptr]; - if (ptra == NULL && ptrb == NULL) { - return 1; + int32_t idx = ft->layout->first_ptr; + jl_value_t *ptra = ((jl_value_t**)ao)[idx]; + jl_value_t *ptrb = ((jl_value_t**)bo)[idx]; + if ((ptra == NULL) != (ptrb == NULL)) { + return 0; + } + else if (ptra == NULL) { // implies ptrb == NULL + continue; // skip this field (it is #undef) } } if (!ft->layout->haspadding) { diff --git a/test/core.jl b/test/core.jl index 0294a61c7d146..985c15781e45c 100644 --- a/test/core.jl +++ b/test/core.jl @@ -92,6 +92,20 @@ let abcd = ABCDconst(1, 2, 3, 4) @test (1, 2, "not constant", 4) === (abcd.a, abcd.b, abcd.c, abcd.d) end +# test `===` handling null pointer in struct #44712 +struct N44712 + a::Some{Any} + b::Int + N44712() = new() +end +let a = Int[0, 1], b = Int[0, 2] + GC.@preserve a b begin + @test unsafe_load(Ptr{N44712}(pointer(a))) !== unsafe_load(Ptr{N44712}(pointer(b))) + end +end + +# another possible issue in #44712 +@test (("", 0),) !== (("", 1),) f47(x::Vector{Vector{T}}) where {T} = 0 @test_throws MethodError f47(Vector{Vector}()) From c4a53a8412b8abafe389d07e76a6e1f6f82cfb9c Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Tue, 29 Mar 2022 06:11:38 +0900 Subject: [PATCH 06/68] inference: implement missing effects propagation from `abstract_invoke` (#44764) Fix #44763 (cherry picked from commit 9d35089b32ed11edd39c114d3efc4e91e710f50e) --- base/compiler/abstractinterpretation.jl | 29 ++++++++++--------------- base/compiler/types.jl | 1 + test/compiler/inference.jl | 9 ++++++++ 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index e7a38b258e9bb..cf39d622bf72f 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -1490,25 +1490,26 @@ end function abstract_invoke(interp::AbstractInterpreter, (; fargs, argtypes)::ArgInfo, sv::InferenceState) ft′ = argtype_by_index(argtypes, 2) ft = widenconst(ft′) - ft === Bottom && return CallMeta(Bottom, false) + ft === Bottom && return CallMeta(Bottom, false), EFFECTS_THROWN (types, isexact, isconcrete, istype) = instanceof_tfunc(argtype_by_index(argtypes, 3)) - types === Bottom && return CallMeta(Bottom, false) - isexact || return CallMeta(Any, false) + types === Bottom && return CallMeta(Bottom, false), EFFECTS_THROWN + isexact || return CallMeta(Any, false), Effects() argtype = argtypes_to_type(argtype_tail(argtypes, 4)) nargtype = typeintersect(types, argtype) - nargtype === Bottom && return CallMeta(Bottom, false) - nargtype isa DataType || return CallMeta(Any, false) # other cases are not implemented below - isdispatchelem(ft) || return CallMeta(Any, false) # check that we might not have a subtype of `ft` at runtime, before doing supertype lookup below + nargtype === Bottom && return CallMeta(Bottom, false), EFFECTS_THROWN + nargtype isa DataType || return CallMeta(Any, false), Effects() # other cases are not implemented below + isdispatchelem(ft) || return CallMeta(Any, false), Effects() # check that we might not have a subtype of `ft` at runtime, before doing supertype lookup below ft = ft::DataType types = rewrap_unionall(Tuple{ft, unwrap_unionall(types).parameters...}, types)::Type nargtype = Tuple{ft, nargtype.parameters...} argtype = Tuple{ft, argtype.parameters...} match, valid_worlds, overlayed = findsup(types, method_table(interp)) - match === nothing && return CallMeta(Any, false) + match === nothing && return CallMeta(Any, false), Effects() update_valid_age!(sv, valid_worlds) method = match.method (ti, env::SimpleVector) = ccall(:jl_type_intersection_with_env, Any, (Any, Any), nargtype, method.sig)::SimpleVector (; rt, edge) = result = abstract_call_method(interp, method, ti, env, false, sv) + effects = result.edge_effects edge !== nothing && add_backedge!(edge::MethodInstance, sv) match = MethodMatch(ti, env, method, argtype <: method.sig) res = nothing @@ -1526,10 +1527,10 @@ function abstract_invoke(interp::AbstractInterpreter, (; fargs, argtypes)::ArgIn const_result = nothing if const_call_result !== nothing if const_call_result.rt ⊑ rt - (; rt, const_result) = const_call_result + (; rt, effects, const_result) = const_call_result end end - return CallMeta(from_interprocedural!(rt, sv, arginfo, sig), InvokeCallInfo(match, const_result)) + return CallMeta(from_interprocedural!(rt, sv, arginfo, sig), InvokeCallInfo(match, const_result)), effects end function invoke_rewrite(xs::Vector{Any}) @@ -1550,14 +1551,8 @@ function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), if f === _apply_iterate return abstract_apply(interp, argtypes, sv, max_methods) elseif f === invoke - call = abstract_invoke(interp, arginfo, sv) - if call.info === false - if call.rt === Bottom - tristate_merge!(sv, Effects(EFFECTS_TOTAL; nothrow=ALWAYS_FALSE)) - else - tristate_merge!(sv, Effects()) - end - end + call, effects = abstract_invoke(interp, arginfo, sv) + tristate_merge!(sv, effects) return call elseif f === modifyfield! tristate_merge!(sv, Effects()) # TODO diff --git a/base/compiler/types.jl b/base/compiler/types.jl index 5e2c7ed46a98d..50ae8ecba6705 100644 --- a/base/compiler/types.jl +++ b/base/compiler/types.jl @@ -59,6 +59,7 @@ function Effects( end const EFFECTS_TOTAL = Effects(ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, false) +const EFFECTS_THROWN = Effects(ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_FALSE, ALWAYS_TRUE, false) const EFFECTS_UNKNOWN = Effects(TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, true) function Effects(e::Effects = EFFECTS_UNKNOWN; diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index e6f113f2b9062..bf4f8391012b7 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -4055,3 +4055,12 @@ end @test Tuple{} <: code_typed(f_boundscheck_elim, Tuple{Int})[1][2] @test !Core.Compiler.builtin_nothrow(Core.get_binding_type, Any[Rational{Int}, Core.Const(:foo)], Any) + +# https://github.com/JuliaLang/julia/issues/44763 +global x44763::Int = 0 +increase_x44763!(n) = (global x44763; x44763 += n) +invoke44763(x) = Base.@invoke increase_x44763!(x) +@test Base.return_types() do + invoke44763(42) +end |> only === Int +@test x44763 == 0 From e71e722f58c87b640b94ad4428a66cc1e83f2917 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Wed, 30 Mar 2022 20:12:11 +0900 Subject: [PATCH 07/68] effects: change the `overlayed::Bool` property to `nonoverlayed::Bool` (#44786) The current naming of `overlayed` is a bit confusing since `overlayed === true`, which is the conservative default value, actually means "it _may_ be overlayed" while `overlayed === false` means "this is absolutely not overlayed". I think it should be named as `nonoverlayed`, and then a query name like `is_nonoverlayed` would be more sensible than the alternative `is_overlayed`, which actually should be something like `can_be_overlayed`. --- base/compiler/abstractinterpretation.jl | 77 +++++++++++++------------ base/compiler/inferencestate.jl | 2 +- base/compiler/methodtable.jl | 10 ++-- base/compiler/ssair/show.jl | 2 +- base/compiler/tfuncs.jl | 40 +++++-------- base/compiler/types.jl | 25 ++++---- src/julia.h | 4 +- 7 files changed, 77 insertions(+), 83 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index cf39d622bf72f..06d1a2c7d3e58 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -39,25 +39,23 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), # function has not seen any side effects, we would like to make sure there # aren't any in the throw block either to enable other optimizations. add_remark!(interp, sv, "Skipped call in throw block") - overlayed = true - if isoverlayed(method_table(interp)) - if !sv.ipo_effects.overlayed - # as we may want to concrete-evaluate this frame in cases when there are - # no overlayed calls, try an additional effort now to check if this call - # isn't overlayed rather than just handling it conservatively - matches = find_matching_methods(arginfo.argtypes, atype, method_table(interp), - InferenceParams(interp).MAX_UNION_SPLITTING, max_methods) - if !isa(matches, FailedMethodMatch) - overlayed = matches.overlayed - end + nonoverlayed = false + if isoverlayed(method_table(interp)) && sv.ipo_effects.nonoverlayed + # as we may want to concrete-evaluate this frame in cases when there are + # no overlayed calls, try an additional effort now to check if this call + # isn't overlayed rather than just handling it conservatively + matches = find_matching_methods(arginfo.argtypes, atype, method_table(interp), + InferenceParams(interp).MAX_UNION_SPLITTING, max_methods) + if !isa(matches, FailedMethodMatch) + nonoverlayed = matches.nonoverlayed end else - overlayed = false + nonoverlayed = true end # At this point we are guaranteed to end up throwing on this path, # which is all that's required for :consistent-cy. Of course, we don't # know anything else about this statement. - tristate_merge!(sv, Effects(; consistent=ALWAYS_TRUE, overlayed)) + tristate_merge!(sv, Effects(; consistent=ALWAYS_TRUE, nonoverlayed)) return CallMeta(Any, false) end @@ -80,11 +78,11 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), any_const_result = false const_results = Union{InferenceResult,Nothing,ConstResult}[] multiple_matches = napplicable > 1 - if matches.overlayed + if !matches.nonoverlayed # currently we don't have a good way to execute the overlayed method definition, # so we should give up pure/concrete eval when any of the matched methods is overlayed f = nothing - tristate_merge!(sv, Effects(EFFECTS_TOTAL; overlayed=true)) + tristate_merge!(sv, Effects(EFFECTS_TOTAL; nonoverlayed=false)) end val = pure_eval_call(interp, f, applicable, arginfo, sv) @@ -204,7 +202,9 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), end if seen != napplicable - tristate_merge!(sv, Effects(; overlayed=false)) # already accounted for method overlay above + # there may be unanalyzed effects within unseen dispatch candidate, + # but we can still ignore nonoverlayed effect here since we already accounted for it + tristate_merge!(sv, EFFECTS_UNKNOWN) elseif isa(matches, MethodMatches) ? (!matches.fullmatch || any_ambig(matches)) : (!_all(b->b, matches.fullmatches) || any_ambig(matches)) # Account for the fact that we may encounter a MethodError with a non-covered or ambiguous signature. @@ -243,7 +243,7 @@ struct MethodMatches valid_worlds::WorldRange mt::Core.MethodTable fullmatch::Bool - overlayed::Bool + nonoverlayed::Bool end any_ambig(info::MethodMatchInfo) = info.results.ambig any_ambig(m::MethodMatches) = any_ambig(m.info) @@ -255,7 +255,7 @@ struct UnionSplitMethodMatches valid_worlds::WorldRange mts::Vector{Core.MethodTable} fullmatches::Vector{Bool} - overlayed::Bool + nonoverlayed::Bool end any_ambig(m::UnionSplitMethodMatches) = _any(any_ambig, m.info.matches) @@ -270,7 +270,7 @@ function find_matching_methods(argtypes::Vector{Any}, @nospecialize(atype), meth valid_worlds = WorldRange() mts = Core.MethodTable[] fullmatches = Bool[] - overlayed = false + nonoverlayed = true for i in 1:length(split_argtypes) arg_n = split_argtypes[i]::Vector{Any} sig_n = argtypes_to_type(arg_n) @@ -281,8 +281,8 @@ function find_matching_methods(argtypes::Vector{Any}, @nospecialize(atype), meth if result === missing return FailedMethodMatch("For one of the union split cases, too many methods matched") end - matches, overlayedᵢ = result - overlayed |= overlayedᵢ + matches, overlayed = result + nonoverlayed &= !overlayed push!(infos, MethodMatchInfo(matches)) for m in matches push!(applicable, m) @@ -309,7 +309,7 @@ function find_matching_methods(argtypes::Vector{Any}, @nospecialize(atype), meth valid_worlds, mts, fullmatches, - overlayed) + nonoverlayed) else mt = ccall(:jl_method_table_for, Any, (Any,), atype) if mt === nothing @@ -329,7 +329,7 @@ function find_matching_methods(argtypes::Vector{Any}, @nospecialize(atype), meth matches.valid_worlds, mt, fullmatch, - overlayed) + !overlayed) end end @@ -702,7 +702,7 @@ function concrete_eval_eligible(interp::AbstractInterpreter, @nospecialize(f), result::MethodCallResult, arginfo::ArgInfo, sv::InferenceState) # disable concrete-evaluation since this function call is tainted by some overlayed # method and currently there is no direct way to execute overlayed methods - isoverlayed(method_table(interp)) && result.edge_effects.overlayed && return false + isoverlayed(method_table(interp)) && !result.edge_effects.nonoverlayed && return false return f !== nothing && result.edge !== nothing && is_total_or_error(result.edge_effects) && @@ -1490,13 +1490,13 @@ end function abstract_invoke(interp::AbstractInterpreter, (; fargs, argtypes)::ArgInfo, sv::InferenceState) ft′ = argtype_by_index(argtypes, 2) ft = widenconst(ft′) - ft === Bottom && return CallMeta(Bottom, false), EFFECTS_THROWN + ft === Bottom && return CallMeta(Bottom, false), EFFECTS_THROWS (types, isexact, isconcrete, istype) = instanceof_tfunc(argtype_by_index(argtypes, 3)) - types === Bottom && return CallMeta(Bottom, false), EFFECTS_THROWN + types === Bottom && return CallMeta(Bottom, false), EFFECTS_THROWS isexact || return CallMeta(Any, false), Effects() argtype = argtypes_to_type(argtype_tail(argtypes, 4)) nargtype = typeintersect(types, argtype) - nargtype === Bottom && return CallMeta(Bottom, false), EFFECTS_THROWN + nargtype === Bottom && return CallMeta(Bottom, false), EFFECTS_THROWS nargtype isa DataType || return CallMeta(Any, false), Effects() # other cases are not implemented below isdispatchelem(ft) || return CallMeta(Any, false), Effects() # check that we might not have a subtype of `ft` at runtime, before doing supertype lookup below ft = ft::DataType @@ -1530,6 +1530,7 @@ function abstract_invoke(interp::AbstractInterpreter, (; fargs, argtypes)::ArgIn (; rt, effects, const_result) = const_call_result end end + effects = Effects(effects; nonoverlayed=!overlayed) return CallMeta(from_interprocedural!(rt, sv, arginfo, sig), InvokeCallInfo(match, const_result)), effects end @@ -1575,12 +1576,12 @@ function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), end end end - tristate_merge!(sv, Effects(; overlayed=false)) # TODO + tristate_merge!(sv, EFFECTS_UNKNOWN) # TODO return CallMeta(Any, false) elseif f === TypeVar # Manually look through the definition of TypeVar to # make sure to be able to get `PartialTypeVar`s out. - tristate_merge!(sv, Effects(; overlayed=false)) # TODO + tristate_merge!(sv, EFFECTS_UNKNOWN) # TODO (la < 2 || la > 4) && return CallMeta(Union{}, false) n = argtypes[2] ub_var = Const(Any) @@ -1593,17 +1594,17 @@ function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), end return CallMeta(typevar_tfunc(n, lb_var, ub_var), false) elseif f === UnionAll - tristate_merge!(sv, Effects(; overlayed=false)) # TODO + tristate_merge!(sv, EFFECTS_UNKNOWN) # TODO return CallMeta(abstract_call_unionall(argtypes), false) elseif f === Tuple && la == 2 - tristate_merge!(sv, Effects(; overlayed=false)) # TODO + tristate_merge!(sv, EFFECTS_UNKNOWN) # TODO aty = argtypes[2] ty = isvarargtype(aty) ? unwrapva(aty) : widenconst(aty) if !isconcretetype(ty) return CallMeta(Tuple, false) end elseif is_return_type(f) - tristate_merge!(sv, Effects(; overlayed=false)) # TODO + tristate_merge!(sv, EFFECTS_UNKNOWN) # TODO return return_type_tfunc(interp, argtypes, sv) elseif la == 2 && istopfunction(f, :!) # handle Conditional propagation through !Bool @@ -1946,21 +1947,21 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e), effects.effect_free ? ALWAYS_TRUE : TRISTATE_UNKNOWN, effects.nothrow ? ALWAYS_TRUE : TRISTATE_UNKNOWN, effects.terminates_globally ? ALWAYS_TRUE : TRISTATE_UNKNOWN, - #=overlayed=#false + #=nonoverlayed=#true )) else - tristate_merge!(sv, Effects(; overlayed=false)) + tristate_merge!(sv, EFFECTS_UNKNOWN) end elseif ehead === :cfunction - tristate_merge!(sv, Effects(; overlayed=false)) + tristate_merge!(sv, EFFECTS_UNKNOWN) t = e.args[1] isa(t, Type) || (t = Any) abstract_eval_cfunction(interp, e, vtypes, sv) elseif ehead === :method - tristate_merge!(sv, Effects(; overlayed=false)) + tristate_merge!(sv, EFFECTS_UNKNOWN) t = (length(e.args) == 1) ? Any : Nothing elseif ehead === :copyast - tristate_merge!(sv, Effects(; overlayed=false)) + tristate_merge!(sv, EFFECTS_UNKNOWN) t = abstract_eval_value(interp, e.args[1], vtypes, sv) if t isa Const && t.val isa Expr # `copyast` makes copies of Exprs @@ -2286,7 +2287,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) effect_free=ALWAYS_FALSE, nothrow=TRISTATE_UNKNOWN)) elseif !isa(lhs, SSAValue) - tristate_merge!(frame, Effects(; overlayed=false)) + tristate_merge!(frame, EFFECTS_UNKNOWN) end elseif hd === :method stmt = stmt::Expr diff --git a/base/compiler/inferencestate.jl b/base/compiler/inferencestate.jl index db6ab574e3859..c920a0a64e17f 100644 --- a/base/compiler/inferencestate.jl +++ b/base/compiler/inferencestate.jl @@ -134,7 +134,7 @@ mutable struct InferenceState #=parent=#nothing, #=cached=#cache === :global, #=inferred=#false, #=dont_work_on_me=#false, - #=ipo_effects=#Effects(consistent, ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, false, inbounds_taints_consistency), + #=ipo_effects=#Effects(EFFECTS_TOTAL; consistent, inbounds_taints_consistency), interp) result.result = frame cache !== :no && push!(get_inference_cache(interp), result) diff --git a/base/compiler/methodtable.jl b/base/compiler/methodtable.jl index da493cf9a9ef5..f2e7a4ec2cb2c 100644 --- a/base/compiler/methodtable.jl +++ b/base/compiler/methodtable.jl @@ -46,7 +46,7 @@ getindex(result::MethodLookupResult, idx::Int) = getindex(result.matches, idx):: Find all methods in the given method table `view` that are applicable to the given signature `sig`. If no applicable methods are found, an empty result is returned. If the number of applicable methods exceeded the specified limit, `missing` is returned. -`overlayed` indicates if any matching method is defined in an overlayed method table. +`overlayed` indicates if any of the matching methods comes from an overlayed method table. """ function findall(@nospecialize(sig::Type), table::InternalMethodTable; limit::Int=Int(typemax(Int32))) result = _findall(sig, nothing, table.world, limit) @@ -101,15 +101,15 @@ It is possible that no method is an upper bound of `sig`, or it is possible that among the upper bounds, there is no least element. In both cases `nothing` is returned. -`overlayed` indicates if the matching method is defined in an overlayed method table. +`overlayed` indicates if any of the matching methods comes from an overlayed method table. """ function findsup(@nospecialize(sig::Type), table::InternalMethodTable) - return (_findsup(sig, nothing, table.world)..., false) + return (_findsup(sig, nothing, table.world)..., true) end function findsup(@nospecialize(sig::Type), table::OverlayMethodTable) match, valid_worlds = _findsup(sig, table.mt, table.world) - match !== nothing && return match, valid_worlds, true + match !== nothing && return match, valid_worlds, false # fall back to the internal method table fallback_match, fallback_valid_worlds = _findsup(sig, nothing, table.world) return ( @@ -117,7 +117,7 @@ function findsup(@nospecialize(sig::Type), table::OverlayMethodTable) WorldRange( max(valid_worlds.min_world, fallback_valid_worlds.min_world), min(valid_worlds.max_world, fallback_valid_worlds.max_world)), - false) + true) end function _findsup(@nospecialize(sig::Type), mt::Union{Nothing,Core.MethodTable}, world::UInt) diff --git a/base/compiler/ssair/show.jl b/base/compiler/ssair/show.jl index 76cbcbd4d5d7d..f4c826a45156f 100644 --- a/base/compiler/ssair/show.jl +++ b/base/compiler/ssair/show.jl @@ -803,7 +803,7 @@ function Base.show(io::IO, e::Core.Compiler.Effects) print(io, ',') printstyled(io, string(tristate_letter(e.terminates), 't'); color=tristate_color(e.terminates)) print(io, ')') - e.overlayed && printstyled(io, '′'; color=:red) + e.nonoverlayed || printstyled(io, '′'; color=:red) end @specialize diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index 25f26a20b115a..a92a2bc977262 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -1756,11 +1756,11 @@ function builtin_effects(f::Builtin, argtypes::Vector{Any}, rt) if (f === Core.getfield || f === Core.isdefined) && length(argtypes) >= 3 # consistent if the argtype is immutable if isvarargtype(argtypes[2]) - return Effects(; effect_free=ALWAYS_TRUE, terminates=ALWAYS_TRUE, overlayed=false) + return Effects(; effect_free=ALWAYS_TRUE, terminates=ALWAYS_TRUE, nonoverlayed=true) end s = widenconst(argtypes[2]) if isType(s) || !isa(s, DataType) || isabstracttype(s) - return Effects(; effect_free=ALWAYS_TRUE, terminates=ALWAYS_TRUE, overlayed=false) + return Effects(; effect_free=ALWAYS_TRUE, terminates=ALWAYS_TRUE, nonoverlayed=true) end s = s::DataType ipo_consistent = !ismutabletype(s) @@ -1785,13 +1785,10 @@ function builtin_effects(f::Builtin, argtypes::Vector{Any}, rt) end effect_free = contains_is(_EFFECT_FREE_BUILTINS, f) || contains_is(_PURE_BUILTINS, f) - return Effects( - ipo_consistent ? ALWAYS_TRUE : ALWAYS_FALSE, - effect_free ? ALWAYS_TRUE : ALWAYS_FALSE, - nothrow ? ALWAYS_TRUE : TRISTATE_UNKNOWN, - #=terminates=#ALWAYS_TRUE, - #=overlayed=#false, - ) + return Effects(EFFECTS_TOTAL; + consistent = ipo_consistent ? ALWAYS_TRUE : ALWAYS_FALSE, + effect_free = effect_free ? ALWAYS_TRUE : ALWAYS_FALSE, + nothrow = nothrow ? ALWAYS_TRUE : TRISTATE_UNKNOWN) end function builtin_nothrow(@nospecialize(f), argtypes::Array{Any, 1}, @nospecialize(rt)) @@ -1957,24 +1954,19 @@ function intrinsic_effects(f::IntrinsicFunction, argtypes::Vector{Any}) return Effects() end - ipo_consistent = !(f === Intrinsics.pointerref || # this one is volatile - f === Intrinsics.arraylen || # this one is volatile + ipo_consistent = !( + f === Intrinsics.pointerref || # this one is volatile + f === Intrinsics.arraylen || # this one is volatile f === Intrinsics.sqrt_llvm_fast || # this one may differ at runtime (by a few ulps) - f === Intrinsics.have_fma || # this one depends on the runtime environment - f === Intrinsics.cglobal) # cglobal lookup answer changes at runtime - + f === Intrinsics.have_fma || # this one depends on the runtime environment + f === Intrinsics.cglobal) # cglobal lookup answer changes at runtime effect_free = !(f === Intrinsics.pointerset) + nothrow = !isvarargtype(argtypes[end]) && intrinsic_nothrow(f, argtypes[2:end]) - nothrow = isvarargtype(argtypes[end]) ? false : - intrinsic_nothrow(f, argtypes[2:end]) - - return Effects( - ipo_consistent ? ALWAYS_TRUE : ALWAYS_FALSE, - effect_free ? ALWAYS_TRUE : ALWAYS_FALSE, - nothrow ? ALWAYS_TRUE : TRISTATE_UNKNOWN, - #=terminates=#ALWAYS_TRUE, - #=overlayed=#false, - ) + return Effects(EFFECTS_TOTAL; + consistent = ipo_consistent ? ALWAYS_TRUE : ALWAYS_FALSE, + effect_free = effect_free ? ALWAYS_TRUE : ALWAYS_FALSE, + nothrow = nothrow ? ALWAYS_TRUE : TRISTATE_UNKNOWN) end # TODO: this function is a very buggy and poor model of the return_type function diff --git a/base/compiler/types.jl b/base/compiler/types.jl index 50ae8ecba6705..492cc22dfe3e1 100644 --- a/base/compiler/types.jl +++ b/base/compiler/types.jl @@ -38,7 +38,7 @@ struct Effects effect_free::TriState nothrow::TriState terminates::TriState - overlayed::Bool + nonoverlayed::Bool # This effect is currently only tracked in inference and modified # :consistent before caching. We may want to track it in the future. inbounds_taints_consistency::Bool @@ -48,33 +48,34 @@ function Effects( effect_free::TriState, nothrow::TriState, terminates::TriState, - overlayed::Bool) + nonoverlayed::Bool) return Effects( consistent, effect_free, nothrow, terminates, - overlayed, + nonoverlayed, false) end -const EFFECTS_TOTAL = Effects(ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, false) -const EFFECTS_THROWN = Effects(ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_FALSE, ALWAYS_TRUE, false) -const EFFECTS_UNKNOWN = Effects(TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, true) +const EFFECTS_TOTAL = Effects(ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, true) +const EFFECTS_THROWS = Effects(ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_FALSE, ALWAYS_TRUE, true) +const EFFECTS_UNKNOWN = Effects(TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, true) # mostly unknown, but it's not overlayed at least (e.g. it's not a call) +const EFFECTS_UNKNOWN′ = Effects(TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, false) # unknown, really -function Effects(e::Effects = EFFECTS_UNKNOWN; +function Effects(e::Effects = EFFECTS_UNKNOWN′; consistent::TriState = e.consistent, effect_free::TriState = e.effect_free, nothrow::TriState = e.nothrow, terminates::TriState = e.terminates, - overlayed::Bool = e.overlayed, + nonoverlayed::Bool = e.nonoverlayed, inbounds_taints_consistency::Bool = e.inbounds_taints_consistency) return Effects( consistent, effect_free, nothrow, terminates, - overlayed, + nonoverlayed, inbounds_taints_consistency) end @@ -97,7 +98,7 @@ function encode_effects(e::Effects) (e.effect_free.state << 2) | (e.nothrow.state << 4) | (e.terminates.state << 6) | - (UInt32(e.overlayed) << 8) + (UInt32(e.nonoverlayed) << 8) end function decode_effects(e::UInt32) return Effects( @@ -119,7 +120,7 @@ function tristate_merge(old::Effects, new::Effects) old.nothrow, new.nothrow), tristate_merge( old.terminates, new.terminates), - old.overlayed | new.overlayed, + old.nonoverlayed & new.nonoverlayed, old.inbounds_taints_consistency | new.inbounds_taints_consistency) end @@ -169,7 +170,7 @@ mutable struct InferenceResult arginfo#=::Union{Nothing,Tuple{ArgInfo,InferenceState}}=# = nothing) argtypes, overridden_by_const = matching_cache_argtypes(linfo, arginfo) return new(linfo, argtypes, overridden_by_const, Any, nothing, - WorldRange(), Effects(; overlayed=false), Effects(; overlayed=false), nothing) + WorldRange(), Effects(), Effects(), nothing) end end diff --git a/src/julia.h b/src/julia.h index b15008dee91e3..f88371e561ca3 100644 --- a/src/julia.h +++ b/src/julia.h @@ -400,7 +400,7 @@ typedef struct _jl_code_instance_t { uint8_t ipo_effect_free:2; uint8_t ipo_nothrow:2; uint8_t ipo_terminates:2; - uint8_t ipo_overlayed:1; + uint8_t ipo_nonoverlayed:1; } ipo_purity_flags; }; union { @@ -410,7 +410,7 @@ typedef struct _jl_code_instance_t { uint8_t effect_free:2; uint8_t nothrow:2; uint8_t terminates:2; - uint8_t overlayed:1; + uint8_t nonoverlayed:1; } purity_flags; }; jl_value_t *argescapes; // escape information of call arguments From 0b1be4bc056874224c5a852c890f50243de1e4de Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Thu, 31 Mar 2022 16:28:03 +0900 Subject: [PATCH 08/68] effects: add reflection utility for the new effect analysis (#44785) This commit adds new reflection utility named `Base.infer_effects` that works in the same way as `Base.return_types` but returns inferred effects instead. It would be helpful to test that certain method call has an expected effects. For example, we can now remove `Base.@pure` annotation from the definition of `BroadcastStyle(a::A, b::B) where {A<:AbstractArrayStyle{M},B<:AbstractArrayStyle{N}} where {M,N}` and checks it's still eligible for concrete evaluation like this (see for the context): ```julia julia> import Base.Broadcast: AbstractArrayStyle, DefaultArrayStyle, Unknown julia> function BroadcastStyle(a::A, b::B) where {A<:AbstractArrayStyle{M},B<:AbstractArrayStyle{N}} where {M,N} if Base.typename(A) === Base.typename(B) return A(Val(max(M, N))) end return Unknown() end BroadcastStyle (generic function with 1 method) julia> # test that the above definition is eligible for concrete evaluation @test Base.infer_effects(BroadcastStyle, (DefaultArrayStyle{1},DefaultArrayStyle{2},)) |> Core.Compiler.is_total_or_error Test Passed ``` Co-authored-by: Takafumi Arakaki --- base/compiler/abstractinterpretation.jl | 4 ++-- base/compiler/typeinfer.jl | 15 +++++++++--- base/compiler/types.jl | 20 ++++++++++------ base/reflection.jl | 29 ++++++++++++++++++++++ test/reflection.jl | 32 +++++++++++++++++++++++++ 5 files changed, 88 insertions(+), 12 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 06d1a2c7d3e58..aea731c43d1dc 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -40,7 +40,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), # aren't any in the throw block either to enable other optimizations. add_remark!(interp, sv, "Skipped call in throw block") nonoverlayed = false - if isoverlayed(method_table(interp)) && sv.ipo_effects.nonoverlayed + if isoverlayed(method_table(interp)) && is_nonoverlayed(sv.ipo_effects) # as we may want to concrete-evaluate this frame in cases when there are # no overlayed calls, try an additional effort now to check if this call # isn't overlayed rather than just handling it conservatively @@ -702,7 +702,7 @@ function concrete_eval_eligible(interp::AbstractInterpreter, @nospecialize(f), result::MethodCallResult, arginfo::ArgInfo, sv::InferenceState) # disable concrete-evaluation since this function call is tainted by some overlayed # method and currently there is no direct way to execute overlayed methods - isoverlayed(method_table(interp)) && !result.edge_effects.nonoverlayed && return false + isoverlayed(method_table(interp)) && !is_nonoverlayed(result.edge_effects) && return false return f !== nothing && result.edge !== nothing && is_total_or_error(result.edge_effects) && diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index c72d03a226472..7f7770e6f9f4b 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -891,15 +891,24 @@ end # compute an inferred AST and return type function typeinf_code(interp::AbstractInterpreter, method::Method, @nospecialize(atype), sparams::SimpleVector, run_optimizer::Bool) + frame = typeinf_frame(interp, method, atype, sparams, run_optimizer) + frame === nothing && return nothing, Any + frame.inferred || return nothing, Any + code = frame.src + rt = widenconst(ignorelimited(frame.result.result)) + return code, rt +end + +# compute an inferred frame +function typeinf_frame(interp::AbstractInterpreter, method::Method, @nospecialize(atype), sparams::SimpleVector, run_optimizer::Bool) mi = specialize_method(method, atype, sparams)::MethodInstance ccall(:jl_typeinf_begin, Cvoid, ()) result = InferenceResult(mi) frame = InferenceState(result, run_optimizer ? :global : :no, interp) - frame === nothing && return (nothing, Any) + frame === nothing && return nothing typeinf(interp, frame) ccall(:jl_typeinf_end, Cvoid, ()) - frame.inferred || return (nothing, Any) - return (frame.src, widenconst(ignorelimited(result.result))) + return frame end # compute (and cache) an inferred AST and return type diff --git a/base/compiler/types.jl b/base/compiler/types.jl index 492cc22dfe3e1..b5ae060dd5394 100644 --- a/base/compiler/types.jl +++ b/base/compiler/types.jl @@ -79,19 +79,25 @@ function Effects(e::Effects = EFFECTS_UNKNOWN′; inbounds_taints_consistency) end +is_consistent(effects::Effects) = effects.consistent === ALWAYS_TRUE +is_effect_free(effects::Effects) = effects.effect_free === ALWAYS_TRUE +is_nothrow(effects::Effects) = effects.nothrow === ALWAYS_TRUE +is_terminates(effects::Effects) = effects.terminates === ALWAYS_TRUE +is_nonoverlayed(effects::Effects) = effects.nonoverlayed + is_total_or_error(effects::Effects) = - effects.consistent === ALWAYS_TRUE && - effects.effect_free === ALWAYS_TRUE && - effects.terminates === ALWAYS_TRUE + is_consistent(effects) && + is_effect_free(effects) && + is_terminates(effects) is_total(effects::Effects) = is_total_or_error(effects) && - effects.nothrow === ALWAYS_TRUE + is_nothrow(effects) is_removable_if_unused(effects::Effects) = - effects.effect_free === ALWAYS_TRUE && - effects.terminates === ALWAYS_TRUE && - effects.nothrow === ALWAYS_TRUE + is_effect_free(effects) && + is_terminates(effects) && + is_nothrow(effects) function encode_effects(e::Effects) return (e.consistent.state << 0) | diff --git a/base/reflection.jl b/base/reflection.jl index 484dc8b586664..d15b3e84cc43d 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -1306,6 +1306,35 @@ function return_types(@nospecialize(f), @nospecialize(types=default_tt(f)); return rt end +function infer_effects(@nospecialize(f), @nospecialize(types=default_tt(f)); + world = get_world_counter(), + interp = Core.Compiler.NativeInterpreter(world)) + ccall(:jl_is_in_pure_context, Bool, ()) && error("code reflection cannot be used from generated functions") + types = to_tuple_type(types) + if isa(f, Core.Builtin) + args = Any[types.parameters...] + rt = Core.Compiler.builtin_tfunction(interp, f, args, nothing) + return Core.Compiler.builtin_effects(f, args, rt) + else + effects = Core.Compiler.EFFECTS_TOTAL + matches = _methods(f, types, -1, world)::Vector + if isempty(matches) + # although this call is known to throw MethodError (thus `nothrow=ALWAYS_FALSE`), + # still mark it `TRISTATE_UNKNOWN` just in order to be consistent with a result + # derived by the effect analysis, which can't prove guaranteed throwness at this moment + return Core.Compiler.Effects(effects; nothrow=Core.Compiler.TRISTATE_UNKNOWN) + end + for match in matches + match = match::Core.MethodMatch + frame = Core.Compiler.typeinf_frame(interp, + match.method, match.spec_types, match.sparams, #=run_optimizer=#false) + frame === nothing && return Core.Compiler.Effects() + effects = Core.Compiler.tristate_merge(effects, frame.ipo_effects) + end + return effects + end +end + """ print_statement_costs(io::IO, f, types) diff --git a/test/reflection.jl b/test/reflection.jl index b1a5b6eb822a3..10973f4679380 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -964,3 +964,35 @@ end @eval m f4(a) = return @test Base.default_tt(m.f4) == Tuple end + +Base.@assume_effects :terminates_locally function issue41694(x::Int) + res = 1 + 1 < x < 20 || throw("bad") + while x > 1 + res *= x + x -= 1 + end + return res +end +maybe_effectful(x::Int) = 42 +maybe_effectful(x::Any) = unknown_operation() +function f_no_methods end + +@testset "infer_effects" begin + @test Base.infer_effects(issue41694, (Int,)) |> Core.Compiler.is_terminates + @test Base.infer_effects((Int,)) do x + issue41694(x) + end |> Core.Compiler.is_terminates + @test Base.infer_effects(issue41694) |> Core.Compiler.is_terminates # use `default_tt` + let effects = Base.infer_effects(maybe_effectful, (Any,)) # union split + @test !Core.Compiler.is_consistent(effects) + @test !Core.Compiler.is_effect_free(effects) + @test !Core.Compiler.is_nothrow(effects) + @test !Core.Compiler.is_terminates(effects) + @test !Core.Compiler.is_nonoverlayed(effects) + end + @test Base.infer_effects(f_no_methods) |> !Core.Compiler.is_nothrow + # builtins + @test Base.infer_effects(typeof, (Any,)) |> Core.Compiler.is_total + @test Base.infer_effects(===, (Any,Any)) |> Core.Compiler.is_total +end From 1693503783d137ef08ededcef9ad397a188f6091 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 1 Apr 2022 10:01:45 +0900 Subject: [PATCH 09/68] follow up #44786, fix `findsup` to return correct `overlayed` value (#44804) --- base/compiler/methodtable.jl | 6 +++--- test/compiler/AbstractInterpreter.jl | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/base/compiler/methodtable.jl b/base/compiler/methodtable.jl index f2e7a4ec2cb2c..7aa686009c1af 100644 --- a/base/compiler/methodtable.jl +++ b/base/compiler/methodtable.jl @@ -104,12 +104,12 @@ In both cases `nothing` is returned. `overlayed` indicates if any of the matching methods comes from an overlayed method table. """ function findsup(@nospecialize(sig::Type), table::InternalMethodTable) - return (_findsup(sig, nothing, table.world)..., true) + return (_findsup(sig, nothing, table.world)..., false) end function findsup(@nospecialize(sig::Type), table::OverlayMethodTable) match, valid_worlds = _findsup(sig, table.mt, table.world) - match !== nothing && return match, valid_worlds, false + match !== nothing && return match, valid_worlds, true # fall back to the internal method table fallback_match, fallback_valid_worlds = _findsup(sig, nothing, table.world) return ( @@ -117,7 +117,7 @@ function findsup(@nospecialize(sig::Type), table::OverlayMethodTable) WorldRange( max(valid_worlds.min_world, fallback_valid_worlds.min_world), min(valid_worlds.max_world, fallback_valid_worlds.max_world)), - true) + false) end function _findsup(@nospecialize(sig::Type), mt::Union{Nothing,Core.MethodTable}, world::UInt) diff --git a/test/compiler/AbstractInterpreter.jl b/test/compiler/AbstractInterpreter.jl index 8d5a50112cbf9..9d1be42891042 100644 --- a/test/compiler/AbstractInterpreter.jl +++ b/test/compiler/AbstractInterpreter.jl @@ -43,6 +43,8 @@ CC.method_table(interp::MTOverlayInterp) = CC.OverlayMethodTable(CC.get_world_co strangesin(x) = sin(x) @overlay OverlayedMT strangesin(x::Float64) = iszero(x) ? nothing : cos(x) + +# inference should use the overlayed method table @test Base.return_types((Float64,); interp=MTOverlayInterp()) do x strangesin(x) end |> only === Union{Float64,Nothing} @@ -50,6 +52,22 @@ end |> only === Union{Float64,Nothing} Base.@invoke strangesin(x::Float64) end |> only === Union{Float64,Nothing} +# effect analysis should figure out that the overlayed method is used +@test Base.infer_effects((Float64,); interp=MTOverlayInterp()) do x + strangesin(x) +end |> !Core.Compiler.is_nonoverlayed +@test Base.infer_effects((Any,); interp=MTOverlayInterp()) do x + Base.@invoke strangesin(x::Float64) +end |> !Core.Compiler.is_nonoverlayed + +# but it should never apply for the native compilation +@test Base.infer_effects((Float64,)) do x + strangesin(x) +end |> Core.Compiler.is_nonoverlayed +@test Base.infer_effects((Any,)) do x + Base.@invoke strangesin(x::Float64) +end |> Core.Compiler.is_nonoverlayed + # fallback to the internal method table @test Base.return_types((Int,); interp=MTOverlayInterp()) do x cos(x) From 5ff15cbe319c58c476a6004dc074a61e236f3dce Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 1 Apr 2022 13:54:43 +0900 Subject: [PATCH 10/68] effects: replaces usages of `ALWAYS_FALSE` with `TRISTATE_UNKNOWN` (#44808) Although they are basically equivalent in the current implementation, it would be more correct conceptually if we propagate `TRISTATE_UNKNOWN` instead of `ALWAYS_TRUE`, as it is hard or impossible to derive a global conclusion from a local analysis on each statement in most places. This commit also adds a docstring that simply explains the design of the effect analysis. --- base/compiler/abstractinterpretation.jl | 18 ++++++++------- base/compiler/tfuncs.jl | 8 +++---- base/compiler/types.jl | 30 ++++++++++++++++++++++++- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index aea731c43d1dc..5252e41244ccc 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -1858,7 +1858,7 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e), t = Bottom tristate_merge!(sv, Effects(EFFECTS_TOTAL; # consistent = ALWAYS_TRUE, # N.B depends on !ismutabletype(t) above - nothrow = ALWAYS_FALSE)) + nothrow = TRISTATE_UNKNOWN)) @goto t_computed elseif !isa(at, Const) allconst = false @@ -1887,8 +1887,8 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e), is_nothrow = false end tristate_merge!(sv, Effects(EFFECTS_TOTAL; - consistent = !ismutabletype(t) ? ALWAYS_TRUE : ALWAYS_FALSE, - nothrow = is_nothrow ? ALWAYS_TRUE : ALWAYS_FALSE)) + consistent = !ismutabletype(t) ? ALWAYS_TRUE : TRISTATE_UNKNOWN, + nothrow = is_nothrow ? ALWAYS_TRUE : TRISTATE_UNKNOWN)) elseif ehead === :splatnew t, isexact = instanceof_tfunc(abstract_eval_value(interp, e.args[1], vtypes, sv)) is_nothrow = false # TODO: More precision @@ -1906,8 +1906,8 @@ function abstract_eval_statement(interp::AbstractInterpreter, @nospecialize(e), end end tristate_merge!(sv, Effects(EFFECTS_TOTAL; - consistent = ismutabletype(t) ? ALWAYS_FALSE : ALWAYS_TRUE, - nothrow = is_nothrow ? ALWAYS_TRUE : ALWAYS_FALSE)) + consistent = ismutabletype(t) ? TRISTATE_UNKNOWN : ALWAYS_TRUE, + nothrow = is_nothrow ? ALWAYS_TRUE : TRISTATE_UNKNOWN)) elseif ehead === :new_opaque_closure tristate_merge!(sv, Effects()) # TODO t = Union{} @@ -2031,9 +2031,11 @@ function abstract_eval_global(M::Module, s::Symbol, frame::InferenceState) ty = abstract_eval_global(M, s) isa(ty, Const) && return ty if isdefined(M,s) - tristate_merge!(frame, Effects(EFFECTS_TOTAL; consistent=ALWAYS_FALSE)) + tristate_merge!(frame, Effects(EFFECTS_TOTAL; consistent=TRISTATE_UNKNOWN)) else - tristate_merge!(frame, Effects(EFFECTS_TOTAL; consistent=ALWAYS_FALSE, nothrow=ALWAYS_FALSE)) + tristate_merge!(frame, Effects(EFFECTS_TOTAL; + consistent=TRISTATE_UNKNOWN, + nothrow=TRISTATE_UNKNOWN)) end return ty end @@ -2284,7 +2286,7 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState) changes = StateUpdate(lhs, VarState(t, false), changes, false) elseif isa(lhs, GlobalRef) tristate_merge!(frame, Effects(EFFECTS_TOTAL; - effect_free=ALWAYS_FALSE, + effect_free=TRISTATE_UNKNOWN, nothrow=TRISTATE_UNKNOWN)) elseif !isa(lhs, SSAValue) tristate_merge!(frame, EFFECTS_UNKNOWN) diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index a92a2bc977262..23a18ae8d8aeb 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -1786,8 +1786,8 @@ function builtin_effects(f::Builtin, argtypes::Vector{Any}, rt) effect_free = contains_is(_EFFECT_FREE_BUILTINS, f) || contains_is(_PURE_BUILTINS, f) return Effects(EFFECTS_TOTAL; - consistent = ipo_consistent ? ALWAYS_TRUE : ALWAYS_FALSE, - effect_free = effect_free ? ALWAYS_TRUE : ALWAYS_FALSE, + consistent = ipo_consistent ? ALWAYS_TRUE : TRISTATE_UNKNOWN, + effect_free = effect_free ? ALWAYS_TRUE : TRISTATE_UNKNOWN, nothrow = nothrow ? ALWAYS_TRUE : TRISTATE_UNKNOWN) end @@ -1964,8 +1964,8 @@ function intrinsic_effects(f::IntrinsicFunction, argtypes::Vector{Any}) nothrow = !isvarargtype(argtypes[end]) && intrinsic_nothrow(f, argtypes[2:end]) return Effects(EFFECTS_TOTAL; - consistent = ipo_consistent ? ALWAYS_TRUE : ALWAYS_FALSE, - effect_free = effect_free ? ALWAYS_TRUE : ALWAYS_FALSE, + consistent = ipo_consistent ? ALWAYS_TRUE : TRISTATE_UNKNOWN, + effect_free = effect_free ? ALWAYS_TRUE : TRISTATE_UNKNOWN, nothrow = nothrow ? ALWAYS_TRUE : TRISTATE_UNKNOWN) end diff --git a/base/compiler/types.jl b/base/compiler/types.jl index b5ae060dd5394..94b6a1c4c8743 100644 --- a/base/compiler/types.jl +++ b/base/compiler/types.jl @@ -33,6 +33,34 @@ function tristate_merge(old::TriState, new::TriState) return new end +""" + effects::Effects + +Represents computational effects of a method call. + +The effects are composed of the following set of different properties: +- `effects.consistent::TriState`: this method is guaranteed to return or terminate consistently +- `effect_free::TriState`: this method is free from externally semantically visible side effects +- `nothrow::TriState`: this method is guaranteed to not throw an exception +- `terminates::TriState`: this method is guaranteed to terminate +- `nonoverlayed::Bool`: indicates that any methods that may be called within this method + are not defined in an [overlayed method table](@ref OverlayMethodTable) +See [`Base.@assume_effects`](@ref) for more detailed explanation on the definitions of these properties. + +Along the abstract interpretation, `Effects` at each statement are analyzed locally and +they are merged into the single global `Effects` that represents the entire effects of +the analyzed method (see `tristate_merge!`). +Each effect property is represented as tri-state and managed separately. +The tri-state consists of `ALWAYS_TRUE`, `TRISTATE_UNKNOWN` and `ALWAYS_FALSE`. +An effect property is initialized with `ALWAYS_TRUE` and then transitioned towards +`TRISTATE_UNKNOWN` or `ALWAYS_FALSE`. When we find a statement that has some effect, +`ALWAYS_TRUE` is propagated if that effect is known to _always_ happen, otherwise +`TRISTATE_UNKNOWN` is propagated. If a property is known to be `ALWAYS_FALSE`, +there is no need to do additional analysis as it can not be refined anyway. +Note that however, within the current data-flow analysis design, it is hard to derive a global +conclusion from a local analysis on each statement, and as a result, the effect analysis +usually propagates `TRISTATE_UNKNOWN` currently. +""" struct Effects consistent::TriState effect_free::TriState @@ -59,7 +87,7 @@ function Effects( end const EFFECTS_TOTAL = Effects(ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_TRUE, true) -const EFFECTS_THROWS = Effects(ALWAYS_TRUE, ALWAYS_TRUE, ALWAYS_FALSE, ALWAYS_TRUE, true) +const EFFECTS_THROWS = Effects(ALWAYS_TRUE, ALWAYS_TRUE, TRISTATE_UNKNOWN, ALWAYS_TRUE, true) const EFFECTS_UNKNOWN = Effects(TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, true) # mostly unknown, but it's not overlayed at least (e.g. it's not a call) const EFFECTS_UNKNOWN′ = Effects(TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, TRISTATE_UNKNOWN, false) # unknown, really From 87ea300f25ceefbec30e9d9d0ebfb2b51e9a5042 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Sat, 2 Apr 2022 07:39:36 +0900 Subject: [PATCH 11/68] add new `:total_may_throw` utility setting for `@assume_effects` (#44775) This setting is particularly useful since it allows the compiler to evaluate a call of the applied method when all the call arguments are fully known, no matter if the call results in an error or not. This commit also adds some more explanations on the difference between `@pure` and `@assume_effects`. --- base/compiler/abstractinterpretation.jl | 2 +- base/compiler/types.jl | 4 +- base/expr.jl | 61 +++++++++++++++++++++---- test/compiler/inline.jl | 6 +-- 4 files changed, 59 insertions(+), 14 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 5252e41244ccc..1b7a02dbb01f9 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -705,7 +705,7 @@ function concrete_eval_eligible(interp::AbstractInterpreter, isoverlayed(method_table(interp)) && !is_nonoverlayed(result.edge_effects) && return false return f !== nothing && result.edge !== nothing && - is_total_or_error(result.edge_effects) && + is_concrete_eval_eligible(result.edge_effects) && is_all_const_arg(arginfo) end diff --git a/base/compiler/types.jl b/base/compiler/types.jl index 94b6a1c4c8743..75c2da979179d 100644 --- a/base/compiler/types.jl +++ b/base/compiler/types.jl @@ -113,13 +113,13 @@ is_nothrow(effects::Effects) = effects.nothrow === ALWAYS_TRUE is_terminates(effects::Effects) = effects.terminates === ALWAYS_TRUE is_nonoverlayed(effects::Effects) = effects.nonoverlayed -is_total_or_error(effects::Effects) = +is_concrete_eval_eligible(effects::Effects) = is_consistent(effects) && is_effect_free(effects) && is_terminates(effects) is_total(effects::Effects) = - is_total_or_error(effects) && + is_concrete_eval_eligible(effects) && is_nothrow(effects) is_removable_if_unused(effects::Effects) = diff --git a/base/expr.jl b/base/expr.jl index 38e89d284c989..9e603885a0037 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -337,12 +337,17 @@ end """ @pure ex - @pure(ex) `@pure` gives the compiler a hint for the definition of a pure function, helping for type inference. -This macro is intended for internal compiler use and may be subject to changes. +!!! warning + This macro is intended for internal compiler use and may be subject to changes. + +!!! warning + In Julia 1.8 and higher, it is favorable to use [`@assume_effects`](@ref) instead of `@pure`. + This is because `@assume_effects` allows a finer grained control over Julia's purity + modeling and the effect system enables a wider range of optimizations. """ macro pure(ex) esc(isa(ex, Expr) ? pushmeta!(ex, :pure) : ex) @@ -350,7 +355,6 @@ end """ @constprop setting ex - @constprop(setting, ex) `@constprop` controls the mode of interprocedural constant propagation for the annotated function. Two `setting`s are supported: @@ -373,11 +377,35 @@ end """ @assume_effects setting... ex - @assume_effects(setting..., ex) `@assume_effects` overrides the compiler's effect modeling for the given method. `ex` must be a method definition or `@ccall` expression. +```jldoctest +julia> Base.@assume_effects :terminates_locally function pow(x) + # this :terminates_locally allows `pow` to be constant-folded + res = 1 + 1 < x < 20 || error("bad pow") + while x > 1 + res *= x + x -= 1 + end + return res + end +pow (generic function with 1 method) + +julia> code_typed() do + pow(12) + end +1-element Vector{Any}: + CodeInfo( +1 ─ return 479001600 +) => Int64 + +julia> Base.@assume_effects :total_may_throw @ccall jl_type_intersection(Vector{Int}::Any, Vector{<:Integer}::Any)::Any +Vector{Int64} (alias for Array{Int64, 1}) +``` + !!! warning Improper use of this macro causes undefined behavior (including crashes, incorrect answers, or other hard to track bugs). Use with care and only if @@ -512,11 +540,26 @@ This `setting` combines the following other assertions: - `:terminates_globally` and is a convenient shortcut. +--- +# `:total_may_throw` + +This `setting` combines the following other assertions: +- `:consistent` +- `:effect_free` +- `:terminates_globally` +and is a convenient shortcut. + !!! note - `@assume_effects :total` is similar to `@Base.pure` with the primary + This setting is particularly useful since it allows the compiler to evaluate a call of + the applied method when all the call arguments are fully known to be constant, no matter + if the call results in an error or not. + + `@assume_effects :total_may_throw` is similar to [`@pure`](@ref) with the primary distinction that the `:consistent`-cy requirement applies world-age wise rather than globally as described above. However, in particular, a method annotated - `@Base.pure` is always `:total`. + `@pure` should always be `:total` or `:total_may_throw`. + Another advantage is that effects introduced by `@assume_effects` are propagated to + callers interprocedurally while a purity defined by `@pure` is not. """ macro assume_effects(args...) (consistent, effect_free, nothrow, terminates_globally, terminates_locally) = @@ -537,12 +580,14 @@ macro assume_effects(args...) terminates_locally = true elseif setting === :total consistent = effect_free = nothrow = terminates_globally = true + elseif setting === :total_may_throw + consistent = effect_free = terminates_globally = true else throw(ArgumentError("@assume_effects $setting not supported")) end end ex = args[end] - isa(ex, Expr) || throw(ArgumentError("Bad expression `$ex` in @constprop [settings] ex")) + isa(ex, Expr) || throw(ArgumentError("Bad expression `$ex` in `@assume_effects [settings] ex`")) if ex.head === :macrocall && ex.args[1] == Symbol("@ccall") ex.args[1] = GlobalRef(Base, Symbol("@ccall_effects")) insert!(ex.args, 3, Core.Compiler.encode_effects_override(Core.Compiler.EffectsOverride( @@ -725,7 +770,7 @@ end """ @generated f - @generated(f) + `@generated` is used to annotate a function which will be generated. In the body of the generated function, only types of arguments can be read (not the values). The function returns a quoted expression evaluated when the diff --git a/test/compiler/inline.jl b/test/compiler/inline.jl index 59604bd317f2e..c3a8499b621a5 100644 --- a/test/compiler/inline.jl +++ b/test/compiler/inline.jl @@ -1088,11 +1088,11 @@ recur_termination22(x) = x * recur_termination21(x-1) recur_termination21(12) + recur_termination22(12) end -const ___CONST_DICT___ = Dict{Any,Any}(:a => 1, :b => 2) -Base.@assume_effects :consistent :effect_free :terminates_globally consteval( +const ___CONST_DICT___ = Dict{Any,Any}(Symbol(c) => i for (i, c) in enumerate('a':'z')) +Base.@assume_effects :total_may_throw concrete_eval( f, args...; kwargs...) = f(args...; kwargs...) @test fully_eliminated() do - consteval(getindex, ___CONST_DICT___, :a) + concrete_eval(getindex, ___CONST_DICT___, :a) end # https://github.com/JuliaLang/julia/issues/44732 From 520c089b2b81e6f5754acd1613534506272490bf Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Fri, 8 Apr 2022 10:42:38 +0900 Subject: [PATCH 12/68] remove `isvarargtype` assertions from `subtype` and `intersect` (#44761) fix #44735 --- base/reflection.jl | 4 ++-- src/subtype.c | 2 -- test/subtype.jl | 6 ++++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/base/reflection.jl b/base/reflection.jl index d15b3e84cc43d..223d5fab040e3 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -675,12 +675,12 @@ struct type with no fields. issingletontype(@nospecialize(t)) = (@_pure_meta; isa(t, DataType) && isdefined(t, :instance)) """ - typeintersect(T, S) + typeintersect(T::Type, S::Type) Compute a type that contains the intersection of `T` and `S`. Usually this will be the smallest such type or one close to it. """ -typeintersect(@nospecialize(a), @nospecialize(b)) = (@_pure_meta; ccall(:jl_type_intersection, Any, (Any, Any), a, b)) +typeintersect(@nospecialize(a), @nospecialize(b)) = (@_pure_meta; ccall(:jl_type_intersection, Any, (Any, Any), a::Type, b::Type)) morespecific(@nospecialize(a), @nospecialize(b)) = ccall(:jl_type_morespecific, Cint, (Any, Any), a, b) != 0 diff --git a/src/subtype.c b/src/subtype.c index eb668645552d7..c43d307e6d421 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -1258,7 +1258,6 @@ static int subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int param) } if (jl_is_unionall(y)) return subtype_unionall(x, (jl_unionall_t*)y, e, 1, param); - assert(!jl_is_vararg(x) && !jl_is_vararg(y)); if (jl_is_datatype(x) && jl_is_datatype(y)) { if (x == y) return 1; if (y == (jl_value_t*)jl_any_type) return 1; @@ -3107,7 +3106,6 @@ static jl_value_t *intersect(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int pa } if (jl_is_unionall(y)) return intersect_unionall(x, (jl_unionall_t*)y, e, 1, param); - assert(!jl_is_vararg(x) && !jl_is_vararg(y)); if (jl_is_datatype(x) && jl_is_datatype(y)) { jl_datatype_t *xd = (jl_datatype_t*)x, *yd = (jl_datatype_t*)y; if (param < 2) { diff --git a/test/subtype.jl b/test/subtype.jl index 3eca685aee84c..eff2c021b481f 100644 --- a/test/subtype.jl +++ b/test/subtype.jl @@ -1976,3 +1976,9 @@ end @testintersect(Tuple{Type{Pair{_A, S} where S<:AbstractArray{<:_A, 2}}, Dict} where _A, Tuple{Type{Pair{_A, S} where S<:AbstractArray{<:_A, 2}} where _A, Union{Array, Pair}}, Bottom) + +# https://github.com/JuliaLang/julia/issues/44735 +@test_throws TypeError(:typeassert, Type, Vararg{Int}) typeintersect(Vararg{Int}, Int) +@test_throws TypeError(:typeassert, Type, Vararg{Int}) typeintersect(Int, Vararg{Int}) +@test_throws TypeError(:typeassert, Type, 1) typeintersect(1, Int) +@test_throws TypeError(:typeassert, Type, 1) typeintersect(Int, 1) From 27fc73ef84501d4bd0ef432af37efb1e8cd896f7 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Wed, 13 Apr 2022 16:20:55 +0900 Subject: [PATCH 13/68] inference: properly compare vararg-tuple `PartialStruct`s fix #44965 --- base/compiler/typelimits.jl | 2 +- test/compiler/inference.jl | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/base/compiler/typelimits.jl b/base/compiler/typelimits.jl index d25c77deb6d2e..2c5adb92e5a09 100644 --- a/base/compiler/typelimits.jl +++ b/base/compiler/typelimits.jl @@ -313,7 +313,7 @@ function issimplertype(@nospecialize(typea), @nospecialize(typeb)) if typea isa PartialStruct aty = widenconst(typea) for i = 1:length(typea.fields) - ai = typea.fields[i] + ai = unwrapva(typea.fields[i]) bi = fieldtype(aty, i) is_lattice_equal(ai, bi) && continue tni = _typename(widenconst(ai)) diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index bf4f8391012b7..9624c35bac13d 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -4056,6 +4056,11 @@ end @test !Core.Compiler.builtin_nothrow(Core.get_binding_type, Any[Rational{Int}, Core.Const(:foo)], Any) +# https://github.com/JuliaLang/julia/issues/44965 +let t = Core.Compiler.tuple_tfunc(Any[Core.Const(42), Vararg{Any}]) + @test Core.Compiler.issimplertype(t, t) +end + # https://github.com/JuliaLang/julia/issues/44763 global x44763::Int = 0 increase_x44763!(n) = (global x44763; x44763 += n) From 72f36437efb5a38bd90907569d03429155b175fc Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Wed, 13 Apr 2022 16:22:32 +0900 Subject: [PATCH 14/68] inference: more `Vararg` handling Just found this case from code review, but this `Vararg` should be widened. --- base/compiler/abstractinterpretation.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 1b7a02dbb01f9..cf483b4f4ff73 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -1270,7 +1270,6 @@ function abstract_apply(interp::AbstractInterpreter, argtypes::Vector{Any}, sv:: # This is vararg, we're not gonna be able to do any inling, # drop the info info = nothing - tail = tuple_tail_elem(unwrapva(ct[end]), cti) push!(ctypes´, push!(ct[1:(end - 1)], tail)) else @@ -1290,8 +1289,9 @@ function abstract_apply(interp::AbstractInterpreter, argtypes::Vector{Any}, sv:: lct = length(ct) # truncate argument list at the first Vararg for i = 1:lct-1 - if isvarargtype(ct[i]) - ct[i] = tuple_tail_elem(ct[i], ct[(i+1):lct]) + cti = ct[i] + if isvarargtype(cti) + ct[i] = tuple_tail_elem(unwrapva(cti), ct[(i+1):lct]) resize!(ct, i) break end From 9d56f9a4cfbdf2826c2a1f7ca4387553963489cb Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Tue, 19 Apr 2022 10:54:36 +0900 Subject: [PATCH 15/68] inference: don't forget to add backedge for effect-free frame (#45017) Now our abstract interpretation system caches two separate lattice properties return type and call effects, and the previous optimization to avoid adding backedge for a frame whose return type is `Any` turns out to be insufficient -- now we also need to make sure that the inferred effects don't provide any useful IPO information. Example: ```julia julia> const CONST_DICT = let d = Dict() for c in 'A':'z' push!(d, c => Int(c)) end d end; julia> Base.@assume_effects :total_may_throw getcharid(c) = CONST_DICT[c]; julia> @noinline callf(f, args...) = f(args...); julia> function entry_to_be_invalidated(c) return callf(getcharid, c) end; julia> @test Base.infer_effects((Char,)) do x entry_to_be_invalidated(x) end |> Core.Compiler.is_concrete_eval_eligible Test Passed julia> @test fully_eliminated(; retval=97) do entry_to_be_invalidated('a') end Test Passed julia> getcharid(c) = CONST_DICT[c]; # now this is not eligible for concrete evaluation julia> @test Base.infer_effects((Char,)) do x entry_to_be_invalidated(x) end |> !Core.Compiler.is_concrete_eval_eligible Test Failed at REPL[10]:1 Expression: Base.infer_effects((Char,)) do x entry_to_be_invalidated(x) end |> !(Core.Compiler.is_concrete_eval_eligible) ERROR: There was an error during testing julia> @test !fully_eliminated() do entry_to_be_invalidated('a') end Test Failed at REPL[11]:1 Expression: !(fully_eliminated() do entry_to_be_invalidated('a') end) ERROR: There was an error during testing ``` --- base/compiler/abstractinterpretation.jl | 32 +++++++++++++++---------- test/compiler/inference.jl | 28 ++++++++++++++++++++++ 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index cf483b4f4ff73..b65a9751d394f 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -78,17 +78,19 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), any_const_result = false const_results = Union{InferenceResult,Nothing,ConstResult}[] multiple_matches = napplicable > 1 + fargs = arginfo.fargs + all_effects = EFFECTS_TOTAL if !matches.nonoverlayed # currently we don't have a good way to execute the overlayed method definition, # so we should give up pure/concrete eval when any of the matched methods is overlayed f = nothing - tristate_merge!(sv, Effects(EFFECTS_TOTAL; nonoverlayed=false)) + all_effects = Effects(all_effects; nonoverlayed=false) end + # try pure-evaluation val = pure_eval_call(interp, f, applicable, arginfo, sv) val !== nothing && return CallMeta(val, MethodResultPure(info)) # TODO: add some sort of edge(s) - fargs = arginfo.fargs for i in 1:napplicable match = applicable[i]::MethodMatch method = match.method @@ -124,7 +126,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), (; effects, const_result) = const_call_result end end - tristate_merge!(sv, effects) + all_effects = tristate_merge(all_effects, effects) push!(const_results, const_result) any_const_result |= const_result !== nothing this_rt = tmerge(this_rt, rt) @@ -173,7 +175,7 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), (; effects, const_result) = const_call_result end end - tristate_merge!(sv, effects) + all_effects = tristate_merge(all_effects, effects) push!(const_results, const_result) any_const_result |= const_result !== nothing end @@ -204,11 +206,11 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), if seen != napplicable # there may be unanalyzed effects within unseen dispatch candidate, # but we can still ignore nonoverlayed effect here since we already accounted for it - tristate_merge!(sv, EFFECTS_UNKNOWN) + all_effects = tristate_merge(all_effects, EFFECTS_UNKNOWN) elseif isa(matches, MethodMatches) ? (!matches.fullmatch || any_ambig(matches)) : (!_all(b->b, matches.fullmatches) || any_ambig(matches)) # Account for the fact that we may encounter a MethodError with a non-covered or ambiguous signature. - tristate_merge!(sv, Effects(EFFECTS_TOTAL; nothrow=TRISTATE_UNKNOWN)) + all_effects = Effects(all_effects; nothrow=TRISTATE_UNKNOWN) end rettype = from_interprocedural!(rettype, sv, arginfo, conditionals) @@ -223,13 +225,14 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(f), # and avoid keeping track of a more complex result type. rettype = Any end - add_call_backedges!(interp, rettype, edges, matches, atype, sv) + add_call_backedges!(interp, rettype, all_effects, edges, matches, atype, sv) if !isempty(sv.pclimitations) # remove self, if present delete!(sv.pclimitations, sv) for caller in sv.callers_in_cycle delete!(sv.pclimitations, caller) end end + tristate_merge!(sv, all_effects) return CallMeta(rettype, info) end @@ -444,11 +447,16 @@ function conditional_argtype(@nospecialize(rt), @nospecialize(sig), argtypes::Ve end end -function add_call_backedges!(interp::AbstractInterpreter, @nospecialize(rettype), edges::Vector{MethodInstance}, - matches::Union{MethodMatches,UnionSplitMethodMatches}, @nospecialize(atype), - sv::InferenceState) - # for `NativeInterpreter`, we don't add backedges when a new method couldn't refine (widen) this type - rettype === Any && return +function add_call_backedges!(interp::AbstractInterpreter, + @nospecialize(rettype), all_effects::Effects, + edges::Vector{MethodInstance}, matches::Union{MethodMatches,UnionSplitMethodMatches}, @nospecialize(atype), + sv::InferenceState) + if rettype === Any && all_effects === Effects() + # for `NativeInterpreter`, we don't need to add backedges when: + # - a new method couldn't refine (widen) this type and + # - the effects don't provide any useful information for interprocedural optimization + return + end for edge in edges add_backedge!(edge, sv) end diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index 9624c35bac13d..e36ec80b9e728 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -7,6 +7,8 @@ isdispatchelem(@nospecialize x) = !isa(x, Type) || Core.Compiler.isdispatchelem( using Random, Core.IR using InteractiveUtils: code_llvm +include("irutils.jl") + f39082(x::Vararg{T}) where {T <: Number} = x[1] let ast = only(code_typed(f39082, Tuple{Vararg{Rational}}))[1] @test ast.slottypes == Any[Const(f39082), Tuple{Vararg{Rational}}] @@ -4069,3 +4071,29 @@ invoke44763(x) = Base.@invoke increase_x44763!(x) invoke44763(42) end |> only === Int @test x44763 == 0 + +# backedge insertion for Any-typed, effect-free frame +const CONST_DICT = let d = Dict() + for c in 'A':'z' + push!(d, c => Int(c)) + end + d +end +Base.@assume_effects :total_may_throw getcharid(c) = CONST_DICT[c] +@noinline callf(f, args...) = f(args...) +function entry_to_be_invalidated(c) + return callf(getcharid, c) +end +@test Base.infer_effects((Char,)) do x + entry_to_be_invalidated(x) +end |> Core.Compiler.is_concrete_eval_eligible +@test fully_eliminated(; retval=97) do + entry_to_be_invalidated('a') +end +getcharid(c) = CONST_DICT[c] # now this is not eligible for concrete evaluation +@test Base.infer_effects((Char,)) do x + entry_to_be_invalidated(x) +end |> !Core.Compiler.is_concrete_eval_eligible +@test !fully_eliminated() do + entry_to_be_invalidated('a') +end From d49e0651b46a2ff52608484d868c7e198742803a Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Wed, 16 Mar 2022 05:38:41 -0500 Subject: [PATCH 16/68] Improve inference for string operations (#44500) This reduces invalidations when loading packages that define new AbstractString types. (cherry picked from commit 515a24240fcebc246d2bddbebd91c3800fc1fb3a) --- base/binaryplatforms.jl | 4 ++- base/io.jl | 10 ++++---- base/iostream.jl | 3 ++- base/logging.jl | 2 +- base/process.jl | 25 ++++++++++--------- base/show.jl | 4 +-- base/strings/substring.jl | 13 +++++++++- stdlib/REPL/src/LineEdit.jl | 2 +- stdlib/REPL/src/TerminalMenus/AbstractMenu.jl | 2 +- stdlib/p7zip_jll/src/p7zip_jll.jl | 2 +- 10 files changed, 41 insertions(+), 26 deletions(-) diff --git a/base/binaryplatforms.jl b/base/binaryplatforms.jl index 87c7f029f0563..e2dda00bf58e7 100644 --- a/base/binaryplatforms.jl +++ b/base/binaryplatforms.jl @@ -667,7 +667,7 @@ const libstdcxx_version_mapping = Dict{String,String}( Parses a string platform triplet back into a `Platform` object. """ -function Base.parse(::Type{Platform}, triplet::AbstractString; validate_strict::Bool = false) +function Base.parse(::Type{Platform}, triplet::String; validate_strict::Bool = false) # Helper function to collapse dictionary of mappings down into a regex of # named capture groups joined by "|" operators c(mapping) = string("(",join(["(?<$k>$v)" for (k, v) in mapping], "|"), ")") @@ -751,6 +751,8 @@ function Base.parse(::Type{Platform}, triplet::AbstractString; validate_strict:: end throw(ArgumentError("Platform `$(triplet)` is not an officially supported platform")) end +Base.parse(::Type{Platform}, triplet::AbstractString; kwargs...) = + parse(Platform, convert(String, triplet)::String; kwargs...) function Base.tryparse(::Type{Platform}, triplet::AbstractString) try diff --git a/base/io.jl b/base/io.jl index 69972249be02f..3fac2287bdabf 100644 --- a/base/io.jl +++ b/base/io.jl @@ -445,7 +445,7 @@ wait_close(io::AbstractPipe) = (wait_close(pipe_writer(io)::IO); wait_close(pipe # Exception-safe wrappers (io = open(); try f(io) finally close(io)) -write(filename::AbstractString, a1, args...) = open(io->write(io, a1, args...), filename, "w") +write(filename::AbstractString, a1, args...) = open(io->write(io, a1, args...), convert(String, filename)::String, "w") """ read(filename::AbstractString, args...) @@ -457,9 +457,9 @@ Open a file and read its contents. `args` is passed to `read`: this is equivalen Read the entire contents of a file as a string. """ -read(filename::AbstractString, args...) = open(io->read(io, args...), filename) +read(filename::AbstractString, args...) = open(io->read(io, args...), convert(String, filename)::String) -read(filename::AbstractString, ::Type{T}) where {T} = open(io->read(io, T), filename) +read(filename::AbstractString, ::Type{T}) where {T} = open(io->read(io, T), convert(String, filename)::String) """ read!(stream::IO, array::AbstractArray) @@ -469,7 +469,7 @@ Read binary data from an I/O stream or file, filling in `array`. """ function read! end -read!(filename::AbstractString, a) = open(io->read!(io, a), filename) +read!(filename::AbstractString, a) = open(io->read!(io, a), convert(String, filename)::String) """ readuntil(stream::IO, delim; keep::Bool = false) @@ -496,7 +496,7 @@ julia> readuntil("my_file.txt", '.', keep = true) julia> rm("my_file.txt") ``` """ -readuntil(filename::AbstractString, args...; kw...) = open(io->readuntil(io, args...; kw...), filename) +readuntil(filename::AbstractString, args...; kw...) = open(io->readuntil(io, args...; kw...), convert(String, filename)::String) """ readline(io::IO=stdin; keep::Bool=false) diff --git a/base/iostream.jl b/base/iostream.jl index 0af0e244cf357..23dfb53256e82 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -272,7 +272,7 @@ safe multi-threaded access. !!! compat "Julia 1.5" The `lock` argument is available as of Julia 1.5. """ -function open(fname::AbstractString; lock = true, +function open(fname::String; lock = true, read :: Union{Bool,Nothing} = nothing, write :: Union{Bool,Nothing} = nothing, create :: Union{Bool,Nothing} = nothing, @@ -299,6 +299,7 @@ function open(fname::AbstractString; lock = true, end return s end +open(fname::AbstractString; kwargs...) = open(convert(String, fname)::String; kwargs...) """ open(filename::AbstractString, [mode::AbstractString]; lock = true) -> IOStream diff --git a/base/logging.jl b/base/logging.jl index 9e40f2231b035..1b8e5e0f54192 100644 --- a/base/logging.jl +++ b/base/logging.jl @@ -457,7 +457,7 @@ end msg = try "Exception while generating log record in module $_module at $filepath:$line" catch ex - "Exception handling log message: $ex" + LazyString("Exception handling log message: ", ex) end bt = real ? catch_backtrace() : backtrace() handle_message( diff --git a/base/process.jl b/base/process.jl index cbc251ed6dbca..876eb74fbd413 100644 --- a/base/process.jl +++ b/base/process.jl @@ -74,7 +74,8 @@ function _uv_hook_close(proc::Process) nothing end -const SpawnIOs = Vector{Any} # convenience name for readability +const SpawnIO = Union{IO, RawFD, OS_HANDLE} +const SpawnIOs = Vector{SpawnIO} # convenience name for readability function as_cpumask(cpus::Vector{UInt16}) n = max(Int(maximum(cpus)), Int(ccall(:uv_cpumask_size, Cint, ()))) @@ -129,7 +130,7 @@ end return pp end -_spawn(cmds::AbstractCmd) = _spawn(cmds, Any[]) +_spawn(cmds::AbstractCmd) = _spawn(cmds, SpawnIO[]) # optimization: we can spawn `Cmd` directly without allocating the ProcessChain function _spawn(cmd::Cmd, stdios::SpawnIOs) @@ -213,7 +214,7 @@ end # open the child end of each element of `stdios`, and initialize the parent end function setup_stdios(f, stdios::SpawnIOs) nstdio = length(stdios) - open_io = Vector{Any}(undef, nstdio) + open_io = SpawnIOs(undef, nstdio) close_io = falses(nstdio) try for i in 1:nstdio @@ -324,19 +325,19 @@ close_stdio(stdio) = close(stdio) # - An Filesystem.File or IOStream object to redirect the output to # - A FileRedirect, containing a string specifying a filename to be opened for the child -spawn_opts_swallow(stdios::StdIOSet) = Any[stdios...] -spawn_opts_inherit(stdios::StdIOSet) = Any[stdios...] +spawn_opts_swallow(stdios::StdIOSet) = SpawnIO[stdios...] +spawn_opts_inherit(stdios::StdIOSet) = SpawnIO[stdios...] spawn_opts_swallow(in::Redirectable=devnull, out::Redirectable=devnull, err::Redirectable=devnull) = - Any[in, out, err] + SpawnIO[in, out, err] # pass original descriptors to child processes by default, because we might # have already exhausted and closed the libuv object for our standard streams. # ref issue #8529 spawn_opts_inherit(in::Redirectable=RawFD(0), out::Redirectable=RawFD(1), err::Redirectable=RawFD(2)) = - Any[in, out, err] + SpawnIO[in, out, err] function eachline(cmd::AbstractCmd; keep::Bool=false) out = PipeEndpoint() - processes = _spawn(cmd, Any[devnull, out, stderr]) + processes = _spawn(cmd, SpawnIO[devnull, out, stderr]) # if the user consumes all the data, also check process exit status for success ondone = () -> (success(processes) || pipeline_error(processes); nothing) return EachLine(out, keep=keep, ondone=ondone)::EachLine @@ -384,20 +385,20 @@ function open(cmds::AbstractCmd, stdio::Redirectable=devnull; write::Bool=false, stdio === devnull || throw(ArgumentError("no stream can be specified for `stdio` in read-write mode")) in = PipeEndpoint() out = PipeEndpoint() - processes = _spawn(cmds, Any[in, out, stderr]) + processes = _spawn(cmds, SpawnIO[in, out, stderr]) processes.in = in processes.out = out elseif read out = PipeEndpoint() - processes = _spawn(cmds, Any[stdio, out, stderr]) + processes = _spawn(cmds, SpawnIO[stdio, out, stderr]) processes.out = out elseif write in = PipeEndpoint() - processes = _spawn(cmds, Any[in, stdio, stderr]) + processes = _spawn(cmds, SpawnIO[in, stdio, stderr]) processes.in = in else stdio === devnull || throw(ArgumentError("no stream can be specified for `stdio` in no-access mode")) - processes = _spawn(cmds, Any[devnull, devnull, stderr]) + processes = _spawn(cmds, SpawnIO[devnull, devnull, stderr]) end return processes end diff --git a/base/show.jl b/base/show.jl index 8359690034c23..999a73047f6b2 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1919,7 +1919,7 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int, quote_level::In na = length(func_args) if (na == 2 || (na > 2 && isa(func, Symbol) && func in (:+, :++, :*)) || (na == 3 && func === :(:))) && all(a -> !isa(a, Expr) || a.head !== :..., func_args) - sep = func === :(:) ? "$func" : " $func " + sep = func === :(:) ? "$func" : " " * convert(String, string(func))::String * " " # if func::Any, avoid string interpolation (invalidation) if func_prec <= prec show_enclosed_list(io, '(', func_args, sep, ')', indent, func_prec, quote_level, true) @@ -2289,7 +2289,7 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int, quote_level::In elseif head === :meta && nargs == 1 && args[1] === :pop_loc print(io, "# meta: pop location") elseif head === :meta && nargs == 2 && args[1] === :pop_loc - print(io, "# meta: pop locations ($(args[2]))") + print(io, "# meta: pop locations ($(args[2]::Int))") # print anything else as "Expr(head, args...)" else unhandled = true diff --git a/base/strings/substring.jl b/base/strings/substring.jl index 5142cf65fe9c5..86efbc1f6df3e 100644 --- a/base/strings/substring.jl +++ b/base/strings/substring.jl @@ -55,6 +55,11 @@ convert(::Type{SubString{S}}, s::AbstractString) where {S<:AbstractString} = SubString(convert(S, s)) convert(::Type{T}, s::T) where {T<:SubString} = s +# Regex match allows only Union{String, SubString{String}} so define conversion to this type +convert(::Type{Union{String, SubString{String}}}, s::String) = s +convert(::Type{Union{String, SubString{String}}}, s::SubString{String}) = s +convert(::Type{Union{String, SubString{String}}}, s::AbstractString) = convert(String, s) + function String(s::SubString{String}) parent = s.string copy = GC.@preserve parent unsafe_string(pointer(parent, s.offset+1), s.ncodeunits) @@ -229,7 +234,13 @@ function string(a::Union{Char, String, SubString{String}, Symbol}...) out = _string_n(n) offs = 1 for v in a - offs += __unsafe_string!(out, v, offs) + if v isa Char + offs += __unsafe_string!(out, v, offs) + elseif v isa String || v isa SubString{String} + offs += __unsafe_string!(out, v, offs) + else + offs += __unsafe_string!(out, v::Symbol, offs) + end end return out end diff --git a/stdlib/REPL/src/LineEdit.jl b/stdlib/REPL/src/LineEdit.jl index cf09cb1966bf1..89f57383d5e48 100644 --- a/stdlib/REPL/src/LineEdit.jl +++ b/stdlib/REPL/src/LineEdit.jl @@ -1646,7 +1646,7 @@ end throw_eager_redirection_cycle(key::Union{Char, String}) = error("Eager redirection cycle detected for key ", repr(key)) throw_could_not_find_redirected_value(key::Union{Char, String}) = - error("Could not find redirected value ", repl(key)) + error("Could not find redirected value ", repr(key)) function keymap_unify(keymaps) ret = Dict{Char,Any}() diff --git a/stdlib/REPL/src/TerminalMenus/AbstractMenu.jl b/stdlib/REPL/src/TerminalMenus/AbstractMenu.jl index b87a9c8c26464..127d0cd88a2cf 100644 --- a/stdlib/REPL/src/TerminalMenus/AbstractMenu.jl +++ b/stdlib/REPL/src/TerminalMenus/AbstractMenu.jl @@ -192,7 +192,7 @@ function request(term::REPL.Terminals.TTYTerminal, m::AbstractMenu; cursor::Unio REPL.Terminals.raw!(term, true) true catch err - suppress_output || @warn("TerminalMenus: Unable to enter raw mode: $err") + suppress_output || @warn "TerminalMenus: Unable to enter raw mode: " exception=(err, catch_backtrace()) false end # hide the cursor diff --git a/stdlib/p7zip_jll/src/p7zip_jll.jl b/stdlib/p7zip_jll/src/p7zip_jll.jl index 99b346017ad97..4320003b282f7 100644 --- a/stdlib/p7zip_jll/src/p7zip_jll.jl +++ b/stdlib/p7zip_jll/src/p7zip_jll.jl @@ -35,7 +35,7 @@ else const pathsep = ':' end -function adjust_ENV!(env::Dict, PATH::String, LIBPATH::String, adjust_PATH::Bool, adjust_LIBPATH::Bool) +function adjust_ENV!(env::Dict{keytype(Base.EnvDict),valtype(Base.EnvDict)}, PATH::String, LIBPATH::String, adjust_PATH::Bool, adjust_LIBPATH::Bool) if adjust_LIBPATH LIBPATH_base = get(env, LIBPATH_env, expanduser(LIBPATH_default)) if !isempty(LIBPATH_base) From 477dfb98e93e38c1f2dec220bc65b12182ceccf9 Mon Sep 17 00:00:00 2001 From: Daniel Karrasch Date: Tue, 29 Mar 2022 17:44:31 +0200 Subject: [PATCH 17/68] Fix performance issue with diagonal multiplication (#44651) (cherry picked from commit 03af78108afe6058f54cf766d8981d26a11c51f7) --- stdlib/LinearAlgebra/src/diagonal.jl | 99 ++++++++++++++++++---------- stdlib/LinearAlgebra/src/generic.jl | 3 +- 2 files changed, 65 insertions(+), 37 deletions(-) diff --git a/stdlib/LinearAlgebra/src/diagonal.jl b/stdlib/LinearAlgebra/src/diagonal.jl index 5d17049cfa4e1..748b165eca7fd 100644 --- a/stdlib/LinearAlgebra/src/diagonal.jl +++ b/stdlib/LinearAlgebra/src/diagonal.jl @@ -249,8 +249,8 @@ end (*)(D::Diagonal, A::AbstractMatrix) = mul!(similar(A, promote_op(*, eltype(A), eltype(D.diag)), size(A)), D, A) -rmul!(A::AbstractMatrix, D::Diagonal) = mul!(A, A, D) -lmul!(D::Diagonal, B::AbstractVecOrMat) = mul!(B, D, B) +rmul!(A::AbstractMatrix, D::Diagonal) = @inline mul!(A, A, D) +lmul!(D::Diagonal, B::AbstractVecOrMat) = @inline mul!(B, D, B) #TODO: It seems better to call (D' * adjA')' directly? function *(adjA::Adjoint{<:Any,<:AbstractMatrix}, D::Diagonal) @@ -285,35 +285,80 @@ function *(D::Diagonal, transA::Transpose{<:Any,<:AbstractMatrix}) end @inline function __muldiag!(out, D::Diagonal, B, alpha, beta) - if iszero(beta) - out .= (D.diag .* B) .*ₛ alpha + require_one_based_indexing(out) + if iszero(alpha) + _rmul_or_fill!(out, beta) else - out .= (D.diag .* B) .*ₛ alpha .+ out .* beta + if iszero(beta) + @inbounds for j in axes(B, 2) + @simd for i in axes(B, 1) + out[i,j] = D.diag[i] * B[i,j] * alpha + end + end + else + @inbounds for j in axes(B, 2) + @simd for i in axes(B, 1) + out[i,j] = D.diag[i] * B[i,j] * alpha + out[i,j] * beta + end + end + end end return out end - @inline function __muldiag!(out, A, D::Diagonal, alpha, beta) - if iszero(beta) - out .= (A .* permutedims(D.diag)) .*ₛ alpha + require_one_based_indexing(out) + if iszero(alpha) + _rmul_or_fill!(out, beta) else - out .= (A .* permutedims(D.diag)) .*ₛ alpha .+ out .* beta + if iszero(beta) + @inbounds for j in axes(A, 2) + dja = D.diag[j] * alpha + @simd for i in axes(A, 1) + out[i,j] = A[i,j] * dja + end + end + else + @inbounds for j in axes(A, 2) + dja = D.diag[j] * alpha + @simd for i in axes(A, 1) + out[i,j] = A[i,j] * dja + out[i,j] * beta + end + end + end end return out end - @inline function __muldiag!(out::Diagonal, D1::Diagonal, D2::Diagonal, alpha, beta) - if iszero(beta) - out.diag .= (D1.diag .* D2.diag) .*ₛ alpha + d1 = D1.diag + d2 = D2.diag + if iszero(alpha) + _rmul_or_fill!(out.diag, beta) else - out.diag .= (D1.diag .* D2.diag) .*ₛ alpha .+ out.diag .* beta + if iszero(beta) + @inbounds @simd for i in eachindex(out.diag) + out.diag[i] = d1[i] * d2[i] * alpha + end + else + @inbounds @simd for i in eachindex(out.diag) + out.diag[i] = d1[i] * d2[i] * alpha + out.diag[i] * beta + end + end + end + return out +end +@inline function __muldiag!(out, D1::Diagonal, D2::Diagonal, alpha, beta) + require_one_based_indexing(out) + mA = size(D1, 1) + d1 = D1.diag + d2 = D2.diag + _rmul_or_fill!(out, beta) + if !iszero(alpha) + @inbounds @simd for i in 1:mA + out[i,i] += d1[i] * d2[i] * alpha + end end return out end - -# only needed for ambiguity resolution, as mul! is explicitly defined for these arguments -@inline __muldiag!(out, D1::Diagonal, D2::Diagonal, alpha, beta) = - mul!(out, D1, D2, alpha, beta) @inline function _muldiag!(out, A, B, alpha, beta) _muldiag_size_check(out, A, B) @@ -340,24 +385,8 @@ end @inline mul!(C::Diagonal, Da::Diagonal, Db::Diagonal, alpha::Number, beta::Number) = _muldiag!(C, Da, Db, alpha, beta) -function mul!(C::AbstractMatrix, Da::Diagonal, Db::Diagonal, alpha::Number, beta::Number) - _muldiag_size_check(C, Da, Db) - require_one_based_indexing(C) - mA = size(Da, 1) - da = Da.diag - db = Db.diag - _rmul_or_fill!(C, beta) - if iszero(beta) - @inbounds @simd for i in 1:mA - C[i,i] = Ref(da[i] * db[i]) .*ₛ alpha - end - else - @inbounds @simd for i in 1:mA - C[i,i] += Ref(da[i] * db[i]) .*ₛ alpha - end - end - return C -end +mul!(C::AbstractMatrix, Da::Diagonal, Db::Diagonal, alpha::Number, beta::Number) = + _muldiag!(C, Da, Db, alpha, beta) _init(op, A::AbstractArray{<:Number}, B::AbstractArray{<:Number}) = (_ -> zero(typeof(op(oneunit(eltype(A)), oneunit(eltype(B)))))) diff --git a/stdlib/LinearAlgebra/src/generic.jl b/stdlib/LinearAlgebra/src/generic.jl index 676e965652e8f..08c77b22d7bbb 100644 --- a/stdlib/LinearAlgebra/src/generic.jl +++ b/stdlib/LinearAlgebra/src/generic.jl @@ -8,8 +8,7 @@ # inside this function. function *ₛ end Broadcast.broadcasted(::typeof(*ₛ), out, beta) = - iszero(beta::Number) ? false : - isone(beta::Number) ? broadcasted(identity, out) : broadcasted(*, out, beta) + iszero(beta::Number) ? false : broadcasted(*, out, beta) """ MulAddMul(alpha, beta) From efb28d21189189446b43a014101b564746036da0 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 29 Mar 2022 14:55:47 -0400 Subject: [PATCH 18/68] profile: fix async deadlock (#44781) Acquiring this lock in many implementations could result in deadlock, even with an existing reader. Add a TLS check for reentry before, instead of relying on the implementation specifics, to avoid any issues. (cherry picked from commit 7df454bdd678aa84365ce34780bb34e9a2730e75) --- src/debuginfo.cpp | 48 +++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 263e506355d69..0e246160a3c16 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -47,21 +47,34 @@ typedef object::SymbolRef SymRef; // while holding this lock. // Certain functions in this file might be called from an unmanaged thread // and cannot have any interaction with the julia runtime -static uv_rwlock_t threadsafe; +// They also may be re-entrant, and operating while threads are paused, so we +// separately manage the re-entrant count behavior for safety across platforms +// Note that we cannot safely upgrade read->write +static uv_rwlock_t debuginfo_asyncsafe; +static pthread_key_t debuginfo_asyncsafe_held; void jl_init_debuginfo(void) { - uv_rwlock_init(&threadsafe); + uv_rwlock_init(&debuginfo_asyncsafe); + if (pthread_key_create(&debuginfo_asyncsafe_held, NULL)) + jl_error("fatal: pthread_key_create failed"); } extern "C" JL_DLLEXPORT void jl_lock_profile_impl(void) { - uv_rwlock_rdlock(&threadsafe); + uintptr_t held = (uintptr_t)pthread_getspecific(debuginfo_asyncsafe_held); + if (held++ == 0) + uv_rwlock_rdlock(&debuginfo_asyncsafe); + pthread_setspecific(debuginfo_asyncsafe_held, (void*)held); } extern "C" JL_DLLEXPORT void jl_unlock_profile_impl(void) { - uv_rwlock_rdunlock(&threadsafe); + uintptr_t held = (uintptr_t)pthread_getspecific(debuginfo_asyncsafe_held); + assert(held); + if (--held == 0) + uv_rwlock_rdunlock(&debuginfo_asyncsafe); + pthread_setspecific(debuginfo_asyncsafe_held, (void*)held); } // some actions aren't signal (especially profiler) safe so we acquire a lock @@ -69,7 +82,8 @@ extern "C" JL_DLLEXPORT void jl_unlock_profile_impl(void) template static void jl_profile_atomic(T f) { - uv_rwlock_wrlock(&threadsafe); + assert(0 == (uintptr_t)pthread_getspecific(debuginfo_asyncsafe_held)); + uv_rwlock_wrlock(&debuginfo_asyncsafe); #ifndef _OS_WINDOWS_ sigset_t sset; sigset_t oset; @@ -80,7 +94,7 @@ static void jl_profile_atomic(T f) #ifndef _OS_WINDOWS_ pthread_sigmask(SIG_SETMASK, &oset, NULL); #endif - uv_rwlock_wrunlock(&threadsafe); + uv_rwlock_wrunlock(&debuginfo_asyncsafe); } @@ -188,12 +202,12 @@ class JITObjectRegistry public: jl_method_instance_t *lookupLinfo(size_t pointer) JL_NOTSAFEPOINT { - uv_rwlock_rdlock(&threadsafe); + jl_lock_profile_impl(); auto region = linfomap.lower_bound(pointer); jl_method_instance_t *linfo = NULL; if (region != linfomap.end() && pointer < region->first + region->second.first) linfo = region->second.second; - uv_rwlock_rdunlock(&threadsafe); + jl_unlock_profile_impl(); return linfo; } @@ -448,9 +462,10 @@ static int lookup_pointer( // DWARFContext/DWARFUnit update some internal tables during these queries, so // a lock is needed. - uv_rwlock_wrlock(&threadsafe); + assert(0 == (uintptr_t)pthread_getspecific(debuginfo_asyncsafe_held)); + uv_rwlock_wrlock(&debuginfo_asyncsafe); auto inlineInfo = context->getInliningInfoForAddress(makeAddress(Section, pointer + slide), infoSpec); - uv_rwlock_wrunlock(&threadsafe); + uv_rwlock_wrunlock(&debuginfo_asyncsafe); int fromC = (*frames)[0].fromC; int n_frames = inlineInfo.getNumberOfFrames(); @@ -473,9 +488,9 @@ static int lookup_pointer( info = inlineInfo.getFrame(i); } else { - uv_rwlock_wrlock(&threadsafe); + uv_rwlock_wrlock(&debuginfo_asyncsafe); info = context->getLineInfoForAddress(makeAddress(Section, pointer + slide), infoSpec); - uv_rwlock_wrunlock(&threadsafe); + uv_rwlock_wrunlock(&debuginfo_asyncsafe); } jl_frame_t *frame = &(*frames)[i]; @@ -1148,7 +1163,8 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, object::SectionRef *Section, llvm::DIContext **context) JL_NOTSAFEPOINT { int found = 0; - uv_rwlock_wrlock(&threadsafe); + assert(0 == (uintptr_t)pthread_getspecific(debuginfo_asyncsafe_held)); + uv_rwlock_wrlock(&debuginfo_asyncsafe); std::map &objmap = jl_jit_object_registry.getObjectMap(); std::map::iterator fit = objmap.lower_bound(fptr); @@ -1164,7 +1180,7 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, } found = 1; } - uv_rwlock_wrunlock(&threadsafe); + uv_rwlock_wrunlock(&debuginfo_asyncsafe); return found; } @@ -1613,13 +1629,13 @@ extern "C" JL_DLLEXPORT uint64_t jl_getUnwindInfo_impl(uint64_t dwAddr) { // Might be called from unmanaged thread - uv_rwlock_rdlock(&threadsafe); + jl_lock_profile_impl(); std::map &objmap = jl_jit_object_registry.getObjectMap(); std::map::iterator it = objmap.lower_bound(dwAddr); uint64_t ipstart = 0; // ip of the start of the section (if found) if (it != objmap.end() && dwAddr < it->first + it->second.SectionSize) { ipstart = (uint64_t)(uintptr_t)(*it).first; } - uv_rwlock_rdunlock(&threadsafe); + jl_unlock_profile_impl(); return ipstart; } From ddc620f729983ebf33ff6acff2c6537940f3faf9 Mon Sep 17 00:00:00 2001 From: Thomas van Doornmalen <60822431+Taaitaaiger@users.noreply.github.com> Date: Mon, 4 Apr 2022 05:57:30 +0200 Subject: [PATCH 19/68] restore MSVC support to julia.h file (#44842) Put back a few lines that were removed by PR $42703 because they are required to successfully embed Julia with MSVC. (cherry picked from commit 209aad1ddfad16d689de677597181005c927cdda) --- src/support/dtypes.h | 8 +++++++- src/support/platform.h | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/support/dtypes.h b/src/support/dtypes.h index 2df897c7ba554..9e3e17015e135 100644 --- a/src/support/dtypes.h +++ b/src/support/dtypes.h @@ -116,7 +116,13 @@ # define NOINLINE_DECL(f) f __attribute__((noinline)) #endif -#if defined(__GNUC__) +#ifdef _COMPILER_MICROSOFT_ +# ifdef _P64 +# define JL_ATTRIBUTE_ALIGN_PTRSIZE(x) __declspec(align(8)) x +# else +# define JL_ATTRIBUTE_ALIGN_PTRSIZE(x) __declspec(align(4)) x +# endif +#elif defined(__GNUC__) # define JL_ATTRIBUTE_ALIGN_PTRSIZE(x) x __attribute__ ((aligned (sizeof(void*)))) #else # define JL_ATTRIBUTE_ALIGN_PTRSIZE(x) diff --git a/src/support/platform.h b/src/support/platform.h index bb960f54d3c4e..cf65fa01423fe 100644 --- a/src/support/platform.h +++ b/src/support/platform.h @@ -37,6 +37,8 @@ #define _COMPILER_CLANG_ #elif defined(__GNUC__) #define _COMPILER_GCC_ +#elif defined(_MSC_VER) +#define _COMPILER_MICROSOFT_ #else #error Unsupported compiler #endif From 2a0f4de814dc5bd23cec0571d51300c14ae1c51d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Mon, 4 Apr 2022 22:11:25 +0200 Subject: [PATCH 20/68] [Zlib_jll] Update to v1.2.12+3 (#44810) Note: this is the first build of the real upstream version 1.2.12 which was released a few days ago. (cherry picked from commit 81e7cfc0b7edaf8da3d892290da5fd20dc2e2a65) --- deps/checksums/zlib | 68 ++++++++++++++++---------------- deps/zlib.version | 4 +- stdlib/Zlib_jll/Project.toml | 2 +- stdlib/Zlib_jll/test/runtests.jl | 2 +- 4 files changed, 38 insertions(+), 38 deletions(-) diff --git a/deps/checksums/zlib b/deps/checksums/zlib index 6b14164c8318f..d524a3f588a18 100644 --- a/deps/checksums/zlib +++ b/deps/checksums/zlib @@ -1,34 +1,34 @@ -Zlib.v1.2.12+1.aarch64-apple-darwin.tar.gz/md5/6e255e13279855a99dae7d4ccf206069 -Zlib.v1.2.12+1.aarch64-apple-darwin.tar.gz/sha512/d160928dc6cad6bbc9fce36ea0d807c1f432aae375e6a032b0fd58d18640d02fc50c25233b32f8b73f3fc3488a091cf57418ad04498160441e3d7e4aa79302fe -Zlib.v1.2.12+1.aarch64-linux-gnu.tar.gz/md5/ff0ce9d6dec1c1b07114ed48f2bcfc88 -Zlib.v1.2.12+1.aarch64-linux-gnu.tar.gz/sha512/fdcea5e1fccc93641d0c372b6ba041c33c006e84ca6ba532bd9d6bb5ac449379daf27c5e1b95df3a6a57d3c24a363f12e55d5fb92184f1130606000e045a0d9b -Zlib.v1.2.12+1.aarch64-linux-musl.tar.gz/md5/900884b5eb02307665c1e6244f9d4be8 -Zlib.v1.2.12+1.aarch64-linux-musl.tar.gz/sha512/c3cbf7b41566af260a6e4ff2a2206b7f88439f0925609c72f822876eff384e3656e6bcd12131eac47d4177e5a1359ea6ebedbae949682c1d307607588ebfd80c -Zlib.v1.2.12+1.armv6l-linux-gnueabihf.tar.gz/md5/2766764794ae29ff4dc97c42faebbd91 -Zlib.v1.2.12+1.armv6l-linux-gnueabihf.tar.gz/sha512/341262c50ba5117ea93afb4acf3a036ee40a83d9b46b13a8360f36d74561c152d9ffa807887f4c452c65e91cae98df44fed861014ce26c4293ee0f45bafcb87e -Zlib.v1.2.12+1.armv6l-linux-musleabihf.tar.gz/md5/9037801d9524b3912acb5a5d3abfaa87 -Zlib.v1.2.12+1.armv6l-linux-musleabihf.tar.gz/sha512/6984076b0262e7ef19f08e6e83aa855eb6b60ae478dcad985d360b38f52ea6cc0fbf4e5c7723c007b722b01dc70ae378f6d487ddbe934e84ab4376de2688ce86 -Zlib.v1.2.12+1.armv7l-linux-gnueabihf.tar.gz/md5/627bcdf4216e9fb7020dcc50f71402e2 -Zlib.v1.2.12+1.armv7l-linux-gnueabihf.tar.gz/sha512/575000bed533f223ef2551ebdb7b431a743f83bf248edaf0a05ba00d33cf7848481952b325d7e18fdce3b91d2f0ec6fd02b24fb8cfa812f8a511f924a192fd1c -Zlib.v1.2.12+1.armv7l-linux-musleabihf.tar.gz/md5/11c79b0221d07986eeaf016650667059 -Zlib.v1.2.12+1.armv7l-linux-musleabihf.tar.gz/sha512/7f0415e8ebad6690621906885f72d3660962279e4ef57893334406a92f3eb9f6dac177d7430da0f4ae1ab0cabf185b33dbb347e054c35498e94e45771dd4b05a -Zlib.v1.2.12+1.i686-linux-gnu.tar.gz/md5/fc024f3aa4fffb298b6059adc7db6911 -Zlib.v1.2.12+1.i686-linux-gnu.tar.gz/sha512/cb219ecd89adda98f84914a4bc9355ba363bd942c7cd16adba70aa3f8ac37d1f7f812df942294a8eb3fa5ed474ee59126a567dea1f536467087fa27eb66c41b1 -Zlib.v1.2.12+1.i686-linux-musl.tar.gz/md5/5473f0c5ae14d4c34bc51c6ad583f21e -Zlib.v1.2.12+1.i686-linux-musl.tar.gz/sha512/c6380f1b22866dbfb8baaf724bcc33f2db3602741d3ffcdd61a6831740f1e4e4344b4ac4ac020054df06ebefac235f56a034a1d7cbc40e6c19d2e953945725c2 -Zlib.v1.2.12+1.i686-w64-mingw32.tar.gz/md5/1119dbaf451c691028522e43e2ca7f20 -Zlib.v1.2.12+1.i686-w64-mingw32.tar.gz/sha512/366d3ef55e3b448176388f8d92c6ffe00e68f7ae62b67ad1ceedb73984ba30b16c8a086807f61e87caa8262e8ea1cb7799b49d22b0269dcee7735d3ea36df6aa -Zlib.v1.2.12+1.powerpc64le-linux-gnu.tar.gz/md5/127bf2fbb739f52d1d455d9b8dd0b08e -Zlib.v1.2.12+1.powerpc64le-linux-gnu.tar.gz/sha512/cd647435a5ca819180f662f288106ce49521ad75501b7c95ad912f008caa264531f8b62ccc042c0f8f2cb1a728d89d84fef395c9f3797b0f9f111c1f8b8ce1b9 -Zlib.v1.2.12+1.x86_64-apple-darwin.tar.gz/md5/5740e0da15acce6234d54b56bc462529 -Zlib.v1.2.12+1.x86_64-apple-darwin.tar.gz/sha512/1b973091f381cd2d1403685fcc7ca69f31019e2bab6a031cc934bffdf339775bbd529fb375996bdade090ff4cfcf6f2aec6cb9891b91a5b21c3f847f159748a0 -Zlib.v1.2.12+1.x86_64-linux-gnu.tar.gz/md5/750e79f7ad235ee94088ad297c407e36 -Zlib.v1.2.12+1.x86_64-linux-gnu.tar.gz/sha512/ae995d9069eda2ac602eb53cd6d86c22d0d5e353504d1a6525a33efb99628fa4abd40d0dcc16f0927c409d5c57b6f7d63208d2aae01474665f9f93114bd1388a -Zlib.v1.2.12+1.x86_64-linux-musl.tar.gz/md5/bb62d2d9f6800c36183d2f2e6e094f42 -Zlib.v1.2.12+1.x86_64-linux-musl.tar.gz/sha512/d2ba384a1d31cf0f3cb6bc843d43005c39a72007954bc58bfa24c5d6d65af10ae2969670baecd854c8074f94424288f3fb29f735c9226f7f8a2df49eb62e6033 -Zlib.v1.2.12+1.x86_64-unknown-freebsd.tar.gz/md5/21dfda8d26dbe76c914216e79d7847d6 -Zlib.v1.2.12+1.x86_64-unknown-freebsd.tar.gz/sha512/2cd7be4070dbf20ab1c46553a9e3f84c98bf8e8fc72bf2eb4678630e648cb9ad02cae5e004f3c2a69216e2782d9bba43eac6a45a480f6fe58d1091a9fbba93ff -Zlib.v1.2.12+1.x86_64-w64-mingw32.tar.gz/md5/140ddbeeaf27867aeeeec118682e879d -Zlib.v1.2.12+1.x86_64-w64-mingw32.tar.gz/sha512/f61f3d1eb7e7960b2fdbc1d68f22526a06ba598cd821261e7ba3819e00daee4c5b5427f9c03277b57b7226860142f0071410c0583535ca4e4b9acbe5ee4b5ade -zlib-cacf7f1d4e3d44d871b605da3b647f07d718623f.tar.gz/md5/93d10d4dd040f14ae63417070d1346e8 -zlib-cacf7f1d4e3d44d871b605da3b647f07d718623f.tar.gz/sha512/a1e9c5a2963266a582192d0fe88c179f5239245f11c4df4427dda755ad77d31e1fcf045d7d3fe49141090f4ff8da13d9a2e8d8d317fe6460a5f3e9bdea29b883 +Zlib.v1.2.12+3.aarch64-apple-darwin.tar.gz/md5/2258883a6412fbdac0b807afd133834f +Zlib.v1.2.12+3.aarch64-apple-darwin.tar.gz/sha512/6e82b57646dfe2b86978d51cb4401d565d00d6bdcfabe09ceb888ad8979bd1398fd9ea7652542f149d88c120110f6c3baa919616f01410e9238a5199f50f5dda +Zlib.v1.2.12+3.aarch64-linux-gnu.tar.gz/md5/663aa0d0791b92464e4822a130ac7fa9 +Zlib.v1.2.12+3.aarch64-linux-gnu.tar.gz/sha512/e50f00d92600a78b2f540e0e8e1dce435d0d0499ea80ce3c3cd0e11c8e3b5b1a97eadca9ac863f597cee369e80bcd50ec1c0a0e0f1a87bb0ff94bbaf453dea2d +Zlib.v1.2.12+3.aarch64-linux-musl.tar.gz/md5/471179a2364d59abb6426b378ea4e195 +Zlib.v1.2.12+3.aarch64-linux-musl.tar.gz/sha512/35208e4be5966343ecb2b78471a3e1a947489f83c828b562db3508506dd0493eae3318c7eb3a6b599e911416795023193df862fbb6fcc7389d44710dc30f16a8 +Zlib.v1.2.12+3.armv6l-linux-gnueabihf.tar.gz/md5/53601c0201dadc8c9ff038167d5c4277 +Zlib.v1.2.12+3.armv6l-linux-gnueabihf.tar.gz/sha512/19744283bb412a656b934347cb7a1d121fbaf7e5f9b1aac373ddf2466567b731817a2e72e3a4d993ca7e5b5eb1fd9bb9c24d0126778367b28bdb94721649298b +Zlib.v1.2.12+3.armv6l-linux-musleabihf.tar.gz/md5/f7c923955fc600785aae455807e63c8b +Zlib.v1.2.12+3.armv6l-linux-musleabihf.tar.gz/sha512/623cd1758465c9e40b0dad93981ae93097a03f4aa67487b7e1c7240be2d780d86f35f8db96743c35bbb329d572741b58e73735a2b1cfb9e18e77f4dbcc714063 +Zlib.v1.2.12+3.armv7l-linux-gnueabihf.tar.gz/md5/5ce0fe42f67e09de047626424d61bc82 +Zlib.v1.2.12+3.armv7l-linux-gnueabihf.tar.gz/sha512/322e32d6fe6cd7a3334f5146f8980d4f1fc85b9a1c60271659ba8b4bbfdec314f8d9e8c6c0719248f5dd18e3daefd946811a3dcc74fa3ae5505d6dd653e65309 +Zlib.v1.2.12+3.armv7l-linux-musleabihf.tar.gz/md5/5115c374df90393cb895dd45c77275c4 +Zlib.v1.2.12+3.armv7l-linux-musleabihf.tar.gz/sha512/b04b4f42220833b99923a3ff349e4a05ad9f67c2b62d4848de37c833b287420b1dbec8a039c09d2a95ab6b68a62c6dcbacb4ba7cc069a4e90a11f8592719d2b8 +Zlib.v1.2.12+3.i686-linux-gnu.tar.gz/md5/37e0186f765fada0d76b9cd6f28c8d5d +Zlib.v1.2.12+3.i686-linux-gnu.tar.gz/sha512/1239675bbf46c6243131585283b0fc23baa32e68226fbb2f0b7a833c8979e2df33590947daade533e37bafe21838a10198e9f9de99e094c21fba6b218b2fceab +Zlib.v1.2.12+3.i686-linux-musl.tar.gz/md5/a0d92af6481929eed3a9fec3dbb2e622 +Zlib.v1.2.12+3.i686-linux-musl.tar.gz/sha512/b448590129ef251083b675c3d7494a90151a03297fd9883efb70bde032d106f16f2ec7c28508d9b4a0d0e5a0be0bdb4bcf0d1a9e4b2ade034a6d6cfc4916536e +Zlib.v1.2.12+3.i686-w64-mingw32.tar.gz/md5/cc38d9ec5430e2ed7fed4792c7ac9551 +Zlib.v1.2.12+3.i686-w64-mingw32.tar.gz/sha512/85ad3babb42682d7b2b69513a30fd5e992a56436dcd7e2a44800bf1bc30d60d09aff5769cfaeefd4f5668e7973a0c2d4ad4d28559ea5f28c1c5419ed595eae57 +Zlib.v1.2.12+3.powerpc64le-linux-gnu.tar.gz/md5/8f57d8c31d2355c64a05db0412462d58 +Zlib.v1.2.12+3.powerpc64le-linux-gnu.tar.gz/sha512/9a0208c7a4dbf71b6f7e1ccaf05e3f3a422507cf0431b6482aab1a7b1bea41bd135320567f7dba6666f37c26f48cb3a627f1a1ebd39bf5c2d61148aadf62a986 +Zlib.v1.2.12+3.x86_64-apple-darwin.tar.gz/md5/5d15bb591d26d24aa9d6c9c8cf3df097 +Zlib.v1.2.12+3.x86_64-apple-darwin.tar.gz/sha512/7d8b0ec5a46a85cef3c5de451823c5cfa73b5b7c5ac98699065bbc5692af556195664908cd5c35184b7a9586fc0adab41fc0f76ee8599ca09a740cf49b9be113 +Zlib.v1.2.12+3.x86_64-linux-gnu.tar.gz/md5/25df63b9e6cbef14b0f0bf2a9eec5d14 +Zlib.v1.2.12+3.x86_64-linux-gnu.tar.gz/sha512/2660b762d816491e6b877020d8dd4a1cf1b171d6232dd5e0f47c6ee7b15504b006cc8f051434df778e0910130ef7456e30d531464470d3c4a2502e8f9fd19e76 +Zlib.v1.2.12+3.x86_64-linux-musl.tar.gz/md5/3f0c85d248711608141046d15b2da339 +Zlib.v1.2.12+3.x86_64-linux-musl.tar.gz/sha512/e4256b1b9520d5b0d97fa7e7ca6f6b9aa2583c6e5f14967392d54e48f27e242461f77e522743b229ab9b333eec5fd51f6d7b1559b566bd68ca0741b05b96df3c +Zlib.v1.2.12+3.x86_64-unknown-freebsd.tar.gz/md5/e67dae1456645930c9e2b2fef6f805c8 +Zlib.v1.2.12+3.x86_64-unknown-freebsd.tar.gz/sha512/5915ec48ae80be829c36a71e2ce580d2d14b7a9824c8f279ad5c69fea62d9a03345b665f224b9dde0bc4b808af246f89ec4f932d47a14236bc3b7db7651e5bec +Zlib.v1.2.12+3.x86_64-w64-mingw32.tar.gz/md5/89b152b3de0068c7c2580b87ad529ed3 +Zlib.v1.2.12+3.x86_64-w64-mingw32.tar.gz/sha512/df4b585f6501f45bc85e8d00c1b03c482d70d3491081246f9e9f9560f90c5f6057b1174a81e653f725209323cd743cf05d3e1aba1385afd26cb6f8c50186f818 +zlib-21767c654d31d2dccdde4330529775c6c5fd5389.tar.gz/md5/1fb2320f871561306bc87b3894727b45 +zlib-21767c654d31d2dccdde4330529775c6c5fd5389.tar.gz/sha512/2ad1e728f97a81b65d24fe5bef66658c94222d717a3486a0d11682b61563d7eaaa578f7457078881e8ed8c91b87aec11634d4a64021546e23a3ecabb3285197a diff --git a/deps/zlib.version b/deps/zlib.version index e363169315051..0b16a7f662dd1 100644 --- a/deps/zlib.version +++ b/deps/zlib.version @@ -1,2 +1,2 @@ -ZLIB_BRANCH=v1.2.11 -ZLIB_SHA1=cacf7f1d4e3d44d871b605da3b647f07d718623f +ZLIB_BRANCH=v1.2.12 +ZLIB_SHA1=21767c654d31d2dccdde4330529775c6c5fd5389 diff --git a/stdlib/Zlib_jll/Project.toml b/stdlib/Zlib_jll/Project.toml index cafaf9c1b577c..77e1da5f9c22e 100644 --- a/stdlib/Zlib_jll/Project.toml +++ b/stdlib/Zlib_jll/Project.toml @@ -1,6 +1,6 @@ name = "Zlib_jll" uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.2.12+1" +version = "1.2.12+3" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/stdlib/Zlib_jll/test/runtests.jl b/stdlib/Zlib_jll/test/runtests.jl index e6adc6b7c951f..cc9e64188a0aa 100644 --- a/stdlib/Zlib_jll/test/runtests.jl +++ b/stdlib/Zlib_jll/test/runtests.jl @@ -3,5 +3,5 @@ using Test, Zlib_jll @testset "Zlib_jll" begin - @test VersionNumber(unsafe_string(ccall((:zlibVersion, libz), Cstring, ()))) == v"1.2.11" + @test VersionNumber(unsafe_string(ccall((:zlibVersion, libz), Cstring, ()))) == v"1.2.12" end From 7cfc43fde18d8ddcf0f6f7aa0cf91c6d86f49b83 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 7 Apr 2022 13:48:04 -0400 Subject: [PATCH 21/68] fix missing field type initialization vars (#44797) We were accidentally passing the start of the list instead of the end of the list, resulting in some values passing through uninitialized. Fix #42297 regression (cherry picked from commit 5f2abf6cfaa49220f08900d6a7a4422b94e32187) --- src/jltypes.c | 4 ++-- test/core.jl | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/jltypes.c b/src/jltypes.c index bb1cdd6c49b97..1686a4b57e500 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -1934,7 +1934,7 @@ void jl_reinstantiate_inner_types(jl_datatype_t *t) // can throw! for (i = 0; i < n; i++) env[i].val = jl_svecref(ndt->parameters, i); - ndt->super = (jl_datatype_t*)inst_type_w_((jl_value_t*)t->super, env, &top, 1); + ndt->super = (jl_datatype_t*)inst_type_w_((jl_value_t*)t->super, &env[n - 1], &top, 1); jl_gc_wb(ndt, ndt->super); } @@ -1944,7 +1944,7 @@ void jl_reinstantiate_inner_types(jl_datatype_t *t) // can throw! for (i = 0; i < n; i++) env[i].val = jl_svecref(ndt->parameters, i); assert(ndt->types == NULL); - ndt->types = inst_ftypes(t->types, env, &top); + ndt->types = inst_ftypes(t->types, &env[n - 1], &top); jl_gc_wb(ndt, ndt->types); if (ndt->isconcretetype) { // cacheable jl_compute_field_offsets(ndt); diff --git a/test/core.jl b/test/core.jl index 985c15781e45c..ad2eac873baaf 100644 --- a/test/core.jl +++ b/test/core.jl @@ -337,6 +337,15 @@ end #struct S22624{A,B,C} <: Ref{S22624{Int64,A}}; end @test_broken @isdefined S22624 +# issue #42297 +mutable struct Node42297{T, V} + value::V + next::Union{Node42297{T, T}, Node42297{T, Val{T}}, Nothing} + Node42297{T}(value) where {T} = new{T, typeof(value)}(value, nothing) +end +@test fieldtype(Node42297{Int,Val{Int}}, 1) === Val{Int} +@test fieldtype(Node42297{Int,Int}, 1) === Int + # issue #3890 mutable struct A3890{T1} x::Matrix{Complex{T1}} From 09e14a4c7713c8a74f3c2ec4adf48a22de3f7645 Mon Sep 17 00:00:00 2001 From: Max Horn Date: Mon, 11 Apr 2022 01:28:41 +0200 Subject: [PATCH 22/68] Fix "anonymous types declared in an anonymous union" warnings (#44807) They look like this: ``` CC src/processor.o In file included from /Users/mhorn/Projekte/Julia/julia.master/src/processor.cpp:10: In file included from ./processor.h:5: ./julia.h:395:9: warning: anonymous types declared in an anonymous union are an extension [-Wnested-anon-types] struct { ^ ./julia.h:405:9: warning: anonymous types declared in an anonymous union are an extension [-Wnested-anon-types] struct { ^ 2 warnings generated. ``` and come from code that was introduced by @keno in PR #43852. But it turns out that the union is not used at all! So I'm simply removing the offending union. Perhaps it is needed for some future work, but it should be trivial to add it back if needed. If that happens, I suggest a comment is added that explain why this looks similar to but has different layout compared to the `typedef _jl_purity_overrides_t` also in `julia.h`. (cherry picked from commit bb91e627a3967b764f2f7163c282e10cfa41929c) --- src/julia.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/julia.h b/src/julia.h index f88371e561ca3..695d816aa19c1 100644 --- a/src/julia.h +++ b/src/julia.h @@ -233,6 +233,7 @@ typedef struct _jl_line_info_node_t { intptr_t inlined_at; } jl_line_info_node_t; +// the following mirrors `struct EffectsOverride` in `base/compiler/types.jl` typedef union __jl_purity_overrides_t { struct { uint8_t ipo_consistent : 1; @@ -393,6 +394,8 @@ typedef struct _jl_code_instance_t { //TODO: uint8_t absolute_max; // whether true max world is unknown // purity results +#ifdef JL_USE_ANON_UNIONS_FOR_PURITY_FLAGS + // see also encode_effects() and decode_effects() in `base/compiler/types.jl`, union { uint32_t ipo_purity_bits; struct { @@ -413,6 +416,10 @@ typedef struct _jl_code_instance_t { uint8_t nonoverlayed:1; } purity_flags; }; +#else + uint32_t ipo_purity_bits; + uint32_t purity_bits; +#endif jl_value_t *argescapes; // escape information of call arguments // compilation state cache From 1f6dc034f269e4877ad512951f5f897b401afdff Mon Sep 17 00:00:00 2001 From: Ian Atol Date: Mon, 11 Apr 2022 18:54:13 -0700 Subject: [PATCH 23/68] Fix 44921 by properly setting Vboxed field (#44942) (cherry picked from commit 4c858f8be31c667348bb8419e40477edd7f4a38d) --- src/codegen.cpp | 17 +++++++++-------- test/compiler/codegen.jl | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 89107f99413dc..a16c854444677 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1790,14 +1790,15 @@ static jl_cgval_t convert_julia_type_union(jl_codectx_t &ctx, const jl_cgval_t & boxv = ctx.builder.CreateSelect( ctx.builder.CreateAnd(wasboxed, isboxed), v.Vboxed, boxv); } + Value *slotv; + MDNode *tbaa; if (v.V == NULL) { // v.V might be NULL if it was all ghost objects before - return jl_cgval_t(boxv, NULL, false, typ, new_tindex, ctx.tbaa()); + slotv = NULL; + tbaa = ctx.tbaa().tbaa_const; } else { Value *isboxv = ctx.builder.CreateIsNotNull(boxv); - Value *slotv; - MDNode *tbaa; if (v.ispointer()) { slotv = v.V; tbaa = v.tbaa; @@ -1810,12 +1811,12 @@ static jl_cgval_t convert_julia_type_union(jl_codectx_t &ctx, const jl_cgval_t & slotv = ctx.builder.CreateSelect(isboxv, decay_derived(ctx, boxv), decay_derived(ctx, emit_bitcast(ctx, slotv, boxv->getType()))); - jl_cgval_t newv = jl_cgval_t(slotv, NULL, false, typ, new_tindex, ctx.tbaa()); - assert(boxv->getType() == ctx.types().T_prjlvalue); - newv.Vboxed = boxv; - newv.tbaa = tbaa; - return newv; } + jl_cgval_t newv = jl_cgval_t(slotv, NULL, false, typ, new_tindex, ctx.tbaa()); + assert(boxv->getType() == ctx.types().T_prjlvalue); + newv.Vboxed = boxv; + newv.tbaa = tbaa; + return newv; } } else { diff --git a/test/compiler/codegen.jl b/test/compiler/codegen.jl index ec89ac9cd72a4..fa332447e394c 100644 --- a/test/compiler/codegen.jl +++ b/test/compiler/codegen.jl @@ -702,6 +702,23 @@ function f42645() end @test ((f42645()::B42645).y::A42645{Int}).x +struct A44921{T} + x::T +end +function f44921(a) + if a == :x + A44921(_f) # _f purposefully undefined + elseif a == :p + g44921(a) + end +end +function g44921(a) + if !@isdefined _f # just needs to be some non constprop-able condition + A44921(()) + end +end +@test f44921(:p) isa A44921 + # issue #43123 @noinline cmp43123(a::Some, b::Some) = something(a) === something(b) @noinline cmp43123(a, b) = a[] === b[] From a89784700c1ba174a60c976a99484aa329d5aeaf Mon Sep 17 00:00:00 2001 From: Will Kimmerer Date: Mon, 11 Apr 2022 22:06:42 -0400 Subject: [PATCH 24/68] distributed docs (#44940) (cherry picked from commit b5bbb9f5f3f1d2a46118dd2b6b944221999f2cae) --- doc/src/manual/distributed-computing.md | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/doc/src/manual/distributed-computing.md b/doc/src/manual/distributed-computing.md index abaf47a53b39c..73c7bd8b1ee00 100644 --- a/doc/src/manual/distributed-computing.md +++ b/doc/src/manual/distributed-computing.md @@ -1258,20 +1258,21 @@ in future releases. ## Noteworthy external packages Outside of Julia parallelism there are plenty of external packages that should be mentioned. -For example [MPI.jl](https://github.com/JuliaParallel/MPI.jl) is a Julia wrapper for the `MPI` protocol, or -[DistributedArrays.jl](https://github.com/JuliaParallel/Distributedarrays.jl), as presented in [Shared Arrays](@ref). +For example [MPI.jl](https://github.com/JuliaParallel/MPI.jl) is a Julia wrapper for the `MPI` protocol, [Dagger.jl](https://github.com/JuliaParallel/Dagger.jl) provides functionality similar to Python's [Dask](https://dask.org/), and +[DistributedArrays.jl](https://github.com/JuliaParallel/Distributedarrays.jl) provides array operations distributed across workers, as presented in [Shared Arrays](@ref). + A mention must be made of Julia's GPU programming ecosystem, which includes: -1. Low-level (C kernel) based operations [OpenCL.jl](https://github.com/JuliaGPU/OpenCL.jl) and [CUDAdrv.jl](https://github.com/JuliaGPU/CUDAdrv.jl) which are respectively an OpenCL interface and a CUDA wrapper. +1. [CUDA.jl](https://github.com/JuliaGPU/CUDA.jl) wraps the various CUDA libraries and supports compiling Julia kernels for Nvidia GPUs. -2. Low-level (Julia Kernel) interfaces like [CUDAnative.jl](https://github.com/JuliaGPU/CUDAnative.jl) which is a Julia native CUDA implementation. +2. [oneAPI.jl](https://github.com/JuliaGPU/oneAPI.jl) wraps the oneAPI unified programming model, and supports executing Julia kernels on supported accelerators. Currently only Linux is supported. -3. High-level vendor-specific abstractions like [CuArrays.jl](https://github.com/JuliaGPU/CuArrays.jl) and [CLArrays.jl](https://github.com/JuliaGPU/CLArrays.jl) +3. [AMDGPU.jl](https://github.com/JuliaGPU/AMDGPU.jl) wraps the AMD ROCm libraries and supports compiling Julia kernels for AMD GPUs. Currently only Linux is supported. -4. High-level libraries like [ArrayFire.jl](https://github.com/JuliaComputing/ArrayFire.jl) and [GPUArrays.jl](https://github.com/JuliaGPU/GPUArrays.jl) +4. High-level libraries like [KernelAbstractions.jl](https://github.com/JuliaGPU/KernelAbstractions.jl), [Tullio.jl](https://github.com/mcabbott/Tullio.jl) and [ArrayFire.jl](https://github.com/JuliaComputing/ArrayFire.jl). -In the following example we will use both `DistributedArrays.jl` and `CuArrays.jl` to distribute an array across multiple +In the following example we will use both `DistributedArrays.jl` and `CUDA.jl` to distribute an array across multiple processes by first casting it through `distribute()` and `CuArray()`. Remember when importing `DistributedArrays.jl` to import it across all processes using [`@everywhere`](@ref) @@ -1284,7 +1285,7 @@ julia> addprocs() julia> @everywhere using DistributedArrays -julia> using CuArrays +julia> using CUDA julia> B = ones(10_000) ./ 2; @@ -1322,9 +1323,8 @@ true julia> typeof(cuC) CuArray{Float64,1} ``` -Keep in mind that some Julia features are not currently supported by CUDAnative.jl[^2] , especially some functions like `sin` will need to be replaced with `CUDAnative.sin`(cc: @maleadt). -In the following example we will use both `DistributedArrays.jl` and `CuArrays.jl` to distribute an array across multiple +In the following example we will use both `DistributedArrays.jl` and `CUDA.jl` to distribute an array across multiple processes and call a generic function on it. ```julia @@ -1407,6 +1407,3 @@ mpirun -np 4 ./julia example.jl introduced a new set of communication mechanisms, collectively referred to as Remote Memory Access (RMA). The motivation for adding rma to the MPI standard was to facilitate one-sided communication patterns. For additional information on the latest MPI standard, see . - -[^2]: - [Julia GPU man pages](https://juliagpu.github.io/CUDAnative.jl/stable/man/usage.html#Julia-support-1) From 099768422df92ce79d2dfcbb599056e9b2de2a26 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 13 Apr 2022 16:26:36 -0400 Subject: [PATCH 25/68] asyncevents: fix missing GC root and race (#44956) The event might have triggered on another thread before we observed it here, or it might have gotten finalized before it got triggered. Either outcome can result in a lost event. (I observed the later situation occurring locally during the Dates test once). (cherry picked from commit 8cc00ffd1c7851003acec419e01a4896f9ed88ad) --- base/asyncevent.jl | 67 +++++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/base/asyncevent.jl b/base/asyncevent.jl index 0736bd463111f..1c52b7cf7ee48 100644 --- a/base/asyncevent.jl +++ b/base/asyncevent.jl @@ -45,13 +45,22 @@ the async condition object itself. """ function AsyncCondition(cb::Function) async = AsyncCondition() - t = @task while _trywait(async) - cb(async) - isopen(async) || return + t = @task begin + unpreserve_handle(async) + while _trywait(async) + cb(async) + isopen(async) || return + end + end + # here we are mimicking parts of _trywait, in coordination with task `t` + preserve_handle(async) + @lock async.cond begin + if async.set + schedule(t) + else + _wait2(async.cond, t) + end end - lock(async.cond) - _wait2(async.cond, t) - unlock(async.cond) return async end @@ -115,6 +124,7 @@ function _trywait(t::Union{Timer, AsyncCondition}) # full barrier now for AsyncCondition t isa Timer || Core.Intrinsics.atomic_fence(:acquire_release) else + t.isopen || return false t.handle == C_NULL && return false iolock_begin() set = t.set @@ -123,14 +133,12 @@ function _trywait(t::Union{Timer, AsyncCondition}) lock(t.cond) try set = t.set - if !set - if t.handle != C_NULL - iolock_end() - set = wait(t.cond) - unlock(t.cond) - iolock_begin() - lock(t.cond) - end + if !set && t.isopen && t.handle != C_NULL + iolock_end() + set = wait(t.cond) + unlock(t.cond) + iolock_begin() + lock(t.cond) end finally unlock(t.cond) @@ -266,19 +274,28 @@ julia> begin """ function Timer(cb::Function, timeout::Real; interval::Real=0.0) timer = Timer(timeout, interval=interval) - t = @task while _trywait(timer) - try - cb(timer) - catch err - write(stderr, "Error in Timer:\n") - showerror(stderr, err, catch_backtrace()) - return + t = @task begin + unpreserve_handle(timer) + while _trywait(timer) + try + cb(timer) + catch err + write(stderr, "Error in Timer:\n") + showerror(stderr, err, catch_backtrace()) + return + end + isopen(timer) || return + end + end + # here we are mimicking parts of _trywait, in coordination with task `t` + preserve_handle(timer) + @lock timer.cond begin + if timer.set + schedule(t) + else + _wait2(timer.cond, t) end - isopen(timer) || return end - lock(timer.cond) - _wait2(timer.cond, t) - unlock(timer.cond) return timer end From 81c881914aeeca375ab43ce146e96858952c2f54 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 14 Apr 2022 04:19:56 -0400 Subject: [PATCH 26/68] [REPL] Fix a REPL test failure by removing an erroneous space in test (#44972) * [REPL] remove erroneous space in test Introduced by https://github.com/JuliaLang/julia/pull/33805 (74f2de13727373b44da76a65ceb5297889d3e482). * Revert "Temporarily move the `REPL` test suite to node 1, to buy us time until we fix the underlying bugs (#44961)" This reverts commit 322fd706a1739daf4776fd050df756e6670ba958. (cherry picked from commit fbec395fd049948618442569aa3a7708ceed2013) --- stdlib/REPL/test/repl.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index 05f583c807165..f34b00a8f0595 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -264,7 +264,7 @@ fake_repl(options = REPL.Options(confirm_exit=false,hascolor=true)) do stdin_wri write(stdin_write, ";") readuntil(stdout_read, "shell> ") Base.print_shell_escaped(stdin_write, Base.julia_cmd().exec..., special=Base.shell_special) - write(stdin_write, """ -e "println(\\"HI\\")\" """) + write(stdin_write, """ -e "println(\\"HI\\")\"""") readuntil(stdout_read, ")\"") proc_stdout_read, proc_stdout = redirect_stdout() get_stdout = @async read(proc_stdout_read, String) From fe0faad28d26da88ce559550a36919d5f16a2781 Mon Sep 17 00:00:00 2001 From: Bart Janssens Date: Fri, 15 Apr 2022 00:47:21 +0200 Subject: [PATCH 27/68] Fix embedding with MSVC (#44976) (cherry picked from commit c589e0d59541b1240f5bf77c7ce8c22594179e1b) --- src/support/dtypes.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/support/dtypes.h b/src/support/dtypes.h index 9e3e17015e135..d49ae0b22b5f9 100644 --- a/src/support/dtypes.h +++ b/src/support/dtypes.h @@ -27,6 +27,16 @@ #define WIN32_LEAN_AND_MEAN #include +#if defined(_COMPILER_MICROSOFT_) && !defined(_SSIZE_T_) && !defined(_SSIZE_T_DEFINED) + +/* See https://github.com/JuliaLang/julia/pull/44587 */ +typedef intptr_t ssize_t; +#define SSIZE_MAX INTPTR_MAX +#define _SSIZE_T_ +#define _SSIZE_T_DEFINED + +#endif /* defined(_COMPILER_MICROSOFT_) && !defined(_SSIZE_T_) && !defined(_SSIZE_T_DEFINED) */ + #if !defined(_COMPILER_GCC_) #define strtoull _strtoui64 From 26a25e217e55a5b041f668588d5afe4216427d5d Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Tue, 19 Apr 2022 04:40:13 -0400 Subject: [PATCH 28/68] Update Example header in admonition to render PDF better, see #44866. (#45026) (cherry picked from commit d29d2d622ae416c3e5ba82cb55ef0535a75fe8e8) --- base/floatfuncs.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/base/floatfuncs.jl b/base/floatfuncs.jl index d1164005d3e44..75cc0f43e4ace 100644 --- a/base/floatfuncs.jl +++ b/base/floatfuncs.jl @@ -97,9 +97,8 @@ julia> round(357.913; sigdigits=4, base=2) Rounding to specified digits in bases other than 2 can be inexact when operating on binary floating point numbers. For example, the [`Float64`](@ref) value represented by `1.15` is actually *less* than 1.15, yet will be - rounded to 1.2. + rounded to 1.2. For example: - # Examples ```jldoctest; setup = :(using Printf) julia> x = 1.15 1.15 From cc5743ce357060cc629a1a6085ac9e6ca1dc5628 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 30 Mar 2022 05:06:31 +0200 Subject: [PATCH 29/68] Initialize jl_page_size when not calling julia_init. (#44699) If we load libjulia-codegen directly, e.g. from opt or llc, we don't call julia_init but init_llvm. Co-authored-by: Keno Fischer (cherry picked from commit f562ad06f49917ef26902f06f6ad1bcb19522d54) --- src/codegen.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/codegen.cpp b/src/codegen.cpp index a16c854444677..cefaa6c8a9fda 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -8185,6 +8185,7 @@ extern "C" void jl_init_llvm(void) { jl_f_donotdelete_addr, new JuliaFunction{XSTR(jl_f_donotdelete), get_func_sig, get_donotdelete_func_attrs} } }; + jl_page_size = jl_getpagesize(); jl_default_debug_info_kind = (int) DICompileUnit::DebugEmissionKind::FullDebug; imaging_mode = jl_options.image_codegen || (jl_generating_output() && !jl_options.incremental); jl_default_cgparams.generic_context = jl_nothing; From f13039e70aefca417bbaf62eb143d2bc47845e82 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 19 Apr 2022 15:06:48 +0200 Subject: [PATCH 30/68] Change type of donotdelete intrinsics --- src/codegen.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index cefaa6c8a9fda..c88ed18cacb87 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -465,6 +465,10 @@ static Type *get_pjlvalue(LLVMContext &C) { return JuliaType::get_pjlvalue_ty(C) static FunctionType *get_func_sig(LLVMContext &C) { return JuliaType::get_jlfunc_ty(C); } +static FunctionType *get_donotdelete_sig(LLVMContext &C) { + return FunctionType::get(getVoidTy(C), true); +} + static AttributeList get_func_attrs(LLVMContext &C) { return AttributeList::get(C, @@ -8182,7 +8186,7 @@ extern "C" void jl_init_llvm(void) { jl_f_arrayset_addr, new JuliaFunction{XSTR(jl_f_arrayset), get_func_sig, get_func_attrs} }, { jl_f_arraysize_addr, new JuliaFunction{XSTR(jl_f_arraysize), get_func_sig, get_func_attrs} }, { jl_f_apply_type_addr, new JuliaFunction{XSTR(jl_f_apply_type), get_func_sig, get_func_attrs} }, - { jl_f_donotdelete_addr, new JuliaFunction{XSTR(jl_f_donotdelete), get_func_sig, get_donotdelete_func_attrs} } + { jl_f_donotdelete_addr, new JuliaFunction{XSTR(jl_f_donotdelete), get_donotdelete_sig, get_donotdelete_func_attrs} } }; jl_page_size = jl_getpagesize(); From 5073c4ec6e1b080f1d1d3166f4d8ebdac0a6946e Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Tue, 19 Apr 2022 15:08:48 +0200 Subject: [PATCH 31/68] Fix attributes for donotdelete intrinsic The change in #44793 changed the signature of the donotdelete intrinsic, but didn't change the corresponding attributes, leaving a `nonnull` attribute on a `void` return, which fails the verifier. --- src/codegen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index c88ed18cacb87..531b35bfedb79 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -485,7 +485,7 @@ static AttributeList get_donotdelete_func_attrs(LLVMContext &C) FnAttrs = FnAttrs.addAttribute(C, Attribute::NoUnwind); return AttributeList::get(C, FnAttrs, - Attributes(C, {Attribute::NonNull}), + Attributes(C, {}), None); } From bbb53ac0e972c700aebf619529a633ee5312f0d2 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Wed, 20 Apr 2022 14:22:47 +0900 Subject: [PATCH 32/68] inference: relax backedge optimization condition (#45030) Follows up #45017. We can relax the condition in `add_call_backedges!` for backedge optimization by ignoring the `nonoverlayed` property when the `AbstractInterpreter` doesn't use overlayed method table at all, since the property will never be tainted anyway even if we add a new method later. --- base/compiler/abstractinterpretation.jl | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index b65a9751d394f..91063ee8d338c 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -451,11 +451,18 @@ function add_call_backedges!(interp::AbstractInterpreter, @nospecialize(rettype), all_effects::Effects, edges::Vector{MethodInstance}, matches::Union{MethodMatches,UnionSplitMethodMatches}, @nospecialize(atype), sv::InferenceState) - if rettype === Any && all_effects === Effects() - # for `NativeInterpreter`, we don't need to add backedges when: - # - a new method couldn't refine (widen) this type and - # - the effects don't provide any useful information for interprocedural optimization - return + # we don't need to add backedges when: + # - a new method couldn't refine (widen) this type and + # - the effects are known to not provide any useful IPO information + if rettype === Any + if !isoverlayed(method_table(interp)) + # we can ignore the `nonoverlayed` property if `interp` doesn't use + # overlayed method table at all since it will never be tainted anyway + all_effects = Effects(all_effects; nonoverlayed=false) + end + if all_effects === Effects() + return + end end for edge in edges add_backedge!(edge, sv) From c6dc6d1464208516d1f66108c3257e009fed7cf2 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Wed, 16 Mar 2022 05:37:30 -0500 Subject: [PATCH 33/68] Fix some inference failures in Base.require call graph (#44628) Discovered via JET. This also causes `make_aliases` to always return a `Vector{SimpleVector}` as its first argument. Co-authored-by: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Co-authored-by: Jameson Nash Co-authored-by: Kristoffer Carlsson (cherry picked from commit cc606574fb440242b66f40fae170a083c28fd52c) --- base/cmd.jl | 2 +- base/deprecated.jl | 9 +++++++-- base/loading.jl | 15 ++++++++++----- base/process.jl | 2 +- base/show.jl | 32 +++++++++++++++++--------------- base/stat.jl | 2 +- base/toml_parser.jl | 2 +- 7 files changed, 38 insertions(+), 26 deletions(-) diff --git a/base/cmd.jl b/base/cmd.jl index ab639a2b185c8..ecabb5c32b1d0 100644 --- a/base/cmd.jl +++ b/base/cmd.jl @@ -130,7 +130,7 @@ function show(io::IO, cmd::Cmd) print(io, '`') if print_cpus print(io, ", ") - show(io, collect(Int, cmd.cpus)) + show(io, collect(Int, something(cmd.cpus))) print(io, ")") end print_env && (print(io, ","); show(io, cmd.env)) diff --git a/base/deprecated.jl b/base/deprecated.jl index 1af38ebcdac1c..09d051334b6c9 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -88,8 +88,13 @@ function depwarn(msg, funcsym; force::Bool=false) _module=begin bt = backtrace() frame, caller = firstcaller(bt, funcsym) - # TODO: Is it reasonable to attribute callers without linfo to Core? - caller.linfo isa Core.MethodInstance ? caller.linfo.def.module : Core + linfo = caller.linfo + if linfo isa Core.MethodInstance + def = linfo.def + def isa Module ? def : def.module + else + Core # TODO: Is it reasonable to attribute callers without linfo to Core? + end end, _file=String(caller.file), _line=caller.line, diff --git a/base/loading.jl b/base/loading.jl index 1f3297df36448..0dadb558a789c 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -596,7 +596,8 @@ end function is_v1_format_manifest(raw_manifest::Dict) if haskey(raw_manifest, "manifest_format") - if raw_manifest["manifest_format"] isa Dict && haskey(raw_manifest["manifest_format"], "uuid") + mf = raw_manifest["manifest_format"] + if mf isa Dict && haskey(mf, "uuid") # the off-chance where an old format manifest has a dep called "manifest_format" return true end @@ -612,7 +613,7 @@ function get_deps(raw_manifest::Dict) return raw_manifest else # if the manifest has no deps, there won't be a `deps` field - return get(Dict{String, Any}, raw_manifest, "deps") + return get(Dict{String, Any}, raw_manifest, "deps")::Dict{String, Any} end end @@ -893,6 +894,7 @@ const TIMING_IMPORTS = Threads.Atomic{Int}(0) if staledeps === true continue end + staledeps = staledeps::Vector{Any} try touch(path_to_try) # update timestamp of precompilation file catch # file might be read-only and then we fail to update timestamp, which is fine @@ -1289,8 +1291,11 @@ include_string(m::Module, txt::AbstractString, fname::AbstractString="string") = function source_path(default::Union{AbstractString,Nothing}="") s = current_task().storage - if s !== nothing && haskey(s::IdDict{Any,Any}, :SOURCE_PATH) - return s[:SOURCE_PATH]::Union{Nothing,String} + if s !== nothing + s = s::IdDict{Any,Any} + if haskey(s, :SOURCE_PATH) + return s[:SOURCE_PATH]::Union{Nothing,String} + end end return default end @@ -1877,7 +1882,7 @@ function get_preferences_hash(uuid::Union{UUID, Nothing}, prefs_list::Vector{Str for name in prefs_list prefs_value = get(prefs, name, nothing) if prefs_value !== nothing - h = hash(prefs_value, h) + h = hash(prefs_value, h)::UInt end end # We always return a `UInt64` so that our serialization format is stable diff --git a/base/process.jl b/base/process.jl index 876eb74fbd413..57c4e0ebd874a 100644 --- a/base/process.jl +++ b/base/process.jl @@ -90,7 +90,7 @@ end @noinline function _spawn_primitive(file, cmd::Cmd, stdio::SpawnIOs) loop = eventloop() cpumask = cmd.cpus - cpumask === nothing || (cpumask = as_cpumask(cmd.cpus)) + cpumask === nothing || (cpumask = as_cpumask(cpumask)) GC.@preserve stdio begin iohandles = Tuple{Cint, UInt}[ # assuming little-endian layout let h = rawhandle(io) diff --git a/base/show.jl b/base/show.jl index 999a73047f6b2..607f13d3778e7 100644 --- a/base/show.jl +++ b/base/show.jl @@ -727,11 +727,11 @@ function show_typealias(io::IO, @nospecialize(x::Type)) end function make_typealiases(@nospecialize(x::Type)) - Any === x && return Core.svec(), Union{} - x <: Tuple && return Core.svec(), Union{} + aliases = SimpleVector[] + Any === x && return aliases, Union{} + x <: Tuple && return aliases, Union{} mods = modulesof!(Set{Module}(), x) Core in mods && push!(mods, Base) - aliases = SimpleVector[] vars = Dict{Symbol,TypeVar}() xenv = UnionAll[] each = Any[] @@ -783,23 +783,24 @@ function make_typealiases(@nospecialize(x::Type)) end end if isempty(aliases) - return Core.svec(), Union{} + return aliases, Union{} end - sort!(aliases, by = x -> x[4], rev = true) # heuristic sort by "best" environment + sort!(aliases, by = x -> x[4]::Tuple{Int,Int}, rev = true) # heuristic sort by "best" environment let applied = Union{} applied1 = Union{} keep = SimpleVector[] prev = (0, 0) for alias in aliases - if alias[4][1] < 2 + alias4 = alias[4]::Tuple{Int,Int} + if alias4[1] < 2 if !(alias[3] <: applied) applied1 = Union{applied1, alias[3]} push!(keep, alias) end - elseif alias[4] == prev || !(alias[3] <: applied) + elseif alias4 == prev || !(alias[3] <: applied) applied = applied1 = Union{applied1, alias[3]} push!(keep, alias) - prev = alias[4] + prev = alias4 end end return keep, applied1 @@ -825,16 +826,17 @@ function show_unionaliases(io::IO, x::Union) end if first && !tvar && length(aliases) == 1 alias = aliases[1] - wheres = make_wheres(io, alias[2], x) - show_typealias(io, alias[1], x, alias[2], wheres) + env = alias[2]::SimpleVector + wheres = make_wheres(io, env, x) + show_typealias(io, alias[1], x, env, wheres) show_wheres(io, wheres) else for alias in aliases print(io, first ? "Union{" : ", ") first = false - env = alias[2] - wheres = make_wheres(io, alias[2], x) - show_typealias(io, alias[1], x, alias[2], wheres) + env = alias[2]::SimpleVector + wheres = make_wheres(io, env, x) + show_typealias(io, alias[1], x, env, wheres) show_wheres(io, wheres) end if tvar @@ -879,7 +881,7 @@ end show(io::IO, @nospecialize(x::Type)) = _show_type(io, inferencebarrier(x)) function _show_type(io::IO, @nospecialize(x::Type)) if print_without_params(x) - show_type_name(io, unwrap_unionall(x).name) + show_type_name(io, (unwrap_unionall(x)::DataType).name) return elseif get(io, :compact, true) && show_typealias(io, x) return @@ -1006,7 +1008,7 @@ function show_datatype(io::IO, x::DataType, wheres::Vector{TypeVar}=TypeVar[]) end else show_type_name(io, x.name) - show_typeparams(io, parameters, unwrap_unionall(x.name.wrapper).parameters, wheres) + show_typeparams(io, parameters, (unwrap_unionall(x.name.wrapper)::DataType).parameters, wheres) end end diff --git a/base/stat.jl b/base/stat.jl index f8d28cadf0c72..f38a82634dc2f 100644 --- a/base/stat.jl +++ b/base/stat.jl @@ -146,7 +146,7 @@ show(io::IO, ::MIME"text/plain", st::StatStruct) = show_statstruct(io, st, false macro stat_call(sym, arg1type, arg) return quote - stat_buf = zeros(UInt8, ccall(:jl_sizeof_stat, Int32, ())) + stat_buf = zeros(UInt8, Int(ccall(:jl_sizeof_stat, Int32, ()))) r = ccall($(Expr(:quote, sym)), Int32, ($(esc(arg1type)), Ptr{UInt8}), $(esc(arg)), stat_buf) if !(r in (0, Base.UV_ENOENT, Base.UV_ENOTDIR, Base.UV_EINVAL)) uv_error(string("stat(", repr($(esc(arg))), ")"), r) diff --git a/base/toml_parser.jl b/base/toml_parser.jl index 66db0e5695551..c19572c2ebaed 100644 --- a/base/toml_parser.jl +++ b/base/toml_parser.jl @@ -1165,7 +1165,7 @@ function parse_string_continue(l::Parser, multiline::Bool, quoted::Bool)::Err{St end function take_chunks(l::Parser, unescape::Bool)::String - nbytes = sum(length, l.chunks) + nbytes = sum(length, l.chunks; init=0) str = Base._string_n(nbytes) offset = 1 for chunk in l.chunks From b1e1c52deeb125576e06f917453eb9e0df31912c Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 20 Apr 2022 11:51:32 -0400 Subject: [PATCH 34/68] gc-ext: only sweep unmarked objects (#45035) This prior conditional was a fixed constant branch, so this seems more like the intent. (cherry picked from commit ac51add4a610668835bb12db4913af1c2c16aaf8) --- src/gc.c | 13 +++++++------ test/gcext/gcext.c | 6 ++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/gc.c b/src/gc.c index 609c2009bf103..6942d153616b9 100644 --- a/src/gc.c +++ b/src/gc.c @@ -552,20 +552,21 @@ JL_DLLEXPORT void jl_finalize_th(jl_task_t *ct, jl_value_t *o) arraylist_free(&copied_list); } +// explicitly scheduled objects for the sweepfunc callback static void gc_sweep_foreign_objs_in_list(arraylist_t *objs) { size_t p = 0; for (size_t i = 0; i < objs->len; i++) { - jl_value_t *v = (jl_value_t *)(objs->items[i]); - jl_datatype_t *t = (jl_datatype_t *)(jl_typeof(v)); + jl_value_t *v = (jl_value_t*)(objs->items[i]); + jl_datatype_t *t = (jl_datatype_t*)(jl_typeof(v)); const jl_datatype_layout_t *layout = t->layout; jl_fielddescdyn_t *desc = (jl_fielddescdyn_t*)jl_dt_layout_fields(layout); - if (!gc_ptr_tag(v, 1)) { + + int bits = jl_astaggedvalue(v)->bits.gc; + if (!gc_marked(bits)) desc->sweepfunc(v); - } - else { + else objs->items[p++] = v; - } } objs->len = p; } diff --git a/test/gcext/gcext.c b/test/gcext/gcext.c index 2b380c43feccb..b66f21bb660ee 100644 --- a/test/gcext/gcext.c +++ b/test/gcext/gcext.c @@ -561,8 +561,10 @@ void sweep_stack_data(jl_value_t *p) { obj_sweeps++; dynstack_t *stk = (dynstack_t *)p; - if (stk->size > stk->capacity) - jl_error("internal error during sweeping"); + if (stk->size > stk->capacity) { + assert(0 && "internal error during sweeping"); + abort(); + } } // Safely execute Julia code From 68ddba354fa9bb779301866fc1de694deb8bdb0c Mon Sep 17 00:00:00 2001 From: woclass Date: Sun, 24 Apr 2022 06:15:37 +0800 Subject: [PATCH 35/68] [doc] Add a Cover page to PDF doc (#45034) * Add cover page * use tikz logo * Add "Release Notes" to devdocs Co-authored-by: Viral B. Shah (cherry picked from commit 8a3b4ab77c95cd8b387f6819cecc2e768e1edaa1) --- doc/make.jl | 4 +- doc/src/assets/cover-splash.tex | 353 ++++++++++++++++++++++++++++++++ doc/src/assets/cover.tex | 46 +++++ doc/src/assets/custom.sty | 45 ++++ doc/src/assets/logo.tex | 142 +++++++++++++ doc/src/assets/preamble.tex | 48 +++++ 6 files changed, 636 insertions(+), 2 deletions(-) create mode 100644 doc/src/assets/cover-splash.tex create mode 100644 doc/src/assets/cover.tex create mode 100644 doc/src/assets/custom.sty create mode 100644 doc/src/assets/logo.tex create mode 100644 doc/src/assets/preamble.tex diff --git a/doc/make.jl b/doc/make.jl index f814ba43382e4..b33173398467d 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -175,8 +175,8 @@ const PAGES = [ "Manual" => ["index.md", Manual...], "Base" => BaseDocs, "Standard Library" => StdlibDocs, - "Developer Documentation" => DevDocs, - hide("NEWS.md"), + # Add "Release Notes" to devdocs + "Developer Documentation" => [DevDocs..., hide("NEWS.md")], ] else const PAGES = [ diff --git a/doc/src/assets/cover-splash.tex b/doc/src/assets/cover-splash.tex new file mode 100644 index 0000000000000..10409a14d5742 --- /dev/null +++ b/doc/src/assets/cover-splash.tex @@ -0,0 +1,353 @@ +%% Direct translation of the backsplash image in "JuliaLang/www.julialang.org" +%% And cropping of the image +%% https://github.com/JuliaLang/www.julialang.org/blob/main/_assets/infra/backsplash-min-0.5.svg + + +\newcommand{\splashScaleFactor}{0.6} +\newcommand{\whiteMaskTransparency}{0.5} +\newcommand{\triangleTransparency}{0.6} +\begin{tikzpicture}[x=1,y=1,yscale=-\splashScaleFactor,xscale=\splashScaleFactor,draw=white] +% Clipping +\clip (510,15) rectangle (1570,350); +% Cropping +\useasboundingbox(510,0) rectangle (1570,350.0); +% gary background +% \draw[fill=splash_gary,opacity=0] (510.0,0.0)--++(1057.5,0.0)--++(0.0,350.0)--++(-1057.5,0.0)--cycle; + +%% Draw triangles +\draw[fill=julia_red,opacity=\triangleTransparency] + (991.9,11.4)--++(51.5,19.7)--++(-56.2,25.3) + ++(56.8,56.9)--++(-47.4,-27.5)--++(48.4,-52.2) + (990.9,9.0)--++(-40.6,1.4)--++(125.4,-21.0) + (969.5,111.5)--++(35.3,20.8)--++(-45.2,3.6) + (952.4,205.8)--++(14.0,55.9)--++(-44.7,-29.8) + ++(33.8,-160.3)--++(12.1,37.4)--++(-63.5,-1.9) + (946.8,335.7)--++(-6.5,-33.3)--++(58.2,17.6) + (920.0,235.8)--++(17.8,64.6)--++(-39.3,15.0) + ++(-0.7,3.0)--++(45.6,18.6)--++(-51.2,22.3) + (910.2,17.7)--++(43.6,51.2)--++(-72.1,-0.9) + (901.8,109.5)--++(53.8,27.4)--++(-62.3,24.5) + (885.4,204.6)--++(32.3,26.8)--++(-47.1,9.2) + (879.5,70.3)--++(19.3,35.8)--++(-54.1,-22.0) + (870.0,317.0)--++(0.0,-69.4)--++(25.6,68.5) + (868.7,319.7)--++(20.5,40.0)--++(-53.1,-8.4) + (867.4,190.9)--++(-12.9,-43.1)--++(35.8,15.6) + (866.0,193.8)--++(1.9,45.8)--++(-63.6,-57.1) + (815.2,79.7)--++(-16.2,-17.1)--++(65.6,-10.8) + ++(3.4,-40.0)--++(0.9,35.9)--++(-50.7,-41.5) + (801.8,362.7)--++(-7.3,-40.3)--++(37.5,29.3) + ++(-16.2,-267.4)--++(35.8,61.3)--++(-41.5,22.6) + (796.4,60.5)--++(-49.1,-26.4)--++(66.1,-27.4) + (765.0,352.3)--++(-0.9,-61.7)--++(27.6,29.5) + (758.0,131.6)--++(40.8,48.4)--++(-80.7,-0.9) + ++(66.2,78.4)--++(-38.6,-22.6)--++(53.7,-50.9) + (741.8,98.9)--++(14.7,30.4)--++(-45.1,7.4) + (720.7,321.9)--++(38.2,30.3)--++(-40.8,-13.9) + (716.2,182.1)--++(19.5,30.7)--++(-50.2,38.1) + (708.3,140.2)--++(6.5,39.0)--++(-52.0,9.3) + ++(4.7,130.8)--++(46.6,19.6)--++(-64.3,20.5) + ++(33.5,-102.7)--++(34.9,62.2)--++(-50.0,-1.9) + (697.1,1.3)--++(3.7,57.5)--++(-37.1,-47.3) + ++(20.3,86.2)--++(-30.6,-59.4)--++(47.4,24.1) + (660.1,193.2)--++(21.3,59.2)--++(-35.1,-12.0) + (650.6,34.4)--++(-49.2,-41.9)--++(59.2,19.1) + (641.9,103.0)--++(40.0,-1.9)--++(-45.6,59.5) + (640.4,99.6)--++(-46.4,-58.7)--++(55.9,-3.8) + (611.6,316.3)--++(34.5,43.8)--++(-43.8,2.8) + (600.4,137.3)--++(-7.1,-17.7)--++(43.4,-15.0) + (596.6,210.6)--++(44.0,28.8)--++(-49.4,-7.2) + (591.0,41.9)--++(0.0,71.1)--++(-19.8,-57.6) + (551.4,303.2)--++(-11.2,-49.5)--++(47.7,-18.7) + (541.9,104.3)--++(47.4,14.9)--++(-51.2,25.1) + (540.0,214.9)--++(-1.9,-66.3)--++(55.9,59.6) + ++(-2.7,-170.7)--++(-44.8,-53.2)--++(49.5,6.5) + (512.0,192.8)--++(-28.2,-58.3)--++(51.7,12.2) + ++(-23.0,50.0)--++(24.9,19.4)--++(-44.3,38.8) + (503.7,105.9)--++(-15.7,-57.3)--++(50.9,53.6) + (487.0,42.5)--(487.0,6.7)--++(53.2,-21.1) + (1602.3,36.6)--++(12.3,58.5)--++(-57.5,-32.1) + (1545.2,19.0)--++(-49.1,-29.1)--++(117.4,19.1) + (1538.8,89.3)--++(67.7,71.6)--++(-73.5,-3.8) + (1537.9,366.1)--++(-2.9,-80.5)--++(70.9,33.6) + (1532.0,159.8)--++(18.4,27.6)--++(-47.0,12.9) + ++(34.3,-114.6)--++(-48.3,-30.1)--++(63.8,8.2) + (1501.7,203.7)--++(63.0,41.4)--++(-71.5,0.9) + (1499.9,137.4)--++(-43.1,-47.7)--++(59.6,25.7) + (1493.0,315.9)--++(0.0,-66.7)--++(39.5,34.8) + (1491.7,319.6)--++(15.7,27.7)--++(-60.1,12.0) + (1469.9,265.9)--++(20.1,49.8)--++(-21.0,-12.2) + (1445.7,193.2)--++(21.8,66.4)--++(-57.8,-16.1) + ++(44.0,-158.6)--++(-7.4,-46.2)--++(37.0,15.7) + (1433.7,139.9)--++(-33.8,-45.1)--++(51.7,-6.6) + ++(-7.4,-52.1)--++(-30.1,1.8)--++(38.3,-51.9) + (1418.7,179.9)--++(-19.1,-27.3)--++(32.7,-9.1) + (1407.3,246.3)--++(7.5,63.2)--++(-49.1,-17.0) + (1384.6,-17.4)--++(24.9,53.5)--++(-49.9,-34.2) + (1367.2,191.1)--++(-28.2,-58.4)--++(57.4,19.8) + (1364.4,296.2)--++(23.2,49.3)--++(-35.3,17.7) + (1301.7,135.9)--++(-4.3,-16.4)--++(35.3,11.2) + (1297.0,116.5)--++(-1.9,-78.9)--++(52.3,57.0) + ++(-49.6,-80.8)--++(48.0,38.2)--++(-50.7,-17.8) + (1296.9,251.9)--++(-8.5,-57.6)--++(69.0,52.0) + (1295.1,254.1)--++(7.4,59.0)--++(-37.8,-54.3) + ++(23.2,74.8)--++(-32.1,-28.5)--++(46.3,12.5) + (1288.0,336.1)--++(55.2,26.3)--++(-77.0,-15.8) + (1258.6,34.5)--++(-14.0,-47.5)--++(50.3,25.2) + ++(-8.4,177.8)--++(-32.7,-39.3)--++(45.8,-12.2) + (1255.1,194.9)--++(6.6,60.1)--++(-60.1,-69.5) + ++(49.7,116.1)--++(-66.9,-32.0)--++(77.2,-10.4) + ++(-10.2,45.7)--++(10.3,41.3)--++(-66.6,13.1) + (1249.1,118.0)--++(1.8,31.1)--++(-43.9,1.8) + (1187.6,-8.5)--++(17.0,65.3)--++(-53.0,-5.7) + (1182.3,316.6)--++(8.3,42.4)--++(-40.6,-24.0) + (1180.3,267.6)--++(-31.7,-12.1)--++(48.5,-67.1) + (1180.1,271.2)--++(1.8,42.2)--++(-31.2,-14.7) + (1150.7,162.1)--++(-7.6,-81.0)--++(57.2,7.6) + (1146.0,256.9)--++(1.9,39.9)--++(-56.7,-44.6) + ++(55.8,81.3)--++(-43.2,-14.7)--++(44.1,-18.4) + (1141.6,77.9)--++(-32.5,-38.8)--++(39.7,13.5) + ++(-40.4,87.8)--++(39.0,24.2)--++(-60.4,13.9) + (1108.9,-14.3)--++(21.7,26.3)--++(-23.5,22.6) + (1108.3,136.4)--++(-14.1,-53.4)--++(45.9,-1.9) + (1066.9,169.8)--++(16.8,10.6)--++(-19.4,35.3) + ++(-15.7,97.5)--++(-10.4,-69.9)--++(48.2,8.5) + (1066.5,166.5)--++(-19.7,-49.7)--++(59.1,22.5) + (1058.4,4.5)--++(43.9,31.1)--++(-54.9,-5.5) + (1048.8,317.1)--++(48.4,2.7)--++(-55.7,27.4) + (1037.4,240.4)--++(-15.5,-33.7)--++(39.1,13.6) + (1007.4,135.4)--++(11.4,68.6)--++(-63.8,-0.9) + (1005.9,249.3)--++(40.4,65.8)--++(-43.3,3.8); + +\draw[fill=julia_purple,opacity=\triangleTransparency] + (995.4,84.1)--++(-8.2,-25.5)--++(54.7,-24.6) + (969.5,108.2)--++(-11.9,-36.5)--++(35.6,14.6) + (967.9,265.8)--++(31.9,52.5)--++(-59.1,-17.8) + ++(28.0,-37.9)--++(-14.0,-55.9)--++(48.5,39.1) + (952.1,202.2)--++(-58.0,-39.0)--++(62.7,-24.7) + (891.2,161.6)--++(-36.3,-15.8)--++(44.7,-35.4) + ++(-19.6,-43.5)--++(-8.8,-16.7)--++(35.1,-29.8) + (884.3,201.4)--++(-15.7,-8.7)--++(21.8,-26.2) + (841.1,84.9)--++(9.9,55.8)--++(-34.2,-58.4) + ++(-7.5,89.0)--++(47.9,19.0)--++(-54.5,-9.9) + (815.2,4.0)--++(-37.7,-10.6)--++(69.6,14.2) + ++(-45.4,175.3)--++(65.2,58.5)--++(-80.6,16.3) + (784.3,261.5)--++(7.3,55.5)--++(-27.3,-29.1) + ++(-18.4,-252.3)--++(49.1,26.4)--++(-51.9,33.1) + (762.0,290.0)--++(0.9,62.9)--++(-41.3,-32.9) + (758.3,128.4)--++(-14.9,-30.7)--++(67.9,-14.9) + (744.5,233.2)--++(-6.3,-19.7)--++(57.2,-28.6) + (744.2,32.2)--++(-43.9,-31.7)--++(49.5,-12.1) + (709.1,136.3)--++(-23.3,-35.4)--++(54.0,-2.8) + (635.0,165.5)--++(23.1,24.0)--++(-59.1,17.6) + (611.3,311.1)--++(-19.9,-76.8)--++(52.2,7.6) + (591.5,120.4)--++(7.1,17.8)--++(-56.0,6.2) + (551.3,307.9)--++(47.1,54.6)--++(-64.0,-12.2) + (540.0,251.5)--++(0.0,-34.1)--++(47.1,15.7) + ++(-16.6,-179.6)--++(-23.7,-65.5)--++(43.7,51.9) + (538.7,256.1)--++(11.1,49.2)--++(-38.0,8.4) + ++(29.1,-212.2)--++(-52.2,-55.0)--++(79.7,9.5) + (1608.2,159.7)--++(-67.6,-71.4)--++(74.2,9.5) + (1567.8,247.7)--++(37.8,69.1)--++(-70.0,-33.1) + (1555.7,61.5)--++(-9.3,-40.1)--++(54.0,14.0) + (1518.1,113.1)--++(-30.4,-56.3)--++(48.9,30.4) + (1517.6,117.1)--++(12.6,38.6)--++(-28.8,-17.1) + ++(7.4,207.0)--++(-15.5,-27.4)--++(38.4,-31.1) + (1484.1,52.5)--++(-37.4,-15.9)--++(43.0,-45.8) + ++(-20.2,268.7)--++(-20.9,-63.7)--++(41.9,51.0) + (1446.6,189.1)--++(-11.3,-46.1)--++(62.1,-2.8) + (1443.9,358.7)--++(-26.3,-46.9)--++(48.8,-6.6) + (1419.8,183.3)--++(24.6,8.2)--++(-34.6,48.3) + (1412.5,41.1)--++(38.5,45.1)--++(-51.7,6.6) + (1367.7,195.4)--++(38.3,47.7)--++(-44.9,2.8) + ++(-2.4,-242.2)--++(50.9,34.8)--++(-57.4,15.1) + (1359.1,248.1)--++(3.7,43.1)--++(-63.7,-37.4) + ++(37.6,-120.9)--++(28.6,59.1)--++(-76.3,-1.0) + (1351.9,57.0)--++(43.4,36.0)--++(-45.2,0.9) + (1336.4,129.8)--++(-37.5,-11.9)--++(48.5,-21.0) + (1304.0,318.5)--++(42.3,43.2)--++(-56.7,-27.0) + (1293.0,36.0)--++(1.9,77.2)--++(-35.3,-76.3) + (1256.9,192.9)--++(-3.6,-39.8)--++(31.7,38.0) + (1252.9,148.1)--++(-1.8,-31.1)--++(42.0,1.8) + (1204.9,153.0)--++(43.5,-1.8)--++(-48.0,30.8) + (1201.2,86.2)--++(-47.4,-32.8)--++(51.1,5.5) + ++(1.4,-3.7)--++(-17.0,-65.3)--++(52.0,-3.8) + (1201.1,91.1)--++(1.9,60.1)--++(-50.7,12.2) + (1184.0,313.8)--++(-1.9,-43.2)--++(66.7,31.9) + (1155.9,186.0)--++(41.1,-0.9)--++(-48.6,67.3) + (1149.3,166.2)--++(4.4,17.7)--++(-62.0,-4.4) + (1132.3,10.8)--++(-22.0,-26.6)--++(73.4,5.5) + (1131.7,13.7)--++(16.5,36.6)--++(-40.3,-13.7) + (1101.8,317.6)--++(-12.3,-64.2)--++(57.6,45.3) + (1092.5,80.0)--++(-44.0,-47.8)--++(56.2,5.6) + (1092.4,83.6)--++(14.1,53.7)--++(-59.4,-22.6) + (1062.9,221.4)--++(22.7,28.1)--++(-46.3,-8.2) + (1057.8,2.0)--++(-50.7,5.3)--++(88.5,-20.4) + (1036.2,243.1)--++(10.1,68.2)--++(-39.6,-64.5) + ++(32.7,100.5)--++(-34.7,-26.5)--++(42.0,-3.7) + (1020.7,203.2)--++(-11.4,-68.2)--++(54.9,33.1) + (1008.6,131.6)--++(-12.0,-43.5)--++(46.3,26.8) + (1001.4,321.2)--++(14.9,40.1)--++(-67.1,-24.2); + +\draw[fill=julia_green,opacity=\triangleTransparency] + (994.5,87.8)--++(11.9,43.0)--++(-35.7,-21.1) + ++(-11.7,28.1)--++(46.9,-3.8)--++(-51.6,66.6) + ++(-8.2,137.3)--++(66.0,23.8)--++(-116.4,-1.8) + (985.4,59.3)--++(8.0,24.9)--++(-34.7,-14.2) + (985.2,55.9)--++(-47.7,-43.0)--++(52.4,-1.9) + (957.5,135.6)--++(-52.1,-26.5)--++(62.1,1.8) + (935.0,11.0)--++(-170.3,-23.1)--++(330.5,-3.7) + (934.3,13.1)--++(18.8,52.0)--++(-41.3,-48.4) + (901.4,106.5)--++(-19.7,-36.5)--++(71.2,0.9) + (896.9,313.7)--++(-26.5,-71.0)--++(48.3,-9.5) + (892.7,164.7)--++(56.9,38.3)--++(-63.5,-0.9) + (890.4,357.7)--++(-19.8,-38.8)--++(25.2,-0.9) + (870.9,47.9)--++(-0.9,-35.8)--++(37.6,4.6) + (869.8,51.8)--++(8.8,16.7)--++(-33.4,13.2) + ++(24.6,155.7)--++(-1.7,-42.6)--++(15.7,8.7) + (868.0,243.2)--++(0.0,72.9)--++(-80.5,-56.6) + ++(28.0,-252.8)--++(52.1,42.7)--++(-69.2,11.4) + (852.4,147.5)--++(13.1,43.8)--++(-54.1,-21.5) + (833.9,350.7)--++(-38.1,-29.8)--++(70.7,-1.9) + (800.9,179.4)--++(-35.9,-42.6)--++(42.6,33.4) + ++(-55.8,-181.9)--++(60.1,16.9)--++(-65.7,27.2) + (796.8,63.3)--++(16.2,17.1)--++(-65.8,14.4) + (763.3,285.9)--++(-17.4,-48.7)--++(37.7,22.0) + (736.5,214.7)--++(6.3,19.7)--++(-54.6,17.0) + (720.3,318.5)--++(-35.0,-62.5)--++(75.7,32.2) + (717.0,340.1)--++(41.8,14.2)--++(-103.1,5.3) + (716.6,177.7)--++(-6.5,-38.9)--++(45.3,-7.4) + (702.9,60.2)--++(-3.8,-58.2)--++(44.1,31.9) + (702.3,63.6)--++(37.1,32.5)--++(-53.9,2.8) + (683.9,101.7)--++(23.5,35.8)--++(-69.6,24.5) + ++(45.3,89.4)--++(-21.8,-60.5)--++(53.0,-9.5) + (666.9,314.1)--++(-20.4,-71.6)--++(35.3,12.1) + (662.2,12.9)--++(36.2,46.1)--++(-46.1,-23.5) + (651.5,39.2)--++(30.9,59.9)--++(-40.2,1.9) + (647.8,359.1)--++(-34.6,-43.9)--++(52.3,3.7) + (644.4,239.4)--++(-46.1,-30.1)--++(60.2,-17.9) + (634.2,162.2)--++(-32.6,-23.3)--++(38.2,-34.4) + (595.1,206.6)--++(-56.0,-59.8)--++(59.8,-6.6) + (589.7,235.7)--++(20.0,77.1)--++(-57.1,-7.6) + (569.8,57.3)--++(20.7,60.1)--++(-47.9,-15.0) + (538.0,218.2)--++(0.0,33.9)--++(-44.0,4.6) + ++(42.2,-113.1)--++(-31.2,-35.8)--++(34.9,-3.7) + (532.8,348.8)--++(-21.2,-33.2)--++(37.8,-8.3) + (496.9,336.0)--++(30.8,13.7)--++(-86.5,-4.3) + (486.2,4.8)--++(-20.6,-17.9)--++(72.7,-2.7) + (1607.2,320.8)--++(4.7,48.2)--++(-71.8,-1.9) + ++(15.2,-302.8)--++(55.7,31.0)--++(-71.2,-9.1) + (1602.6,33.9)--++(-52.5,-13.6)--++(67.9,-10.0) + (1588.5,189.0)--++(-32.4,-1.8)--++(49.9,-22.8) + (1551.3,189.2)--++(15.0,54.5)--++(-62.9,-41.3) + (1544.3,21.5)--++(9.4,40.3)--++(-65.6,-8.4) + (1533.2,288.9)--++(2.8,77.1)--++(-25.7,-18.4) + (1531.4,153.0)--++(-12.3,-37.8)--++(17.6,-24.6) + (1502.0,198.9)--++(-0.9,-58.1)--++(29.5,17.5) + (1491.0,248.8)--++(0.0,64.1)--++(-20.8,-51.4) + (1484.8,55.7)--++(31.0,57.3)--++(-61.0,-26.3) + (1454.8,-16.9)--++(34.2,5.5)--++(-42.5,45.2) + (1453.4,88.9)--++(44.4,49.2)--++(-62.4,2.8) + ++(56.0,103.8)--++(-42.9,-52.2)--++(51.3,10.3) + (1444.2,38.1)--++(7.3,45.6)--++(-37.4,-43.8) + (1433.8,145.4)--++(10.8,44.1)--++(-24.3,-8.1) + (1416.8,308.9)--++(-7.6,-63.5)--++(57.9,16.1) + (1412.2,37.0)--++(-25.6,-55.0)--++(65.5,0.9) + (1399.0,150.7)--(1399,97.0)--++(33.3,44.4) + (1398.0,153.7)--++(19.3,27.6)--++(-47.9,10.1) + (1397.4,92.2)--++(-44.3,-36.7)--++(57.5,-15.1) + ++(-21.4,303.7)--++(-23.3,-49.4)--++(48.4,16.8) + (1389.105,347.066)--++(48.442,13.211)--++(-81.91,3.524) + (1364.8,290.6)--++(-3.7,-42.7)--++(44.5,-2.8) + (1359.2,245.2)--++(-69.2,-52.1)--++(75.8,0.9) + (1350.4,362.9)--++(-44.6,-45.5)--++(56.9,-22.8) + (1350.2,53.1)--++(-50.9,-40.5)--++(57.4,-9.4) + (1349.4,97.0)--++(45.4,52.9)--++(-56.6,-19.5) + (1301.8,137.9)--++(32.7,-5.5)--++(-45.5,55.5) + (1297.1,10.9)--++(-50.1,-25.0)--++(131.6,-3.7) + (1295.4,119.7)--++(4.4,16.6)--++(-42.9,11.4) + ++(6.5,112.7)--++(38.2,54.9)--++(-48.4,-13.0) + (1286.6,196.1)--++(8.3,56.1)--++(-30.4,4.6) + ++(28.6,-222.7)--++(-31.4,0.9)--++(34.0,-20.9) + (1263.0,348.0)--++(76.7,15.7)--++(-138.7,-3.5) + (1251.2,151.7)--++(3.7,41.1)--++(-53.2,-9.3) + (1249.3,113.4)--++(-41.7,-54.9)--++(49.3,-20.8) + ++(-57.3,148.6)--++(61.5,71.1)--++(-78.8,10.6) + (1204.9,150.0)--++(-1.9,-60.2)--++(45.2,26.3) + (1192.6,359.0)--++(-8.5,-43.2)--++(64.8,-11.3) + (1155.8,184.0)--++(-4.3,-17.4)--++(42.5,16.5) + ++(-43.4,-129.5)--++(47.4,32.8)--++(-54.7,-7.3) + (1150.1,49.9)--++(-16.8,-37.3)--++(52.2,-21.5) + ++(-37.5,345.1)--++(40.4,23.9)--++(-72.5,1.8) + (1149.9,296.8)--++(-1.8,-39.3)--++(31.1,11.9) + ++(-29.2,31.2)--++(30.9,14.5)--++(-31.8,18.2) + (1141.3,83.2)--++(7.5,79.9)--++(-39.5,-24.4) + ++(36.8,116.3)--++(-55.9,-4.7)--++(63.4,-62.5) + (1106.4,39.1)--++(33.5,40.0)--++(-45.6,1.9) + (1105.1,35.1)--++(-45.0,-31.9)--++(46.9,-18.7) + (1100.4,320.4)--++(11.3,41.3)--++(-68.5,-13.1) + (1087.5,253.7)--++(12.3,64.2)--++(-50.0,-2.8) + (1084.2,183.6)--++(2.7,64.4)--++(-22.7,-28.1) + ++(-17.2,-186.4)--++(44.5,48.3)--++(-45.5,31.3) + (1045.5,29.7)--++(-50.2,-19.2)--++(61.1,-6.4) + (1044.5,116.4)--++(19.5,49.2)--++(-53.9,-32.5) + (1020.1,207.6)--++(15.4,33.5)--++(-29.0,3.6) + ++(12.0,116.7)--++(-14.4,-38.6)--++(34.1,26.0) + (1001.1,316.6)--++(-31.7,-52.2)--++(34.5,-16.8); + +\draw[fill=julia_blue,opacity=\triangleTransparency] + (956.6,68.7)--++(-19.3,-53.3)--++(46.8,42.2) + (939.4,298.8)--++(-17.8,-64.5)--++(44.9,29.9) + (938.2,302.4)--++(6.4,33.0)--++(-44.9,-18.3) + (920.0,230.7)--++(-32.2,-26.7)--++(62.5,0.9) + (869.1,10.0)--++(-88.1,-17.9)--++(137.3,18.7) + (853.7,144.2)--++(-10.4,-58.6)--++(55.7,22.7) + (841.6,82.9)--++(-23.4,-2.6)--++(47.6,-26.8) + (808.1,168.1)--++(-48.6,-38.1)--++(54.3,-46.7) + (793.9,319.0)--++(-7.6,-57.9)--++(79.7,56.0) + (792.4,322.3)--++(7.3,40.4)--++(-34.0,-8.3) + (743.4,236.2)--++(17.9,49.9)--++(-75.4,-32.0) + (741.1,94.9)--++(-37.5,-32.8)--++(40.3,-26.2) + (737.4,211.7)--++(-19.5,-30.7)--++(79.0,0.9) + (716.2,337.6)--++(-43.8,-18.4)--++(46.4,1.8) + (662.0,10.0)--++(-55.8,-18.0)--++(86.8,8.6) + (660.0,188.6)--++(-23.2,-24.1)--++(68.7,-24.1) + (644.8,243.8)--++(20.9,73.0)--++(-53.1,-3.8) + (600.4,361.8)--++(-47.0,-54.5)--++(56.3,7.5) + ++(-8.9,-174.0)--++(32.6,23.3)--++(-36.4,42.0) + (597.8,-8.0)--++(50.7,43.2)--++(-55.4,3.8) + (593.0,117.6)--++(0.0,-74.7)--++(46.3,58.6) + (589.3,231.7)--++(-45.8,-15.3)--++(51.2,-6.3) + (543.6,-15.1)--++(24.9,68.9)--++(-80.4,-9.6) + (536.1,149.9)--++(1.8,64.0)--++(-24.7,-19.2) + ++(-3.1,117.6)--++(-17.8,-53.4)--++(45.0,-4.7) + (510.0,316.9)--++(20.4,31.9)--++(-31.9,-14.2) + ++(4.5,-226.0)--++(31.2,35.8)--++(-50.4,-11.9) + (1552.4,186.7)--++(-18.4,-27.6)--++(70.9,3.7) + (1533.9,282.6)--++(-39.3,-34.6)--++(71.1,-0.9) + ++(2.4,-4.3)--++(-14.8,-53.7)--++(34.3,1.9) + (1508.8,349.1)--++(24.7,17.7)--++(-82.2,-6.2) + (1499.0,141.3)--++(0.9,59.5)--++(-51.9,-10.4) + (1491.9,-10.3)--++(51.2,30.3)--++(-56.8,32.2) + (1468.5,305.4)--++(21.8,12.7)--++(-43.7,39.1) + (1467.0,303.1)--++(-48.1,6.5)--++(49.0,-46.2) + (1415.9,312.8)--++(26.1,46.6)--++(-51.3,-14.0) + (1407.4,241.7)--++(-38.6,-48.0)--++(49.0,-10.4) + (1397.0,95.0)--++(0.0,54.3)--++(-45.9,-53.3) + (1357.6,1.1)--++(-49.6,8.1)--++(71.5,-25.2) + (1348.1,92.6)--++(-50.7,-55.3)--++(52.5,18.4) + (1304.8,315.6)--++(-7.6,-60.7)--++(64.5,37.9) + (1263.5,253.6)--++(-6.4,-58.7)--++(28.4,-1.8) + ++(-21.9,152.5)--++(-9.9,-39.7)--++(32.5,28.9) + (1258.6,39.7)--++(35.8,77.2)--++(-43.3,-1.9) + (1242.8,-12.3)--++(14.0,47.7)--++(-48.7,20.6) + (1206.7,60.5)--++(39.5,52.1)--++(-43.1,-25.1) + (1198.2,182.6)--++(-44.9,-17.4)--++(49.4,-11.9) + (1113.6,361.3)--++(-11.1,-40.7)--++(43.5,14.8) + (1088.9,248.7)--++(-2.9,-67.6)--++(66.7,4.8) + (1084.6,178.6)--++(-16.8,-10.6)--++(37.2,-25.7) + (1062.1,218.6)--++(-40.0,-14.0)--++(42.8,-34.5) + (1004.5,244.3)--++(-48.7,-39.3)--++(62.7,0.9); + +% White Mask +\draw[fill=white,opacity=\whiteMaskTransparency] (500,0)--(1560,0)--(1560,360)--(500,360)--cycle; +\end{tikzpicture} diff --git a/doc/src/assets/cover.tex b/doc/src/assets/cover.tex new file mode 100644 index 0000000000000..67b77e520acd3 --- /dev/null +++ b/doc/src/assets/cover.tex @@ -0,0 +1,46 @@ +%% ============================================================================ +%% Custom tex styles, including this file, add a custom cover to the document. +%% +%% These custom styles include: +%% - `cover.tex`: This file, The main definition of the cover, +%% used to replace the default `\maketitle` command. +%% - `custom.sty`: Load the macro package required for the cover, +%% define the background image style, etc. +%% - `preamble.tex`: Replace the default preamble for inserting a custom cover. +%% - `logo.tex`: logo of julia. +%% - `cover-splash.tex`: Background image of the cover title, +%% from julia's homepage. +%% ============================================================================ + +%% ---- reset page geometry for cover page +\newgeometry{left=2cm,right=2cm,bottom=3cm} +% ref: memman@v3.7q, P65, "4.1. Styling the titling" +% http://mirrors.ctan.org/macros/latex/contrib/memoir/memman.pdf +\begin{titlingpage} + % set background image + \BgThispage + \vspace*{2.2cm} + + %% Centering content + \begin{center} + %% Main Heading + \textcolor{black}{ \MainHeading \DocMainTitle } + \vfill + + %% logo + % logo scale factor + \newcommand{\scaleFactor}{0.5} + \input{./assets/logo} + \\[1.5cm] + % git tag or doc version + { \SecondaryHeading V\JuliaVersion\ } + \vfill + + { \HUGE \DocAuthors } + \\[0.5cm] + % build time + { \huge \today } + \end{center} +\end{titlingpage} +\restoregeometry +%% ---- restore geometry diff --git a/doc/src/assets/custom.sty b/doc/src/assets/custom.sty new file mode 100644 index 0000000000000..f257d2d3d2174 --- /dev/null +++ b/doc/src/assets/custom.sty @@ -0,0 +1,45 @@ +%% Load the macro package required for the cover. + + +%% pkg for make cover page BEGIN ---------------------------------------------- +% Load `geometry' to modify margins later +\usepackage{geometry} +% "some": use \BgThispage to change background +% ref: background@v2.1,# 2.1 Options, "pages=" +% http://mirrors.ctan.org/macros/latex/contrib/background/background.pdf +\usepackage[pages=some]{background} + +%% Color definitions for Julia +%% https://github.com/JuliaLang/julia-logo-graphics#color-definitions +\definecolor{julia_blue} {HTML}{4063D8} +\definecolor{julia_green} {HTML}{389826} +\definecolor{julia_purple}{HTML}{9558B2} +\definecolor{julia_red} {HTML}{CB3C33} +\definecolor{splash_gary} {HTML}{1A1A33} + +% ---- define heading background +% ref: background.pdf, #2.1 Options +\backgroundsetup{ +scale=1, % scaling factor +angle=0, % counterclockwise angle +opacity=1, % transparency +contents={ +%% Place the background image `title-bg' in the right place via `tikz'. +% tikz option "remember picture", "overlay" +% ref: pgfmanual@3.1.9a, #17.13.1 Referencing a Node in a Different Picture\ +% http://mirrors.ctan.org/graphics/pgf/base/doc/pgfmanual.pdf +\begin{tikzpicture}[remember picture,overlay,draw=white] + \draw [path picture={ + % ref: pgfmanual, 15.6, "Predefined node path picture bounding box" + \node at (path picture bounding box.center){ + \input{assets/cover-splash} + };}] (-0.5\paperwidth,4cm) rectangle (0.5\paperwidth,11cm); + % Put picture to right place + % ref: pgfmanual, #2.6 Rectangle Path Construction +\end{tikzpicture} +}}% + +% ---- Heading font style +\DeclareFixedFont{\MainHeading}{T1}{phv}{b}{n}{1.5cm} +\DeclareFixedFont{\SecondaryHeading}{T1}{phv}{b}{n}{0.8cm} +%% cover page END ------------------------------------------------------------- diff --git a/doc/src/assets/logo.tex b/doc/src/assets/logo.tex new file mode 100644 index 0000000000000..a19022140d17f --- /dev/null +++ b/doc/src/assets/logo.tex @@ -0,0 +1,142 @@ + +%% Direct translation of the Julia logo definition code in Luxor.jl +%% https://github.com/JuliaGraphics/Luxor.jl/blob/master/src/juliagraphics.jl#L62 +\begin{tikzpicture}[x=1,y=1,yscale=-\scaleFactor,xscale=\scaleFactor] +% Blue circle in "j" +\path[fill=julia_blue] (77.953125, 68.08984375) .. + controls (77.953125, 77.7578125) and (70.1171875, 85.58984375) .. (60.453125, 85.58984375) .. + controls (50.7890625, 85.58984375) and (42.953125, 77.7578125) .. (42.953125, 68.08984375) .. + controls (42.953125, 58.42578125) and (50.7890625, 50.58984375) .. (60.453125, 50.58984375) .. + controls (70.1171875, 50.58984375) and (77.953125, 58.42578125) .. (77.953125, 68.08984375); + +% Letter "j" +\path[fill=black] (72.87109375, 177.3125) .. + controls (72.87109375, 184.84765625) and (72.0234375, 190.93359375) .. (70.328125, 195.56640625) .. + controls (68.6328125, 200.203125) and (66.22265625, 203.80078125) .. (63.09375, 206.36328125) .. + controls (59.96875, 208.92578125) and (56.21875, 210.640625) .. (51.84765625, 211.5078125) .. + controls (47.4765625, 212.37109375) and (42.61328125, 212.8046875) .. (37.265625, 212.8046875) .. + controls (30.02734375, 212.8046875) and (24.48828125, 211.67578125) .. (20.6484375, 209.4140625) .. + controls (16.8046875, 207.15234375) and (14.8828125, 204.44140625) .. (14.8828125, 201.2734375) .. + controls (14.8828125, 198.63671875) and (15.953125, 196.4140625) .. (18.1015625, 194.60546875) .. + controls (20.25, 192.796875) and (23.1328125, 191.89453125) .. (26.75, 191.89453125) .. + controls (29.46484375, 191.89453125) and (31.6328125, 192.62890625) .. (33.25, 194.09765625) .. + controls (34.87109375, 195.56640625) and (36.2109375, 197.01953125) .. (37.265625, 198.44921875) .. + controls (38.46875, 200.03125) and (39.48828125, 201.0859375) .. (40.31640625, 201.61328125) .. + controls (41.14453125, 202.140625) and (41.8984375, 202.40625) .. (42.578125, 202.40625) .. + controls (44.0078125, 202.40625) and (45.1015625, 201.55859375) .. (45.85546875, 199.86328125) .. + controls (46.609375, 198.16796875) and (46.984375, 194.87109375) .. (46.984375, 189.97265625) -- + (46.984375, 97.05078125) -- + (72.87109375, 89.9296875) -- + cycle; + +% Letter "u" +\path[fill=black] (109.73828125, 92.4140625) -- + (109.73828125, 152.21484375) .. + controls (109.73828125, 153.875) and (110.05859375, 155.4375) .. (110.69921875, 156.90625) .. + controls (111.33984375, 158.375) and (112.2265625, 159.640625) .. (113.35546875, 160.6953125) .. + controls (114.48828125, 161.75) and (115.8046875, 162.59765625) .. (117.3125, 163.23828125) .. + controls (118.8203125, 163.87890625) and (120.44140625, 164.19921875) .. (122.17578125, 164.19921875) .. + controls (124.1328125, 164.19921875) and (126.359375, 163.1015625) .. (129.0703125, 161.203125) .. + controls (133.36328125, 158.1953125) and (135.96484375, 156.12890625) .. (135.96484375, 153.68359375) .. + controls (135.96484375, 153.09765625) and (135.96484375, 92.4140625) .. (135.96484375, 92.4140625) -- + (161.73828125, 92.4140625) -- + (161.73828125, 177.3125) -- + (135.96484375, 177.3125) -- + (135.96484375, 169.3984375) .. + controls (132.57421875, 172.26171875) and (128.95703125, 174.55859375) .. (125.11328125, 176.29296875) .. + controls (121.26953125, 178.02734375) and (117.5390625, 178.89453125) .. (113.921875, 178.89453125) .. + controls (109.703125, 178.89453125) and (105.78125, 178.1953125) .. (102.1640625, 176.80078125) .. + controls (98.546875, 175.40625) and (95.3828125, 173.50390625) .. (92.671875, 171.09375) .. + controls (89.95703125, 168.68359375) and (87.828125, 165.85546875) .. (86.28125, 162.61328125) .. + controls (84.73828125, 159.375) and (83.96484375, 155.90625) .. (83.96484375, 152.21484375) -- + (83.96484375, 92.4140625) -- + cycle; + +% Letter "l" +\path[fill=black] (197.8828125, 177.3125) -- + (172.22265625, 177.3125) -- + (172.22265625, 58.27734375) -- + (197.8828125, 51.15625) -- + cycle; + +%% "i" + 3 circles +% Green circle +\path[fill=julia_green] (261.30078125, 31.671875) .. + controls (261.30078125, 41.3359375) and (253.46484375, 49.171875) .. (243.80078125, 49.171875) .. + controls (234.1328125, 49.171875) and (226.30078125, 41.3359375) .. (226.30078125, 31.671875) .. + controls (226.30078125, 22.0078125) and (234.1328125, 14.171875) .. (243.80078125, 14.171875) .. + controls (253.46484375, 14.171875) and (261.30078125, 22.0078125) .. (261.30078125, 31.671875); + +% Purple circle +\path[fill=julia_purple] (282.3203125, 68.08984375) .. + controls (282.3203125, 77.7578125) and (274.484375, 85.58984375) .. (264.8203125, 85.58984375) .. + controls (255.15625, 85.58984375) and (247.3203125, 77.7578125) .. (247.3203125, 68.08984375) .. + controls (247.3203125, 58.42578125) and (255.15625, 50.58984375) .. (264.8203125, 50.58984375) .. + controls (274.484375, 50.58984375) and (282.3203125, 58.42578125) .. (282.3203125, 68.08984375); + +% Red circle in "i" +\path[fill=julia_red] (240.2734375, 68.08984375) .. + controls (240.2734375, 77.7578125) and (232.4375, 85.58984375) .. (222.7734375, 85.58984375) .. + controls (213.10546875, 85.58984375) and (205.2734375, 77.7578125) .. (205.2734375, 68.08984375) .. + controls (205.2734375, 58.42578125) and (213.10546875, 50.58984375) .. (222.7734375, 50.58984375) .. + controls (232.4375, 50.58984375) and (240.2734375, 58.42578125) .. (240.2734375, 68.08984375); + +% Letter "i" +\path[fill=black] (208.6015625, 97.05078125) -- + (234.375, 89.9296875) -- + (234.375, 177.3125) -- + (208.6015625, 177.3125) -- + cycle; + +% Letter "a" +\path[fill=black,nonzero rule] (288.2265625, 133.44921875) .. + controls (285.73828125, 134.5078125) and (283.23046875, 135.73046875) .. (280.70703125, 137.125) .. + controls (278.18359375, 138.51953125) and (275.8828125, 140.046875) .. (273.8125, 141.703125) .. + controls (271.73828125, 143.359375) and (270.0625, 145.1328125) .. (268.78125, 147.015625) .. + controls (267.5, 148.8984375) and (266.859375, 150.859375) .. (266.859375, 152.89453125) .. + controls (266.859375, 154.4765625) and (267.06640625, 156.00390625) .. (267.48046875, 157.47265625) .. + controls (267.89453125, 158.94140625) and (268.48046875, 160.203125) .. (269.234375, 161.2578125) .. + controls (269.98828125, 162.3125) and (270.81640625, 163.16015625) .. (271.72265625, 163.80078125) .. + controls (272.625, 164.44140625) and (273.60546875, 164.76171875) .. (274.66015625, 164.76171875) .. + controls (276.76953125, 164.76171875) and (278.8984375, 164.12109375) .. (281.046875, 162.83984375) .. + controls (283.1953125, 161.55859375) and (285.5859375, 159.94140625) .. (288.2265625, 157.98046875) -- + cycle + (314.109375, 177.3125) -- + (288.2265625, 177.3125) -- + (288.2265625, 170.52734375) .. + controls (286.79296875, 171.734375) and (285.3984375, 172.84765625) .. (284.04296875, 173.86328125) .. + controls (282.6875, 174.87890625) and (281.16015625, 175.765625) .. (279.46484375, 176.51953125) .. + controls (277.76953125, 177.2734375) and (275.8671875, 177.85546875) .. (273.75390625, 178.2734375) .. + controls (271.64453125, 178.6875) and (269.15625, 178.89453125) .. (266.296875, 178.89453125) .. + controls (262.375, 178.89453125) and (258.8515625, 178.328125) .. (255.7265625, 177.19921875) .. + controls (252.59765625, 176.06640625) and (249.94140625, 174.5234375) .. (247.7578125, 172.5625) .. + controls (245.5703125, 170.60546875) and (243.89453125, 168.28515625) .. (242.7265625, 165.609375) .. + controls (241.55859375, 162.9375) and (240.97265625, 160.015625) .. (240.97265625, 156.8515625) .. + controls (240.97265625, 153.609375) and (241.59375, 150.671875) .. (242.83984375, 148.03125) .. + controls (244.08203125, 145.39453125) and (245.77734375, 143.0234375) .. (247.92578125, 140.91015625) .. + controls (250.07421875, 138.80078125) and (252.578125, 136.91796875) .. (255.44140625, 135.2578125) .. + controls (258.3046875, 133.6015625) and (261.37890625, 132.07421875) .. (264.65625, 130.6796875) .. + controls (267.93359375, 129.28515625) and (271.34375, 128.0078125) .. (274.88671875, 126.83984375) .. + controls (278.42578125, 125.671875) and (281.93359375, 124.55859375) .. (285.3984375, 123.50390625) -- + (288.2265625, 122.82421875) -- + (288.2265625, 114.4609375) .. + controls (288.2265625, 109.03515625) and (287.1875, 105.19140625) .. (285.1171875, 102.9296875) .. + controls (283.04296875, 100.66796875) and (280.2734375, 99.5390625) .. (276.80859375, 99.5390625) .. + controls (272.73828125, 99.5390625) and (269.91015625, 100.51953125) .. (268.328125, 102.4765625) .. + controls (266.74609375, 104.4375) and (265.953125, 106.80859375) .. (265.953125, 109.59765625) .. + controls (265.953125, 111.1796875) and (265.78515625, 112.7265625) .. (265.4453125, 114.234375) .. + controls (265.109375, 115.7421875) and (264.5234375, 117.05859375) .. (263.6953125, 118.19140625) .. + controls (262.8671875, 119.3203125) and (261.6796875, 120.2265625) .. (260.1328125, 120.90234375) .. + controls (258.58984375, 121.58203125) and (256.6484375, 121.921875) .. (254.3125, 121.921875) .. + controls (250.6953125, 121.921875) and (247.7578125, 120.8828125) .. (245.49609375, 118.8125) .. + controls (243.234375, 116.73828125) and (242.10546875, 114.12109375) .. (242.10546875, 110.953125) .. + controls (242.10546875, 108.015625) and (243.1015625, 105.28515625) .. (245.09765625, 102.76171875) .. + controls (247.09765625, 100.234375) and (249.7890625, 98.06640625) .. (253.18359375, 96.26171875) .. + controls (256.57421875, 94.44921875) and (260.4921875, 93.01953125) .. (264.9375, 91.96484375) .. + controls (269.3828125, 90.91015625) and (274.09375, 90.3828125) .. (279.06640625, 90.3828125) .. + controls (285.171875, 90.3828125) and (290.4296875, 90.9296875) .. (294.83984375, 92.01953125) .. + controls (299.24609375, 93.11328125) and (302.8828125, 94.67578125) .. (305.74609375, 96.7109375) .. + controls (308.609375, 98.74609375) and (310.71875, 101.1953125) .. (312.07421875, 104.05859375) .. + controls (313.43359375, 106.921875) and (314.109375, 110.12890625) .. (314.109375, 113.66796875) -- + cycle; +\end{tikzpicture} diff --git a/doc/src/assets/preamble.tex b/doc/src/assets/preamble.tex new file mode 100644 index 0000000000000..fe26add788c5b --- /dev/null +++ b/doc/src/assets/preamble.tex @@ -0,0 +1,48 @@ +%% Copied from the default preamble of `Documenter.jl`. +%% +%% With patch: +%% - inserting a custom cover + + +%% Default preamble BEGIN +\documentclass[oneside, a4paper]{memoir} + +\usepackage{./documenter} +\usepackage{./custom} + + +%% TOC settings +% -- TOC depth +% value: [part, chapter, section, subsection, +% subsubsection, paragraph, subparagraph] +\settocdepth{section} % show "part+chapter+section" in TOC +% -- TOC spacing +% ref: https://tex.stackexchange.com/questions/60317/toc-spacing-in-memoir +% doc: memoir/memman.pdf +% - Figure 9.2: Layout of a ToC +% - Table 9.3: Value of K in macros for styling entries +\makeatletter +% {part} to {chaper} +\setlength{\cftbeforepartskip}{1.5em \@plus \p@} +% {chaper} to {chaper} +\setlength{\cftbeforechapterskip}{0.0em \@plus \p@} +% Chapter num to chapter title spacing (Figure 9.2@memman) +\setlength{\cftchapternumwidth}{2.5em \@plus \p@} +% indent before section number +\setlength{\cftsectionindent}{2.5em \@plus \p@} +% Section num to section title spacing (Figure 9.2@memman) +\setlength{\cftsectionnumwidth}{4.0em \@plus \p@} +\makeatother + +%% Main document begin +\begin{document} + +\frontmatter +%% ---- Custom cover page +% \maketitle +\input{assets/cover.tex} % insert cover page +%% ---- Custom cover page +\cleardoublepage % makes the next page a odd-numbered page +\tableofcontents +\mainmatter +%% preamble END From eadbeec06c88fba301ce43e50d8e261bb5dd68d3 Mon Sep 17 00:00:00 2001 From: Orestis Ousoultzoglou Date: Sun, 24 Apr 2022 01:26:29 +0300 Subject: [PATCH 36/68] correct man-custom-indices ref in devdocs (#45071) (cherry picked from commit d8346ccf4141703687e4b9274511fbd22f0c3961) --- doc/src/devdocs/boundscheck.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/devdocs/boundscheck.md b/doc/src/devdocs/boundscheck.md index f840a0283ea15..258528dbd5960 100644 --- a/doc/src/devdocs/boundscheck.md +++ b/doc/src/devdocs/boundscheck.md @@ -52,7 +52,7 @@ end ``` Which quietly assumes 1-based indexing and therefore exposes unsafe memory access when used -with [`OffsetArrays`](@ref man-custom-indice): +with [`OffsetArrays`](@ref man-custom-indices): ```julia-repl julia> using OffsetArrays From c2ce2a1b9e480488adf29a2ce592e45d5d541f2e Mon Sep 17 00:00:00 2001 From: Orestis Ousoultzoglou Date: Sun, 24 Apr 2022 04:46:34 +0300 Subject: [PATCH 37/68] Correct typo in devdocs gc-sa (#45072) (cherry picked from commit 45abec434ae5732cb9fb04f3fa720597c568b448) --- doc/src/devdocs/gc-sa.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/devdocs/gc-sa.md b/doc/src/devdocs/gc-sa.md index 85d16c1e4e195..d2517374398c3 100644 --- a/doc/src/devdocs/gc-sa.md +++ b/doc/src/devdocs/gc-sa.md @@ -2,7 +2,7 @@ ## Running the analysis -The analyzer plugin that drives the anlysis ships with julia. Its +The analyzer plugin that drives the analysis ships with julia. Its source code can be found in `src/clangsa`. Running it requires the clang dependency to be build. Set the `BUILD_LLVM_CLANG` variable in your Make.user in order to build an appropriate version of clang. From 515a5ce658088522a60bcf731c1b435cd75c99e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Fri, 29 Apr 2022 10:06:45 +0100 Subject: [PATCH 38/68] [deps] Fix compillation of LLVM with system zlib (#45119) (cherry picked from commit 5e74df5f0ee91c66b95e92c35de65bb02d3e6e9e) --- deps/llvm.mk | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deps/llvm.mk b/deps/llvm.mk index 5afef0b83ba3c..19b684f709dd0 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -233,10 +233,14 @@ $$(LLVM_BUILDDIR_withtype)/build-compiled: $$(SRCCACHE)/$$(LLVM_SRC_DIR)/$1.patc LLVM_PATCH_PREV := $$(SRCCACHE)/$$(LLVM_SRC_DIR)/$1.patch-applied endef +ifeq ($(USE_SYSTEM_ZLIB), 0) +$(LLVM_BUILDDIR_withtype)/build-configured: | $(build_prefix)/manifest/zlib +endif + # NOTE: LLVM 12 and 13 have their patches applied to JuliaLang/llvm-project # declare that all patches must be applied before running ./configure -$(LLVM_BUILDDIR_withtype)/build-configured: | $(LLVM_PATCH_PREV) $(build_prefix)/manifest/zlib +$(LLVM_BUILDDIR_withtype)/build-configured: | $(LLVM_PATCH_PREV) $(LLVM_BUILDDIR_withtype)/build-configured: $(SRCCACHE)/$(LLVM_SRC_DIR)/source-extracted | $(llvm_python_workaround) mkdir -p $(dir $@) From f5e292a18dac66c2b08bc3d328314a23933d760b Mon Sep 17 00:00:00 2001 From: Jishnu Bhattacharya Date: Mon, 2 May 2022 12:11:21 +0400 Subject: [PATCH 39/68] Create a copy while evaluating eigvals(::Diagonal) (#45048) (cherry picked from commit b9d82808b8a3bd383b01d85ba3038b56e18c74e0) --- stdlib/LinearAlgebra/src/diagonal.jl | 2 +- stdlib/LinearAlgebra/test/diagonal.jl | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/stdlib/LinearAlgebra/src/diagonal.jl b/stdlib/LinearAlgebra/src/diagonal.jl index 748b165eca7fd..b06ccdd77cc57 100644 --- a/stdlib/LinearAlgebra/src/diagonal.jl +++ b/stdlib/LinearAlgebra/src/diagonal.jl @@ -704,7 +704,7 @@ function pinv(D::Diagonal{T}, tol::Real) where T end #Eigensystem -eigvals(D::Diagonal{<:Number}; permute::Bool=true, scale::Bool=true) = D.diag +eigvals(D::Diagonal{<:Number}; permute::Bool=true, scale::Bool=true) = copy(D.diag) eigvals(D::Diagonal; permute::Bool=true, scale::Bool=true) = [eigvals(x) for x in D.diag] #For block matrices, etc. eigvecs(D::Diagonal) = Matrix{eltype(D)}(I, size(D)) diff --git a/stdlib/LinearAlgebra/test/diagonal.jl b/stdlib/LinearAlgebra/test/diagonal.jl index 6efed3b7d9cff..8bc84d93c6348 100644 --- a/stdlib/LinearAlgebra/test/diagonal.jl +++ b/stdlib/LinearAlgebra/test/diagonal.jl @@ -465,6 +465,13 @@ end @test sort([eigvals(D)...;], by=LinearAlgebra.eigsortby) ≈ eigvals([D.diag[1] zeros(3,2); zeros(2,3) D.diag[2]]) end +@testset "eigvals should return a copy of the diagonal" begin + D = Diagonal([1, 2, 3]) + lam = eigvals(D) + D[3,3] = 4 # should not affect lam + @test lam == [1, 2, 3] +end + @testset "eigmin (#27847)" begin for _ in 1:100 d = randn(rand(1:10)) From 6d4b8d033db1848dadfca2fbbae7a4adab48b63b Mon Sep 17 00:00:00 2001 From: Yuto Horikawa Date: Tue, 3 May 2022 04:31:36 +0900 Subject: [PATCH 40/68] Fix bug in `pinv` (#45009) (cherry picked from commit b4eb88a71f8c2d8343b21d8fdd1ec403073a222c) --- stdlib/LinearAlgebra/src/dense.jl | 7 ++-- stdlib/LinearAlgebra/test/pinv.jl | 61 ++++++++++++++----------------- 2 files changed, 31 insertions(+), 37 deletions(-) diff --git a/stdlib/LinearAlgebra/src/dense.jl b/stdlib/LinearAlgebra/src/dense.jl index 249010adb4e5c..182277cfbf9a4 100644 --- a/stdlib/LinearAlgebra/src/dense.jl +++ b/stdlib/LinearAlgebra/src/dense.jl @@ -1449,12 +1449,13 @@ function pinv(A::AbstractMatrix{T}; atol::Real = 0.0, rtol::Real = (eps(real(flo return similar(A, Tout, (n, m)) end if isdiag(A) - ind = diagind(A) - dA = view(A, ind) + indA = diagind(A) + dA = view(A, indA) maxabsA = maximum(abs, dA) tol = max(rtol * maxabsA, atol) B = fill!(similar(A, Tout, (n, m)), 0) - B[ind] .= (x -> abs(x) > tol ? pinv(x) : zero(x)).(dA) + indB = diagind(B) + B[indB] .= (x -> abs(x) > tol ? pinv(x) : zero(x)).(dA) return B end SVD = svd(A) diff --git a/stdlib/LinearAlgebra/test/pinv.jl b/stdlib/LinearAlgebra/test/pinv.jl index d3eafb26797a9..c7268865a0505 100644 --- a/stdlib/LinearAlgebra/test/pinv.jl +++ b/stdlib/LinearAlgebra/test/pinv.jl @@ -63,39 +63,23 @@ function tridiag(T::Type, m::Integer, n::Integer) end tridiag(m::Integer, n::Integer) = tridiag(Float64, m::Integer, n::Integer) -function randn_float64(m::Integer, n::Integer) - a=randn(m,n) - b = Matrix{Float64}(undef, m, n) - for i=1:n - for j=1:m - b[j,i]=convert(Float64,a[j,i]) - end - end - return b -end - -function randn_float32(m::Integer, n::Integer) - a=randn(m,n) - b = Matrix{Float32}(undef, m, n) - for i=1:n - for j=1:m - b[j,i]=convert(Float32,a[j,i]) - end - end - return b -end - +function test_pinv(a,tol1,tol2) + m,n = size(a) -function test_pinv(a,m,n,tol1,tol2,tol3) apinv = @inferred pinv(a) - + @test size(apinv) == (n,m) @test norm(a*apinv*a-a)/norm(a) ≈ 0 atol=tol1 - x0 = randn(n); b = a*x0; x = apinv*b + @test norm(apinv*a*apinv-apinv)/norm(apinv) ≈ 0 atol=tol1 + b = a*randn(n) + x = apinv*b @test norm(a*x-b)/norm(b) ≈ 0 atol=tol1 - apinv = pinv(a,sqrt(eps(real(one(eltype(a)))))) + apinv = @inferred pinv(a,sqrt(eps(real(one(eltype(a)))))) + @test size(apinv) == (n,m) @test norm(a*apinv*a-a)/norm(a) ≈ 0 atol=tol2 - x0 = randn(n); b = a*x0; x = apinv*b + @test norm(apinv*a*apinv-apinv)/norm(apinv) ≈ 0 atol=tol2 + b = a*randn(n) + x = apinv*b @test norm(a*x-b)/norm(b) ≈ 0 atol=tol2 end @@ -104,28 +88,25 @@ end default_tol = (real(one(eltya))) * max(m,n) * 10 tol1 = 1e-2 tol2 = 1e-5 - tol3 = 1e-5 if real(eltya) == Float32 tol1 = 1e0 tol2 = 1e-2 - tol3 = 1e-2 end @testset "dense/ill-conditioned matrix" begin - ### a = randn_float64(m,n) * hilb(eltya,n) a = hilb(eltya, m, n) - test_pinv(a, m, n, tol1, tol2, tol3) + test_pinv(a, tol1, tol2) end @testset "dense/diagonal matrix" begin a = onediag(eltya, m, n) - test_pinv(a, m, n, default_tol, default_tol, default_tol) + test_pinv(a, default_tol, default_tol) end @testset "dense/tri-diagonal matrix" begin a = tridiag(eltya, m, n) - test_pinv(a, m, n, default_tol, tol2, default_tol) + test_pinv(a, default_tol, tol2) end @testset "Diagonal matrix" begin a = onediag_sparse(eltya, m) - test_pinv(a, m, m, default_tol, default_tol, default_tol) + test_pinv(a, default_tol, default_tol) end @testset "Vector" begin a = rand(eltya, m) @@ -164,6 +145,18 @@ end @test C ≈ ones(2,2) end + @testset "non-square diagonal matrices" begin + A = eltya[1 0 ; 0 1 ; 0 0] + B = pinv(A) + @test A*B*A ≈ A + @test B*A*B ≈ B + + A = eltya[1 0 0 ; 0 1 0] + B = pinv(A) + @test A*B*A ≈ A + @test B*A*B ≈ B + end + if eltya <: LinearAlgebra.BlasReal @testset "sub-normal numbers/vectors/matrices" begin a = pinv(floatmin(eltya)/100) From f89f049a7ec1d601186bfcb87e2b756a51fce967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Wed, 4 May 2022 16:57:19 +0100 Subject: [PATCH 41/68] [deps] Remove Python workaround in LLVM build, no longer necessary (#45176) (cherry picked from commit 862018b20dcef1b79568b2833991713556f1a1c1) --- deps/llvm.mk | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/deps/llvm.mk b/deps/llvm.mk index 19b684f709dd0..7c4c2d60dfd9a 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -194,16 +194,6 @@ ifeq ($(BUILD_LLDB),0) LLVM_CMAKE += -DLLVM_TOOL_LLDB_BUILD=OFF endif -# LLDB still relies on plenty of python 2.x infrastructure, without checking -llvm_python_location=$(shell /usr/bin/env python2 -c 'import sys; print(sys.executable)') -llvm_python_workaround=$(SRCCACHE)/python2_path -$(llvm_python_workaround): - mkdir -p $@ - -python -c 'import sys; sys.exit(not sys.version_info > (3, 0))' && \ - /usr/bin/env python2 -c 'import sys; sys.exit(not sys.version_info < (3, 0))' && \ - ln -sf $(llvm_python_location) "$@/python" && \ - ln -sf $(llvm_python_location)-config "$@/python-config" - LLVM_CMAKE += -DCMAKE_EXE_LINKER_FLAGS="$(LLVM_LDFLAGS)" \ -DCMAKE_SHARED_LINKER_FLAGS="$(LLVM_LDFLAGS)" @@ -242,32 +232,27 @@ endif # declare that all patches must be applied before running ./configure $(LLVM_BUILDDIR_withtype)/build-configured: | $(LLVM_PATCH_PREV) -$(LLVM_BUILDDIR_withtype)/build-configured: $(SRCCACHE)/$(LLVM_SRC_DIR)/source-extracted | $(llvm_python_workaround) +$(LLVM_BUILDDIR_withtype)/build-configured: $(SRCCACHE)/$(LLVM_SRC_DIR)/source-extracted mkdir -p $(dir $@) cd $(dir $@) && \ - export PATH=$(llvm_python_workaround):"$$PATH" && \ $(CMAKE) $(SRCCACHE)/$(LLVM_SRC_DIR)/llvm $(CMAKE_GENERATOR_COMMAND) $(CMAKE_COMMON) $(LLVM_CMAKE) \ || { echo '*** To install a newer version of cmake, run contrib/download_cmake.sh ***' && false; } echo 1 > $@ -$(LLVM_BUILDDIR_withtype)/build-compiled: $(LLVM_BUILDDIR_withtype)/build-configured | $(llvm_python_workaround) +$(LLVM_BUILDDIR_withtype)/build-compiled: $(LLVM_BUILDDIR_withtype)/build-configured cd $(LLVM_BUILDDIR_withtype) && \ - export PATH=$(llvm_python_workaround):"$$PATH" && \ $(if $(filter $(CMAKE_GENERATOR),make), \ $(MAKE), \ $(CMAKE) --build .) echo 1 > $@ -$(LLVM_BUILDDIR_withtype)/build-checked: $(LLVM_BUILDDIR_withtype)/build-compiled | $(llvm_python_workaround) +$(LLVM_BUILDDIR_withtype)/build-checked: $(LLVM_BUILDDIR_withtype)/build-compiled ifeq ($(OS),$(BUILD_OS)) cd $(LLVM_BUILDDIR_withtype) && \ - export PATH=$(llvm_python_workaround):"$$PATH" && \ $(CMAKE) --build . --target check endif echo 1 > $@ -$(build_prefix)/manifest/llvm: | $(llvm_python_workaround) - LLVM_INSTALL = \ cd $1 && mkdir -p $2$$(build_depsbindir) && \ cp -r $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit $2$$(build_depsbindir)/ && \ From e0e24c12df15d5fc87b19e10c2fcfb81ec29440e Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 6 May 2022 22:18:40 -0400 Subject: [PATCH 42/68] Fix different .cachepath initialization points for precompile load (#45149) For quite some time I've been observing Revise randomly not pick up changes to my packages. Usually I just write that off to Revise getting into a bad state and restarting Julia fixes it. Today, I got very annoyed by this again and decided to finally track it down. What turns out to happen here is the packages in question are those which: A. Are being precompiled on load B. Use Requires C. Have an `@require`'d-dependency already loaded D. The change to be revised is made in a file not loaded via Requires.jl. In this case the `__init__` of the package triggers Requires, which in turn calls back to Revise, which tries to start watching the package. However, on the `compilecache` path (but not on the path where we just find a pre-existing cache file), we used to not set the .cachepath property of `pkgorigins` until after the `__init__` callbacks run, causing Revise not to be able to see those files. Infuriatingly, restarting julia fixes this because it just loads the .ji file that was just compiled. Fix this by unifying the point at which the .cachepath is set, always setting it just prior to the module initialization callback. (cherry picked from commit 58ab4c73088933dc53fd3e28325172bf97a0da8e) --- base/loading.jl | 24 ++++++++++++------------ test/precompile.jl | 14 +++++++++++++- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 0dadb558a789c..9a8a0b50e2166 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -786,7 +786,7 @@ end # these return either the array of modules loaded from the path / content given # or an Exception that describes why it couldn't be loaded # and it reconnects the Base.Docs.META -function _include_from_serialized(path::String, depmods::Vector{Any}) +function _include_from_serialized(pkg::PkgId, path::String, depmods::Vector{Any}) sv = ccall(:jl_restore_incremental, Any, (Cstring, Any), path, depmods) if isa(sv, Exception) return sv @@ -802,6 +802,11 @@ function _include_from_serialized(path::String, depmods::Vector{Any}) register_root_module(M) end end + + # Register this cache path now - If Requires.jl is loaded, Revise may end + # up looking at the cache path during the init callback. + get!(PkgOrigin, pkgorigins, pkg).cachepath = path + inits = sv[2]::Vector{Any} if !isempty(inits) unlock(require_lock) # temporarily _unlock_ during these callbacks @@ -856,7 +861,7 @@ function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt64, modpath::U return nothing end -function _require_from_serialized(path::String) +function _require_from_serialized(pkg::PkgId, path::String) # loads a precompile cache file, ignoring stale_cachfile tests # load all of the dependent modules first local depmodnames @@ -877,7 +882,7 @@ function _require_from_serialized(path::String) depmods[i] = dep::Module end # then load the file - return _include_from_serialized(path, depmods) + return _include_from_serialized(pkg, path, depmods) end # use an Int counter so that nested @time_imports calls all remain open @@ -915,7 +920,7 @@ const TIMING_IMPORTS = Threads.Atomic{Int}(0) if staledeps === true continue end - restored = _include_from_serialized(path_to_try, staledeps) + restored = _include_from_serialized(pkg, path_to_try, staledeps) if isa(restored, Exception) @debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored else @@ -1083,10 +1088,7 @@ require(uuidkey::PkgId) = @lock require_lock _require_prelocked(uuidkey) function _require_prelocked(uuidkey::PkgId) just_loaded_pkg = false if !root_module_exists(uuidkey) - cachefile = _require(uuidkey) - if cachefile !== nothing - get!(PkgOrigin, pkgorigins, uuidkey).cachepath = cachefile - end + _require(uuidkey) # After successfully loading, notify downstream consumers run_package_callbacks(uuidkey) just_loaded_pkg = true @@ -1208,11 +1210,11 @@ function _require(pkg::PkgId) end # fall-through to loading the file locally else - m = _require_from_serialized(cachefile) + m = _require_from_serialized(pkg, cachefile) if isa(m, Exception) @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m else - return cachefile + return end end end @@ -2001,8 +2003,6 @@ get_compiletime_preferences(::Nothing) = String[] @debug "Rejecting cache file $cachefile because preferences hash does not match 0x$(string(prefs_hash, base=16)) != 0x$(string(curr_prefs_hash, base=16))" return true end - - get!(PkgOrigin, pkgorigins, id).cachepath = cachefile end return depmods # fresh cachefile diff --git a/test/precompile.jl b/test/precompile.jl index d39dcd9f7ccb8..5162d026628b7 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -313,7 +313,7 @@ precompile_test_harness(false) do dir # the module doesn't reload from the image: @test_warn "@ccallable was already defined for this method name" begin @test_logs (:warn, "Replacing module `$Foo_module`") begin - ms = Base._require_from_serialized(cachefile) + ms = Base._require_from_serialized(Base.PkgId(Foo), cachefile) @test isa(ms, Array{Any,1}) end end @@ -1278,3 +1278,15 @@ end @test any(mi -> mi.specTypes.parameters[2] === Any, mis) @test all(mi -> isa(mi.cache, Core.CodeInstance), mis) end + +# Test that the cachepath is available in pkgorigins during the +# __init__ callback +precompile_test_harness("__init__ cachepath") do load_path + write(joinpath(load_path, "InitCachePath.jl"), + """ + module InitCachePath + __init__() = Base.pkgorigins[Base.PkgId(InitCachePath)] + end + """) + @test isa((@eval (using InitCachePath; InitCachePath)), Module) +end From 66ba1ad933da3444e2910431259054f214e34c14 Mon Sep 17 00:00:00 2001 From: Simon Byrne Date: Sat, 7 May 2022 22:02:20 -0700 Subject: [PATCH 43/68] Avoid race condition in cleaning up cache files (#45214) If multiple processes attempt to clean up cache files at the same time, a race condition can result, e.g. https://buildkite.com/clima/climaatmos-ci/builds/812#6a961e99-e2a1-488b-a116-2a45dee26d38/102-104 (cherry picked from commit d4acead9130f97f0135bee604d94e1f67dabc70f) --- base/loading.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/loading.jl b/base/loading.jl index 9a8a0b50e2166..c1ddabda6c469 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1541,7 +1541,7 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in cachefiles = filter!(x -> startswith(x, entryfile * "_"), readdir(cachepath)) if length(cachefiles) >= MAX_NUM_PRECOMPILE_FILES[] idx = findmin(mtime.(joinpath.(cachepath, cachefiles)))[2] - rm(joinpath(cachepath, cachefiles[idx])) + rm(joinpath(cachepath, cachefiles[idx]); force=true) end end From fb45a1af00661be77a4fae13d33fd39a4ae96dbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=A4ck?= Date: Mon, 9 May 2022 16:55:56 +0200 Subject: [PATCH 44/68] Consider additional default ssh keys for LibGit2. (#44767) (cherry picked from commit b6b0874a7341158067abe450ddb0b3f17d4efd64) --- stdlib/LibGit2/src/callbacks.jl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/stdlib/LibGit2/src/callbacks.jl b/stdlib/LibGit2/src/callbacks.jl index 5da032d3143e2..6228f442df37f 100644 --- a/stdlib/LibGit2/src/callbacks.jl +++ b/stdlib/LibGit2/src/callbacks.jl @@ -91,12 +91,15 @@ function authenticate_ssh(libgit2credptr::Ptr{Ptr{Cvoid}}, p::CredentialPayload, cred.user = unsafe_string(username_ptr) end - cred.prvkey = Base.get(ENV, "SSH_KEY_PATH") do - default = joinpath(homedir(), ".ssh", "id_rsa") - if isempty(cred.prvkey) && isfile(default) - default - else - cred.prvkey + if haskey(ENV, "SSH_KEY_PATH") + cred.prvkey = ENV["SSH_KEY_PATH"] + elseif isempty(cred.prvkey) + for keytype in ("rsa", "ecdsa") + private_key_file = joinpath(homedir(), ".ssh", "id_$keytype") + if isfile(private_key_file) + cred.prvkey = private_key_file + break + end end end From a380056c0e7b0ab2993144a83ca9a74a485cb9ca Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 9 May 2022 12:06:47 -0400 Subject: [PATCH 45/68] syntax: keyword sorter location should just be definition line (#45199) fix #45171 (cherry picked from commit 5c557b22c73d5de977cd1c007ef075c56ff8afcf) --- src/julia-syntax.scm | 7 +++++-- src/utils.scm | 8 ++++++++ test/backtrace.jl | 13 +++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 7538b89d32cb3..e07f47df73a8f 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -529,7 +529,7 @@ ,(method-def-expr- name positional-sparams pargl-all `(block - ,@(without-generated prologue) + ,@(keep-first linenum? (without-generated prologue)) ,(let (;; call mangled(vals..., [rest_kw,] pargs..., [vararg]...) (ret `(return (call ,mangled ,@(if ordered-defaults keynames vals) @@ -548,7 +548,10 @@ ,(if (any kwarg? pargl) (gensy) UNUSED) (call (core kwftype) ,ftype)) ,kw ,@pargl ,@vararg) `(block - ,@(filter linenum? prologue) + ,@(let ((lnns (filter linenum? prologue))) + (if (pair? lnns) + (list (car lnns)) + '())) ;; nospecialize meta for just positional args ,@(map (lambda (m) `(meta ,(cadr m) ,@(filter (lambda (v) (not (memq v keynames))) diff --git a/src/utils.scm b/src/utils.scm index 7be6b2999a90c..97464b9a14e5a 100644 --- a/src/utils.scm +++ b/src/utils.scm @@ -104,3 +104,11 @@ (begin (put! tbl (car xs) i) (loop (cdr xs) (+ i 1))))) tbl)) + +;; keep at most the first element matching a given predicate +(define (keep-first pred lst) + (cond ((null? lst) lst) + ((pred (car lst)) + (cons (car lst) (filter (lambda (x) (not (pred x))) (cdr lst)))) + (else + (cons (car lst) (keep-first pred (cdr lst)))))) diff --git a/test/backtrace.jl b/test/backtrace.jl index 35b607137a5c2..c0abad5146b39 100644 --- a/test/backtrace.jl +++ b/test/backtrace.jl @@ -224,6 +224,19 @@ let trace = try @test trace[1].line == 2 end +# issue #45171 +linenum = @__LINE__; function f45171(;kwarg = true) + 1 + error() +end +let trace = try + f45171() + catch + stacktrace(catch_backtrace()) + end + @test trace[3].line == linenum +end + # issue #29695 (see also test for #28442) let code = """ f29695(c) = g29695(c) From 51f8ac2b2d87219c2ca468edaa9b57008e25ab42 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 9 May 2022 10:55:08 -0400 Subject: [PATCH 46/68] fix #45162, function arg not specialized if only called with keywords (#45198) (cherry picked from commit c62ea269067415db4ad5deeb28df87585de0ac4c) --- src/julia-syntax.scm | 5 +++++ test/syntax.jl | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index e07f47df73a8f..ed5a25961df88 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -3254,6 +3254,11 @@ (let ((vi (get tab (cadr e) #f))) (if vi (vinfo:set-called! vi #t)) + ;; calls to functions with keyword args go through `kwfunc` first + (if (and (length= e 3) (equal? (cadr e) '(core kwfunc))) + (let ((vi2 (get tab (caddr e) #f))) + (if vi2 + (vinfo:set-called! vi2 #t)))) (for-each (lambda (x) (analyze-vars x env captvars sp tab)) (cdr e)))) ((decl) diff --git a/test/syntax.jl b/test/syntax.jl index 8320060678d16..df5ed7babbffd 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -3274,3 +3274,7 @@ end # issue 44723 demo44723()::Any = Base.Experimental.@opaque () -> true ? 1 : 2 @test demo44723()() == 1 + +# issue #45162 +f45162(f) = f(x=1) +@test first(methods(f45162)).called != 0 From b3dde45e943c476d8d6e63227f0db1b861be1299 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 11 Mar 2022 16:59:07 -0800 Subject: [PATCH 47/68] Fix a concurrency bug in `iterate(::Dict)` (#44534) (cherry picked from commit 6be86a380b09d0f02404140cb042f1ffb06c3442) --- NEWS.md | 4 ++++ base/dict.jl | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 73ca691ab7c3f..3e9eece2260ea 100644 --- a/NEWS.md +++ b/NEWS.md @@ -97,6 +97,10 @@ New library functions New library features -------------------- +* A known concurrency issue of `iterate` methods on `Dict` and other derived objects such + as `keys(::Dict)`, `values(::Dict)`, and `Set` is fixed. These methods of `iterate` can + now be called on a dictionary or set shared by arbitrary tasks provided that there are no + tasks mutating the dictionary or set ([#44534]). * `@time` and `@timev` now take an optional description to allow annotating the source of time reports, e.g. `@time "Evaluating foo" foo()` ([#42431]). * `range` accepts either `stop` or `length` as a sole keyword argument ([#39241]). diff --git a/base/dict.jl b/base/dict.jl index dabdfa5c34773..220840ed0e5ea 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -702,7 +702,7 @@ end @propagate_inbounds _iterate(t::Dict{K,V}, i) where {K,V} = i == 0 ? nothing : (Pair{K,V}(t.keys[i],t.vals[i]), i == typemax(Int) ? 0 : i+1) @propagate_inbounds function iterate(t::Dict) - _iterate(t, skip_deleted_floor!(t)) + _iterate(t, skip_deleted(t, t.idxfloor)) end @propagate_inbounds iterate(t::Dict, i) = _iterate(t, skip_deleted(t, i)) From 293b51387ea144fcefa6c709fae2f2606fba917f Mon Sep 17 00:00:00 2001 From: KristofferC Date: Mon, 16 May 2022 10:01:58 +0200 Subject: [PATCH 48/68] Bump Documenter to get the new pdf improvements for the manual. --- doc/Manifest.toml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/doc/Manifest.toml b/doc/Manifest.toml index f38f11a7778bb..ffd798ec59a90 100644 --- a/doc/Manifest.toml +++ b/doc/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.8.0-DEV.1335" +julia_version = "1.7.2" manifest_format = "2.0" project_hash = "e0c77beb18dc1f6cce661ebd60658c0c1a77390f" @@ -24,9 +24,9 @@ version = "0.8.6" [[deps.Documenter]] deps = ["ANSIColoredPrinters", "Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] -git-tree-sha1 = "75c6cf9d99e0efc79b724f5566726ad3ad010a01" +git-tree-sha1 = "122d031e8dcb2d3e767ed434bc4d1ae1788b5a7f" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "0.27.12" +version = "0.27.17" [[deps.IOCapture]] deps = ["Logging", "Random"] @@ -40,9 +40,9 @@ uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" [[deps.JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37" +git-tree-sha1 = "3c837543ddb02250ef42f4738347454f95079d4e" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.2" +version = "0.21.3" [[deps.LibGit2]] deps = ["Base64", "NetworkOptions", "Printf", "SHA"] @@ -60,13 +60,12 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" -version = "1.2.0" [[deps.Parsers]] deps = ["Dates"] -git-tree-sha1 = "92f91ba9e5941fc781fecf5494ac1da87bdac775" +git-tree-sha1 = "1285416549ccfcdf0c50d4997a94331e88d68413" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.2.0" +version = "2.3.1" [[deps.Printf]] deps = ["Unicode"] @@ -82,7 +81,6 @@ uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" From b11258aaf765dec6d1b5e53fd6f4f3d7921a4d9d Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 11 May 2022 11:23:59 -0400 Subject: [PATCH 49/68] codegen: add handling for undefined phinode values (#45155) The optimization pass often uses values for phi values (and thus by extension, also for pi, phic and upsilon values) that are invalid. We make sure that these have a null pointer, so that we can detect that case at runtime (at the cost of slightly worse code generation for them), but it means we need to be very careful to check for that. This is identical to #39747, which added the equivalent code to the other side of the conditional there, but missed some additional relevant, but rare, cases that are observed to be possible. The `emit_isa_and_defined` is derived from the LLVM name for this operation: `isa_and_nonnull`. Secondly, we also optimize `emit_unionmove` to change a bad IR case to a better IR form. Fix #44501 (cherry picked from commit 72b80e2b158b9d4016e70791efdc81fead933881) --- src/cgutils.cpp | 118 ++++++++++++++++++---------------- src/codegen.cpp | 30 +++++---- src/llvm-late-gc-lowering.cpp | 3 +- 3 files changed, 82 insertions(+), 69 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index b219498315905..7eb2fa11d209d 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -761,7 +761,7 @@ static bool is_uniontype_allunboxed(jl_value_t *typ) return for_each_uniontype_small([&](unsigned, jl_datatype_t*) {}, typ, counter); } -static Value *emit_typeof_boxed(jl_codectx_t &ctx, const jl_cgval_t &p); +static Value *emit_typeof_boxed(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybenull=false); static unsigned get_box_tindex(jl_datatype_t *jt, jl_value_t *ut) { @@ -814,15 +814,9 @@ static LoadInst *emit_nthptr_recast(jl_codectx_t &ctx, Value *v, ssize_t n, MDNo } static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &v); +static Value *emit_typeof(jl_codectx_t &ctx, Value *v, bool maybenull); -// Returns ctx.types().T_prjlvalue -static Value *emit_typeof(jl_codectx_t &ctx, Value *tt) -{ - assert(tt != NULL && !isa(tt) && "expected a conditionally boxed value"); - return ctx.builder.CreateCall(prepare_call(jl_typeof_func), {tt}); -} - -static jl_cgval_t emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p) +static jl_cgval_t emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybenull) { // given p, compute its type if (p.constant) @@ -835,7 +829,7 @@ static jl_cgval_t emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p) return mark_julia_const(ctx, jl_typeof(tp)); } } - return mark_julia_type(ctx, emit_typeof(ctx, p.V), true, jl_datatype_type); + return mark_julia_type(ctx, emit_typeof(ctx, p.V, maybenull), true, jl_datatype_type); } if (p.TIndex) { Value *tindex = ctx.builder.CreateAnd(p.TIndex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x7f)); @@ -870,7 +864,7 @@ static jl_cgval_t emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p) BasicBlock *mergeBB = BasicBlock::Create(ctx.builder.getContext(), "merge", ctx.f); ctx.builder.CreateCondBr(isnull, boxBB, unboxBB); ctx.builder.SetInsertPoint(boxBB); - auto boxTy = emit_typeof(ctx, p.Vboxed); + auto boxTy = emit_typeof(ctx, p.Vboxed, maybenull); ctx.builder.CreateBr(mergeBB); boxBB = ctx.builder.GetInsertBlock(); // could have changed ctx.builder.SetInsertPoint(unboxBB); @@ -892,9 +886,9 @@ static jl_cgval_t emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p) } // Returns ctx.types().T_prjlvalue -static Value *emit_typeof_boxed(jl_codectx_t &ctx, const jl_cgval_t &p) +static Value *emit_typeof_boxed(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybenull) { - return boxed(ctx, emit_typeof(ctx, p)); + return boxed(ctx, emit_typeof(ctx, p, maybenull)); } static Value *emit_datatype_types(jl_codectx_t &ctx, Value *dt) @@ -1129,6 +1123,23 @@ static Value *emit_nullcheck_guard2(jl_codectx_t &ctx, Value *nullcheck1, }); } +// Returns typeof(v), or null if v is a null pointer at run time and maybenull is true. +// This is used when the value might have come from an undefined value (a PhiNode), +// yet we try to read its type to compute a union index when moving the value (a PiNode). +// Returns a ctx.types().T_prjlvalue typed Value +static Value *emit_typeof(jl_codectx_t &ctx, Value *v, bool maybenull) +{ + assert(v != NULL && !isa(v) && "expected a conditionally boxed value"); + Function *typeof = prepare_call(jl_typeof_func); + if (maybenull) + return emit_guarded_test(ctx, null_pointer_cmp(ctx, v), Constant::getNullValue(typeof->getReturnType()), [&] { + // e.g. emit_typeof(ctx, v) + return ctx.builder.CreateCall(typeof, {v}); + }); + return ctx.builder.CreateCall(typeof, {v}); +} + + static void emit_type_error(jl_codectx_t &ctx, const jl_cgval_t &x, Value *type, const std::string &msg) { Value *msg_val = stringConstPtr(ctx.emission_context, ctx.builder, msg); @@ -1256,7 +1267,7 @@ static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, BasicBlock *postBB = BasicBlock::Create(ctx.builder.getContext(), "post_isa", ctx.f); ctx.builder.CreateCondBr(isboxed, isaBB, postBB); ctx.builder.SetInsertPoint(isaBB); - Value *istype_boxed = ctx.builder.CreateICmpEQ(emit_typeof(ctx, x.Vboxed), + Value *istype_boxed = ctx.builder.CreateICmpEQ(emit_typeof(ctx, x.Vboxed, false), track_pjlvalue(ctx, literal_pointer_val(ctx, intersected_type))); ctx.builder.CreateBr(postBB); isaBB = ctx.builder.GetInsertBlock(); // could have changed @@ -1312,6 +1323,20 @@ static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 0)), false); } +// If this might have been sourced from a PhiNode object, it is possible our +// Vboxed pointer itself is null (undef) at runtime even if we thought we should +// know exactly the type of the bytes that should have been inside. +// +// n.b. It is also possible the value is a ghost of some sort, and we will +// declare that the pointer is legal (for zero bytes) even though it might be undef. +static Value *emit_isa_and_defined(jl_codectx_t &ctx, const jl_cgval_t &val, jl_value_t *typ) +{ + return emit_nullcheck_guard(ctx, val.ispointer() ? val.V : nullptr, [&] { + return emit_isa(ctx, val, typ, nullptr).first; + }); +} + + static void emit_typecheck(jl_codectx_t &ctx, const jl_cgval_t &x, jl_value_t *type, const std::string &msg) { Value *istype; @@ -2885,42 +2910,16 @@ static Value *compute_box_tindex(jl_codectx_t &ctx, Value *datatype, jl_value_t return tindex; } -// Returns typeof(v), or null if v is a null pointer at run time. -// This is used when the value might have come from an undefined variable, -// yet we try to read its type to compute a union index when moving the value. -static Value *emit_typeof_or_null(jl_codectx_t &ctx, Value *v) -{ - BasicBlock *nonnull = BasicBlock::Create(ctx.builder.getContext(), "nonnull", ctx.f); - BasicBlock *postBB = BasicBlock::Create(ctx.builder.getContext(), "postnull", ctx.f); - Value *isnull = ctx.builder.CreateICmpEQ(v, Constant::getNullValue(v->getType())); - ctx.builder.CreateCondBr(isnull, postBB, nonnull); - BasicBlock *entry = ctx.builder.GetInsertBlock(); - ctx.builder.SetInsertPoint(nonnull); - Value *typof = emit_typeof(ctx, v); - ctx.builder.CreateBr(postBB); - nonnull = ctx.builder.GetInsertBlock(); // could have changed - ctx.builder.SetInsertPoint(postBB); - PHINode *ti = ctx.builder.CreatePHI(typof->getType(), 2); - ti->addIncoming(Constant::getNullValue(typof->getType()), entry); - ti->addIncoming(typof, nonnull); - return ti; -} - // get the runtime tindex value, assuming val is already converted to type typ if it has a TIndex -static Value *compute_tindex_unboxed(jl_codectx_t &ctx, const jl_cgval_t &val, jl_value_t *typ) +static Value *compute_tindex_unboxed(jl_codectx_t &ctx, const jl_cgval_t &val, jl_value_t *typ, bool maybenull=false) { if (val.typ == jl_bottom_type) return UndefValue::get(getInt8Ty(ctx.builder.getContext())); if (val.constant) return ConstantInt::get(getInt8Ty(ctx.builder.getContext()), get_box_tindex((jl_datatype_t*)jl_typeof(val.constant), typ)); - if (val.TIndex) return ctx.builder.CreateAnd(val.TIndex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x7f)); - Value *typof; - if (val.isboxed && !jl_is_concrete_type(val.typ) && !jl_is_type_type(val.typ)) - typof = emit_typeof_or_null(ctx, val.V); - else - typof = emit_typeof_boxed(ctx, val); + Value *typof = emit_typeof_boxed(ctx, val, maybenull); return compute_box_tindex(ctx, typof, val.typ, typ); } @@ -3102,14 +3101,17 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con Value *src_ptr = data_pointer(ctx, src); unsigned nb = jl_datatype_size(typ); unsigned alignment = julia_alignment(typ); - Value *nbytes = ConstantInt::get(getSizeTy(ctx.builder.getContext()), nb); - if (skip) { - // TODO: this Select is very bad for performance, but is necessary to work around LLVM bugs with the undef option that we want to use: - // select copy dest -> dest to simulate an undef value / conditional copy - // src_ptr = ctx.builder.CreateSelect(skip, dest, src_ptr); - nbytes = ctx.builder.CreateSelect(skip, Constant::getNullValue(getSizeTy(ctx.builder.getContext())), nbytes); - } - emit_memcpy(ctx, dest, tbaa_dst, src_ptr, src.tbaa, nbytes, alignment, isVolatile); + // TODO: this branch may be bad for performance, but is necessary to work around LLVM bugs with the undef option that we want to use: + // select copy dest -> dest to simulate an undef value / conditional copy + // if (skip) src_ptr = ctx.builder.CreateSelect(skip, dest, src_ptr); + auto f = [&] { + (void)emit_memcpy(ctx, dest, tbaa_dst, src_ptr, src.tbaa, nb, alignment, isVolatile); + return nullptr; + }; + if (skip) + emit_guarded_test(ctx, skip, nullptr, f); + else + f(); } } } @@ -3162,12 +3164,16 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con } else { assert(src.isboxed && "expected boxed value for sizeof/alignment computation"); - Value *datatype = emit_typeof_boxed(ctx, src); - Value *copy_bytes = emit_datatype_size(ctx, datatype); - if (skip) { - copy_bytes = ctx.builder.CreateSelect(skip, ConstantInt::get(copy_bytes->getType(), 0), copy_bytes); - } - emit_memcpy(ctx, dest, tbaa_dst, src, copy_bytes, /*TODO: min-align*/1, isVolatile); + auto f = [&] { + Value *datatype = emit_typeof_boxed(ctx, src); + Value *copy_bytes = emit_datatype_size(ctx, datatype); + emit_memcpy(ctx, dest, tbaa_dst, src, copy_bytes, /*TODO: min-align*/1, isVolatile); + return nullptr; + }; + if (skip) + emit_guarded_test(ctx, skip, nullptr, f); + else + f(); } } diff --git a/src/codegen.cpp b/src/codegen.cpp index 531b35bfedb79..cccb4b12cfc1d 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1743,7 +1743,7 @@ static jl_cgval_t convert_julia_type_union(jl_codectx_t &ctx, const jl_cgval_t & if (!union_isaBB) { union_isaBB = BasicBlock::Create(ctx.builder.getContext(), "union_isa", ctx.f); ctx.builder.SetInsertPoint(union_isaBB); - union_box_dt = emit_typeof_or_null(ctx, v.Vboxed); + union_box_dt = emit_typeof(ctx, v.Vboxed, skip != NULL); post_union_isaBB = ctx.builder.GetInsertBlock(); } }; @@ -1842,7 +1842,6 @@ static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_ return ghostValue(ctx, typ); Value *new_tindex = NULL; if (jl_is_concrete_type(typ)) { - assert(skip == nullptr && "skip only valid for union type return"); if (v.TIndex && !jl_is_pointerfree(typ)) { // discovered that this union-split type must actually be isboxed if (v.Vboxed) { @@ -1850,14 +1849,20 @@ static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_ } else { // type mismatch: there weren't any boxed values in the union - CreateTrap(ctx.builder); + if (skip) + *skip = ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 1); + else + CreateTrap(ctx.builder); return jl_cgval_t(ctx.builder.getContext()); } } if (jl_is_concrete_type(v.typ) && !jl_is_kind(v.typ)) { if (jl_is_concrete_type(typ) && !jl_is_kind(typ)) { // type mismatch: changing from one leaftype to another - CreateTrap(ctx.builder); + if (skip) + *skip = ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 1); + else + CreateTrap(ctx.builder); return jl_cgval_t(ctx.builder.getContext()); } } @@ -2802,7 +2807,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } else if (f == jl_builtin_typeof && nargs == 1) { - *ret = emit_typeof(ctx, argv[1]); + *ret = emit_typeof(ctx, argv[1], false); return true; } @@ -3944,7 +3949,7 @@ static jl_cgval_t emit_sparam(jl_codectx_t &ctx, size_t i) ctx.spvals_ptr, i + sizeof(jl_svec_t) / sizeof(jl_value_t*)); Value *sp = tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, bp, Align(sizeof(void*)))); - Value *isnull = ctx.builder.CreateICmpNE(emit_typeof(ctx, sp), + Value *isnull = ctx.builder.CreateICmpNE(emit_typeof(ctx, sp, false), track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jl_tvar_type))); jl_unionall_t *sparam = (jl_unionall_t*)ctx.linfo->def.method->sig; for (size_t j = 0; j < i; j++) { @@ -4018,7 +4023,7 @@ static jl_cgval_t emit_isdefined(jl_codectx_t &ctx, jl_value_t *sym) ctx.spvals_ptr, i + sizeof(jl_svec_t) / sizeof(jl_value_t*)); Value *sp = tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, bp, Align(sizeof(void*)))); - isnull = ctx.builder.CreateICmpNE(emit_typeof(ctx, sp), + isnull = ctx.builder.CreateICmpNE(emit_typeof(ctx, sp, false), track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jl_tvar_type))); } else { @@ -4191,7 +4196,7 @@ static void emit_vi_assignment_unboxed(jl_codectx_t &ctx, jl_varinfo_t &vi, Valu } } else { - emit_unionmove(ctx, vi.value.V, ctx.tbaa().tbaa_stack, rval_info, isboxed, vi.isVolatile); + emit_unionmove(ctx, vi.value.V, ctx.tbaa().tbaa_stack, rval_info, /*skip*/isboxed, vi.isVolatile); } } } @@ -4655,7 +4660,8 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval) jl_error("GotoIfNot in value position"); } if (jl_is_pinode(expr)) { - return convert_julia_type(ctx, emit_expr(ctx, jl_fieldref_noalloc(expr, 0)), jl_fieldref_noalloc(expr, 1)); + Value *skip = NULL; + return convert_julia_type(ctx, emit_expr(ctx, jl_fieldref_noalloc(expr, 0)), jl_fieldref_noalloc(expr, 1), &skip); } if (!jl_is_expr(expr)) { int needroot = true; @@ -7504,7 +7510,7 @@ static std::pair, jl_llvm_functions_t> else { // must be careful to emit undef here (rather than a bitcast or // load of val) if the runtime type of val isn't phiType - Value *isvalid = emit_isa(ctx, val, phiType, NULL).first; + Value *isvalid = emit_isa_and_defined(ctx, val, phiType); V = emit_guarded_test(ctx, isvalid, undef_value_for_type(VN->getType()), [&] { return emit_unbox(ctx, VN->getType(), val, phiType); }); @@ -7516,7 +7522,7 @@ static std::pair, jl_llvm_functions_t> // must be careful to emit undef here (rather than a bitcast or // load of val) if the runtime type of val isn't phiType assert(lty != ctx.types().T_prjlvalue); - Value *isvalid = emit_isa(ctx, val, phiType, NULL).first; + Value *isvalid = emit_isa_and_defined(ctx, val, phiType); emit_guarded_test(ctx, isvalid, nullptr, [&] { (void)emit_unbox(ctx, lty, val, phiType, maybe_decay_tracked(ctx, dest), ctx.tbaa().tbaa_stack); return nullptr; @@ -7558,7 +7564,7 @@ static std::pair, jl_llvm_functions_t> RTindex = new_union.TIndex; if (!RTindex) { assert(new_union.isboxed && new_union.Vboxed && "convert_julia_type failed"); - RTindex = compute_tindex_unboxed(ctx, new_union, phiType); + RTindex = compute_tindex_unboxed(ctx, new_union, phiType, true); if (dest) { // If dest is not set, this is a ghost union, the recipient of which // is often not prepared to handle a boxed representation of the ghost. diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index 3586527668135..9f716db3898d5 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -1065,8 +1065,9 @@ void RecursivelyVisit(callback f, Value *V) { if (isa(TheUser)) f(VU); if (isa(TheUser) || isa(TheUser) || - isa(TheUser) || isa(TheUser) || + isa(TheUser) || isa(TheUser) || // TODO: should these be removed from this list? isa(TheUser) || isa(TheUser) || + isa(TheUser) || // ICmpEQ/ICmpNE can be used with ptr types isa(TheUser) || isa(TheUser)) continue; if (isa(TheUser) || isa(TheUser) || isa(TheUser)) { From 834c8e6553cd0a0eeae2aca678fda4845e62715d Mon Sep 17 00:00:00 2001 From: N5N3 <2642243996@qq.com> Date: Mon, 16 May 2022 20:07:03 +0800 Subject: [PATCH 50/68] Use `CartesianIndices(Rsrc)` as the shared iterator. (#45289) There's no performance change, if the `indices`s of `Rdest` and `Rsrc` are all `NTuple{N,<:AbstractUnitRange}`. (cherry picked from commit a91be39b671638b8c48215931eb4ccd948ce33a3) --- base/multidimensional.jl | 12 +++++++----- test/copy.jl | 13 ++++++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index b5e401a7834e7..4be18766a30e6 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -1113,16 +1113,18 @@ function copyto!(dest::AbstractArray{T1,N}, Rdest::CartesianIndices{N}, checkbounds(src, first(Rsrc)) checkbounds(src, last(Rsrc)) src′ = unalias(dest, src) - ΔI = first(Rdest) - first(Rsrc) + CRdest = CartesianIndices(Rdest) + CRsrc = CartesianIndices(Rsrc) + ΔI = first(CRdest) - first(CRsrc) if @generated quote - @nloops $N i (n->Rsrc.indices[n]) begin - @inbounds @nref($N,dest,n->i_n+ΔI[n]) = @nref($N,src′,i) + @nloops $N i (n->CRsrc.indices[n]) begin + @inbounds @nref($N,dest,n->Rdest.indices[n][i_n+ΔI[n]]) = @nref($N,src,n->Rsrc.indices[n][i_n]) end end else - for I in Rsrc - @inbounds dest[I + ΔI] = src′[I] + for I in CRsrc + @inbounds dest[Rdest[I + ΔI]] = src′[Rsrc[I]] end end dest diff --git a/test/copy.jl b/test/copy.jl index 28d34e4756a6b..654e95dec67d5 100644 --- a/test/copy.jl +++ b/test/copy.jl @@ -58,11 +58,14 @@ end @test B == A end let A = reshape(1:6, 3, 2), B = zeros(8,8) - RA = CartesianIndices(axes(A)) - copyto!(B, CartesianIndices((5:7,2:3)), A, RA) - @test B[5:7,2:3] == A - B[5:7,2:3] .= 0 - @test all(x->x==0, B) + RBs = Any[(5:7,2:3), (3:2:7,1:2:3), (6:-1:4,2:-1:1)] + RAs = Any[axes(A), reverse.(axes(A))] + for RB in RBs, RA in RAs + copyto!(B, CartesianIndices(RB), A, CartesianIndices(RA)) + @test B[RB...] == A[RA...] + B[RB...] .= 0 + @test all(iszero, B) + end end end From 3dfa74d2786faed1de5782d472330c4a91f6729f Mon Sep 17 00:00:00 2001 From: N5N3 <2642243996@qq.com> Date: Tue, 17 May 2022 18:54:03 +0800 Subject: [PATCH 51/68] Typo fix. (#45333) (cherry picked from commit eed2dba73d5c781055c23120d357a17c875e685b) --- base/multidimensional.jl | 2 +- test/copy.jl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 4be18766a30e6..cd55b371de5a3 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -1119,7 +1119,7 @@ function copyto!(dest::AbstractArray{T1,N}, Rdest::CartesianIndices{N}, if @generated quote @nloops $N i (n->CRsrc.indices[n]) begin - @inbounds @nref($N,dest,n->Rdest.indices[n][i_n+ΔI[n]]) = @nref($N,src,n->Rsrc.indices[n][i_n]) + @inbounds @nref($N,dest,n->Rdest.indices[n][i_n+ΔI[n]]) = @nref($N,src′,n->Rsrc.indices[n][i_n]) end end else diff --git a/test/copy.jl b/test/copy.jl index 654e95dec67d5..04fda36728e62 100644 --- a/test/copy.jl +++ b/test/copy.jl @@ -67,6 +67,10 @@ end @test all(iszero, B) end end + let A = [reshape(1:6, 3, 2);;] + copyto!(A, CartesianIndices((2:3,2)), A, CartesianIndices((2,2))) + @test A[2:3,:] == [1 4;2 5] + end end @testset "shallow and deep copying" begin From 43ba8aec070f036ec7d541598db904e7679c8e9e Mon Sep 17 00:00:00 2001 From: FX Coudert Date: Tue, 17 May 2022 23:48:34 +0200 Subject: [PATCH 52/68] Resurrect libunwind patches (#45189) Fixes #44499 (cherry picked from commit f18324c856aad346c9040abb0f1b327aace7d081) --- deps/patches/llvm-libunwind-force-dwarf.patch | 179 +++++++++++++++++ ...-libunwind-freebsd-libgcc-api-compat.patch | 107 ++++++++++ .../llvm-libunwind-prologue-epilogue.patch | 183 ++++++++++++++++++ ...ibunwind-revert-monorepo-requirement.patch | 156 +++++++++++++++ deps/unwind.mk | 10 +- 5 files changed, 634 insertions(+), 1 deletion(-) create mode 100644 deps/patches/llvm-libunwind-force-dwarf.patch create mode 100644 deps/patches/llvm-libunwind-freebsd-libgcc-api-compat.patch create mode 100644 deps/patches/llvm-libunwind-prologue-epilogue.patch create mode 100644 deps/patches/llvm-libunwind-revert-monorepo-requirement.patch diff --git a/deps/patches/llvm-libunwind-force-dwarf.patch b/deps/patches/llvm-libunwind-force-dwarf.patch new file mode 100644 index 0000000000000..2f4d31acb8a4a --- /dev/null +++ b/deps/patches/llvm-libunwind-force-dwarf.patch @@ -0,0 +1,179 @@ +An updated version of this libosxunwind commit: + +Author: Keno Fischer +Date: Tue Aug 27 15:01:22 2013 -0400 + + Add option to step with DWARF + +--- +diff -pur a/libunwind/include/libunwind.h b/libunwind/include/libunwind.h +--- a/libunwind/include/libunwind.h 2021-06-28 18:23:38.000000000 +0200 ++++ b/libunwind/include/libunwind.h 2022-05-04 18:44:24.000000000 +0200 +@@ -108,6 +108,7 @@ extern "C" { + + extern int unw_getcontext(unw_context_t *) LIBUNWIND_AVAIL; + extern int unw_init_local(unw_cursor_t *, unw_context_t *) LIBUNWIND_AVAIL; ++extern int unw_init_local_dwarf(unw_cursor_t *, unw_context_t *) LIBUNWIND_AVAIL; + extern int unw_step(unw_cursor_t *) LIBUNWIND_AVAIL; + extern int unw_get_reg(unw_cursor_t *, unw_regnum_t, unw_word_t *) LIBUNWIND_AVAIL; + extern int unw_get_fpreg(unw_cursor_t *, unw_regnum_t, unw_fpreg_t *) LIBUNWIND_AVAIL; +Only in b/libunwind/include: libunwind.h.orig +diff -pur a/libunwind/src/UnwindCursor.hpp b/libunwind/src/UnwindCursor.hpp +--- a/libunwind/src/UnwindCursor.hpp 2021-06-28 18:23:38.000000000 +0200 ++++ b/libunwind/src/UnwindCursor.hpp 2022-05-04 18:45:11.000000000 +0200 +@@ -437,6 +437,9 @@ public: + virtual bool isSignalFrame() { + _LIBUNWIND_ABORT("isSignalFrame not implemented"); + } ++ virtual void setForceDWARF(bool) { ++ _LIBUNWIND_ABORT("setForceDWARF not implemented"); ++ } + virtual bool getFunctionName(char *, size_t, unw_word_t *) { + _LIBUNWIND_ABORT("getFunctionName not implemented"); + } +@@ -894,6 +897,7 @@ public: + virtual void getInfo(unw_proc_info_t *); + virtual void jumpto(); + virtual bool isSignalFrame(); ++ virtual void setForceDWARF(bool force); + virtual bool getFunctionName(char *buf, size_t len, unw_word_t *off); + virtual void setInfoBasedOnIPRegister(bool isReturnAddress = false); + virtual const char *getRegisterName(int num); +@@ -963,7 +967,7 @@ private: + const UnwindInfoSections §s); + int stepWithCompactEncoding() { + #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) +- if ( compactSaysUseDwarf() ) ++ if ( _forceDwarf || compactSaysUseDwarf() ) + return stepWithDwarfFDE(); + #endif + R dummy; +@@ -1198,6 +1202,7 @@ private: + unw_proc_info_t _info; + bool _unwindInfoMissing; + bool _isSignalFrame; ++ bool _forceDwarf; + #if defined(_LIBUNWIND_TARGET_LINUX) && defined(_LIBUNWIND_TARGET_AARCH64) + bool _isSigReturn = false; + #endif +@@ -1207,7 +1212,7 @@ private: + template + UnwindCursor::UnwindCursor(unw_context_t *context, A &as) + : _addressSpace(as), _registers(context), _unwindInfoMissing(false), +- _isSignalFrame(false) { ++ _isSignalFrame(false), _forceDwarf(false) { + static_assert((check_fit, unw_cursor_t>::does_fit), + "UnwindCursor<> does not fit in unw_cursor_t"); + static_assert((alignof(UnwindCursor) <= alignof(unw_cursor_t)), +@@ -1217,7 +1222,8 @@ UnwindCursor::UnwindCursor(unw_con + + template + UnwindCursor::UnwindCursor(A &as, void *) +- : _addressSpace(as), _unwindInfoMissing(false), _isSignalFrame(false) { ++ : _addressSpace(as), _unwindInfoMissing(false), _isSignalFrame(false), ++ _forceDwarf(false) { + memset(&_info, 0, sizeof(_info)); + // FIXME + // fill in _registers from thread arg +@@ -1273,6 +1279,10 @@ template bool U + return _isSignalFrame; + } + ++template void UnwindCursor::setForceDWARF(bool force) { ++ _forceDwarf = force; ++} ++ + #endif // defined(_LIBUNWIND_SUPPORT_SEH_UNWIND) + + #if defined(_LIBUNWIND_ARM_EHABI) +@@ -1941,7 +1951,13 @@ void UnwindCursor::setInfoBasedOnI + // record that we have no unwind info. + if (_info.format == 0) + _unwindInfoMissing = true; ++ #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) ++ if (!(_forceDwarf || compactSaysUseDwarf(&dwarfOffset))) ++ return; ++ #else + return; ++ #endif ++ + } + } + #endif // defined(_LIBUNWIND_SUPPORT_COMPACT_UNWIND) +diff -pur a/libunwind/src/libunwind.cpp b/libunwind/src/libunwind.cpp +--- a/libunwind/src/libunwind.cpp 2021-06-28 18:23:38.000000000 +0200 ++++ b/libunwind/src/libunwind.cpp 2022-05-04 18:44:24.000000000 +0200 +@@ -71,6 +71,7 @@ _LIBUNWIND_HIDDEN int __unw_init_local(u + new (reinterpret_cast *>(cursor)) + UnwindCursor( + context, LocalAddressSpace::sThisAddressSpace); ++ static_assert(sizeof(unw_cursor_t) >= sizeof(UnwindCursor), "libunwind header outdated"); + #undef REGISTER_KIND + AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; + co->setInfoBasedOnIPRegister(); +@@ -79,6 +80,54 @@ _LIBUNWIND_HIDDEN int __unw_init_local(u + } + _LIBUNWIND_WEAK_ALIAS(__unw_init_local, unw_init_local) + ++_LIBUNWIND_HIDDEN int __unw_init_local_dwarf(unw_cursor_t *cursor, ++ unw_context_t *context) { ++ _LIBUNWIND_TRACE_API("__unw_init_local_dwarf(cursor=%p, context=%p)", ++ static_cast(cursor), ++ static_cast(context)); ++#if defined(__i386__) ++# define REGISTER_KIND Registers_x86 ++#elif defined(__x86_64__) ++# define REGISTER_KIND Registers_x86_64 ++#elif defined(__powerpc64__) ++# define REGISTER_KIND Registers_ppc64 ++#elif defined(__ppc__) ++# define REGISTER_KIND Registers_ppc ++#elif defined(__aarch64__) ++# define REGISTER_KIND Registers_arm64 ++#elif defined(__arm__) ++# define REGISTER_KIND Registers_arm ++#elif defined(__or1k__) ++# define REGISTER_KIND Registers_or1k ++#elif defined(__hexagon__) ++# define REGISTER_KIND Registers_hexagon ++#elif defined(__mips__) && defined(_ABIO32) && _MIPS_SIM == _ABIO32 ++# define REGISTER_KIND Registers_mips_o32 ++#elif defined(__mips64) ++# define REGISTER_KIND Registers_mips_newabi ++#elif defined(__mips__) ++# warning The MIPS architecture is not supported with this ABI and environment! ++#elif defined(__sparc__) ++# define REGISTER_KIND Registers_sparc ++#elif defined(__riscv) && __riscv_xlen == 64 ++# define REGISTER_KIND Registers_riscv ++#else ++# error Architecture not supported ++#endif ++ // Use "placement new" to allocate UnwindCursor in the cursor buffer. ++ new (reinterpret_cast *>(cursor)) ++ UnwindCursor( ++ context, LocalAddressSpace::sThisAddressSpace); ++ static_assert(sizeof(unw_cursor_t) >= sizeof(UnwindCursor), "libunwind header outdated"); ++#undef REGISTER_KIND ++ AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor; ++ co->setForceDWARF(true); ++ co->setInfoBasedOnIPRegister(); ++ ++ return UNW_ESUCCESS; ++} ++_LIBUNWIND_WEAK_ALIAS(__unw_init_local_dwarf, unw_init_local_dwarf) ++ + /// Get value of specified register at cursor position in stack frame. + _LIBUNWIND_HIDDEN int __unw_get_reg(unw_cursor_t *cursor, unw_regnum_t regNum, + unw_word_t *value) { +diff -pur a/libunwind/src/libunwind_ext.h b/libunwind/src/libunwind_ext.h +--- a/libunwind/src/libunwind_ext.h 2021-06-28 18:23:38.000000000 +0200 ++++ b/libunwind/src/libunwind_ext.h 2022-05-04 18:44:24.000000000 +0200 +@@ -25,6 +25,7 @@ extern "C" { + + extern int __unw_getcontext(unw_context_t *); + extern int __unw_init_local(unw_cursor_t *, unw_context_t *); ++extern int __unw_init_local_dwarf(unw_cursor_t *, unw_context_t *); + extern int __unw_step(unw_cursor_t *); + extern int __unw_get_reg(unw_cursor_t *, unw_regnum_t, unw_word_t *); + extern int __unw_get_fpreg(unw_cursor_t *, unw_regnum_t, unw_fpreg_t *); diff --git a/deps/patches/llvm-libunwind-freebsd-libgcc-api-compat.patch b/deps/patches/llvm-libunwind-freebsd-libgcc-api-compat.patch new file mode 100644 index 0000000000000..afb4b941d5b92 --- /dev/null +++ b/deps/patches/llvm-libunwind-freebsd-libgcc-api-compat.patch @@ -0,0 +1,107 @@ +Modification of the following patch in the FreeBSD source tree, which +includes LLVM libunwind in contrib/llvm-project/libunwind. + +From 9f287522cec9feac040d7cb845a440a8f6b7b90e Mon Sep 17 00:00:00 2001 +From: Dimitry Andric +Date: Sun, 2 Aug 2020 18:12:14 +0000 +Subject: [PATCH] Reapply r310365 (by emaste): + +libunwind: make __{de,}register_frame compatible with libgcc API + +The libgcc __register_frame and __deregister_frame functions take a +pointer to a set of FDE/CIEs, terminated by an entry where length is 0. + +In Apple's libunwind implementation the pointer is taken to be to a +single FDE. I suspect this was just an Apple bug, compensated by Apple- +specific code in LLVM. + +See lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp and +http://lists.llvm.org/pipermail/llvm-dev/2013-April/061737.html +for more detail. + +This change is based on the LLVM RTDyldMemoryManager.cpp. It should +later be changed to be alignment-safe. + +Reported by: dim +Reviewed by: dim +Sponsored by: The FreeBSD Foundation +Differential Revision: https://reviews.freebsd.org/D8869 + +Reapply r351610: + +Update libunwind custom frame register and deregister functions for +FreeBSD: use the new doubly underscored names for unw_add_dynamic_fde +and unw_remove_dynamic_fde. + +NOTE: this should be upstreamed... +--- + .../libunwind/src/UnwindLevel1-gcc-ext.c | 42 ++++++++++++++++++- + 1 file changed, 41 insertions(+), 1 deletion(-) + +diff --git a/libunwind/src/UnwindLevel1-gcc-ext.c b/libunwind/src/UnwindLevel1-gcc-ext.c +index 310b836d129e5..30f9cabf241f2 100644 +--- a/libunwind/src/UnwindLevel1-gcc-ext.c ++++ b/libunwind/src/UnwindLevel1-gcc-ext.c +@@ -234,6 +234,46 @@ _LIBUNWIND_EXPORT uintptr_t _Unwind_GetIPInfo(struct _Unwind_Context *context, + + #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND) + ++#if defined(__FreeBSD__) ++ ++// Based on LLVM's lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp ++// and XXX should be fixed to be alignment-safe. ++static void processFDE(const char *addr, bool isDeregister) { ++ uint64_t length; ++ while ((length = *((const uint32_t *)addr)) != 0) { ++ const char *p = addr + 4; ++ if (length == 0xffffffff) { ++ length = *((const uint64_t *)p); ++ p += 8; ++ } ++ uint32_t offset = *((const uint32_t *)p); ++ if (offset != 0) { ++ if (isDeregister) ++ __unw_remove_dynamic_fde((unw_word_t)(uintptr_t)addr); ++ else ++ __unw_add_dynamic_fde((unw_word_t)(uintptr_t)addr); ++ } ++ addr = p + length; ++ } ++} ++ ++/// Called by programs with dynamic code generators that want to register ++/// dynamically generated FDEs, with a libgcc-compatible API. ++ ++_LIBUNWIND_EXPORT void __register_frame(const void *addr) { ++ _LIBUNWIND_TRACE_API("__register_frame(%p)", addr); ++ processFDE(addr, false); ++} ++ ++/// Called by programs with dynamic code generators that want to unregister ++/// dynamically generated FDEs, with a libgcc-compatible API. ++_LIBUNWIND_EXPORT void __deregister_frame(const void *addr) { ++ _LIBUNWIND_TRACE_API("__deregister_frame(%p)", addr); ++ processFDE(addr, true); ++} ++ ++#else // defined(__FreeBSD__) ++ + /// Called by programs with dynamic code generators that want + /// to register a dynamically generated FDE. + /// This function has existed on Mac OS X since 10.4, but +@@ -243,7 +283,6 @@ _LIBUNWIND_EXPORT void __register_frame(const void *fde) { + __unw_add_dynamic_fde((unw_word_t)(uintptr_t)fde); + } + +- + /// Called by programs with dynamic code generators that want + /// to unregister a dynamically generated FDE. + /// This function has existed on Mac OS X since 10.4, but +@@ -253,6 +292,7 @@ _LIBUNWIND_EXPORT void __deregister_frame(const void *fde) { + __unw_remove_dynamic_fde((unw_word_t)(uintptr_t)fde); + } + ++#endif // defined(__FreeBSD__) + + // The following register/deregister functions are gcc extensions. + // They have existed on Mac OS X, but have never worked because Mac OS X diff --git a/deps/patches/llvm-libunwind-prologue-epilogue.patch b/deps/patches/llvm-libunwind-prologue-epilogue.patch new file mode 100644 index 0000000000000..7dadca728f9cf --- /dev/null +++ b/deps/patches/llvm-libunwind-prologue-epilogue.patch @@ -0,0 +1,183 @@ +An updated version of this libosxunwind commit: + +commit ca57a5b60de4cd1daa42ed2e5d1d4aa3e96a09d1 +Author: Keno Fischer +Date: Mon Aug 26 15:28:08 2013 -0400 + + Add support for unwinding during prologue/epilogue + +--- +diff --git a/libunwind/src/CompactUnwinder.hpp b/libunwind/src/CompactUnwinder.hpp +index 1c3175dff50a..78a658ccbc27 100644 +--- a/libunwind/src/CompactUnwinder.hpp ++++ b/libunwind/src/CompactUnwinder.hpp +@@ -310,6 +310,50 @@ int CompactUnwinder_x86_64::stepWithCompactEncodingRBPFrame( + uint32_t savedRegistersLocations = + EXTRACT_BITS(compactEncoding, UNWIND_X86_64_RBP_FRAME_REGISTERS); + ++ // If we have not stored EBP yet ++ if (functionStart == registers.getIP()) { ++ uint64_t rsp = registers.getSP(); ++ // old esp is ebp less return address ++ registers.setSP(rsp+8); ++ // pop return address into eip ++ registers.setIP(addressSpace.get64(rsp)); ++ ++ return UNW_STEP_SUCCESS; ++ } else if (functionStart + 1 == registers.getIP()) { ++ uint64_t rsp = registers.getSP(); ++ // old esp is ebp less return address ++ registers.setSP(rsp + 16); ++ // pop return address into eip ++ registers.setIP(addressSpace.get64(rsp + 8)); ++ ++ return UNW_STEP_SUCCESS; ++ } ++ ++ // If we're about to return, we've already popped the base pointer ++ uint8_t b = addressSpace.get8(registers.getIP()); ++ ++ // This is a hack to detect VZEROUPPER but in between popq rbp and ret ++ // It's not pretty but it works ++ if (b == 0xC5) { ++ if ((b = addressSpace.get8(registers.getIP() + 1)) == 0xF8 && ++ (b = addressSpace.get8(registers.getIP() + 2)) == 0x77) ++ b = addressSpace.get8(registers.getIP() + 3); ++ else ++ goto skip_ret; ++ } ++ ++ if (b == 0xC3 || b == 0xCB || b == 0xC2 || b == 0xCA) { ++ uint64_t rbp = registers.getSP(); ++ // old esp is ebp less return address ++ registers.setSP(rbp + 16); ++ // pop return address into eip ++ registers.setIP(addressSpace.get64(rbp + 8)); ++ ++ return UNW_STEP_SUCCESS; ++ } ++ ++ skip_ret: ++ + uint64_t savedRegisters = registers.getRBP() - 8 * savedRegistersOffset; + for (int i = 0; i < 5; ++i) { + switch (savedRegistersLocations & 0x7) { +@@ -430,6 +474,118 @@ int CompactUnwinder_x86_64::stepWithCompactEncodingFrameless( + } + } + } ++ ++ // Note that the order of these registers is so that ++ // registersSaved[0] is the one that will be pushed onto the stack last. ++ // Thus, if we want to walk this from the top, we need to go in reverse. ++ assert(regCount <= 6); ++ ++ // check whether we are still in the prologue ++ uint64_t curAddr = functionStart; ++ if (regCount > 0) { ++ for (int8_t i = (int8_t)(regCount) - 1; i >= 0; --i) { ++ if (registers.getIP() == curAddr) { ++ // None of the registers have been modified yet, so we don't need to reload them ++ framelessUnwind(addressSpace, registers.getSP() + 8 * (regCount - (uint64_t)(i + 1)), registers); ++ return UNW_STEP_SUCCESS; ++ } else { ++ assert(curAddr < registers.getIP()); ++ } ++ ++ ++ // pushq %rbp and pushq %rbx is 1 byte. Everything else 2 ++ if ((UNWIND_X86_64_REG_RBP == registersSaved[i]) || ++ (UNWIND_X86_64_REG_RBX == registersSaved[i])) ++ curAddr += 1; ++ else ++ curAddr += 2; ++ } ++ } ++ if (registers.getIP() == curAddr) { ++ // None of the registers have been modified yet, so we don't need to reload them ++ framelessUnwind(addressSpace, registers.getSP() + 8*regCount, registers); ++ return UNW_STEP_SUCCESS; ++ } else { ++ assert(curAddr < registers.getIP()); ++ } ++ ++ ++ // And now for the epilogue ++ { ++ uint8_t i = 0; ++ uint64_t p = registers.getIP(); ++ uint8_t b = 0; ++ ++ while (true) { ++ b = addressSpace.get8(p++); ++ // This is a hack to detect VZEROUPPER but in between the popq's and ret ++ // It's not pretty but it works ++ if (b == 0xC5) { ++ if ((b = addressSpace.get8(p++)) == 0xF8 && (b = addressSpace.get8(p++)) == 0x77) ++ b = addressSpace.get8(p++); ++ else ++ break; ++ } ++ // popq %rbx popq %rbp ++ if (b == 0x5B || b == 0x5D) { ++ i++; ++ } else if (b == 0x41) { ++ b = addressSpace.get8(p++); ++ if (b == 0x5C || b == 0x5D || b == 0x5E || b == 0x5F) ++ i++; ++ else ++ break; ++ } else if (b == 0xC3 || b == 0xCB || b == 0xC2 || b == 0xCA) { ++ // i pop's haven't happened yet ++ uint64_t savedRegisters = registers.getSP() + 8 * i; ++ if (regCount > 0) { ++ for (int8_t j = (int8_t)(regCount) - 1; j >= (int8_t)(regCount) - i; --j) { ++ uint64_t addr = savedRegisters - 8 * (regCount - (uint64_t)(j)); ++ switch (registersSaved[j]) { ++ case UNWIND_X86_64_REG_RBX: ++ registers.setRBX(addressSpace.get64(addr)); ++ break; ++ case UNWIND_X86_64_REG_R12: ++ registers.setR12(addressSpace.get64(addr)); ++ break; ++ case UNWIND_X86_64_REG_R13: ++ registers.setR13(addressSpace.get64(addr)); ++ break; ++ case UNWIND_X86_64_REG_R14: ++ registers.setR14(addressSpace.get64(addr)); ++ break; ++ case UNWIND_X86_64_REG_R15: ++ registers.setR15(addressSpace.get64(addr)); ++ break; ++ case UNWIND_X86_64_REG_RBP: ++ registers.setRBP(addressSpace.get64(addr)); ++ break; ++ default: ++ _LIBUNWIND_DEBUG_LOG("bad register for frameless, encoding=%08X for " ++ "function starting at 0x%llX", ++ encoding, functionStart); ++ _LIBUNWIND_ABORT("invalid compact unwind encoding"); ++ } ++ } ++ } ++ framelessUnwind(addressSpace, savedRegisters, registers); ++ return UNW_STEP_SUCCESS; ++ } else { ++ break; ++ } ++ } ++ } ++ ++ /* ++ 0x10fe2733a: 5b popq %rbx ++ 0x10fe2733b: 41 5c popq %r12 ++ 0x10fe2733d: 41 5d popq %r13 ++ 0x10fe2733f: 41 5e popq %r14 ++ 0x10fe27341: 41 5f popq %r15 ++ 0x10fe27343: 5d popq %rbp ++ */ ++ ++ + uint64_t savedRegisters = registers.getSP() + stackSize - 8 - 8 * regCount; + for (uint32_t i = 0; i < regCount; ++i) { + switch (registersSaved[i]) { diff --git a/deps/patches/llvm-libunwind-revert-monorepo-requirement.patch b/deps/patches/llvm-libunwind-revert-monorepo-requirement.patch new file mode 100644 index 0000000000000..4e3897dfb9801 --- /dev/null +++ b/deps/patches/llvm-libunwind-revert-monorepo-requirement.patch @@ -0,0 +1,156 @@ +Upstream commit 8c03fdf34a659925a3f09c8f54016e47ea1c7519 changed the build such +that it requires living inside the monorepo with libcxx available, only so that +it can reuse a CMake file to simplify some build steps. This patch is a revert +of that commit applied only to libunwind. + +--- +diff --git a/libunwind/CMakeLists.txt b/libunwind/CMakeLists.txt +index 570b8db90653..a383d7d77d6f 100644 +--- a/libunwind/CMakeLists.txt ++++ b/libunwind/CMakeLists.txt +@@ -1,7 +1,3 @@ +-if (NOT IS_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}/../libcxx") +- message(FATAL_ERROR "libunwind requires being built in a monorepo layout with libcxx available") +-endif() +- + #=============================================================================== + # Setup Project + #=============================================================================== +@@ -15,31 +11,103 @@ set(CMAKE_MODULE_PATH + ${CMAKE_MODULE_PATH} + ) + +-set(LIBUNWIND_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +-set(LIBUNWIND_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) +-set(LIBUNWIND_LIBCXX_PATH "${CMAKE_CURRENT_LIST_DIR}/../libcxx" CACHE PATH +- "Specify path to libc++ source.") +- + if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR OR LIBUNWIND_STANDALONE_BUILD) + project(libunwind LANGUAGES C CXX ASM) + ++ # Rely on llvm-config. ++ set(CONFIG_OUTPUT) ++ if(NOT LLVM_CONFIG_PATH) ++ find_program(LLVM_CONFIG_PATH "llvm-config") ++ endif() ++ if (DEFINED LLVM_PATH) ++ set(LLVM_INCLUDE_DIR ${LLVM_INCLUDE_DIR} CACHE PATH "Path to llvm/include") ++ set(LLVM_PATH ${LLVM_PATH} CACHE PATH "Path to LLVM source tree") ++ set(LLVM_MAIN_SRC_DIR ${LLVM_PATH}) ++ set(LLVM_CMAKE_PATH "${LLVM_PATH}/cmake/modules") ++ elseif(LLVM_CONFIG_PATH) ++ message(STATUS "Found LLVM_CONFIG_PATH as ${LLVM_CONFIG_PATH}") ++ set(CONFIG_COMMAND ${LLVM_CONFIG_PATH} "--includedir" "--prefix" "--src-root") ++ execute_process(COMMAND ${CONFIG_COMMAND} ++ RESULT_VARIABLE HAD_ERROR ++ OUTPUT_VARIABLE CONFIG_OUTPUT) ++ if (NOT HAD_ERROR) ++ string(REGEX REPLACE "[ \t]*[\r\n]+[ \t]*" ";" ++ CONFIG_OUTPUT ${CONFIG_OUTPUT}) ++ else() ++ string(REPLACE ";" " " CONFIG_COMMAND_STR "${CONFIG_COMMAND}") ++ message(STATUS "${CONFIG_COMMAND_STR}") ++ message(FATAL_ERROR "llvm-config failed with status ${HAD_ERROR}") ++ endif() ++ ++ list(GET CONFIG_OUTPUT 0 INCLUDE_DIR) ++ list(GET CONFIG_OUTPUT 1 LLVM_OBJ_ROOT) ++ list(GET CONFIG_OUTPUT 2 MAIN_SRC_DIR) ++ ++ set(LLVM_INCLUDE_DIR ${INCLUDE_DIR} CACHE PATH "Path to llvm/include") ++ set(LLVM_BINARY_DIR ${LLVM_OBJ_ROOT} CACHE PATH "Path to LLVM build tree") ++ set(LLVM_MAIN_SRC_DIR ${MAIN_SRC_DIR} CACHE PATH "Path to LLVM source tree") ++ set(LLVM_LIT_PATH "${LLVM_PATH}/utils/lit/lit.py") ++ ++ # --cmakedir is supported since llvm r291218 (4.0 release) ++ execute_process( ++ COMMAND ${LLVM_CONFIG_PATH} --cmakedir ++ RESULT_VARIABLE HAD_ERROR ++ OUTPUT_VARIABLE CONFIG_OUTPUT ++ ERROR_QUIET) ++ if(NOT HAD_ERROR) ++ string(STRIP "${CONFIG_OUTPUT}" LLVM_CMAKE_PATH_FROM_LLVM_CONFIG) ++ file(TO_CMAKE_PATH "${LLVM_CMAKE_PATH_FROM_LLVM_CONFIG}" LLVM_CMAKE_PATH) ++ else() ++ file(TO_CMAKE_PATH "${LLVM_BINARY_DIR}" LLVM_BINARY_DIR_CMAKE_STYLE) ++ set(LLVM_CMAKE_PATH "${LLVM_BINARY_DIR_CMAKE_STYLE}/lib${LLVM_LIBDIR_SUFFIX}/cmake/llvm") ++ endif() ++ else() ++ message(WARNING "UNSUPPORTED LIBUNWIND CONFIGURATION DETECTED: " ++ "llvm-config not found and LLVM_MAIN_SRC_DIR not defined. " ++ "Reconfigure with -DLLVM_CONFIG=path/to/llvm-config " ++ "or -DLLVM_PATH=path/to/llvm-source-root.") ++ endif() ++ ++ if (EXISTS ${LLVM_CMAKE_PATH}) ++ list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_PATH}") ++ include("${LLVM_CMAKE_PATH}/AddLLVM.cmake") ++ include("${LLVM_CMAKE_PATH}/HandleLLVMOptions.cmake") ++ else() ++ message(WARNING "Not found: ${LLVM_CMAKE_PATH}") ++ endif() ++ + set(PACKAGE_NAME libunwind) + set(PACKAGE_VERSION 12.0.1) + set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") + set(PACKAGE_BUGREPORT "llvm-bugs@lists.llvm.org") + +- # Add the CMake module path of libcxx so we can reuse HandleOutOfTreeLLVM.cmake +- set(LIBUNWIND_LIBCXX_CMAKE_PATH "${LIBUNWIND_LIBCXX_PATH}/cmake/Modules") +- list(APPEND CMAKE_MODULE_PATH "${LIBUNWIND_LIBCXX_CMAKE_PATH}") ++ if (EXISTS ${LLVM_MAIN_SRC_DIR}/utils/lit/lit.py) ++ set(LLVM_LIT ${LLVM_MAIN_SRC_DIR}/utils/lit/lit.py) ++ else() ++ # Seek installed Lit. ++ find_program(LLVM_LIT "lit.py" ${LLVM_MAIN_SRC_DIR}/utils/lit ++ DOC "Path to lit.py") ++ endif() + +- # In a standalone build, we don't have llvm to automatically generate the +- # llvm-lit script for us. So we need to provide an explicit directory that +- # the configurator should write the script into. +- set(LIBUNWIND_STANDALONE_BUILD 1) +- set(LLVM_LIT_OUTPUT_DIR "${LIBUNWIND_BINARY_DIR}/bin") ++ if (LLVM_LIT) ++ # Define the default arguments to use with 'lit', and an option for the user ++ # to override. ++ set(LIT_ARGS_DEFAULT "-sv") ++ if (MSVC OR XCODE) ++ set(LIT_ARGS_DEFAULT "${LIT_ARGS_DEFAULT} --no-progress-bar") ++ endif() ++ set(LLVM_LIT_ARGS "${LIT_ARGS_DEFAULT}" CACHE STRING "Default options for lit") ++ ++ # On Win32 hosts, provide an option to specify the path to the GnuWin32 tools. ++ if (WIN32 AND NOT CYGWIN) ++ set(LLVM_LIT_TOOLS_DIR "" CACHE PATH "Path to GnuWin32 tools") ++ endif() ++ else() ++ set(LLVM_INCLUDE_TESTS OFF) ++ endif() + +- # Find the LLVM sources and simulate LLVM CMake options. +- include(HandleOutOfTreeLLVM) ++ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}) ++ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib${LLVM_LIBDIR_SUFFIX}) + else() + set(LLVM_LIT "${CMAKE_SOURCE_DIR}/utils/lit/lit.py") + endif() +@@ -85,8 +153,6 @@ set(LIBUNWIND_TEST_COMPILER_FLAGS "" CACHE STRING + "Additional compiler flags for test programs.") + set(LIBUNWIND_TEST_CONFIG "${CMAKE_CURRENT_SOURCE_DIR}/test/lit.site.cfg.in" CACHE STRING + "The Lit testing configuration to use when running the tests.") +-set(LIBUNWIND_TEST_PARAMS "" CACHE STRING +- "A list of parameters to run the Lit test suite with.") + + if (NOT LIBUNWIND_ENABLE_SHARED AND NOT LIBUNWIND_ENABLE_STATIC) + message(FATAL_ERROR "libunwind must be built as either a shared or static library.") +@@ -113,6 +179,9 @@ set(CMAKE_MODULE_PATH + "${CMAKE_CURRENT_SOURCE_DIR}/cmake" + ${CMAKE_MODULE_PATH}) + ++set(LIBUNWIND_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) ++set(LIBUNWIND_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) ++ + if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE) + set(LIBUNWIND_LIBRARY_DIR ${LLVM_LIBRARY_OUTPUT_INTDIR}/${LLVM_DEFAULT_TARGET_TRIPLE}/c++) + set(LIBUNWIND_INSTALL_LIBRARY_DIR lib${LLVM_LIBDIR_SUFFIX}/${LLVM_DEFAULT_TARGET_TRIPLE}/c++) diff --git a/deps/unwind.mk b/deps/unwind.mk index c794b94d5e636..dbdffdae31996 100644 --- a/deps/unwind.mk +++ b/deps/unwind.mk @@ -95,10 +95,18 @@ $(SRCCACHE)/llvmunwind-$(LLVMUNWIND_VER)/llvm-libunwind-force-dwarf.patch-applie cd $(SRCCACHE)/llvmunwind-$(LLVMUNWIND_VER) && patch -p2 -f < $(SRCDIR)/patches/llvm-libunwind-force-dwarf.patch echo 1 > $@ +$(SRCCACHE)/llvmunwind-$(LLVMUNWIND_VER)/llvm-libunwind-revert-monorepo-requirement.patch-applied: $(SRCCACHE)/llvmunwind-$(LLVMUNWIND_VER)/llvm-libunwind-force-dwarf.patch-applied + cd $(SRCCACHE)/llvmunwind-$(LLVMUNWIND_VER) && patch -p2 -f < $(SRCDIR)/patches/llvm-libunwind-revert-monorepo-requirement.patch + echo 1 > $@ + +$(SRCCACHE)/llvmunwind-$(LLVMUNWIND_VER)/llvm-libunwind-freebsd-libgcc-api-compat.patch-applied: $(SRCCACHE)/llvmunwind-$(LLVMUNWIND_VER)/llvm-libunwind-revert-monorepo-requirement.patch-applied + cd $(SRCCACHE)/llvmunwind-$(LLVMUNWIND_VER) && patch -p2 -f < $(SRCDIR)/patches/llvm-libunwind-freebsd-libgcc-api-compat.patch + echo 1 > $@ + checksum-llvmunwind: $(SRCCACHE)/llvmunwind-$(LLVMUNWIND_VER).tar.xz $(JLCHECKSUM) $< -$(BUILDDIR)/llvmunwind-$(LLVMUNWIND_VER)/build-configured: $(SRCCACHE)/llvmunwind-$(LLVMUNWIND_VER)/source-extracted $(SRCCACHE)/llvmunwind-$(LLVMUNWIND_VER)/llvm-libunwind-force-dwarf.patch-applied +$(BUILDDIR)/llvmunwind-$(LLVMUNWIND_VER)/build-configured: $(SRCCACHE)/llvmunwind-$(LLVMUNWIND_VER)/source-extracted $(SRCCACHE)/llvmunwind-$(LLVMUNWIND_VER)/llvm-libunwind-freebsd-libgcc-api-compat.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ $(CMAKE) $(dir $<) $(LLVMUNWIND_OPTS) From 1f9140f4a02c683ee64775a95e0e6b59058c0cfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Wed, 18 May 2022 03:50:34 +0100 Subject: [PATCH 53/68] [deps] Use newer `config.sub` when building nghttp2 (#45346) (cherry picked from commit 7f84d46ce01b9400a61dd5660c7607a4629abad6) --- deps/nghttp2.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/deps/nghttp2.mk b/deps/nghttp2.mk index 2b8a18728b712..9a161e1b238d3 100644 --- a/deps/nghttp2.mk +++ b/deps/nghttp2.mk @@ -8,6 +8,7 @@ $(SRCCACHE)/nghttp2-$(NGHTTP2_VER).tar.bz2: | $(SRCCACHE) $(SRCCACHE)/nghttp2-$(NGHTTP2_VER)/source-extracted: $(SRCCACHE)/nghttp2-$(NGHTTP2_VER).tar.bz2 $(JLCHECKSUM) $< cd $(dir $<) && $(TAR) -jxf $< + cp $(SRCDIR)/patches/config.sub $(SRCCACHE)/nghttp2-$(NGHTTP2_VER)/config.sub touch -c $(SRCCACHE)/nghttp2-$(NGHTTP2_VER)/configure # old target echo 1 > $@ From 1ccb618cae7ab30c7279cf13b4c76ef558f46203 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 18 May 2022 04:01:28 -0400 Subject: [PATCH 54/68] fix #45024, lost `expected assignment after const` error (#45344) (cherry picked from commit 2d40898a0bb09bbde7dae36cacc6ed5e23c7c0fa) --- src/julia-parser.scm | 18 ++++++++++++++++-- test/syntax.jl | 10 ++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/julia-parser.scm b/src/julia-parser.scm index 97a11df701a37..22d677b8bdaa2 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -1355,11 +1355,19 @@ (list 'where (rewrap-where x (cadr w)) (caddr w)) x)) +(define (parse-struct-field s) + (let ((tok (peek-token s))) + ;; allow `const x` only as a struct field + (if (eq? tok 'const) + (begin (take-token s) + `(const ,(parse-eq s))) + (parse-eq s)))) + (define (parse-struct-def s mut? word) (if (reserved-word? (peek-token s)) (error (string "invalid type name \"" (take-token s) "\""))) (let ((sig (parse-subtype-spec s))) - (begin0 (list 'struct (if mut? '(true) '(false)) sig (parse-block s)) + (begin0 (list 'struct (if mut? '(true) '(false)) sig (parse-block s parse-struct-field)) (expect-end s word)))) ;; consume any number of line endings from a token stream @@ -1456,7 +1464,13 @@ `(const ,expr) expr))) ((const) - `(const ,(parse-eq s))) + (let ((assgn (parse-eq s))) + (if (not (and (pair? assgn) + (or (eq? (car assgn) '=) + (eq? (car assgn) 'global) + (eq? (car assgn) 'local)))) + (error "expected assignment after \"const\"") + `(const ,assgn)))) ((function macro) (let* ((loc (line-number-node s)) diff --git a/test/syntax.jl b/test/syntax.jl index df5ed7babbffd..6ed3e7ca59ad7 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -3278,3 +3278,13 @@ demo44723()::Any = Base.Experimental.@opaque () -> true ? 1 : 2 # issue #45162 f45162(f) = f(x=1) @test first(methods(f45162)).called != 0 + +# issue #45024 +@test_throws ParseError("expected assignment after \"const\"") Meta.parse("const x") +@test_throws ParseError("expected assignment after \"const\"") Meta.parse("const x::Int") +# these cases have always been caught during lowering, since (const (global x)) is not +# ambiguous with the lowered form (const x), but that could probably be changed. +@test Meta.lower(@__MODULE__, :(global const x)) == Expr(:error, "expected assignment after \"const\"") +@test Meta.lower(@__MODULE__, :(global const x::Int)) == Expr(:error, "expected assignment after \"const\"") +@test Meta.lower(@__MODULE__, :(const global x)) == Expr(:error, "expected assignment after \"const\"") +@test Meta.lower(@__MODULE__, :(const global x::Int)) == Expr(:error, "expected assignment after \"const\"") From eb82f1846fdc2d195ab1031fe192b27fd171f148 Mon Sep 17 00:00:00 2001 From: Prem Chintalapudi Date: Sat, 12 Mar 2022 00:57:16 -0500 Subject: [PATCH 55/68] codegen: explicitly handle Float16 intrinsics (#45249) Fixes #44829, until llvm fixes the support for these intrinsics itself Also need to handle vectors, since the vectorizer may have introduced them. Also change our runtime emulation versions to f32 for consistency. (cherry picked from commit f2c627ef8af37c3cf94c19a5403bc6cd796d5031) --- src/APInt-C.cpp | 6 +- src/julia.expmap | 6 - src/julia_internal.h | 14 +- src/llvm-demote-float16.cpp | 276 ++++++++++++++++++++++++++++++------ src/runtime_intrinsics.c | 64 +++++---- 5 files changed, 284 insertions(+), 82 deletions(-) diff --git a/src/APInt-C.cpp b/src/APInt-C.cpp index bc0a62e21dd3e..f06d4362bf958 100644 --- a/src/APInt-C.cpp +++ b/src/APInt-C.cpp @@ -316,7 +316,7 @@ void LLVMByteSwap(unsigned numbits, integerPart *pa, integerPart *pr) { void LLVMFPtoInt(unsigned numbits, void *pa, unsigned onumbits, integerPart *pr, bool isSigned, bool *isExact) { double Val; if (numbits == 16) - Val = __gnu_h2f_ieee(*(uint16_t*)pa); + Val = julia__gnu_h2f_ieee(*(uint16_t*)pa); else if (numbits == 32) Val = *(float*)pa; else if (numbits == 64) @@ -391,7 +391,7 @@ void LLVMSItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPar val = a.roundToDouble(true); } if (onumbits == 16) - *(uint16_t*)pr = __gnu_f2h_ieee(val); + *(uint16_t*)pr = julia__gnu_f2h_ieee(val); else if (onumbits == 32) *(float*)pr = val; else if (onumbits == 64) @@ -408,7 +408,7 @@ void LLVMUItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPar val = a.roundToDouble(false); } if (onumbits == 16) - *(uint16_t*)pr = __gnu_f2h_ieee(val); + *(uint16_t*)pr = julia__gnu_f2h_ieee(val); else if (onumbits == 32) *(float*)pr = val; else if (onumbits == 64) diff --git a/src/julia.expmap b/src/julia.expmap index 2d801dceae044..6717f8d00c621 100644 --- a/src/julia.expmap +++ b/src/julia.expmap @@ -37,12 +37,6 @@ environ; __progname; - /* compiler run-time intrinsics */ - __gnu_h2f_ieee; - __extendhfsf2; - __gnu_f2h_ieee; - __truncdfhf2; - local: *; }; diff --git a/src/julia_internal.h b/src/julia_internal.h index 21c46129b7149..3b37f3b075d67 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1522,8 +1522,18 @@ jl_sym_t *_jl_symbol(const char *str, size_t len) JL_NOTSAFEPOINT; #define JL_GC_ASSERT_LIVE(x) (void)(x) #endif -float __gnu_h2f_ieee(uint16_t param) JL_NOTSAFEPOINT; -uint16_t __gnu_f2h_ieee(float param) JL_NOTSAFEPOINT; +JL_DLLEXPORT float julia__gnu_h2f_ieee(uint16_t param) JL_NOTSAFEPOINT; +JL_DLLEXPORT uint16_t julia__gnu_f2h_ieee(float param) JL_NOTSAFEPOINT; +JL_DLLEXPORT uint16_t julia__truncdfhf2(double param) JL_NOTSAFEPOINT; +//JL_DLLEXPORT double julia__extendhfdf2(uint16_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT int32_t julia__fixhfsi(uint16_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT int64_t julia__fixhfdi(uint16_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT uint32_t julia__fixunshfsi(uint16_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT uint64_t julia__fixunshfdi(uint16_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT uint16_t julia__floatsihf(int32_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT uint16_t julia__floatdihf(int64_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT uint16_t julia__floatunsihf(uint32_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT uint16_t julia__floatundihf(uint64_t n) JL_NOTSAFEPOINT; #ifdef __cplusplus } diff --git a/src/llvm-demote-float16.cpp b/src/llvm-demote-float16.cpp index 3e328424e26d2..a68f8faf2e3a7 100644 --- a/src/llvm-demote-float16.cpp +++ b/src/llvm-demote-float16.cpp @@ -18,6 +18,7 @@ #include "support/dtypes.h" +#include #include #include #include @@ -28,15 +29,193 @@ using namespace llvm; namespace { +inline AttributeSet getFnAttrs(const AttributeList &Attrs) +{ +#if JL_LLVM_VERSION >= 140000 + return Attrs.getFnAttrs(); +#else + return Attrs.getFnAttributes(); +#endif +} + +inline AttributeSet getRetAttrs(const AttributeList &Attrs) +{ +#if JL_LLVM_VERSION >= 140000 + return Attrs.getRetAttrs(); +#else + return Attrs.getRetAttributes(); +#endif +} + +static Instruction *replaceIntrinsicWith(IntrinsicInst *call, Type *RetTy, ArrayRef args) +{ + Intrinsic::ID ID = call->getIntrinsicID(); + assert(ID); + auto oldfType = call->getFunctionType(); + auto nargs = oldfType->getNumParams(); + assert(args.size() > nargs); + SmallVector argTys(nargs); + for (unsigned i = 0; i < nargs; i++) + argTys[i] = args[i]->getType(); + auto newfType = FunctionType::get(RetTy, argTys, oldfType->isVarArg()); + + // Accumulate an array of overloaded types for the given intrinsic + // and compute the new name mangling schema + SmallVector overloadTys; + { + SmallVector Table; + getIntrinsicInfoTableEntries(ID, Table); + ArrayRef TableRef = Table; + auto res = Intrinsic::matchIntrinsicSignature(newfType, TableRef, overloadTys); + assert(res == Intrinsic::MatchIntrinsicTypes_Match); + (void)res; + bool matchvararg = !Intrinsic::matchIntrinsicVarArg(newfType->isVarArg(), TableRef); + assert(matchvararg); + (void)matchvararg; + } + auto newF = Intrinsic::getDeclaration(call->getModule(), ID, overloadTys); + assert(newF->getFunctionType() == newfType); + newF->setCallingConv(call->getCallingConv()); + assert(args.back() == call->getCalledFunction()); + auto newCall = CallInst::Create(newF, args.drop_back(), "", call); + newCall->setTailCallKind(call->getTailCallKind()); + auto old_attrs = call->getAttributes(); + newCall->setAttributes(AttributeList::get(call->getContext(), getFnAttrs(old_attrs), + getRetAttrs(old_attrs), {})); // drop parameter attributes + return newCall; +} + + +static Value* CreateFPCast(Instruction::CastOps opcode, Value *V, Type *DestTy, IRBuilder<> &builder) +{ + Type *SrcTy = V->getType(); + Type *RetTy = DestTy; + if (auto *VC = dyn_cast(V)) { + // The input IR often has things of the form + // fcmp olt half %0, 0xH7C00 + // and we would like to avoid turning that constant into a call here + // if we can simply constant fold it to the new type. + VC = ConstantExpr::getCast(opcode, VC, DestTy, true); + if (VC) + return VC; + } + assert(SrcTy->isVectorTy() == DestTy->isVectorTy()); + if (SrcTy->isVectorTy()) { + unsigned NumElems = cast(SrcTy)->getNumElements(); + assert(cast(DestTy)->getNumElements() == NumElems && "Mismatched cast"); + Value *NewV = UndefValue::get(DestTy); + RetTy = RetTy->getScalarType(); + for (unsigned i = 0; i < NumElems; ++i) { + Value *I = builder.getInt32(i); + Value *Vi = builder.CreateExtractElement(V, I); + Vi = CreateFPCast(opcode, Vi, RetTy, builder); + NewV = builder.CreateInsertElement(NewV, Vi, I); + } + return NewV; + } + auto &M = *builder.GetInsertBlock()->getModule(); + auto &ctx = M.getContext(); + // Pick the Function to call in the Julia runtime + StringRef Name; + switch (opcode) { + case Instruction::FPExt: + // this is exact, so we only need one conversion + assert(SrcTy->isHalfTy()); + Name = "julia__gnu_h2f_ieee"; + RetTy = Type::getFloatTy(ctx); + break; + case Instruction::FPTrunc: + assert(DestTy->isHalfTy()); + if (SrcTy->isFloatTy()) + Name = "julia__gnu_f2h_ieee"; + else if (SrcTy->isDoubleTy()) + Name = "julia__truncdfhf2"; + break; + // All F16 fit exactly in Int32 (-65504 to 65504) + case Instruction::FPToSI: JL_FALLTHROUGH; + case Instruction::FPToUI: + assert(SrcTy->isHalfTy()); + Name = "julia__gnu_h2f_ieee"; + RetTy = Type::getFloatTy(ctx); + break; + case Instruction::SIToFP: JL_FALLTHROUGH; + case Instruction::UIToFP: + assert(DestTy->isHalfTy()); + Name = "julia__gnu_f2h_ieee"; + SrcTy = Type::getFloatTy(ctx); + break; + default: + errs() << Instruction::getOpcodeName(opcode) << ' '; + V->getType()->print(errs()); + errs() << " to "; + DestTy->print(errs()); + errs() << " is an "; + llvm_unreachable("invalid cast"); + } + if (Name.empty()) { + errs() << Instruction::getOpcodeName(opcode) << ' '; + V->getType()->print(errs()); + errs() << " to "; + DestTy->print(errs()); + errs() << " is an "; + llvm_unreachable("illegal cast"); + } + // Coerce the source to the required size and type + auto T_int16 = Type::getInt16Ty(ctx); + if (SrcTy->isHalfTy()) + SrcTy = T_int16; + if (opcode == Instruction::SIToFP) + V = builder.CreateSIToFP(V, SrcTy); + else if (opcode == Instruction::UIToFP) + V = builder.CreateUIToFP(V, SrcTy); + else + V = builder.CreateBitCast(V, SrcTy); + // Call our intrinsic + if (RetTy->isHalfTy()) + RetTy = T_int16; + auto FT = FunctionType::get(RetTy, {SrcTy}, false); + FunctionCallee F = M.getOrInsertFunction(Name, FT); + Value *I = builder.CreateCall(F, {V}); + // Coerce the result to the expected type + if (opcode == Instruction::FPToSI) + I = builder.CreateFPToSI(I, DestTy); + else if (opcode == Instruction::FPToUI) + I = builder.CreateFPToUI(I, DestTy); + else if (opcode == Instruction::FPExt) + I = builder.CreateFPCast(I, DestTy); + else + I = builder.CreateBitCast(I, DestTy); + return I; +} + static bool demoteFloat16(Function &F) { auto &ctx = F.getContext(); - auto T_float16 = Type::getHalfTy(ctx); auto T_float32 = Type::getFloatTy(ctx); SmallVector erase; for (auto &BB : F) { for (auto &I : BB) { + // extend Float16 operands to Float32 + bool Float16 = I.getType()->getScalarType()->isHalfTy(); + for (size_t i = 0; !Float16 && i < I.getNumOperands(); i++) { + Value *Op = I.getOperand(i); + if (Op->getType()->getScalarType()->isHalfTy()) + Float16 = true; + } + if (!Float16) + continue; + + if (auto CI = dyn_cast(&I)) { + if (CI->getOpcode() != Instruction::BitCast) { // aka !CI->isNoopCast(DL) + IRBuilder<> builder(&I); + Value *NewI = CreateFPCast(CI->getOpcode(), I.getOperand(0), I.getType(), builder); + I.replaceAllUsesWith(NewI); + erase.push_back(&I); + } + continue; + } + switch (I.getOpcode()) { case Instruction::FNeg: case Instruction::FAdd: @@ -47,6 +226,9 @@ static bool demoteFloat16(Function &F) case Instruction::FCmp: break; default: + if (auto intrinsic = dyn_cast(&I)) + if (intrinsic->getIntrinsicID()) + break; continue; } @@ -58,61 +240,67 @@ static bool demoteFloat16(Function &F) IRBuilder<> builder(&I); // extend Float16 operands to Float32 - bool OperandsChanged = false; + // XXX: Calls to llvm.fma.f16 may need to go to f64 to be correct? SmallVector Operands(I.getNumOperands()); for (size_t i = 0; i < I.getNumOperands(); i++) { Value *Op = I.getOperand(i); - if (Op->getType() == T_float16) { - Op = builder.CreateFPExt(Op, T_float32); - OperandsChanged = true; + if (Op->getType()->getScalarType()->isHalfTy()) { + Op = CreateFPCast(Instruction::FPExt, Op, Op->getType()->getWithNewType(T_float32), builder); } Operands[i] = (Op); } // recreate the instruction if any operands changed, // truncating the result back to Float16 - if (OperandsChanged) { - Value *NewI; - switch (I.getOpcode()) { - case Instruction::FNeg: - assert(Operands.size() == 1); - NewI = builder.CreateFNeg(Operands[0]); - break; - case Instruction::FAdd: - assert(Operands.size() == 2); - NewI = builder.CreateFAdd(Operands[0], Operands[1]); - break; - case Instruction::FSub: - assert(Operands.size() == 2); - NewI = builder.CreateFSub(Operands[0], Operands[1]); - break; - case Instruction::FMul: - assert(Operands.size() == 2); - NewI = builder.CreateFMul(Operands[0], Operands[1]); - break; - case Instruction::FDiv: - assert(Operands.size() == 2); - NewI = builder.CreateFDiv(Operands[0], Operands[1]); - break; - case Instruction::FRem: - assert(Operands.size() == 2); - NewI = builder.CreateFRem(Operands[0], Operands[1]); - break; - case Instruction::FCmp: - assert(Operands.size() == 2); - NewI = builder.CreateFCmp(cast(&I)->getPredicate(), - Operands[0], Operands[1]); + Value *NewI; + switch (I.getOpcode()) { + case Instruction::FNeg: + assert(Operands.size() == 1); + NewI = builder.CreateFNeg(Operands[0]); + break; + case Instruction::FAdd: + assert(Operands.size() == 2); + NewI = builder.CreateFAdd(Operands[0], Operands[1]); + break; + case Instruction::FSub: + assert(Operands.size() == 2); + NewI = builder.CreateFSub(Operands[0], Operands[1]); + break; + case Instruction::FMul: + assert(Operands.size() == 2); + NewI = builder.CreateFMul(Operands[0], Operands[1]); + break; + case Instruction::FDiv: + assert(Operands.size() == 2); + NewI = builder.CreateFDiv(Operands[0], Operands[1]); + break; + case Instruction::FRem: + assert(Operands.size() == 2); + NewI = builder.CreateFRem(Operands[0], Operands[1]); + break; + case Instruction::FCmp: + assert(Operands.size() == 2); + NewI = builder.CreateFCmp(cast(&I)->getPredicate(), + Operands[0], Operands[1]); + break; + default: + if (auto intrinsic = dyn_cast(&I)) { + // XXX: this is not correct in general + // some obvious failures include llvm.convert.to.fp16.*, llvm.vp.*to*, llvm.experimental.constrained.*to*, llvm.masked.* + Type *RetTy = I.getType(); + if (RetTy->getScalarType()->isHalfTy()) + RetTy = RetTy->getWithNewType(T_float32); + NewI = replaceIntrinsicWith(intrinsic, RetTy, Operands); break; - default: - abort(); } - cast(NewI)->copyMetadata(I); - cast(NewI)->copyFastMathFlags(&I); - if (NewI->getType() != I.getType()) - NewI = builder.CreateFPTrunc(NewI, I.getType()); - I.replaceAllUsesWith(NewI); - erase.push_back(&I); + abort(); } + cast(NewI)->copyMetadata(I); + cast(NewI)->copyFastMathFlags(&I); + if (NewI->getType() != I.getType()) + NewI = CreateFPCast(Instruction::FPTrunc, NewI, I.getType(), builder); + I.replaceAllUsesWith(NewI); + erase.push_back(&I); } } diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index 9525b655dc5e3..f505a9d5e5d5a 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -15,9 +15,6 @@ const unsigned int host_char_bit = 8; // float16 intrinsics -// TODO: use LLVM's compiler-rt on all platforms (Xcode already links compiler-rt) - -#if !defined(_OS_DARWIN_) static inline float half_to_float(uint16_t ival) JL_NOTSAFEPOINT { @@ -188,22 +185,17 @@ static inline uint16_t float_to_half(float param) JL_NOTSAFEPOINT return h; } -JL_DLLEXPORT float __gnu_h2f_ieee(uint16_t param) +JL_DLLEXPORT float julia__gnu_h2f_ieee(uint16_t param) { return half_to_float(param); } -JL_DLLEXPORT float __extendhfsf2(uint16_t param) -{ - return half_to_float(param); -} - -JL_DLLEXPORT uint16_t __gnu_f2h_ieee(float param) +JL_DLLEXPORT uint16_t julia__gnu_f2h_ieee(float param) { return float_to_half(param); } -JL_DLLEXPORT uint16_t __truncdfhf2(double param) +JL_DLLEXPORT uint16_t julia__truncdfhf2(double param) { float res = (float)param; uint32_t resi; @@ -225,7 +217,25 @@ JL_DLLEXPORT uint16_t __truncdfhf2(double param) return float_to_half(res); } -#endif +//JL_DLLEXPORT double julia__extendhfdf2(uint16_t n) { return (double)julia__gnu_h2f_ieee(n); } +//JL_DLLEXPORT int32_t julia__fixhfsi(uint16_t n) { return (int32_t)julia__gnu_h2f_ieee(n); } +//JL_DLLEXPORT int64_t julia__fixhfdi(uint16_t n) { return (int64_t)julia__gnu_h2f_ieee(n); } +//JL_DLLEXPORT uint32_t julia__fixunshfsi(uint16_t n) { return (uint32_t)julia__gnu_h2f_ieee(n); } +//JL_DLLEXPORT uint64_t julia__fixunshfdi(uint16_t n) { return (uint64_t)julia__gnu_h2f_ieee(n); } +//JL_DLLEXPORT uint16_t julia__floatsihf(int32_t n) { return julia__gnu_f2h_ieee((float)n); } +//JL_DLLEXPORT uint16_t julia__floatdihf(int64_t n) { return julia__gnu_f2h_ieee((float)n); } +//JL_DLLEXPORT uint16_t julia__floatunsihf(uint32_t n) { return julia__gnu_f2h_ieee((float)n); } +//JL_DLLEXPORT uint16_t julia__floatundihf(uint64_t n) { return julia__gnu_f2h_ieee((float)n); } +//HANDLE_LIBCALL(F16, F128, __extendhftf2) +//HANDLE_LIBCALL(F16, F80, __extendhfxf2) +//HANDLE_LIBCALL(F80, F16, __truncxfhf2) +//HANDLE_LIBCALL(F128, F16, __trunctfhf2) +//HANDLE_LIBCALL(PPCF128, F16, __trunctfhf2) +//HANDLE_LIBCALL(F16, I128, __fixhfti) +//HANDLE_LIBCALL(F16, I128, __fixunshfti) +//HANDLE_LIBCALL(I128, F16, __floattihf) +//HANDLE_LIBCALL(I128, F16, __floatuntihf) + // run time version of bitcast intrinsic JL_DLLEXPORT jl_value_t *jl_bitcast(jl_value_t *ty, jl_value_t *v) @@ -551,9 +561,9 @@ static inline unsigned select_by_size(unsigned sz) JL_NOTSAFEPOINT } #define fp_select(a, func) \ - sizeof(a) == sizeof(float) ? func##f((float)a) : func(a) + sizeof(a) <= sizeof(float) ? func##f((float)a) : func(a) #define fp_select2(a, b, func) \ - sizeof(a) == sizeof(float) ? func##f(a, b) : func(a, b) + sizeof(a) <= sizeof(float) ? func##f(a, b) : func(a, b) // fast-function generators // @@ -597,11 +607,11 @@ static inline void name(unsigned osize, void *pa, void *pr) JL_NOTSAFEPOINT \ static inline void name(unsigned osize, void *pa, void *pr) JL_NOTSAFEPOINT \ { \ uint16_t a = *(uint16_t*)pa; \ - float A = __gnu_h2f_ieee(a); \ + float A = julia__gnu_h2f_ieee(a); \ if (osize == 16) { \ float R; \ OP(&R, A); \ - *(uint16_t*)pr = __gnu_f2h_ieee(R); \ + *(uint16_t*)pr = julia__gnu_f2h_ieee(R); \ } else { \ OP((uint16_t*)pr, A); \ } \ @@ -625,11 +635,11 @@ static void jl_##name##16(unsigned runtime_nbits, void *pa, void *pb, void *pr) { \ uint16_t a = *(uint16_t*)pa; \ uint16_t b = *(uint16_t*)pb; \ - float A = __gnu_h2f_ieee(a); \ - float B = __gnu_h2f_ieee(b); \ + float A = julia__gnu_h2f_ieee(a); \ + float B = julia__gnu_h2f_ieee(b); \ runtime_nbits = 16; \ float R = OP(A, B); \ - *(uint16_t*)pr = __gnu_f2h_ieee(R); \ + *(uint16_t*)pr = julia__gnu_f2h_ieee(R); \ } // float or integer inputs, bool output @@ -650,8 +660,8 @@ static int jl_##name##16(unsigned runtime_nbits, void *pa, void *pb) JL_NOTSAFEP { \ uint16_t a = *(uint16_t*)pa; \ uint16_t b = *(uint16_t*)pb; \ - float A = __gnu_h2f_ieee(a); \ - float B = __gnu_h2f_ieee(b); \ + float A = julia__gnu_h2f_ieee(a); \ + float B = julia__gnu_h2f_ieee(b); \ runtime_nbits = 16; \ return OP(A, B); \ } @@ -691,12 +701,12 @@ static void jl_##name##16(unsigned runtime_nbits, void *pa, void *pb, void *pc, uint16_t a = *(uint16_t*)pa; \ uint16_t b = *(uint16_t*)pb; \ uint16_t c = *(uint16_t*)pc; \ - float A = __gnu_h2f_ieee(a); \ - float B = __gnu_h2f_ieee(b); \ - float C = __gnu_h2f_ieee(c); \ + float A = julia__gnu_h2f_ieee(a); \ + float B = julia__gnu_h2f_ieee(b); \ + float C = julia__gnu_h2f_ieee(c); \ runtime_nbits = 16; \ float R = OP(A, B, C); \ - *(uint16_t*)pr = __gnu_f2h_ieee(R); \ + *(uint16_t*)pr = julia__gnu_f2h_ieee(R); \ } @@ -1318,7 +1328,7 @@ static inline int fpiseq##nbits(c_type a, c_type b) JL_NOTSAFEPOINT { \ fpiseq_n(float, 32) fpiseq_n(double, 64) #define fpiseq(a,b) \ - sizeof(a) == sizeof(float) ? fpiseq32(a, b) : fpiseq64(a, b) + sizeof(a) <= sizeof(float) ? fpiseq32(a, b) : fpiseq64(a, b) bool_fintrinsic(eq,eq_float) bool_fintrinsic(ne,ne_float) @@ -1367,7 +1377,7 @@ cvt_iintrinsic(LLVMFPtoUI, fptoui) if (!(osize < 8 * sizeof(a))) \ jl_error("fptrunc: output bitsize must be < input bitsize"); \ else if (osize == 16) \ - *(uint16_t*)pr = __gnu_f2h_ieee(a); \ + *(uint16_t*)pr = julia__gnu_f2h_ieee(a); \ else if (osize == 32) \ *(float*)pr = a; \ else if (osize == 64) \ From 945a2db4250ca386dcc673e6f81ebff7da5567cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Mon, 28 Feb 2022 19:33:51 +0000 Subject: [PATCH 56/68] Guard GCC-specific macros with `_COMPILER_GCC_` (#44353) By default Clang on Linux defines the macro `__GNUC__`, so to guard GCC-specific code paths it isn't sufficient to check `#ifdef __GNUC__`. (cherry picked from commit bcc0f70d73bd93775e1e7fd74ef7c46f23616218) --- src/julia_internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/julia_internal.h b/src/julia_internal.h index 3b37f3b075d67..8e7079f75364f 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -422,7 +422,7 @@ jl_svec_t *jl_perm_symsvec(size_t n, ...); // this sizeof(__VA_ARGS__) trick can't be computed until C11, but that only matters to Clang in some situations #if !defined(__clang_analyzer__) && !(defined(_COMPILER_ASAN_ENABLED_) || defined(_COMPILER_TSAN_ENABLED_)) -#ifdef __GNUC__ +#ifdef _COMPILER_GCC_ #define jl_perm_symsvec(n, ...) \ (jl_perm_symsvec)(__extension__({ \ static_assert( \ From 8c6994127a8fbb7170efb34e1c118408ac21925b Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Thu, 19 May 2022 15:56:49 -0400 Subject: [PATCH 57/68] Fix error in validating complex row-first hvncat (#45365) (cherry picked from commit 0f2ed77dca88785c9ae0fb1cf1a77593d1527c18) --- base/abstractarray.jl | 5 ++++- test/abstractarray.jl | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 9c3cb23865dff..e7dda128fcadc 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -2375,6 +2375,9 @@ function _typed_hvncat_dims(::Type{T}, dims::NTuple{N, Int}, row_first::Bool, as # validate shapes for lowest level of concatenation d = findfirst(>(1), dims) if d !== nothing # all dims are 1 + if row_first && d < 3 + d = d == 1 ? 2 : 1 + end nblocks = length(as) ÷ dims[d] for b ∈ 1:nblocks offset = ((b - 1) * dims[d]) @@ -2382,7 +2385,7 @@ function _typed_hvncat_dims(::Type{T}, dims::NTuple{N, Int}, row_first::Bool, as for i ∈ offset .+ (2:dims[d]) for dd ∈ 1:N dd == d && continue - if size(as[startelementi], dd) != size(as[i], dd) + if cat_size(as[startelementi], dd) != cat_size(as[i], dd) throw(ArgumentError("incompatible shape in element $i")) end end diff --git a/test/abstractarray.jl b/test/abstractarray.jl index d650cf67ebf11..cd084c747897c 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -1539,6 +1539,8 @@ using Base: typed_hvncat # Issue 43933 - semicolon precedence mistake should produce an error @test_throws ArgumentError [[1 1]; 2 ;; 3 ; [3 4]] @test_throws ArgumentError [[1 ;;; 1]; 2 ;;; 3 ; [3 ;;; 4]] + + @test [[1 2; 3 4] [5; 6]; [7 8] 9;;;] == [1 2 5; 3 4 6; 7 8 9;;;] end @testset "keepat!" begin From 8c33506975587f268ce6d25c8e4d38fdc0766d00 Mon Sep 17 00:00:00 2001 From: "Viral B. Shah" Date: Fri, 20 May 2022 12:00:29 -0400 Subject: [PATCH 58/68] Apply patch for GMP CVE-2021-43618 (#45375) * Apply patch for GMP CVE-2021-43618 * Update checksums (cherry picked from commit dea980590f41bab820be4c42b42f29639ac8a56c) --- deps/checksums/gmp | 116 +++++++++++++------------- deps/gmp.mk | 8 +- deps/patches/gmp-CVE-2021-43618.patch | 24 ++++++ stdlib/GMP_jll/Project.toml | 2 +- 4 files changed, 90 insertions(+), 60 deletions(-) create mode 100644 deps/patches/gmp-CVE-2021-43618.patch diff --git a/deps/checksums/gmp b/deps/checksums/gmp index 47cee2e34a42f..0c45aa6a00ca9 100644 --- a/deps/checksums/gmp +++ b/deps/checksums/gmp @@ -1,60 +1,60 @@ -GMP.v6.2.1+1.aarch64-apple-darwin.tar.gz/md5/03cb14ac16daabb4a77fe1c78e8e48a9 -GMP.v6.2.1+1.aarch64-apple-darwin.tar.gz/sha512/5b8f974a07f579272981f5ebe44191385a4ce95f58d434a3565ffa827a6d65824cbe4173736b7328630bbccfe6af4242195aec24de3f0aa687e2e32a18a97a5c -GMP.v6.2.1+1.aarch64-linux-gnu-cxx03.tar.gz/md5/0ce7d419a49f2f90033618bdda2588e7 -GMP.v6.2.1+1.aarch64-linux-gnu-cxx03.tar.gz/sha512/16363dedaae116fa0d493182aeadb2ffa7f990f1813e4b47cae3cd61ca71f23b65267ea4e2c698d52bd78d61e12feaa73179d7b86ab6d6df01eeb7b6a9b27958 -GMP.v6.2.1+1.aarch64-linux-gnu-cxx11.tar.gz/md5/011f1cdc39b9e529b4b6ea80f4c33108 -GMP.v6.2.1+1.aarch64-linux-gnu-cxx11.tar.gz/sha512/1ed2139580c5c78578f350ee83dbf9cd0120d9d36e1951438d757f5734cda7931600b3f83bfe0d0d806926636d6aea8048c6b64aa42a22e59310282c2428f417 -GMP.v6.2.1+1.aarch64-linux-musl-cxx03.tar.gz/md5/34f17083a1f142c284b707cc82407b00 -GMP.v6.2.1+1.aarch64-linux-musl-cxx03.tar.gz/sha512/dd32912c31a8422734c2e5d5a37001ac18f0e9de151982583d9dc185e5cc3e45076d737729345cca8e8eaf42993d4102353261a2de245e26a8a9cd86960a2fbf -GMP.v6.2.1+1.aarch64-linux-musl-cxx11.tar.gz/md5/9ba1b822f20f88a1e4c6e81dc8c4fdc1 -GMP.v6.2.1+1.aarch64-linux-musl-cxx11.tar.gz/sha512/d8a4ecd5c35022b9c912c3b4fabe3a4c31258d6a1bd38e4fea13a3da53206a29bfd90f4d602f6e3ee3ee271d84289d1ecdf45534adfabf7e657daef5b5cb0b21 -GMP.v6.2.1+1.armv6l-linux-gnueabihf-cxx03.tar.gz/md5/23e28efa2579d636cb4c80036da5d4ea -GMP.v6.2.1+1.armv6l-linux-gnueabihf-cxx03.tar.gz/sha512/02c8023958fa616c1f944898e686510d449b743d053cfd42f526e9c4fe3ff3dd9de7309694b8537b4bb6dc978085339eb787983ec4ba32dc041448c912a8b982 -GMP.v6.2.1+1.armv6l-linux-gnueabihf-cxx11.tar.gz/md5/bf2a2c4f81f6d04746cc528438f62639 -GMP.v6.2.1+1.armv6l-linux-gnueabihf-cxx11.tar.gz/sha512/1c152abeed24761c775e78a64835f8e61b28b16cbc29a6fde88fa4fdbf2a5782cd62697bd03a552d873995bda58b7bdc081c11ecd5e4badde2dea426e5218116 -GMP.v6.2.1+1.armv6l-linux-musleabihf-cxx03.tar.gz/md5/25cbceed2cf1bb12601fe285c342d6b0 -GMP.v6.2.1+1.armv6l-linux-musleabihf-cxx03.tar.gz/sha512/37d8b21bf59c0c555f2b59d6dca4f486bf1725ae18a7fea9a2f31533c54ebb818b5ddb88ec8aa2b618e0ecad78973659abd1a9f095f64ef65067ab8ed08d7801 -GMP.v6.2.1+1.armv6l-linux-musleabihf-cxx11.tar.gz/md5/8ec72c769625a218c6951abed32b3684 -GMP.v6.2.1+1.armv6l-linux-musleabihf-cxx11.tar.gz/sha512/4cb9ccb97859b0918002b649e1b5e74e1fc89a2daeec6f32d5a06ce0d84217f54d1ee788f472cebeefc73ef52284a3d5607efbed47058b438d2dcbcf9f384ed0 -GMP.v6.2.1+1.armv7l-linux-gnueabihf-cxx03.tar.gz/md5/6f799d6516cc46af28eacf8409634825 -GMP.v6.2.1+1.armv7l-linux-gnueabihf-cxx03.tar.gz/sha512/541c1e03726584ddb672a83becdc9a99c68f5da9a7415750d582753b47774910bf25cee7fe21f5b5c2a80ff8ce87fc327abd45bf54d6cfe821cb202c81b67e43 -GMP.v6.2.1+1.armv7l-linux-gnueabihf-cxx11.tar.gz/md5/17dba9ebcc1bf4637095a98a876375a8 -GMP.v6.2.1+1.armv7l-linux-gnueabihf-cxx11.tar.gz/sha512/648220e632618d23e8611e10fa4bb2e581ed4432e3fff77d0d7349a7585bffa65ae57bf1ce64c550bf6d2acc016f499c0bbbfed8088281445b9d4ecbbf9a64bc -GMP.v6.2.1+1.armv7l-linux-musleabihf-cxx03.tar.gz/md5/79c77b81cc16fd22ad4cef75af7aa220 -GMP.v6.2.1+1.armv7l-linux-musleabihf-cxx03.tar.gz/sha512/0059ba54806ef0ca6621ddcd309a18922c4c7d9d9e214bc6870b6338a9449a472934cc27569402741d41a18dd53a896aae2f68b788f853fd4ea3db63035c8153 -GMP.v6.2.1+1.armv7l-linux-musleabihf-cxx11.tar.gz/md5/87b79bfc5c780e214863d0f0c1944da9 -GMP.v6.2.1+1.armv7l-linux-musleabihf-cxx11.tar.gz/sha512/88dcabcf96d8f2dcc7968333a94adcb8e8a91615b67ca23edf75c3368a89ef60a8deff8e8532d0cd4d5dd5356343b753b0ae0bf88ce7e190639468bf8170939a -GMP.v6.2.1+1.i686-linux-gnu-cxx03.tar.gz/md5/61d39e42ab6fd5844e938605e357b336 -GMP.v6.2.1+1.i686-linux-gnu-cxx03.tar.gz/sha512/8e0d382adf6b45cbf613092cee524551a04096b0bc6fb8893701edae9c1928bda67b5522cae3ef954a882ff73b735190881ade37495d9d1a6db88ed6fbcdc6b1 -GMP.v6.2.1+1.i686-linux-gnu-cxx11.tar.gz/md5/b66b49054426adf3e1d3454a80010d97 -GMP.v6.2.1+1.i686-linux-gnu-cxx11.tar.gz/sha512/b28f22bbfbf796c4e959b1fa3433d46b4cf0dbd402c0497a6d4893c8030aa12fd038da4846d8bce02199f1da9b0158d78f2b4ff2636799ba139602775725ff6d -GMP.v6.2.1+1.i686-linux-musl-cxx03.tar.gz/md5/69ea3b3348813777a1682e41a117d7c3 -GMP.v6.2.1+1.i686-linux-musl-cxx03.tar.gz/sha512/048dd08b5891864e69504baf6328ef5423e0f8e31c5c6cfac552eb51b3ef943af83b7ac654c33e1a0cf061c5832e08eebb9c03dbda6532fbc24e160e99c2aae6 -GMP.v6.2.1+1.i686-linux-musl-cxx11.tar.gz/md5/e7c82091d29a3e5958442c9ec631ad78 -GMP.v6.2.1+1.i686-linux-musl-cxx11.tar.gz/sha512/8574f2e42e181a7bd1cf8aa8056a14d13efe555ee74b14e14aef1bdce7f26ce2afe41b4f85ee20de6823045d5ff38e4dbcebcc7042fff4288af1b7d296202d43 -GMP.v6.2.1+1.i686-w64-mingw32-cxx03.tar.gz/md5/dcef59aa056dcd56e6e36ad49174389f -GMP.v6.2.1+1.i686-w64-mingw32-cxx03.tar.gz/sha512/3cf3096c325ae2baea8b3c3aed4a26d649dc2bb3cf0d979809d9962521422ada3fdcdddbcfc52b27d43b473a1d3ed4a40368cdeb16cac4d32718c604dbc9f388 -GMP.v6.2.1+1.i686-w64-mingw32-cxx11.tar.gz/md5/b772a602b016e73dfc9a93908f51622b -GMP.v6.2.1+1.i686-w64-mingw32-cxx11.tar.gz/sha512/00e06591e2cc44100dca1a8897c72933bf4bd8c3c732daea99a9efa4d0a67f6a8820bf3e5d27583dfddc50d4cda656fa7462a2c453035d03657948f0051dc2fe -GMP.v6.2.1+1.powerpc64le-linux-gnu-cxx03.tar.gz/md5/b31c423855c4c5633b41301e3b424312 -GMP.v6.2.1+1.powerpc64le-linux-gnu-cxx03.tar.gz/sha512/2565176e2bbcb9deab25a91736e8b6de01e7dca619ed1fcc98cebcaaa144eb03f89f4f6d5989aa8454b0d1c7266d1ace690e6deef67c0cf5c3fc1c2ab4d41b43 -GMP.v6.2.1+1.powerpc64le-linux-gnu-cxx11.tar.gz/md5/1ed2494342b5713308f6ffed5fe3863d -GMP.v6.2.1+1.powerpc64le-linux-gnu-cxx11.tar.gz/sha512/c600802c81c77247a24a50ec0695f742177c8c9f090b4c345f9b0cd065b35183f49592a764cdb7b1b6d5ee3722e7dd26672d85db963d1e490731545a36d1e581 -GMP.v6.2.1+1.x86_64-apple-darwin.tar.gz/md5/51e00a2b55e9f81eb62abe23bb5f6fd9 -GMP.v6.2.1+1.x86_64-apple-darwin.tar.gz/sha512/91731427afd8df54b54d87b93006190a8b959438dc591eb5fa44724056911b8bd5588b2b1e70e9da3d8d6e9ce5aaa6fea66b0706f636cb56b3c860e8f3c0550a -GMP.v6.2.1+1.x86_64-linux-gnu-cxx03.tar.gz/md5/3f3a6f15e4e8499470bbe69a9ea885c1 -GMP.v6.2.1+1.x86_64-linux-gnu-cxx03.tar.gz/sha512/2659344ab097cd9542a5946c127a43af6fad05aa1445d69a4978d1a6d9a9f0e0502a5a60c6ca88acccb86d038dd10f2a72a7c2d4dd7ad5383c7d687e9720cc88 -GMP.v6.2.1+1.x86_64-linux-gnu-cxx11.tar.gz/md5/15ee858d8e1f07f18df8a893634d859e -GMP.v6.2.1+1.x86_64-linux-gnu-cxx11.tar.gz/sha512/9d8ffa570eb22a5a908679e06af4dd0ce8c06cf97ff9fd766baeca352a99bcc54b4b71b9c52829ba80043a688f2ed6a33b0302072518f2b16416235d5295ea00 -GMP.v6.2.1+1.x86_64-linux-musl-cxx03.tar.gz/md5/79078a236575994696e7328e34326243 -GMP.v6.2.1+1.x86_64-linux-musl-cxx03.tar.gz/sha512/d4b77a4056a2b0dcb6f789381fff720ab7481cc7edb4672756cb2057ed6475abeb6ea414e6cec3e2450ef7302b647d7d2fc2d9f7de52feddd7767548392e84bb -GMP.v6.2.1+1.x86_64-linux-musl-cxx11.tar.gz/md5/94f822c7521f83652d87fd5f1ad8bb19 -GMP.v6.2.1+1.x86_64-linux-musl-cxx11.tar.gz/sha512/fa4f70f81524d47b65d5cf3ff5abe38a691f09e3297c62f0db2512483702b9af33bc4a3c15f6f1465d6dce4eeb19f665f29872e6dd7caea0806f4c7fd32c2c5a -GMP.v6.2.1+1.x86_64-unknown-freebsd.tar.gz/md5/cdb93a733763e8a4fc29652fda8c8b13 -GMP.v6.2.1+1.x86_64-unknown-freebsd.tar.gz/sha512/ec529f57eb167bfcb367310b375a3cded007cbc386cab9b09faa9fe8f37a443302c674814ada6c82125ad0ce4aebecb75bb61633a21e7a3a00fc928fbe05cb4f -GMP.v6.2.1+1.x86_64-w64-mingw32-cxx03.tar.gz/md5/8b5be9da6a0a293e14ab1d589a622b98 -GMP.v6.2.1+1.x86_64-w64-mingw32-cxx03.tar.gz/sha512/73287b8390cac2ce8afc4565c5218ac739ed8a23c56754f4667570039f022b777284aee25d7857a94ff46fd502ac0fabe46f509a5f870b1aa074f6ed1278dcf1 -GMP.v6.2.1+1.x86_64-w64-mingw32-cxx11.tar.gz/md5/11bcbfc3b65b19d73c3abf92ec46cb6a -GMP.v6.2.1+1.x86_64-w64-mingw32-cxx11.tar.gz/sha512/1dd9a6fe5c4991483a2d46420cd892271d37d9d23c409ed782b7736ab1942cd6c42360efbc308b5684bd5f991c7a96e8d375f3e855dc537bb3089e3402eed110 +GMP.v6.2.1+2.aarch64-apple-darwin.tar.gz/md5/37a4c537149a1d6d7424833294e61dac +GMP.v6.2.1+2.aarch64-apple-darwin.tar.gz/sha512/33dd86279b5b3b08496180c92971c2e7ef84715e9ed3a80071a178ee94de6231ea3cf7b4dd4fa7e0dbd0b386a1a04c4f6b28446e86cb92c100ebb295b2f5ee3a +GMP.v6.2.1+2.aarch64-linux-gnu-cxx03.tar.gz/md5/44ef76b228cdc4cf54e5d4b40a29034d +GMP.v6.2.1+2.aarch64-linux-gnu-cxx03.tar.gz/sha512/255a680c75d3e8ca542dffc47050adfce038e25a12a4131c18dc719d36b364c1a6488ee5743d1c5de445b4bc5ccbb932399f7071083d86fe5bd2befc521cfbfd +GMP.v6.2.1+2.aarch64-linux-gnu-cxx11.tar.gz/md5/0289ffc3621b5d62dc2f9e1b36c41f9f +GMP.v6.2.1+2.aarch64-linux-gnu-cxx11.tar.gz/sha512/f27b82efb5aa1d7eaaed7574d3312969664eac38f45cf40c6de13ca20b256d45481546fc1a402e6c04bee416c842a092a4e57b8df702bbcdc52f742555d07aa7 +GMP.v6.2.1+2.aarch64-linux-musl-cxx03.tar.gz/md5/9ff4c76804f59056b49a9bf5b6a02099 +GMP.v6.2.1+2.aarch64-linux-musl-cxx03.tar.gz/sha512/d86afa10bdc4e20fa259a17ce7d0a5dca2524b42752bc7d5c33e4323973587d234d4c420900deef34670bfce8ab8c6725e7edb45bfd3896b2644a42ec187dfd7 +GMP.v6.2.1+2.aarch64-linux-musl-cxx11.tar.gz/md5/cc9857a965afcdcbc2b378a368360690 +GMP.v6.2.1+2.aarch64-linux-musl-cxx11.tar.gz/sha512/c46bff9fdcbecc71c12914dadb31ee9fd5b4293cb45bda782200daa18d7f7e8b588e0c0f68a39c2fec7cc3d026bcef3620dae35ae2dd3acf2505dcfc084d11bd +GMP.v6.2.1+2.armv6l-linux-gnueabihf-cxx03.tar.gz/md5/5b3343367896e31b29571fe0d2b90390 +GMP.v6.2.1+2.armv6l-linux-gnueabihf-cxx03.tar.gz/sha512/65a501db63c386727aa336d6dbecdff0417628bc9ff7ac1b2161922246d94f8caa71b63fc3789ec6bb10aff03b96d5d0c22c37c82bd95d74e557df8de7e8a09c +GMP.v6.2.1+2.armv6l-linux-gnueabihf-cxx11.tar.gz/md5/cc04dda18412fa11f228e66eb5a03aad +GMP.v6.2.1+2.armv6l-linux-gnueabihf-cxx11.tar.gz/sha512/49fdd452fe8f0129ee06795e04a0cc0238132f9d6f60a124dd2c7395fabbb71f005c16d95fdc00d87f8bf82b048cc54e07f162fbc38223c644854cc72c4d26b0 +GMP.v6.2.1+2.armv6l-linux-musleabihf-cxx03.tar.gz/md5/675599595f3dedb8ca11151168da7110 +GMP.v6.2.1+2.armv6l-linux-musleabihf-cxx03.tar.gz/sha512/eedcdc2230fd81d613d54be356679a97b59491f5f9a17c518239b5504c3dd5da15721d553f57ae21f1c55d253e808e7afd1d1651b8c666379c55c7b48f71217e +GMP.v6.2.1+2.armv6l-linux-musleabihf-cxx11.tar.gz/md5/9a74abbc46439ae8268ca926f0045691 +GMP.v6.2.1+2.armv6l-linux-musleabihf-cxx11.tar.gz/sha512/6329506f7a886d0dd907b051d6cbab1bd0cd21b2d5715f55402bf9ad6cb1ae33e058931bdf6cba17658b0e455f9e4fb7f9aad274755a159106cfe1c4d1ea328a +GMP.v6.2.1+2.armv7l-linux-gnueabihf-cxx03.tar.gz/md5/8c20e0def927a202f2d23aed78aadb4a +GMP.v6.2.1+2.armv7l-linux-gnueabihf-cxx03.tar.gz/sha512/b7f42efae6fce864c9e07714056444ba74befb9cc9a766ffe14e676240f23f83d3241b1bf3a8f4a282acbdc197287fffb27dadedf3055505ad63bb0b9df573c6 +GMP.v6.2.1+2.armv7l-linux-gnueabihf-cxx11.tar.gz/md5/423a625816b3c52efa6021e76f6009b7 +GMP.v6.2.1+2.armv7l-linux-gnueabihf-cxx11.tar.gz/sha512/21cbbfd647d4a7c884344dc66e0fd83d654d22c3338669539e8eab515bdc6bbd772b47f949d28280789e4343e9a8d6319a73dc9e11c23da381b8a452ef7fb098 +GMP.v6.2.1+2.armv7l-linux-musleabihf-cxx03.tar.gz/md5/7d67f981538d7a69ab1e458a54bf56f4 +GMP.v6.2.1+2.armv7l-linux-musleabihf-cxx03.tar.gz/sha512/8aefbcddc326d4ef289dcdba8d3bd56a5f9656a7be30c83b4dbd9a0b8ee26a963c6a2f4294c94b8a8f2f712f1e1c9e17b8b9dcc9967d64294ca466e51656f7c7 +GMP.v6.2.1+2.armv7l-linux-musleabihf-cxx11.tar.gz/md5/ed8713b71636ea75fcc0c9fbc4a8618d +GMP.v6.2.1+2.armv7l-linux-musleabihf-cxx11.tar.gz/sha512/d7f50d06a256fd9176d5fbf682ff599a5ffba62bb35fb37321ab41e88970921a9d9fa4531bd74e73e471c7e15fcae568d0536d3e32a2b2d7f81dc9cd1f0c039f +GMP.v6.2.1+2.i686-linux-gnu-cxx03.tar.gz/md5/875f0bc57172788cb80ca2b80ff3065f +GMP.v6.2.1+2.i686-linux-gnu-cxx03.tar.gz/sha512/808a3c2422b5168260dbf7a3875d5c8151e10b20a8ec87a66bf08f71ad7cf5de20fb7a4f3457c3ab2b4ffc9627764c743baa96f409629c70f2233ea7a5b628b9 +GMP.v6.2.1+2.i686-linux-gnu-cxx11.tar.gz/md5/09ae13f2a6a0dc317d2bca5700d2bf59 +GMP.v6.2.1+2.i686-linux-gnu-cxx11.tar.gz/sha512/9c986e2904247de937e30c05b29e0179986d7747b217468c59bc56af6d4c48d4575f24dace521dc8d66d84230eebd695fe0538972bfd744182ca940a23a9239c +GMP.v6.2.1+2.i686-linux-musl-cxx03.tar.gz/md5/45f53fd95dd69a6ee6b43463976b5aa6 +GMP.v6.2.1+2.i686-linux-musl-cxx03.tar.gz/sha512/4df57d6c88f0ff86e0ee78da8f6ad02decf7a38884ae8c785c114e0e38e791b733e0d046c90712327c08645dd40b7f0391fcb3258cb3bfb8b6a62c59c27d6e83 +GMP.v6.2.1+2.i686-linux-musl-cxx11.tar.gz/md5/8b15988bfb1ba0543eefab73b3ac3439 +GMP.v6.2.1+2.i686-linux-musl-cxx11.tar.gz/sha512/e32dec7ded9bf6fc26033df83521481dde851c68d7cc45efaabeded7603417cdc5016de45f78a956b69aaed00a55a91aa8b1cd5bbe5431b01074dafce2c47751 +GMP.v6.2.1+2.i686-w64-mingw32-cxx03.tar.gz/md5/4138d0b5185f722aef4e1f215f381275 +GMP.v6.2.1+2.i686-w64-mingw32-cxx03.tar.gz/sha512/255d4ecf178b9440b667c56e542baa4422d731f83a67accd41b76268274c2344fbbf94979fddbbd1f6b5751bac2d228a8ef49a93365de78c1772146edd1b4845 +GMP.v6.2.1+2.i686-w64-mingw32-cxx11.tar.gz/md5/606b4b453af25ded1323aee9e085c132 +GMP.v6.2.1+2.i686-w64-mingw32-cxx11.tar.gz/sha512/8605b764ff6e5d81767432fd8e70c25c5ad76f2cac7c2b3d6ed0596df692300973803487c970a896a0a316d46de3e3cae31b21d4e11fe2961e228cd389da13da +GMP.v6.2.1+2.powerpc64le-linux-gnu-cxx03.tar.gz/md5/3fbd157df4ae738da6820b26fb75e75e +GMP.v6.2.1+2.powerpc64le-linux-gnu-cxx03.tar.gz/sha512/6e64c5c4e393c0001bd7085e627126134b5999c2d8df2fa9b72c9f9835d6b0f0ad440a2f58fe6537ec446a517f8df2667881871fce9b4d61c356d2b52080d641 +GMP.v6.2.1+2.powerpc64le-linux-gnu-cxx11.tar.gz/md5/35608e3166278d52a482d7e19313eca6 +GMP.v6.2.1+2.powerpc64le-linux-gnu-cxx11.tar.gz/sha512/a9550fe2b94e0e111a487159c0cd8fb6f1a21b8941ada7bb281572079dbbece921f80b0275bcc8f88117ecc72e7f8e93219350f5444b67295620db1aa9ae947d +GMP.v6.2.1+2.x86_64-apple-darwin.tar.gz/md5/b5004a436660a2533b94b41c592b686c +GMP.v6.2.1+2.x86_64-apple-darwin.tar.gz/sha512/b7b4dc8025ce304c5b899084f42c8f5aad5bbe03509bada17dbe6be952f98306729180a22b5d0a095692f349406db0b98f99f5e3f2be5f2165825e6f7f7d1813 +GMP.v6.2.1+2.x86_64-linux-gnu-cxx03.tar.gz/md5/47ba899c9ac714a4594f999d845f45cf +GMP.v6.2.1+2.x86_64-linux-gnu-cxx03.tar.gz/sha512/99624ec71865d6285ab409ef54f4cf12ba246de6233de56a2fb9f70806574891539efed32e711202003570c157918fde8d53534c695fd5b8476e0d4e0ecd1bd4 +GMP.v6.2.1+2.x86_64-linux-gnu-cxx11.tar.gz/md5/3b0c1258ecafcaf96e549f9b979420ee +GMP.v6.2.1+2.x86_64-linux-gnu-cxx11.tar.gz/sha512/b94d8f25d23597f96cc0cf0aebd1708755a8714ec4a481108add852b77addc737d3d8feba566ec410db019698ca2de826583b1a6105f0d2188679e7f72331df0 +GMP.v6.2.1+2.x86_64-linux-musl-cxx03.tar.gz/md5/061cfe5f416c1365e98d6b1ed89abd63 +GMP.v6.2.1+2.x86_64-linux-musl-cxx03.tar.gz/sha512/b6847f7ff599fa811851788a6ec6ce69ba02dbb3672d0a64b03b7056b35215536b059287709b3d207bc977094e994a7d744061b7ecf95886510285489bb89578 +GMP.v6.2.1+2.x86_64-linux-musl-cxx11.tar.gz/md5/81911acbc0c3607338c6455b1798cab8 +GMP.v6.2.1+2.x86_64-linux-musl-cxx11.tar.gz/sha512/e007441194abc5c80d9521a17e2ab9e6fb54f319571f4045fec2f7464ffaa99652d3252416c15d110dbf9deaad2c1dc94f81c638e28ce620cf543f554eb7d1e0 +GMP.v6.2.1+2.x86_64-unknown-freebsd.tar.gz/md5/ef7173194848e8d00d73ef05fc520f0e +GMP.v6.2.1+2.x86_64-unknown-freebsd.tar.gz/sha512/512c3cf8fb951fe0ef7b1715b78202d0bdf5844fe33e16c4674a19e6335440fb5352d7bde71fce83e8e373efe43281d05b160b11657a582a9d3a0201ce97a189 +GMP.v6.2.1+2.x86_64-w64-mingw32-cxx03.tar.gz/md5/882c6749f217f5a691b744ef728ad089 +GMP.v6.2.1+2.x86_64-w64-mingw32-cxx03.tar.gz/sha512/53424ad8a9dcfb8e0e738d4521b2ab1c75aaf54668a54a76b8bcab2404308e69b531dc25b3dc18bc8eaa7ebd9e2914d6624c5d371e6c0ecb9e8d24aa575e99ab +GMP.v6.2.1+2.x86_64-w64-mingw32-cxx11.tar.gz/md5/bcdd7bcbc69161744397d249a9c82e45 +GMP.v6.2.1+2.x86_64-w64-mingw32-cxx11.tar.gz/sha512/b7f8fb4f5aaf5034d4d2f60e29cc7b5e06c13d4b677af30f30831e1fc95925a575275ebffda36efcc09e29ccd78ba56475c1be3ad0627e28862057764f1ef74e gmp-6.2.1.tar.bz2/md5/28971fc21cf028042d4897f02fd355ea gmp-6.2.1.tar.bz2/sha512/8904334a3bcc5c896ececabc75cda9dec642e401fb5397c4992c4fabea5e962c9ce8bd44e8e4233c34e55c8010cc28db0545f5f750cbdbb5f00af538dc763be9 diff --git a/deps/gmp.mk b/deps/gmp.mk index a37327d82101e..cbb99aba4a003 100644 --- a/deps/gmp.mk +++ b/deps/gmp.mk @@ -39,10 +39,16 @@ $(SRCCACHE)/gmp-$(GMP_VER)/gmp_alloc_overflow_func.patch-applied: $(SRCCACHE)/gm patch -p1 < $(SRCDIR)/patches/gmp_alloc_overflow_func.patch echo 1 > $@ +$(SRCCACHE)/gmp-$(GMP_VER)/gmp-CVE-2021-43618.patch-applied: $(SRCCACHE)/gmp-$(GMP_VER)/gmp_alloc_overflow_func.patch-applied + cd $(dir $@) && \ + patch -p1 < $(SRCDIR)/patches/gmp-CVE-2021-43618.patch + echo 1 > $@ + $(SRCCACHE)/gmp-$(GMP_VER)/source-patched: \ $(SRCCACHE)/gmp-$(GMP_VER)/gmp-HG-changeset.patch-applied \ $(SRCCACHE)/gmp-$(GMP_VER)/gmp-exception.patch-applied \ - $(SRCCACHE)/gmp-$(GMP_VER)/gmp_alloc_overflow_func.patch-applied + $(SRCCACHE)/gmp-$(GMP_VER)/gmp_alloc_overflow_func.patch-applied \ + $(SRCCACHE)/gmp-$(GMP_VER)/gmp-CVE-2021-43618.patch-applied echo 1 > $@ $(BUILDDIR)/gmp-$(GMP_VER)/build-configured: $(SRCCACHE)/gmp-$(GMP_VER)/source-extracted $(SRCCACHE)/gmp-$(GMP_VER)/source-patched diff --git a/deps/patches/gmp-CVE-2021-43618.patch b/deps/patches/gmp-CVE-2021-43618.patch new file mode 100644 index 0000000000000..a4e420e9219da --- /dev/null +++ b/deps/patches/gmp-CVE-2021-43618.patch @@ -0,0 +1,24 @@ +# Origin: https://gmplib.org/repo/gmp-6.2/rev/561a9c25298e +# HG changeset patch +# User Marco Bodrato +# Date 1634836009 -7200 +# Node ID 561a9c25298e17bb01896801ff353546c6923dbd +# Parent e1fd9db13b475209a864577237ea4b9105b3e96e +mpz/inp_raw.c: Avoid bit size overflows + +diff -r e1fd9db13b47 -r 561a9c25298e mpz/inp_raw.c +--- a/mpz/inp_raw.c Tue Dec 22 23:49:51 2020 +0100 ++++ b/mpz/inp_raw.c Thu Oct 21 19:06:49 2021 +0200 +@@ -88,8 +88,11 @@ + + abs_csize = ABS (csize); + ++ if (UNLIKELY (abs_csize > ~(mp_bitcnt_t) 0 / 8)) ++ return 0; /* Bit size overflows */ ++ + /* round up to a multiple of limbs */ +- abs_xsize = BITS_TO_LIMBS (abs_csize*8); ++ abs_xsize = BITS_TO_LIMBS ((mp_bitcnt_t) abs_csize * 8); + + if (abs_xsize != 0) + { diff --git a/stdlib/GMP_jll/Project.toml b/stdlib/GMP_jll/Project.toml index 0fc262e562da7..510b6f6a49c60 100644 --- a/stdlib/GMP_jll/Project.toml +++ b/stdlib/GMP_jll/Project.toml @@ -1,6 +1,6 @@ name = "GMP_jll" uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" -version = "6.2.1+1" +version = "6.2.1+2" [deps] Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" From dea62ee35878702e0c993c13e09f59f852702066 Mon Sep 17 00:00:00 2001 From: Fabian Zickgraf Date: Sat, 21 May 2022 03:09:47 +0000 Subject: [PATCH 59/68] Use root module when determining UUID in @artifact_str (#45392) Otherwise, overrides do not trigger when using `artifact"..."` inside a submodule. (cherry picked from commit 9b106adcdff120cdfc1fb0c0d6c50b68a787ce95) --- stdlib/Artifacts/src/Artifacts.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stdlib/Artifacts/src/Artifacts.jl b/stdlib/Artifacts/src/Artifacts.jl index 6d3bdb5fb674b..27e352be59270 100644 --- a/stdlib/Artifacts/src/Artifacts.jl +++ b/stdlib/Artifacts/src/Artifacts.jl @@ -524,9 +524,10 @@ function jointail(dir, tail) end function _artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dict, hash, platform, @nospecialize(lazyartifacts)) - if haskey(Base.module_keys, __module__) + moduleroot = Base.moduleroot(__module__) + if haskey(Base.module_keys, moduleroot) # Process overrides for this UUID, if we know what it is - process_overrides(artifact_dict, Base.module_keys[__module__].uuid) + process_overrides(artifact_dict, Base.module_keys[moduleroot].uuid) end # If the artifact exists, we're in the happy path and we can immediately From 823071e0b61135fb2e106459eba3d3ad5933ff69 Mon Sep 17 00:00:00 2001 From: FX Coudert Date: Sat, 21 May 2022 17:27:39 +0200 Subject: [PATCH 60/68] OpenBLAS: Find objconv in its proper path (#45391) (cherry picked from commit 434d340afc7e4cc4d18b09c3858a6c6bc47ba85b) --- deps/openblas.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/openblas.mk b/deps/openblas.mk index d4ee63a543bf0..26d92fc4bd70b 100644 --- a/deps/openblas.mk +++ b/deps/openblas.mk @@ -29,7 +29,7 @@ endif ifeq ($(USE_BLAS64), 1) OPENBLAS_BUILD_OPTS += INTERFACE64=1 SYMBOLSUFFIX="$(OPENBLAS_SYMBOLSUFFIX)" LIBPREFIX="libopenblas$(OPENBLAS_LIBNAMESUFFIX)" ifeq ($(OS), Darwin) -OPENBLAS_BUILD_OPTS += OBJCONV=$(abspath $(build_bindir)/objconv) +OPENBLAS_BUILD_OPTS += OBJCONV=$(abspath $(build_depsbindir)/objconv) $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/build-compiled: | $(build_prefix)/manifest/objconv endif endif From b8943ecdd344d00f4f0a2b5196c38b07e03c7310 Mon Sep 17 00:00:00 2001 From: FX Coudert Date: Sat, 21 May 2022 20:58:12 +0200 Subject: [PATCH 61/68] Update PCRE2 to 10.40 (#45398) (cherry picked from commit 3d6731be2bf54b7b2ab7f33a28cae7c7a91c0d14) --- deps/Versions.make | 2 +- deps/checksums/pcre | 68 ++--- .../pcre2-sljit-apple-silicon-support.patch | 244 ------------------ deps/patches/pcre2-sljit-nomprotect.patch | 17 -- deps/pcre.mk | 13 +- stdlib/PCRE2_jll/Project.toml | 2 +- stdlib/PCRE2_jll/test/runtests.jl | 2 +- 7 files changed, 39 insertions(+), 309 deletions(-) delete mode 100644 deps/patches/pcre2-sljit-apple-silicon-support.patch delete mode 100644 deps/patches/pcre2-sljit-nomprotect.patch diff --git a/deps/Versions.make b/deps/Versions.make index 77d568ee7c6b5..b865922c7f3d9 100644 --- a/deps/Versions.make +++ b/deps/Versions.make @@ -94,7 +94,7 @@ P7ZIP_VER := 16.2.0 P7ZIP_JLL_NAME := p7zip # PCRE -PCRE_VER := 10.36 +PCRE_VER := 10.40 PCRE_JLL_NAME := PCRE2 # SuiteSparse diff --git a/deps/checksums/pcre b/deps/checksums/pcre index 05a06f9844ddf..202265ee58060 100644 --- a/deps/checksums/pcre +++ b/deps/checksums/pcre @@ -1,34 +1,34 @@ -PCRE2.v10.36.0+2.aarch64-apple-darwin.tar.gz/md5/12ac3bee39df3a79f868f6463964953b -PCRE2.v10.36.0+2.aarch64-apple-darwin.tar.gz/sha512/a1a1312931deb7f742f80886188babcf9c179ed3f156626fb23d92633fde896d1ee9b2d72cd99ae4a1f8048971b6d939e9b0b10c455d4eeec24b265968593486 -PCRE2.v10.36.0+2.aarch64-linux-gnu.tar.gz/md5/32240ccddee3040aeedcbe69ea52fcad -PCRE2.v10.36.0+2.aarch64-linux-gnu.tar.gz/sha512/86fb9febd186fcaeec83d2ed336fb060d7e49c7b7efe1bd8a6d2d74023ddbcce04eed5cf0e5d15348313eb2b51cd6b27763c08f7b9cf4eaf9df22d88f9405ef8 -PCRE2.v10.36.0+2.aarch64-linux-musl.tar.gz/md5/06abf8210e597a8669fb371da73865ce -PCRE2.v10.36.0+2.aarch64-linux-musl.tar.gz/sha512/063edaa92e36468a8cf70ca9e25d9004586400a5304c0e91b067788825cbf5354e0190cad951f163e318b65d0f3f915f1944d03de61a5627ead2ead2674d3279 -PCRE2.v10.36.0+2.armv6l-linux-gnueabihf.tar.gz/md5/70ca2acdd5b1524141f15d02d26c3b1c -PCRE2.v10.36.0+2.armv6l-linux-gnueabihf.tar.gz/sha512/377fdc5fd8b771027ffe8c0871e1688f8d991caf930b26b397eae01504af2fad5bdfbe2b3af33f25cf4b5c7bfd73dc77b16b65882a7846803a00edc0968ccef2 -PCRE2.v10.36.0+2.armv6l-linux-musleabihf.tar.gz/md5/860180f0a15ad38fac20590fab177718 -PCRE2.v10.36.0+2.armv6l-linux-musleabihf.tar.gz/sha512/412e7b0355a7bcdecca4ff5f85a1c6af1eeb094a9f07c2e90de105a0e0e6acedcbca146b5c136509ef8b38666f645b0c06fc68676dd8b1b70e2c7af4b070eb3d -PCRE2.v10.36.0+2.armv7l-linux-gnueabihf.tar.gz/md5/12fd561c00fc7fca14e577ed54525740 -PCRE2.v10.36.0+2.armv7l-linux-gnueabihf.tar.gz/sha512/e5655e5c3f96a3a95699be534acbd399bc29873fa1064f50c2d78c43ad8e85a1fbf9039bcb674a88ecdb9bf5b468f9ecdf9a79f0dce5d95996f99d6c700da79a -PCRE2.v10.36.0+2.armv7l-linux-musleabihf.tar.gz/md5/97d5eab8806a1920e6fd30f82db1b754 -PCRE2.v10.36.0+2.armv7l-linux-musleabihf.tar.gz/sha512/827fc45049a4b3adb6de2ab0569e45dd5e8749c09c42e57c579d3d6350f0953f6ad4fae1ba71af7347c9271ffff805a0200b5c9418e7f1894a6bc17a4fe0071c -PCRE2.v10.36.0+2.i686-linux-gnu.tar.gz/md5/d7c9fdbcf3055c4745ea93a9274e16d1 -PCRE2.v10.36.0+2.i686-linux-gnu.tar.gz/sha512/ac0edd5d5910e7948a65c2a5c9fb05d2a6beb3f9bd875ea87433b910444bcba617ac5bc215fa0f101cbd7c5556966de7593080674cfaf28fdc8784e2485cf71b -PCRE2.v10.36.0+2.i686-linux-musl.tar.gz/md5/05ef7559eba68cecbad0f2c75c017640 -PCRE2.v10.36.0+2.i686-linux-musl.tar.gz/sha512/91603d596a1b70bc4a933f9151fc791e09a167e4ad2de442a7ff9c355a329353cc9fb3148cf75639eaef0de3cf4f71212525f1040b0eff63c5d884892814b7af -PCRE2.v10.36.0+2.i686-w64-mingw32.tar.gz/md5/8015e6633bf0f4c359f85445d4a98a9a -PCRE2.v10.36.0+2.i686-w64-mingw32.tar.gz/sha512/527183fcc473c8e3f04622701cf73a55c5df132713e8230cd0bfd484023da594a9e29f5745d384f1e1015b8efac96e88bd985b06af5901b0d3052f90af8d89d6 -PCRE2.v10.36.0+2.powerpc64le-linux-gnu.tar.gz/md5/2ece20fa11fdbae393fb85a41ee1e17d -PCRE2.v10.36.0+2.powerpc64le-linux-gnu.tar.gz/sha512/e6fbc03efed53da43b3b15b31cc0fbd85aaf5cc65564392b8c7bc02695d3a32fe832880d547c37b3a508197a4d4023be0aef910cd36da69a54ee184880cc0438 -PCRE2.v10.36.0+2.x86_64-apple-darwin.tar.gz/md5/26c560dd16b460a1ac7c81807edbacc6 -PCRE2.v10.36.0+2.x86_64-apple-darwin.tar.gz/sha512/ce56bc399e204e4b437d3f398b4e68c33d9c55ec990126523f3be0b14571603eea3b3104e1909deb22eab3f5302da72fcc690d1a279cb85ef598c42a5ef9a8a9 -PCRE2.v10.36.0+2.x86_64-linux-gnu.tar.gz/md5/474dec882abefcb56febddc309ed4682 -PCRE2.v10.36.0+2.x86_64-linux-gnu.tar.gz/sha512/882898c2d6cab8cd5ecf1027388bd08ddd1fec2339b45388786f98c53518bf7ca56f9e2cccb4a5ede953cc85e6c1cc54a5a00f80ece4cbfdc17e5f6116a9976a -PCRE2.v10.36.0+2.x86_64-linux-musl.tar.gz/md5/af6d90c071437c5529306a5bafe6f6aa -PCRE2.v10.36.0+2.x86_64-linux-musl.tar.gz/sha512/92a16960d7514c829a5f372a40472c87c717d49e9694030ae0cb39106d6530f5bb169155a74a416bf340139f9dea231ddc2b7ae6e54fcb935f6a9bf672b5e0c1 -PCRE2.v10.36.0+2.x86_64-unknown-freebsd.tar.gz/md5/97410029c0b6ed5f7fb0d14e1f1215ea -PCRE2.v10.36.0+2.x86_64-unknown-freebsd.tar.gz/sha512/229e910759da2959ddef83ca89e05a050c266b8e755c85dfce6a786658be541911c3b78a0fca7dfdee1b41fbbdccf57da75cf9fe45fd2821dba8d2aaeabfd538 -PCRE2.v10.36.0+2.x86_64-w64-mingw32.tar.gz/md5/39827564bca329768e0380bd79b869fe -PCRE2.v10.36.0+2.x86_64-w64-mingw32.tar.gz/sha512/4579049b99fca3334d726b0ca1f07524d1643a758e375b5b02b8f294ba7d9c2a4130da1a1523de29033233a8848105b3cb660e15bb4a759593405d805ee99883 -pcre2-10.36.tar.bz2/md5/bd7e7421ff3fa2e2d5429229ecfad095 -pcre2-10.36.tar.bz2/sha512/fc2a920562c80c3d31cedd94028fab55314ae0fb168cac7178f286c344a11fc514939edc3b83b8e0b57c872db4e595fd5530fd1d4b8c779be629553e9ec965a3 +PCRE2.v10.40.0+0.aarch64-apple-darwin.tar.gz/md5/3d6b01c094c9e1adad2c1d42a3e7c3a6 +PCRE2.v10.40.0+0.aarch64-apple-darwin.tar.gz/sha512/374f9f35ae7925a6db6249850822d90c56c11b1b49971b76f016203e85bcc14ea6ab7e017b0ad5ce56c47b0715b2a396099749656e7d7291008a2dc8cb393792 +PCRE2.v10.40.0+0.aarch64-linux-gnu.tar.gz/md5/0f4c7daae3c08e5438b0af3299cbb003 +PCRE2.v10.40.0+0.aarch64-linux-gnu.tar.gz/sha512/ee9c6275019ef09a2fd7c6a649ebe184b58dae4e65a9b38159bac596e0427819e086084ca56be0f2f2ad0eb98a50a2511999cb46d5e9d1f03d39b04ade5e270d +PCRE2.v10.40.0+0.aarch64-linux-musl.tar.gz/md5/baf858fd38471dd933312079ebaf065d +PCRE2.v10.40.0+0.aarch64-linux-musl.tar.gz/sha512/3b50f6380673d30d487a3b10e6c58b76ff47fbb5c774f59f15bcc0b92e7740e73ad04c62b86e8eab0c916d4c231449f5279eae37aa401fab1a46c6e11687e806 +PCRE2.v10.40.0+0.armv6l-linux-gnueabihf.tar.gz/md5/9c582d85fe43e205679d2ed8d1ee3df7 +PCRE2.v10.40.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/fb7df17fa39ac93c7af92f4afdcdd120b171682ce172561a65fae3c6e3b1c26c5715b1264007fd12713464cbff406fb19117adaf1d50bd239f0dc53e7842ca8e +PCRE2.v10.40.0+0.armv6l-linux-musleabihf.tar.gz/md5/a9c6c90c69d3de7030bd5015092a1340 +PCRE2.v10.40.0+0.armv6l-linux-musleabihf.tar.gz/sha512/7030aaaac0d275e72f3a36fe5104d11eba9bd1909c3d7126c751c9409f619d25c7735c7d3354b48786aef1ca9f1be48a60e0bd04a04c6b098915e6c4b2935e5f +PCRE2.v10.40.0+0.armv7l-linux-gnueabihf.tar.gz/md5/cc4add9c80f47ac3fb682aca3347aca3 +PCRE2.v10.40.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/4a21795524d3cf8112384d133b47e87738a8c1efa71606fb55f5fabe1cc4108b2921c2efb539506552a2b630398a6770d93c9c541d5123b7a84016aad7a112f0 +PCRE2.v10.40.0+0.armv7l-linux-musleabihf.tar.gz/md5/51c54233c6e536671f2c1af74e1773d5 +PCRE2.v10.40.0+0.armv7l-linux-musleabihf.tar.gz/sha512/3889cf1faacd16779c87ac00317fbc36e54f5a99733b838920add360196edbe388c12421380105a87041d3502e5f4bea74460dedc3d797aafde5cb0f960516d0 +PCRE2.v10.40.0+0.i686-linux-gnu.tar.gz/md5/368342965b12beed2c4c92e60f7dda8f +PCRE2.v10.40.0+0.i686-linux-gnu.tar.gz/sha512/bdb3692412d0b1d07bf302fbd129755e4a53e6b39caf135df912da79088e5db29a788680b282292919c45560a795fab60d043feece63cae2296165a9909ecb57 +PCRE2.v10.40.0+0.i686-linux-musl.tar.gz/md5/79bf801c0d86614ebf95ef83016195e6 +PCRE2.v10.40.0+0.i686-linux-musl.tar.gz/sha512/d35d15ccc8b09a33088efb4bf631cbbb3ff332521f37fdaa5fc106e576a54cb57ad1243dc3db1ab17a8195fd1476889b8d548987437a195267fae7683769da38 +PCRE2.v10.40.0+0.i686-w64-mingw32.tar.gz/md5/930cbf007549542b027a1db72bab0e58 +PCRE2.v10.40.0+0.i686-w64-mingw32.tar.gz/sha512/e9bad56ca6e1871f2bf37c8b2b03ecbc77acd3f4b04c95dd6e63a4cb38487fc3349a97ca7f575c158fde8b948c363af3f7cffc4ad89af9df09e536119a1d743b +PCRE2.v10.40.0+0.powerpc64le-linux-gnu.tar.gz/md5/cebf0e67b6ae67fa841e491bf8955ae0 +PCRE2.v10.40.0+0.powerpc64le-linux-gnu.tar.gz/sha512/e04087f3e3268d389c08068ac8ae45f017e742787f20235eb6e4d32257ae3a3e445c61dc80db5a2c73d3fea5721272ec517c8b3be428d8aca097e691a14eb659 +PCRE2.v10.40.0+0.x86_64-apple-darwin.tar.gz/md5/5ed58d794f55139baac9a1ee50da3647 +PCRE2.v10.40.0+0.x86_64-apple-darwin.tar.gz/sha512/e906c6953be8a894d4cfa1792843e85aef58cf3b87baf4bcba99d19c84bd7d67dfbde85f1ddad42cbd51d2b1fa36797ce2ad79d79b19a792ca886bf52632a919 +PCRE2.v10.40.0+0.x86_64-linux-gnu.tar.gz/md5/db3fd5e855ca47b90d9a1faf58c88279 +PCRE2.v10.40.0+0.x86_64-linux-gnu.tar.gz/sha512/9082201b6519a693cf0038cf667841a0a4e4158698e1b7455ed3e0db1a7796c7303cf105975ddf059a6dbf5865eaf99f33d4e42803364935da7fa9e9c3bcb5b5 +PCRE2.v10.40.0+0.x86_64-linux-musl.tar.gz/md5/ab3456b926864ab27d5a4ce8dd42d1e7 +PCRE2.v10.40.0+0.x86_64-linux-musl.tar.gz/sha512/4b9109d9fadde86b1d76c420cb3e8b884ccba6fa08fec4fb039c384af5f040cf52b3232fbf4921cf680f36e54683b28bdb77e3b2a8943acf974f446e99f93475 +PCRE2.v10.40.0+0.x86_64-unknown-freebsd.tar.gz/md5/ee7679ad09e13f3cf9a2089e761bd718 +PCRE2.v10.40.0+0.x86_64-unknown-freebsd.tar.gz/sha512/cce31108246bdc2947865339a7cdbb7f505baf3b1b94fa6f6d825416149d8bc888a0a55961873f041cb94bba623c27f5ecaef23dda284cc57b76b30987fb6f5b +PCRE2.v10.40.0+0.x86_64-w64-mingw32.tar.gz/md5/8178c12311e6f74bc1155d6d49dfb612 +PCRE2.v10.40.0+0.x86_64-w64-mingw32.tar.gz/sha512/9d03dd7ee07fdce9af7e6995e533c59dc274417c0e39a27ccea397291b17d6865bf9c80bbc7c9aa8e908518ba33873b39b9cbfd36bc7137cb5b7432c5684e073 +pcre2-10.40.tar.bz2/md5/a5cc4e276129c177d4fffb40601019a4 +pcre2-10.40.tar.bz2/sha512/00e7b48a6554b9127cb6fe24c5cacf72783416a9754ec88f62f73c52f46ed72c86c1869e62c91a31b2ff2cbafbbedabca44b3f1eb7670bc92f49d8401c7374e8 diff --git a/deps/patches/pcre2-sljit-apple-silicon-support.patch b/deps/patches/pcre2-sljit-apple-silicon-support.patch deleted file mode 100644 index 3aff832ca08fd..0000000000000 --- a/deps/patches/pcre2-sljit-apple-silicon-support.patch +++ /dev/null @@ -1,244 +0,0 @@ -From e87e1ccf93768238db3d6e28d0272980dba707fa Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Carlo=20Marcelo=20Arenas=20Bel=C3=B3n?= -Date: Mon, 30 Nov 2020 01:35:13 -0800 -Subject: [PATCH] macos: add BigSur support to execalloc (#90) - -Apple Silicon requires that pages that will hold JIT code are -marked with MAP_JIT (even if not using the hardened runtime) -and that a call be made to a pthread function before writing -to them, so a special exception could be made to the current -thread[1]; add support for both. - -since the allocator keeps the metadata about chunk/block in the -executable pages, all functions that modify that metadata will -also need to be updated. - -note that since there is no need for an accurate pointer range -with the apple implementation, NULL is passed for the pointers. - -historically, adding MAP_JIT was only recommended when the hardened -runtime was being used as it adds several undocumented restrictions -(like not being able to use JIT pages accross fork()) so the -new codepath won't be used if running in Intel. - -Tested-by: @Keno -Fixes: #51 - -[1] https://developer.apple.com/documentation/apple_silicon/porting_just-in-time_compilers_to_apple_silicon?language=objc ---- - sljit_src/sljitExecAllocator.c | 113 ++++++++++++++++++--------------- - 1 file changed, 63 insertions(+), 50 deletions(-) - -diff --git a/sljit_src/sljitExecAllocator.c b/sljit_src/sljitExecAllocator.c -index 61a32f2..2e1c138 100644 ---- a/sljit_src/sljitExecAllocator.c -+++ b/sljit_src/sljitExecAllocator.c -@@ -79,6 +79,7 @@ - */ - - #ifdef _WIN32 -+#define SLJIT_UPDATE_WX_FLAGS(from, to, enable_exec) - - static SLJIT_INLINE void* alloc_chunk(sljit_uw size) - { -@@ -91,65 +92,76 @@ static SLJIT_INLINE void free_chunk(void *chunk, sljit_uw size) - VirtualFree(chunk, 0, MEM_RELEASE); - } - --#else -- --#ifdef __APPLE__ --#ifdef MAP_ANON --/* Configures TARGET_OS_OSX when appropriate */ --#include -- --#if TARGET_OS_OSX && defined(MAP_JIT) --#include --#endif /* TARGET_OS_OSX && MAP_JIT */ -- --#ifdef MAP_JIT -+#else /* POSIX */ - -+#if defined(__APPLE__) && defined(MAP_JIT) - /* - On macOS systems, returns MAP_JIT if it is defined _and_ we're running on a -- version where it's OK to have more than one JIT block. -+ version where it's OK to have more than one JIT block or where MAP_JIT is -+ required. - On non-macOS systems, returns MAP_JIT if it is defined. - */ -+#include -+#if TARGET_OS_OSX -+#if defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86 -+#ifdef MAP_ANON -+#include -+#include -+ -+#define SLJIT_MAP_JIT (get_map_jit_flag()) -+ - static SLJIT_INLINE int get_map_jit_flag() - { --#if TARGET_OS_OSX -- sljit_sw page_size = get_page_alignment() + 1; -+ sljit_sw page_size; - void *ptr; -+ struct utsname name; - static int map_jit_flag = -1; - -- /* -- The following code is thread safe because multiple initialization -- sets map_jit_flag to the same value and the code has no side-effects. -- Changing the kernel version witout system restart is (very) unlikely. -- */ -- if (map_jit_flag == -1) { -- struct utsname name; -- -+ if (map_jit_flag < 0) { - map_jit_flag = 0; - uname(&name); - -- /* Kernel version for 10.14.0 (Mojave) */ -+ /* Kernel version for 10.14.0 (Mojave) or later */ - if (atoi(name.release) >= 18) { -+ page_size = get_page_alignment() + 1; - /* Only use MAP_JIT if a hardened runtime is used */ -+ ptr = mmap(NULL, page_size, PROT_WRITE | PROT_EXEC, -+ MAP_PRIVATE | MAP_ANON, -1, 0); - -- ptr = mmap(NULL, page_size, PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANON, -1, 0); -- -- if (ptr == MAP_FAILED) { -- map_jit_flag = MAP_JIT; -- } else { -+ if (ptr != MAP_FAILED) - munmap(ptr, page_size); -- } -+ else -+ map_jit_flag = MAP_JIT; - } - } -- - return map_jit_flag; --#else /* !TARGET_OS_OSX */ -- return MAP_JIT; --#endif /* TARGET_OS_OSX */ - } -- --#endif /* MAP_JIT */ - #endif /* MAP_ANON */ --#endif /* __APPLE__ */ -+#else /* !SLJIT_CONFIG_X86 */ -+#if !(defined SLJIT_CONFIG_ARM && SLJIT_CONFIG_ARM) -+#error Unsupported architecture -+#endif /* SLJIT_CONFIG_ARM */ -+#include -+ -+#define SLJIT_MAP_JIT (MAP_JIT) -+#define SLJIT_UPDATE_WX_FLAGS(from, to, enable_exec) \ -+ apple_update_wx_flags(enable_exec) -+ -+static SLJIT_INLINE void apple_update_wx_flags(sljit_s32 enable_exec) -+{ -+ pthread_jit_write_protect_np(enable_exec); -+} -+#endif /* SLJIT_CONFIG_X86 */ -+#else /* !TARGET_OS_OSX */ -+#define SLJIT_MAP_JIT (MAP_JIT) -+#endif /* TARGET_OS_OSX */ -+#endif /* __APPLE__ && MAP_JIT */ -+#ifndef SLJIT_UPDATE_WX_FLAGS -+#define SLJIT_UPDATE_WX_FLAGS(from, to, enable_exec) -+#endif /* !SLJIT_UPDATE_WX_FLAGS */ -+#ifndef SLJIT_MAP_JIT -+#define SLJIT_MAP_JIT (0) -+#endif /* !SLJIT_MAP_JIT */ - - static SLJIT_INLINE void* alloc_chunk(sljit_uw size) - { -@@ -157,12 +169,7 @@ static SLJIT_INLINE void* alloc_chunk(sljit_uw size) - const int prot = PROT_READ | PROT_WRITE | PROT_EXEC; - - #ifdef MAP_ANON -- -- int flags = MAP_PRIVATE | MAP_ANON; -- --#ifdef MAP_JIT -- flags |= get_map_jit_flag(); --#endif -+ int flags = MAP_PRIVATE | MAP_ANON | SLJIT_MAP_JIT; - - retval = mmap(NULL, size, prot, flags, -1, 0); - #else /* !MAP_ANON */ -@@ -173,14 +180,15 @@ static SLJIT_INLINE void* alloc_chunk(sljit_uw size) - #endif /* MAP_ANON */ - - if (retval == MAP_FAILED) -- retval = NULL; -- else { -- if (mprotect(retval, size, prot) < 0) { -- munmap(retval, size); -- retval = NULL; -- } -+ return NULL; -+ -+ if (mprotect(retval, size, prot) < 0) { -+ munmap(retval, size); -+ return NULL; - } - -+ SLJIT_UPDATE_WX_FLAGS(retval, (uint8_t *)retval + size, 0); -+ - return retval; - } - -@@ -189,7 +197,7 @@ static SLJIT_INLINE void free_chunk(void *chunk, sljit_uw size) - munmap(chunk, size); - } - --#endif -+#endif /* windows */ - - /* --------------------------------------------------------------------- */ - /* Common functions */ -@@ -261,6 +269,7 @@ SLJIT_API_FUNC_ATTRIBUTE void* sljit_malloc_exec(sljit_uw size) - while (free_block) { - if (free_block->size >= size) { - chunk_size = free_block->size; -+ SLJIT_UPDATE_WX_FLAGS(NULL, NULL, 0); - if (chunk_size > size + 64) { - /* We just cut a block from the end of the free block. */ - chunk_size -= size; -@@ -326,6 +335,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) - allocated_size -= header->size; - - /* Connecting free blocks together if possible. */ -+ SLJIT_UPDATE_WX_FLAGS(NULL, NULL, 0); - - /* If header->prev_size == 0, free_block will equal to header. - In this case, free_block->header.size will be > 0. */ -@@ -358,6 +368,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_exec(void* ptr) - } - } - -+ SLJIT_UPDATE_WX_FLAGS(NULL, NULL, 1); - SLJIT_ALLOCATOR_UNLOCK(); - } - -@@ -367,6 +378,7 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) - struct free_block* next_free_block; - - SLJIT_ALLOCATOR_LOCK(); -+ SLJIT_UPDATE_WX_FLAGS(NULL, NULL, 0); - - free_block = free_blocks; - while (free_block) { -@@ -381,5 +393,6 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_unused_memory_exec(void) - } - - SLJIT_ASSERT((total_size && free_blocks) || (!total_size && !free_blocks)); -+ SLJIT_UPDATE_WX_FLAGS(NULL, NULL, 1); - SLJIT_ALLOCATOR_UNLOCK(); - } --- -2.30.0 - diff --git a/deps/patches/pcre2-sljit-nomprotect.patch b/deps/patches/pcre2-sljit-nomprotect.patch deleted file mode 100644 index 3c2df1808630b..0000000000000 --- a/deps/patches/pcre2-sljit-nomprotect.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/sljit_src/sljitExecAllocator.c b/sljit_src/sljitExecAllocator.c -index 2e1c138..bae8cd6 100644 ---- a/sljit_src/sljitExecAllocator.c -+++ b/sljit_src/sljitExecAllocator.c -@@ -182,10 +182,12 @@ static SLJIT_INLINE void* alloc_chunk(sljit_uw size) - if (retval == MAP_FAILED) - return NULL; - -+#ifdef SLIJT_WX_OS_NEEDSCHEK - if (mprotect(retval, size, prot) < 0) { - munmap(retval, size); - return NULL; - } -+#endif - - SLJIT_UPDATE_WX_FLAGS(retval, (uint8_t *)retval + size, 0); - diff --git a/deps/pcre.mk b/deps/pcre.mk index 053a773e5609e..2fd8ad48b199f 100644 --- a/deps/pcre.mk +++ b/deps/pcre.mk @@ -6,26 +6,17 @@ PCRE_CFLAGS := -O3 PCRE_LDFLAGS := $(RPATH_ESCAPED_ORIGIN) $(SRCCACHE)/pcre2-$(PCRE_VER).tar.bz2: | $(SRCCACHE) - $(JLDOWNLOAD) $@ https://github.com/PhilipHazel/pcre2/releases/download/pcre2-$(PCRE_VER)/pcre2-$(PCRE_VER).tar.bz2 + $(JLDOWNLOAD) $@ https://github.com/PCRE2Project/pcre2/releases/download/pcre2-$(PCRE_VER)/pcre2-$(PCRE_VER).tar.bz2 $(SRCCACHE)/pcre2-$(PCRE_VER)/source-extracted: $(SRCCACHE)/pcre2-$(PCRE_VER).tar.bz2 $(JLCHECKSUM) $< cd $(dir $<) && $(TAR) jxf $(notdir $<) - cp $(SRCDIR)/patches/config.sub $(SRCCACHE)/pcre2-$(PCRE_VER)/config.sub echo 1 > $@ checksum-pcre: $(SRCCACHE)/pcre2-$(PCRE_VER).tar.bz2 $(JLCHECKSUM) $< -$(SRCCACHE)/pcre2-$(PCRE_VER)/pcre2-sljit-apple-silicon-support.patch-applied: $(SRCCACHE)/pcre2-$(PCRE_VER)/source-extracted - cd $(SRCCACHE)/pcre2-$(PCRE_VER) && patch -d src/sljit -p2 -f < $(SRCDIR)/patches/pcre2-sljit-apple-silicon-support.patch - echo 1 > $@ - -$(SRCCACHE)/pcre2-$(PCRE_VER)/pcre2-sljit-nomprotect.patch-applied: $(SRCCACHE)/pcre2-$(PCRE_VER)/pcre2-sljit-apple-silicon-support.patch-applied - cd $(SRCCACHE)/pcre2-$(PCRE_VER) && patch -d src/sljit -p2 -f < $(SRCDIR)/patches/pcre2-sljit-nomprotect.patch - echo 1 > $@ - -$(BUILDDIR)/pcre2-$(PCRE_VER)/build-configured: $(SRCCACHE)/pcre2-$(PCRE_VER)/source-extracted $(SRCCACHE)/pcre2-$(PCRE_VER)/pcre2-sljit-apple-silicon-support.patch-applied $(SRCCACHE)/pcre2-$(PCRE_VER)/pcre2-sljit-nomprotect.patch-applied +$(BUILDDIR)/pcre2-$(PCRE_VER)/build-configured: $(SRCCACHE)/pcre2-$(PCRE_VER)/source-extracted mkdir -p $(dir $@) cd $(dir $@) && \ $(dir $<)/configure $(CONFIGURE_COMMON) --enable-jit --includedir=$(build_includedir) CFLAGS="$(CFLAGS) $(PCRE_CFLAGS) -g -O0" LDFLAGS="$(LDFLAGS) $(PCRE_LDFLAGS)" diff --git a/stdlib/PCRE2_jll/Project.toml b/stdlib/PCRE2_jll/Project.toml index b7718fcf79f48..187eddb2a5541 100644 --- a/stdlib/PCRE2_jll/Project.toml +++ b/stdlib/PCRE2_jll/Project.toml @@ -1,6 +1,6 @@ name = "PCRE2_jll" uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" -version = "10.36.0+2" +version = "10.40.0+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/stdlib/PCRE2_jll/test/runtests.jl b/stdlib/PCRE2_jll/test/runtests.jl index b2446e7e5caab..21e7e7db7286b 100644 --- a/stdlib/PCRE2_jll/test/runtests.jl +++ b/stdlib/PCRE2_jll/test/runtests.jl @@ -6,5 +6,5 @@ using Test, Libdl, PCRE2_jll vstr = zeros(UInt8, 32) @test ccall((:pcre2_config_8, libpcre2_8), Cint, (UInt32, Ref{UInt8}), 11, vstr) > 0 vn = VersionNumber(split(unsafe_string(pointer(vstr)), " ")[1]) - @test vn == v"10.36.0" + @test vn == v"10.40.0" end From a02630ed8f6ccf52eea58e27875751a5491fc3f0 Mon Sep 17 00:00:00 2001 From: Jerry Ling Date: Sun, 22 May 2022 09:33:43 -0400 Subject: [PATCH 62/68] set default blas num threads to Sys.CPU_THREADS / 2 (#45412) Set default blas num threads to Sys.CPU_THREADS / 2 in absence of OPENBLAS_NUM_THREADS Co-authored-by: SamuraiAku <61489439+SamuraiAku@users.noreply.github.com> (cherry picked from commit 390503ebe26074646d880b43ef0f4cf17db37df0) --- stdlib/LinearAlgebra/src/LinearAlgebra.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stdlib/LinearAlgebra/src/LinearAlgebra.jl b/stdlib/LinearAlgebra/src/LinearAlgebra.jl index 7417e6256cef5..e8c2de0ac056a 100644 --- a/stdlib/LinearAlgebra/src/LinearAlgebra.jl +++ b/stdlib/LinearAlgebra/src/LinearAlgebra.jl @@ -581,6 +581,10 @@ function __init__() end # register a hook to disable BLAS threading Base.at_disable_library_threading(() -> BLAS.set_num_threads(1)) + + if !haskey(ENV, "OPENBLAS_NUM_THREADS") + BLAS.set_num_threads(max(1, Sys.CPU_THREADS ÷ 2)) + end end end # module LinearAlgebra From 6acc46b5ff8263c9dea6237fd3dc84944f0e502a Mon Sep 17 00:00:00 2001 From: Perry Fraser Date: Mon, 23 May 2022 03:31:50 -0400 Subject: [PATCH 63/68] Don't error when transposing a single character (#45420) This fixes an issue where an error would be thrown in the REPL if you tried to transpose an input that was a single character while your cursor was to the right of that character (e.g., "A|"). To fix this, let's move left once before we check for if we're at the start of a line. This does change behavior slightly in that the cursor can move left once without actually transposing anything, but this seems to match what Emacs does with M-x transpose-chars in an equivalent situation. (cherry picked from commit 9dd993e0604ab19f51e61c0f8d7f339599352e95) --- stdlib/REPL/src/LineEdit.jl | 3 ++- stdlib/REPL/test/lineedit.jl | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/stdlib/REPL/src/LineEdit.jl b/stdlib/REPL/src/LineEdit.jl index 89f57383d5e48..e3daa4677f077 100644 --- a/stdlib/REPL/src/LineEdit.jl +++ b/stdlib/REPL/src/LineEdit.jl @@ -1085,8 +1085,9 @@ function edit_transpose_chars(s::MIState) end function edit_transpose_chars(buf::IOBuffer) - position(buf) == 0 && return false + # Moving left but not transpoing anything is intentional, and matches Emacs's behavior eof(buf) && char_move_left(buf) + position(buf) == 0 && return false char_move_left(buf) pos = position(buf) a, b = read(buf, Char), read(buf, Char) diff --git a/stdlib/REPL/test/lineedit.jl b/stdlib/REPL/test/lineedit.jl index decad3eb07938..87028e239d5b8 100644 --- a/stdlib/REPL/test/lineedit.jl +++ b/stdlib/REPL/test/lineedit.jl @@ -375,6 +375,16 @@ let buf = IOBuffer() @test content(buf) == "βγαεδ" LineEdit.edit_transpose_chars(buf) @test content(buf) == "βγαδε" + + seek(buf, 0) + @inferred(LineEdit.edit_clear(buf)) + edit_insert(buf, "a") + LineEdit.edit_transpose_chars(buf) + @test content(buf) == "a" + seekend(buf) + LineEdit.edit_transpose_chars(buf) + @test content(buf) == "a" + @test position(buf) == 0 end @testset "edit_word_transpose" begin From 5403c43089764305d28f179d90f802ae2812c121 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Tue, 24 May 2022 14:41:02 +0200 Subject: [PATCH 64/68] bump Pkg version to latest 1.8 --- .../Pkg-7c1544f092a006556ce46dfaf9a93cb408b705d3.tar.gz/md5 | 1 + .../Pkg-7c1544f092a006556ce46dfaf9a93cb408b705d3.tar.gz/sha512 | 1 + .../Pkg-e31a3dc77201e1c7c469f6d4572c521f93fefb20.tar.gz/md5 | 1 - .../Pkg-e31a3dc77201e1c7c469f6d4572c521f93fefb20.tar.gz/sha512 | 1 - stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/Pkg-7c1544f092a006556ce46dfaf9a93cb408b705d3.tar.gz/md5 create mode 100644 deps/checksums/Pkg-7c1544f092a006556ce46dfaf9a93cb408b705d3.tar.gz/sha512 delete mode 100644 deps/checksums/Pkg-e31a3dc77201e1c7c469f6d4572c521f93fefb20.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-e31a3dc77201e1c7c469f6d4572c521f93fefb20.tar.gz/sha512 diff --git a/deps/checksums/Pkg-7c1544f092a006556ce46dfaf9a93cb408b705d3.tar.gz/md5 b/deps/checksums/Pkg-7c1544f092a006556ce46dfaf9a93cb408b705d3.tar.gz/md5 new file mode 100644 index 0000000000000..7b3aa9f19c01b --- /dev/null +++ b/deps/checksums/Pkg-7c1544f092a006556ce46dfaf9a93cb408b705d3.tar.gz/md5 @@ -0,0 +1 @@ +59386028556257c324385840a93bb43d diff --git a/deps/checksums/Pkg-7c1544f092a006556ce46dfaf9a93cb408b705d3.tar.gz/sha512 b/deps/checksums/Pkg-7c1544f092a006556ce46dfaf9a93cb408b705d3.tar.gz/sha512 new file mode 100644 index 0000000000000..fbe82d95808ae --- /dev/null +++ b/deps/checksums/Pkg-7c1544f092a006556ce46dfaf9a93cb408b705d3.tar.gz/sha512 @@ -0,0 +1 @@ +2cd00db324fab2ffe245f9804fc3d1cb8bc733e892d40ec18c02fd69b4c165111e42aacf4654a1d610f933d9bc6868fe9a01e3241116f796ef73fa7d48fd499c diff --git a/deps/checksums/Pkg-e31a3dc77201e1c7c469f6d4572c521f93fefb20.tar.gz/md5 b/deps/checksums/Pkg-e31a3dc77201e1c7c469f6d4572c521f93fefb20.tar.gz/md5 deleted file mode 100644 index f4cc3e5cde47d..0000000000000 --- a/deps/checksums/Pkg-e31a3dc77201e1c7c469f6d4572c521f93fefb20.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -df5033e44bde58e85642eabe9a3a118b diff --git a/deps/checksums/Pkg-e31a3dc77201e1c7c469f6d4572c521f93fefb20.tar.gz/sha512 b/deps/checksums/Pkg-e31a3dc77201e1c7c469f6d4572c521f93fefb20.tar.gz/sha512 deleted file mode 100644 index de5c95167ce9b..0000000000000 --- a/deps/checksums/Pkg-e31a3dc77201e1c7c469f6d4572c521f93fefb20.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -d3630f9fab8b72c9a42d5bb43a7ad4e9e024510b189dd63c581e989960d6478bd6c6c6676f702a0fea8be67c58182a7febd2b63c5934bc86068c7cd4168cdf9b diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 5706cef05e9a5..b33656cecd56a 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = e31a3dc77201e1c7c469f6d4572c521f93fefb20 +PKG_SHA1 = 7c1544f092a006556ce46dfaf9a93cb408b705d3 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 11a1ee2a484afc9d74c34bcd31b07c92c2096337 Mon Sep 17 00:00:00 2001 From: FX Coudert Date: Tue, 24 May 2022 10:24:30 -0600 Subject: [PATCH 65/68] Update p7zip to 17.04 (#45435) Co-authored-by: KristofferC (cherry picked from commit 86f5501f248bb30304fc19a9abc8affee98e42f8) --- deps/Versions.make | 2 +- deps/checksums/p7zip | 69 ++-- deps/p7zip.mk | 34 +- deps/patches/p7zip-12-CVE-2016-9296.patch | 23 -- deps/patches/p7zip-13-CVE-2017-17969.patch | 35 -- ...7zip-15-Enhanced-encryption-strength.patch | 298 ------------------ deps/patches/p7zip-Windows_ErrorMsg.patch | 33 -- stdlib/p7zip_jll/Project.toml | 2 +- 8 files changed, 46 insertions(+), 450 deletions(-) delete mode 100644 deps/patches/p7zip-12-CVE-2016-9296.patch delete mode 100644 deps/patches/p7zip-13-CVE-2017-17969.patch delete mode 100644 deps/patches/p7zip-15-Enhanced-encryption-strength.patch delete mode 100644 deps/patches/p7zip-Windows_ErrorMsg.patch diff --git a/deps/Versions.make b/deps/Versions.make index b865922c7f3d9..7442543716ad4 100644 --- a/deps/Versions.make +++ b/deps/Versions.make @@ -90,7 +90,7 @@ OPENLIBM_JLL_NAME := OpenLibm PATCHELF_VER := 0.13 # p7zip -P7ZIP_VER := 16.2.0 +P7ZIP_VER := 17.04 P7ZIP_JLL_NAME := p7zip # PCRE diff --git a/deps/checksums/p7zip b/deps/checksums/p7zip index 0df5ed96067b1..faeedc1a05d72 100644 --- a/deps/checksums/p7zip +++ b/deps/checksums/p7zip @@ -1,34 +1,35 @@ -p7zip-16.2.0.tar.bz2/md5/a0128d661cfe7cc8c121e73519c54fbf -p7zip-16.2.0.tar.bz2/sha512/d2c4d53817f96bb4c7683f42045198d4cd509cfc9c3e2cb85c8d9dc4ab6dfa7496449edeac4e300ecf986a9cbbc90bd8f8feef8156895d94617c04e507add55f -p7zip.v16.2.1+1.aarch64-apple-darwin.tar.gz/md5/12485086522a08b62dfef503b61af007 -p7zip.v16.2.1+1.aarch64-apple-darwin.tar.gz/sha512/dc9d92b294a65c55d8742b33df0d905a8cd1e80500647b33537fd404167aaa43a01280bb19035a9e4da94dd49c6ee712a0fbf455b9661af487e1c35a9a09eda7 -p7zip.v16.2.1+1.aarch64-linux-gnu.tar.gz/md5/35a760ced992c9cd4c6085e40394299b -p7zip.v16.2.1+1.aarch64-linux-gnu.tar.gz/sha512/da3123601db48cead255240e048e33de401de52cbddddbc1e109dd7b3b36645251008108c7545abaf09e0b2803198ac4067b00a3f0ff7fe31f65a5de4ce49710 -p7zip.v16.2.1+1.aarch64-linux-musl.tar.gz/md5/f795313bc73c9f635a63861126c838eb -p7zip.v16.2.1+1.aarch64-linux-musl.tar.gz/sha512/886b0e9e2476915be2c7106a8fb4547e9326d50fad93d8002ca97f4e35a856fee43a2350b48404f112938da6fc19255cb2dfb31e3112c74551d4a3ccb44a7fbf -p7zip.v16.2.1+1.armv6l-linux-gnueabihf.tar.gz/md5/644ed1b6a5d7bb16407cea5264ef45ce -p7zip.v16.2.1+1.armv6l-linux-gnueabihf.tar.gz/sha512/3cbdb56faca44ac2a3ea4cba35b8913811a2d3602a689496228968fb17c23b191ab3e01b43f619526cd8ea0f33c5a4453d2b5cca7437026e54b2c164acb1e8ee -p7zip.v16.2.1+1.armv6l-linux-musleabihf.tar.gz/md5/219fdda71c08848844b4630e613bf35d -p7zip.v16.2.1+1.armv6l-linux-musleabihf.tar.gz/sha512/419297b14aa820f8f49d6add367fe3a7153be18546e41e9f8bf6bbddada7535301dd3ea524089981046fc739b8094cff9113fb2aeca2947e796a8e6b74414245 -p7zip.v16.2.1+1.armv7l-linux-gnueabihf.tar.gz/md5/919e6508e4b2adb82fa2493a805875e9 -p7zip.v16.2.1+1.armv7l-linux-gnueabihf.tar.gz/sha512/cf8f58ee590e23aa6fe348b639f2b052fbc0ed52ecf7ce1e370f7dc3255e47727ef65a109b14cd045d59201ef8a5b426eb05b167967ce95581a35df7a6b67400 -p7zip.v16.2.1+1.armv7l-linux-musleabihf.tar.gz/md5/8bfb81a9a4d31ac9f05b59c19490461e -p7zip.v16.2.1+1.armv7l-linux-musleabihf.tar.gz/sha512/6b13c1971e7049613aefd4a2bad64d534ffc7293efb037b2da92e23754462fc3872169399f3a9fe34bc337b900ecc4fccc878e3e54067238b3f890c09f8e05f0 -p7zip.v16.2.1+1.i686-linux-gnu.tar.gz/md5/f62eefb6fb2724082933e95d706b232f -p7zip.v16.2.1+1.i686-linux-gnu.tar.gz/sha512/43a669bb64e0318c16feade75ade6e4ac73e056fb33479268e217310fa469a8f535ace13b8ade45495d96d8a540e1c247dcdb8fd7044c8096693f3766f00224f -p7zip.v16.2.1+1.i686-linux-musl.tar.gz/md5/8a80bbfcb8c4a05d6c56539640a7bfaf -p7zip.v16.2.1+1.i686-linux-musl.tar.gz/sha512/38ce14788fbfd964fa446c98c89ecd3854c732f5529406d6d650d8f0ac4a657caeea8ae2985370f5cee129d974a4bafa8cd164fd1c11ae0cad5191e9640534f0 -p7zip.v16.2.1+1.i686-w64-mingw32.tar.gz/md5/d55077826cdfe69747efd4fd53b81e18 -p7zip.v16.2.1+1.i686-w64-mingw32.tar.gz/sha512/71ee03bbb9916eff2e7807ff25d1c1992c209506c4602f570095ee0cd12355ed4590d77dfd090085a109604c4cbad221154bfd55d5fd79bf35c76b3b43c67a25 -p7zip.v16.2.1+1.powerpc64le-linux-gnu.tar.gz/md5/16682edc596bc1f7d6311339644070fb -p7zip.v16.2.1+1.powerpc64le-linux-gnu.tar.gz/sha512/09c3bfbae7c4ab2757fdee0dac4baf71f6fa7b99aab48c5260ed9481c5e7b05317f7a6d466c543ffe46318281011b61c5652fef33466c02a5b24b3c39d92137d -p7zip.v16.2.1+1.x86_64-apple-darwin.tar.gz/md5/6d7873510fca444740ab2f4ae701ae3a -p7zip.v16.2.1+1.x86_64-apple-darwin.tar.gz/sha512/e6fc0c669b62eb2e6f11d07e840ce44beb6c8981750ac4fb5d7401cf00916465f97f8b3a49c73777d893752a7df9bed8bf40068fe7339df88942a21aff4e9d2a -p7zip.v16.2.1+1.x86_64-linux-gnu.tar.gz/md5/2cd2efe4d51967ac8acf24a6f2c80893 -p7zip.v16.2.1+1.x86_64-linux-gnu.tar.gz/sha512/a0fdf061b5d7da97134eee7fc9afb468d8bee01108843814432d318d2b5c6217772e62700a015d5be41010ecf7b613218ed9e8ea6e2da2a24d1e5c15a1734a59 -p7zip.v16.2.1+1.x86_64-linux-musl.tar.gz/md5/f5a312e21abd7f24100e91eefa875c7f -p7zip.v16.2.1+1.x86_64-linux-musl.tar.gz/sha512/034b00d0685da5456b91f45c0b4196e0aa21436e67ecd7a09318a578a814491774ca5c2ce2c49f6b17e1665d9c8a896a0f2f6fca6d3260208ad8be44c1dce656 -p7zip.v16.2.1+1.x86_64-unknown-freebsd.tar.gz/md5/1e647ff7fd8bf2dfdcdd569c743e9c8c -p7zip.v16.2.1+1.x86_64-unknown-freebsd.tar.gz/sha512/e868eb1bab65ff383177ed0e929ff0db084df1f4b144430098f25cb8df788696113fe466ecf756c4ca61439fa8eed8c8a3fc396aec2972bea6ec7b3b0be51baa -p7zip.v16.2.1+1.x86_64-w64-mingw32.tar.gz/md5/70d58fe372550313b18437f58cd249e1 -p7zip.v16.2.1+1.x86_64-w64-mingw32.tar.gz/sha512/1908d3dfd218e33c8e85366e02d920e237111b5fdb8bf028d8f7a2029ec7292c465d4d0ee50f58ef186fa8c83bfe33ea98d0bacdbcbb9c345b71eeb038cbda89 +p7zip.v17.4.0+0.aarch64-apple-darwin.tar.gz/md5/af8134ed9c24b99d69e4edb4d5226ca5 +p7zip.v17.4.0+0.aarch64-apple-darwin.tar.gz/sha512/b8bb6aee60a54cca37568af8b2d9baedd892ba0d4918b93bcb29d74189524af7115901f4fabafb1ca58ed17e97c59846fcdfbd460abc81059806802b0a7be840 +p7zip.v17.4.0+0.aarch64-linux-gnu.tar.gz/md5/20abac5ebb99f31742878013c02f96a3 +p7zip.v17.4.0+0.aarch64-linux-gnu.tar.gz/sha512/6d8ebf895b969b1f707d0c23a19db4cd0dee47957d076e6e389395e09404d55bfcb78bb14bb67bb35b93b6a0072f2b4f097d839503d1ccab62b4ce28939dc71d +p7zip.v17.4.0+0.aarch64-linux-musl.tar.gz/md5/185c979c7419b7ded3832c0f5cfd3b77 +p7zip.v17.4.0+0.aarch64-linux-musl.tar.gz/sha512/722e880c9f111738cb4cde84bf62c36892dbefdba625ae2b9e0fae76a7b1eabfa481a9838fbf9667223f19f62b6f09fcfd42b50c2bff7a65af0fae3616250fc7 +p7zip.v17.4.0+0.armv6l-linux-gnueabihf.tar.gz/md5/dceb37181763f86bf12f8ca473cf3403 +p7zip.v17.4.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/51e409bbcd3c54838cb3219b2476c8b45c8340e0a2fd26cced0d8484ae7f51711723e06e9023fce9ae9a1b51b5fb94aba536428ce2a5c5902b38498a0b3c2b50 +p7zip.v17.4.0+0.armv6l-linux-musleabihf.tar.gz/md5/193ecd888787ea03a500d102a7e33afa +p7zip.v17.4.0+0.armv6l-linux-musleabihf.tar.gz/sha512/d525aad33f5ed27dc993f31c6db2996b830716bfac9bc7c49cb462ea3f0b412d0d3267765b9952c85e9c9be31d36d095d55ba89c0fa2c92823d9490372389c95 +p7zip.v17.4.0+0.armv7l-linux-gnueabihf.tar.gz/md5/096f11a7f1af5ff730bb8cfef22e335e +p7zip.v17.4.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/1866ffd0169e0795594aaa70f1af8102ebbd79b3cafaadfb9c6a537dac0cdbb6eb7c31ad5165a975508c1b850744f94b60d9c530d658cdcc5536a474203cff21 +p7zip.v17.4.0+0.armv7l-linux-musleabihf.tar.gz/md5/fef1576982f45d1922582f6f7a7d6665 +p7zip.v17.4.0+0.armv7l-linux-musleabihf.tar.gz/sha512/71061585b32fa1a8e0a403a60c07e9f90586291a9799d7e2d6f7e6ec9f7b0ebf4b45ed080efd87cad82c45f71ec9a14cbcf9134a73bad4f5e3329f23bc6df01a +p7zip.v17.4.0+0.i686-linux-gnu.tar.gz/md5/8818389b3bf00f10c6a39fe0c4a331b4 +p7zip.v17.4.0+0.i686-linux-gnu.tar.gz/sha512/bec2051a258f7e8a762b7cd4324e7b8f00fe5d99d48f05fb3557c41604e8b08af9ab66ab830f4a48086656be41aaf011b2aae0fb530e0ffefec38689f85a3bb5 +p7zip.v17.4.0+0.i686-linux-musl.tar.gz/md5/4ed9c16a65ed1d656aa214013e46eb28 +p7zip.v17.4.0+0.i686-linux-musl.tar.gz/sha512/7a5b3e15d0038bea0de7fc28ce058d7f93b8e04f271e30953a6b52d2b5d71f59d10177033e888a50cf8dfeb4f44bcf3271c9b9d1b28d0122ab2b239decdad446 +p7zip.v17.4.0+0.i686-w64-mingw32.tar.gz/md5/d06cff2ec0b7c8415700587f931ce1ac +p7zip.v17.4.0+0.i686-w64-mingw32.tar.gz/sha512/ed72440f5306a57465a70b00bff33185a83c3e223844a79aa0b0d1fbe30dbd35da75e6188725aa621f5c4574a09527daf1e4893c7c6979ab91b2c09b4979dbcb +p7zip.v17.4.0+0.powerpc64le-linux-gnu.tar.gz/md5/949ca7d111e497b82c9c762e5ac63a6b +p7zip.v17.4.0+0.powerpc64le-linux-gnu.tar.gz/sha512/4842e0d44bf6380100723209596f526181fefe8a81d59c28658d03ea16600e71d010d5c7898b4c943efdd9caaa2301c3fdb0dccb343d631d1734acda1c559f65 +p7zip.v17.4.0+0.x86_64-apple-darwin.tar.gz/md5/2322c7a08f62592ca394a716949008bc +p7zip.v17.4.0+0.x86_64-apple-darwin.tar.gz/sha512/9549f3e1052730ce13414636b32f0d1a9a1ac944a2b622380eac0da144b11fd65d437afe877ba6797d651da9c4ec77f0ebd3e515146caceaa2524829419eda48 +p7zip.v17.4.0+0.x86_64-linux-gnu.tar.gz/md5/a21b12946a62ef3688d5fc965974e8f7 +p7zip.v17.4.0+0.x86_64-linux-gnu.tar.gz/sha512/d32faeac23acf8a023f65350ba1d62bb3d9f904e32570ae03b8fb0a5375758784dd95be8caeecd007cbde40e103854a077e2c817f62afa72491f3b8966deb738 +p7zip.v17.4.0+0.x86_64-linux-musl.tar.gz/md5/c448e872d4ad66beb2d46d9134952f2f +p7zip.v17.4.0+0.x86_64-linux-musl.tar.gz/sha512/92588f4817e145ef655c718dec049e7f43dd93644f43f19cd320643fac5f5b2312837c7a6c3e782e97fd08747311c58ed4657484f8bc778942fc5206ff8ea4e5 +p7zip.v17.4.0+0.x86_64-unknown-freebsd.tar.gz/md5/2cca6259a2eb1b0fea777d566267bf05 +p7zip.v17.4.0+0.x86_64-unknown-freebsd.tar.gz/sha512/92f90e2be4a8b8fcd80a4ceacac8bbab750913526b85f9279f8ee9ed91b77248b5de2d35d0c6241d0ad51fda185f4cb1ead1dcc9d23e2bef35e0b61efe3c3170 +p7zip.v17.4.0+0.x86_64-w64-mingw32.tar.gz/md5/5d272c78d7ffb40da0f333463f3cc098 +p7zip.v17.4.0+0.x86_64-w64-mingw32.tar.gz/sha512/2d999c6df4786cec1bba396b3a651a63740f4b799e9fc11754afd24438076e898daae74b4d3c7072450428e89881991e8884711cd4c349879a00c7aeeb4e1d3e +p7zip-17.04.tar.gz/md5/00acfd6be87848231722d2d53f89e4a5 +p7zip-17.04.tar.gz/sha512/ad176db5b657b1c39584f6792c47978d94f2f1ccb1cf5bdb0f52ab31a7356b3822f4a922152c4253f4aa7e79166ba052b6592530b7a38f548cd555fe9c008be3 + diff --git a/deps/p7zip.mk b/deps/p7zip.mk index 20c85602f767a..8c0d11d74a061 100644 --- a/deps/p7zip.mk +++ b/deps/p7zip.mk @@ -1,36 +1,20 @@ ## p7zip ## ifneq ($(USE_BINARYBUILDER_P7ZIP),1) -# Force optimization for P7ZIP flags (Issue #11668) -$(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.bz2: | $(SRCCACHE) - $(JLDOWNLOAD) $@ https://downloads.sourceforge.net/project/p7zip/p7zip/16.02/p7zip_16.02_src_all.tar.bz2 -$(BUILDDIR)/p7zip-$(P7ZIP_VER)/source-extracted: $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.bz2 +$(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz: | $(SRCCACHE) + $(JLDOWNLOAD) $@ https://github.com/jinfeihan57/p7zip/archive/refs/tags/v$(P7ZIP_VER).tar.gz + +$(BUILDDIR)/p7zip-$(P7ZIP_VER)/source-extracted: $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz $(JLCHECKSUM) $< mkdir -p $(dir $@) - cd $(dir $@) && $(TAR) --strip-components 1 -jxf $< + cd $(dir $@) && $(TAR) --strip-components 1 -zxf $< echo 1 > $@ -checksum-p7zip: $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.bz2 +checksum-p7zip: $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz $(JLCHECKSUM) $< -$(BUILDDIR)/p7zip-$(P7ZIP_VER)/p7zip-12-CVE-2016-9296.patch-applied: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/source-extracted - cd $(dir $@) && patch -p1 -f < $(SRCDIR)/patches/p7zip-12-CVE-2016-9296.patch - echo 1 > $@ - -$(BUILDDIR)/p7zip-$(P7ZIP_VER)/p7zip-13-CVE-2017-17969.patch-applied: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/p7zip-12-CVE-2016-9296.patch-applied - cd $(dir $@) && patch -p1 -f < $(SRCDIR)/patches/p7zip-13-CVE-2017-17969.patch - echo 1 > $@ - -$(BUILDDIR)/p7zip-$(P7ZIP_VER)/p7zip-15-Enhanced-encryption-strength.patch-applied: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/p7zip-13-CVE-2017-17969.patch-applied - cd $(dir $@) && patch -p4 -f < $(SRCDIR)/patches/p7zip-15-Enhanced-encryption-strength.patch - echo 1 > $@ - -$(BUILDDIR)/p7zip-$(P7ZIP_VER)/p7zip-Windows_ErrorMsg.patch-applied: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/p7zip-15-Enhanced-encryption-strength.patch-applied - cd $(dir $@) && patch -p0 -f < $(SRCDIR)/patches/p7zip-Windows_ErrorMsg.patch - echo 1 > $@ - -$(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/p7zip-Windows_ErrorMsg.patch-applied +$(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/source-extracted $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-compiled: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured $(MAKE) -C $(dir $<) $(MAKE_COMMON) CC="$(CC)" CXX="$(CXX)" 7za echo 1 > $@ @@ -49,10 +33,10 @@ clean-p7zip: -$(MAKE) -C $(BUILDDIR)/p7zip-$(P7ZIP_VER) clean distclean-p7zip: - -rm -rf $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.bz2 $(SRCCACHE)/p7zip-$(P7ZIP_VER) $(BUILDDIR)/p7zip-$(P7ZIP_VER) + -rm -rf $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz $(SRCCACHE)/p7zip-$(P7ZIP_VER) $(BUILDDIR)/p7zip-$(P7ZIP_VER) -get-p7zip: $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.bz2 +get-p7zip: $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz extract-p7zip: $(SRCCACHE)/p7zip-$(P7ZIP_VER)/source-extracted configure-p7zip: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured compile-p7zip: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-compiled diff --git a/deps/patches/p7zip-12-CVE-2016-9296.patch b/deps/patches/p7zip-12-CVE-2016-9296.patch deleted file mode 100644 index 42245c92c0aae..0000000000000 --- a/deps/patches/p7zip-12-CVE-2016-9296.patch +++ /dev/null @@ -1,23 +0,0 @@ -From: Robert Luberda -Date: Sat, 19 Nov 2016 08:48:08 +0100 -Subject: Fix nullptr dereference (CVE-2016-9296) - -Patch taken from https://sourceforge.net/p/p7zip/bugs/185/ ---- - CPP/7zip/Archive/7z/7zIn.cpp | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/CPP/7zip/Archive/7z/7zIn.cpp b/CPP/7zip/Archive/7z/7zIn.cpp -index b0c6b98..7c6dde2 100644 ---- a/CPP/7zip/Archive/7z/7zIn.cpp -+++ b/CPP/7zip/Archive/7z/7zIn.cpp -@@ -1097,7 +1097,8 @@ HRESULT CInArchive::ReadAndDecodePackedStreams( - if (CrcCalc(data, unpackSize) != folders.FolderCRCs.Vals[i]) - ThrowIncorrect(); - } -- HeadersSize += folders.PackPositions[folders.NumPackStreams]; -+ if (folders.PackPositions) -+ HeadersSize += folders.PackPositions[folders.NumPackStreams]; - return S_OK; - } - diff --git a/deps/patches/p7zip-13-CVE-2017-17969.patch b/deps/patches/p7zip-13-CVE-2017-17969.patch deleted file mode 100644 index a9787c4a90886..0000000000000 --- a/deps/patches/p7zip-13-CVE-2017-17969.patch +++ /dev/null @@ -1,35 +0,0 @@ -From: =?utf-8?q?Antoine_Beaupr=C3=A9?= -Date: Fri, 2 Feb 2018 11:11:41 +0100 -Subject: Heap-based buffer overflow in 7zip/Compress/ShrinkDecoder.cpp - -Origin: vendor, https://sourceforge.net/p/p7zip/bugs/_discuss/thread/0920f369/27d7/attachment/CVE-2017-17969.patch -Forwarded: https://sourceforge.net/p/p7zip/bugs/_discuss/thread/0920f369/#27d7 -Bug: https://sourceforge.net/p/p7zip/bugs/204/ -Bug-Debian: https://bugs.debian.org/888297 -Bug-Debian-Security: https://security-tracker.debian.org/tracker/CVE-2017-17969 -Reviewed-by: Salvatore Bonaccorso -Last-Update: 2018-02-01 -Applied-Upstream: 18.00-beta ---- - CPP/7zip/Compress/ShrinkDecoder.cpp | 7 ++++++- - 1 file changed, 6 insertions(+), 1 deletion(-) - -diff --git a/CPP/7zip/Compress/ShrinkDecoder.cpp b/CPP/7zip/Compress/ShrinkDecoder.cpp -index 80b7e67..ca37764 100644 ---- a/CPP/7zip/Compress/ShrinkDecoder.cpp -+++ b/CPP/7zip/Compress/ShrinkDecoder.cpp -@@ -121,8 +121,13 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream * - { - _stack[i++] = _suffixes[cur]; - cur = _parents[cur]; -+ if (cur >= kNumItems || i >= kNumItems) -+ break; - } -- -+ -+ if (cur >= kNumItems || i >= kNumItems) -+ break; -+ - _stack[i++] = (Byte)cur; - lastChar2 = (Byte)cur; - diff --git a/deps/patches/p7zip-15-Enhanced-encryption-strength.patch b/deps/patches/p7zip-15-Enhanced-encryption-strength.patch deleted file mode 100644 index ab1cfb9c743fb..0000000000000 --- a/deps/patches/p7zip-15-Enhanced-encryption-strength.patch +++ /dev/null @@ -1,298 +0,0 @@ -From ea31bbe661abef761e49983b56923e6523b9463a Mon Sep 17 00:00:00 2001 -From: aone -Date: Thu, 7 Mar 2019 10:06:16 +0100 -Subject: [PATCH] Enhanced encryption strength from 7-Zip 19.00 - -https://github.com/aonez/Keka/issues/379 -https://sourceforge.net/p/sevenzip/bugs/2176 ---- - .../CPP/7zip/Archive/Wim/WimHandlerOut.cpp | 2 +- - Bin/p7zip/source/CPP/7zip/Crypto/7zAes.cpp | 4 +- - Bin/p7zip/source/CPP/7zip/Crypto/RandGen.cpp | 135 ++++++++++++++++-- - Bin/p7zip/source/CPP/7zip/Crypto/RandGen.h | 19 +++ - Bin/p7zip/source/CPP/7zip/Crypto/WzAes.cpp | 2 +- - .../source/CPP/7zip/Crypto/ZipCrypto.cpp | 2 +- - 6 files changed, 146 insertions(+), 18 deletions(-) - -diff --git a/Bin/p7zip/source/CPP/7zip/Archive/Wim/WimHandlerOut.cpp b/Bin/p7zip/source/CPP/7zip/Archive/Wim/WimHandlerOut.cpp -index 1d198df0..39679883 100644 ---- a/Bin/p7zip/source/CPP/7zip/Archive/Wim/WimHandlerOut.cpp -+++ b/Bin/p7zip/source/CPP/7zip/Archive/Wim/WimHandlerOut.cpp -@@ -671,7 +671,7 @@ void CHeader::SetDefaultFields(bool useLZX) - ChunkSize = kChunkSize; - ChunkSizeBits = kChunkSizeBits; - } -- g_RandomGenerator.Generate(Guid, 16); -+ MY_RAND_GEN(Guid, 16); - PartNumber = 1; - NumParts = 1; - NumImages = 1; -diff --git a/Bin/p7zip/source/CPP/7zip/Crypto/7zAes.cpp b/Bin/p7zip/source/CPP/7zip/Crypto/7zAes.cpp -index d33b562a..2ed69bad 100644 ---- a/Bin/p7zip/source/CPP/7zip/Crypto/7zAes.cpp -+++ b/Bin/p7zip/source/CPP/7zip/Crypto/7zAes.cpp -@@ -164,8 +164,8 @@ STDMETHODIMP CEncoder::ResetInitVector() - { - for (unsigned i = 0; i < sizeof(_iv); i++) - _iv[i] = 0; -- _ivSize = 8; -- g_RandomGenerator.Generate(_iv, _ivSize); -+ _ivSize = 16; -+ MY_RAND_GEN(_iv, _ivSize); - return S_OK; - } - -diff --git a/Bin/p7zip/source/CPP/7zip/Crypto/RandGen.cpp b/Bin/p7zip/source/CPP/7zip/Crypto/RandGen.cpp -index f5ea31f0..a70f4ec8 100644 ---- a/Bin/p7zip/source/CPP/7zip/Crypto/RandGen.cpp -+++ b/Bin/p7zip/source/CPP/7zip/Crypto/RandGen.cpp -@@ -2,14 +2,44 @@ - - #include "StdAfx.h" - -+#include "RandGen.h" -+ -+#ifndef USE_STATIC_SYSTEM_RAND -+ - #ifndef _7ZIP_ST - #include "../../Windows/Synchronization.h" - #endif - --#include "RandGen.h" - --#ifndef _WIN32 -+#ifdef _WIN32 -+ -+#ifdef _WIN64 -+#define USE_STATIC_RtlGenRandom -+#endif -+ -+#ifdef USE_STATIC_RtlGenRandom -+ -+#include -+ -+EXTERN_C_BEGIN -+#ifndef RtlGenRandom -+ #define RtlGenRandom SystemFunction036 -+ BOOLEAN WINAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength); -+#endif -+EXTERN_C_END -+ -+#else -+EXTERN_C_BEGIN -+typedef BOOLEAN (WINAPI * Func_RtlGenRandom)(PVOID RandomBuffer, ULONG RandomBufferLength); -+EXTERN_C_END -+#endif -+ -+ -+#else - #include -+#include -+#include -+#include - #define USE_POSIX_TIME - #define USE_POSIX_TIME2 - #endif -@@ -21,11 +51,9 @@ - #endif - #endif - --// This is not very good random number generator. --// Please use it only for salt. --// First generated data block depends from timer and processID. -+// The seed and first generated data block depend from processID, -+// theadID, timer and system random generator, if available. - // Other generated data blocks depend from previous state --// Maybe it's possible to restore original timer value from generated value. - - #define HASH_UPD(x) Sha256_Update(&hash, (const Byte *)&x, sizeof(x)); - -@@ -34,25 +62,102 @@ void CRandomGenerator::Init() - CSha256 hash; - Sha256_Init(&hash); - -+ unsigned numIterations = 1000; -+ -+ { -+ #ifndef UNDER_CE -+ const unsigned kNumIterations_Small = 100; -+ const unsigned kBufSize = 32; -+ Byte buf[kBufSize]; -+ #endif -+ - #ifdef _WIN32 -+ - DWORD w = ::GetCurrentProcessId(); - HASH_UPD(w); - w = ::GetCurrentThreadId(); - HASH_UPD(w); -+ -+ #ifdef UNDER_CE -+ /* -+ if (CeGenRandom(kBufSize, buf)) -+ { -+ numIterations = kNumIterations_Small; -+ Sha256_Update(&hash, buf, kBufSize); -+ } -+ */ -+ #elif defined(USE_STATIC_RtlGenRandom) -+ if (RtlGenRandom(buf, kBufSize)) -+ { -+ numIterations = kNumIterations_Small; -+ Sha256_Update(&hash, buf, kBufSize); -+ } - #else -+ { -+ HMODULE hModule = ::LoadLibrary(TEXT("Advapi32.dll")); -+ if (hModule) -+ { -+ // SystemFunction036() is real name of RtlGenRandom() function -+ Func_RtlGenRandom my_RtlGenRandom = (Func_RtlGenRandom)GetProcAddress(hModule, "SystemFunction036"); -+ if (my_RtlGenRandom) -+ { -+ if (my_RtlGenRandom(buf, kBufSize)) -+ { -+ numIterations = kNumIterations_Small; -+ Sha256_Update(&hash, buf, kBufSize); -+ } -+ } -+ ::FreeLibrary(hModule); -+ } -+ } -+ #endif -+ -+ #else -+ - pid_t pid = getpid(); - HASH_UPD(pid); - pid = getppid(); - HASH_UPD(pid); -+ -+ { -+ int f = open("/dev/urandom", O_RDONLY); -+ unsigned numBytes = kBufSize; -+ if (f >= 0) -+ { -+ do -+ { -+ int n = read(f, buf, numBytes); -+ if (n <= 0) -+ break; -+ Sha256_Update(&hash, buf, n); -+ numBytes -= n; -+ } -+ while (numBytes); -+ close(f); -+ if (numBytes == 0) -+ numIterations = kNumIterations_Small; -+ } -+ } -+ /* -+ { -+ int n = getrandom(buf, kBufSize, 0); -+ if (n > 0) -+ { -+ Sha256_Update(&hash, buf, n); -+ if (n == kBufSize) -+ numIterations = kNumIterations_Small; -+ } -+ } -+ */ -+ -+ #endif -+ } -+ -+ #ifdef _DEBUG -+ numIterations = 2; - #endif - -- for (unsigned i = 0; i < -- #ifdef _DEBUG -- 2; -- #else -- 1000; -- #endif -- i++) -+ do - { - #ifdef _WIN32 - LARGE_INTEGER v; -@@ -83,6 +188,8 @@ void CRandomGenerator::Init() - Sha256_Update(&hash, _buff, SHA256_DIGEST_SIZE); - } - } -+ while (--numIterations); -+ - Sha256_Final(&hash, _buff); - _needInit = false; - } -@@ -120,3 +227,5 @@ void CRandomGenerator::Generate(Byte *data, unsigned size) - } - - CRandomGenerator g_RandomGenerator; -+ -+#endif -diff --git a/Bin/p7zip/source/CPP/7zip/Crypto/RandGen.h b/Bin/p7zip/source/CPP/7zip/Crypto/RandGen.h -index cfdcd60d..5122ec4b 100644 ---- a/Bin/p7zip/source/CPP/7zip/Crypto/RandGen.h -+++ b/Bin/p7zip/source/CPP/7zip/Crypto/RandGen.h -@@ -5,6 +5,21 @@ - - #include "../../../C/Sha256.h" - -+#ifdef _WIN64 -+// #define USE_STATIC_SYSTEM_RAND -+#endif -+ -+#ifdef USE_STATIC_SYSTEM_RAND -+ -+#ifdef _WIN32 -+#include -+#define MY_RAND_GEN(data, size) RtlGenRandom(data, size) -+#else -+#define MY_RAND_GEN(data, size) getrandom(data, size, 0) -+#endif -+ -+#else -+ - class CRandomGenerator - { - Byte _buff[SHA256_DIGEST_SIZE]; -@@ -18,4 +33,8 @@ public: - - extern CRandomGenerator g_RandomGenerator; - -+#define MY_RAND_GEN(data, size) g_RandomGenerator.Generate(data, size) -+ -+#endif -+ - #endif -diff --git a/Bin/p7zip/source/CPP/7zip/Crypto/WzAes.cpp b/Bin/p7zip/source/CPP/7zip/Crypto/WzAes.cpp -index 4572f06e..d415ab84 100644 ---- a/Bin/p7zip/source/CPP/7zip/Crypto/WzAes.cpp -+++ b/Bin/p7zip/source/CPP/7zip/Crypto/WzAes.cpp -@@ -96,7 +96,7 @@ STDMETHODIMP CBaseCoder::Init() - HRESULT CEncoder::WriteHeader(ISequentialOutStream *outStream) - { - unsigned saltSize = _key.GetSaltSize(); -- g_RandomGenerator.Generate(_key.Salt, saltSize); -+ MY_RAND_GEN(_key.Salt, saltSize); - Init2(); - RINOK(WriteStream(outStream, _key.Salt, saltSize)); - return WriteStream(outStream, _key.PwdVerifComputed, kPwdVerifSize); -diff --git a/Bin/p7zip/source/CPP/7zip/Crypto/ZipCrypto.cpp b/Bin/p7zip/source/CPP/7zip/Crypto/ZipCrypto.cpp -index ae715063..8610297a 100644 ---- a/Bin/p7zip/source/CPP/7zip/Crypto/ZipCrypto.cpp -+++ b/Bin/p7zip/source/CPP/7zip/Crypto/ZipCrypto.cpp -@@ -49,7 +49,7 @@ HRESULT CEncoder::WriteHeader_Check16(ISequentialOutStream *outStream, UInt16 cr - PKZIP 2.0+ used 1 byte CRC check. It's more secure. - We also use 1 byte CRC. */ - -- g_RandomGenerator.Generate(h, kHeaderSize - 1); -+ MY_RAND_GEN(h, kHeaderSize - 1); - // h[kHeaderSize - 2] = (Byte)(crc); - h[kHeaderSize - 1] = (Byte)(crc >> 8); - --- -2.17.1 - diff --git a/deps/patches/p7zip-Windows_ErrorMsg.patch b/deps/patches/p7zip-Windows_ErrorMsg.patch deleted file mode 100644 index 71de3e9f59c86..0000000000000 --- a/deps/patches/p7zip-Windows_ErrorMsg.patch +++ /dev/null @@ -1,33 +0,0 @@ -This fixes the build with Clang 6.0: - - ../../../../CPP/Windows/ErrorMsg.cpp:24:10: error: case value evaluates to -2147024809, which cannot be narrowed to type 'DWORD' (aka 'unsigned int') [-Wc++11-narrowing] - case E_INVALIDARG : txt = "E_INVALIDARG"; break ; - ^ - ../../../../CPP/Common/MyWindows.h:89:22: note: expanded from macro 'E_INVALIDARG' - #define E_INVALIDARG ((HRESULT)0x80070057L) - ^ - -The HRESULT cast in the macro causes the value to be read as signed int. ---- CPP/Windows/ErrorMsg.cpp.orig 2015-01-18 18:20:28 UTC -+++ CPP/Windows/ErrorMsg.cpp -@@ -15,13 +15,13 @@ UString MyFormatMessage(DWORD errorCode) - - switch(errorCode) { - case ERROR_NO_MORE_FILES : txt = "No more files"; break ; -- case E_NOTIMPL : txt = "E_NOTIMPL"; break ; -- case E_NOINTERFACE : txt = "E_NOINTERFACE"; break ; -- case E_ABORT : txt = "E_ABORT"; break ; -- case E_FAIL : txt = "E_FAIL"; break ; -- case STG_E_INVALIDFUNCTION : txt = "STG_E_INVALIDFUNCTION"; break ; -- case E_OUTOFMEMORY : txt = "E_OUTOFMEMORY"; break ; -- case E_INVALIDARG : txt = "E_INVALIDARG"; break ; -+ case (DWORD)(E_NOTIMPL) : txt = "E_NOTIMPL"; break ; -+ case (DWORD)(E_NOINTERFACE) : txt = "E_NOINTERFACE"; break ; -+ case (DWORD)(E_ABORT) : txt = "E_ABORT"; break ; -+ case (DWORD)(E_FAIL) : txt = "E_FAIL"; break ; -+ case (DWORD)(STG_E_INVALIDFUNCTION) : txt = "STG_E_INVALIDFUNCTION"; break ; -+ case (DWORD)(E_OUTOFMEMORY) : txt = "E_OUTOFMEMORY"; break ; -+ case (DWORD)(E_INVALIDARG) : txt = "E_INVALIDARG"; break ; - case ERROR_DIRECTORY : txt = "Error Directory"; break ; - default: - txt = strerror(errorCode); diff --git a/stdlib/p7zip_jll/Project.toml b/stdlib/p7zip_jll/Project.toml index 75e04b6362fdf..4c9bf62ad7ec1 100644 --- a/stdlib/p7zip_jll/Project.toml +++ b/stdlib/p7zip_jll/Project.toml @@ -1,6 +1,6 @@ name = "p7zip_jll" uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" -version = "16.2.1+1" +version = "17.4.0+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" From 7a4cef745168ec9e7143e6e0b28942eb82932624 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 26 Jan 2022 14:51:58 -0500 Subject: [PATCH 66/68] [Distributed] Set stdin to devnull before closing it Distributed closes and destroys stdin, but some tests attempted to explicitly use it, leading to test problems. We previously interpreted this as passing devnull, but this is better to be explicit. (cherry picked from commit 64a86f9bca119db3a64e02294f9da16ed6b50a07) --- stdlib/Distributed/src/cluster.jl | 5 ++++- stdlib/Distributed/test/distributed_exec.jl | 6 +++++- stdlib/REPL/src/TerminalMenus/util.jl | 10 +++++----- .../TerminalMenus/legacytests/old_multiselect_menu.jl | 4 ++-- .../test/TerminalMenus/legacytests/old_radio_menu.jl | 6 +++--- stdlib/REPL/test/TerminalMenus/multiselect_menu.jl | 4 ++-- .../test/TerminalMenus/multiselect_with_skip_menu.jl | 4 ++-- stdlib/REPL/test/TerminalMenus/radio_menu.jl | 8 ++++---- stdlib/REPL/test/TerminalMenus/runtests.jl | 11 ++++++----- 9 files changed, 33 insertions(+), 25 deletions(-) diff --git a/stdlib/Distributed/src/cluster.jl b/stdlib/Distributed/src/cluster.jl index 5e90f231f59b1..b7268a7e053ee 100644 --- a/stdlib/Distributed/src/cluster.jl +++ b/stdlib/Distributed/src/cluster.jl @@ -232,7 +232,10 @@ start_worker(cookie::AbstractString=readline(stdin); kwargs...) = start_worker(s function start_worker(out::IO, cookie::AbstractString=readline(stdin); close_stdin::Bool=true, stderr_to_stdout::Bool=true) init_multi() - close_stdin && close(stdin) # workers will not use it + if close_stdin # workers will not use it + redirect_stdin(devnull) + close(stdin) + end stderr_to_stdout && redirect_stderr(stdout) init_worker(cookie) diff --git a/stdlib/Distributed/test/distributed_exec.jl b/stdlib/Distributed/test/distributed_exec.jl index d2d46aebf0f1b..4819623ccf9e1 100644 --- a/stdlib/Distributed/test/distributed_exec.jl +++ b/stdlib/Distributed/test/distributed_exec.jl @@ -1616,7 +1616,11 @@ cluster_cookie("") for close_stdin in (true, false), stderr_to_stdout in (true, false) local npids = addprocs_with_testenv(RetainStdioTester(close_stdin,stderr_to_stdout)) @test remotecall_fetch(myid, npids[1]) == npids[1] - @test close_stdin != remotecall_fetch(()->isopen(stdin), npids[1]) + if close_stdin + @test remotecall_fetch(()->stdin === devnull && !isreadable(stdin), npids[1]) + else + @test remotecall_fetch(()->stdin !== devnull && isopen(stdin) && isreadable(stdin), npids[1]) + end @test stderr_to_stdout == remotecall_fetch(()->(stderr === stdout), npids[1]) rmprocs(npids) end diff --git a/stdlib/REPL/src/TerminalMenus/util.jl b/stdlib/REPL/src/TerminalMenus/util.jl index 8ad9ec0e4100d..91e336070d2cf 100644 --- a/stdlib/REPL/src/TerminalMenus/util.jl +++ b/stdlib/REPL/src/TerminalMenus/util.jl @@ -17,24 +17,24 @@ readbyte(stream::IO=stdin) = read(stream, Char) # Read the next key from stdin. It is also able to read several bytes for # escaped keys such as the arrow keys, home/end keys, etc. # Escaped keys are returned using the `Key` enum. -readkey(stream::Base.LibuvStream=stdin) = UInt32(_readkey(stream)) -function _readkey(stream::Base.LibuvStream=stdin) +readkey(stream::IO=stdin) = UInt32(_readkey(stream)) +function _readkey(stream::IO=stdin) c = readbyte(stream) # Escape characters if c == '\x1b' - stream.buffer.size < 2 && return '\x1b' + bytesavailable(stream) < 1 && return '\x1b' esc_a = readbyte(stream) esc_a == 'v' && return PAGE_UP # M-v esc_a == '<' && return HOME_KEY # M-< esc_a == '>' && return END_KEY # M-> - stream.buffer.size < 3 && return '\x1b' + bytesavailable(stream) < 1 && return '\x1b' esc_b = readbyte(stream) if esc_a == '[' || esc_a == 'O' if esc_b >= '0' && esc_b <= '9' - stream.buffer.size < 4 && return '\x1b' + bytesavailable(stream) < 1 && return '\x1b' esc_c = readbyte(stream) if esc_c == '~' esc_b == '1' && return HOME_KEY diff --git a/stdlib/REPL/test/TerminalMenus/legacytests/old_multiselect_menu.jl b/stdlib/REPL/test/TerminalMenus/legacytests/old_multiselect_menu.jl index 49dbcc42c3095..9902a992ea081 100644 --- a/stdlib/REPL/test/TerminalMenus/legacytests/old_multiselect_menu.jl +++ b/stdlib/REPL/test/TerminalMenus/legacytests/old_multiselect_menu.jl @@ -33,6 +33,6 @@ TerminalMenus.writeLine(buf, multi_menu, 1, true) # Test SDTIN multi_menu = MultiSelectMenu(string.(1:10), warn=false) -@test simulate_input(Set([1,2]), multi_menu, :enter, :down, :enter, 'd') +@test simulate_input(multi_menu, :enter, :down, :enter, 'd') == Set([1,2]) multi_menu = MultiSelectMenu(["single option"], warn=false) -@test simulate_input(Set([1]), multi_menu, :up, :up, :down, :enter, 'd') +@test simulate_input(multi_menu, :up, :up, :down, :enter, 'd') == Set([1]) diff --git a/stdlib/REPL/test/TerminalMenus/legacytests/old_radio_menu.jl b/stdlib/REPL/test/TerminalMenus/legacytests/old_radio_menu.jl index 9438808a847d6..248d5cd6a3183 100644 --- a/stdlib/REPL/test/TerminalMenus/legacytests/old_radio_menu.jl +++ b/stdlib/REPL/test/TerminalMenus/legacytests/old_radio_menu.jl @@ -36,8 +36,8 @@ TerminalMenus.writeLine(buf, radio_menu, 1, true) # Test using stdin radio_menu = RadioMenu(string.(1:10), warn=false) -@test simulate_input(3, radio_menu, :down, :down, :enter) +@test simulate_input(radio_menu, :down, :down, :enter) == 3 radio_menu = RadioMenu(["single option"], warn=false) -@test simulate_input(1, radio_menu, :up, :up, :down, :up, :enter) +@test simulate_input(radio_menu, :up, :up, :down, :up, :enter) == 1 radio_menu = RadioMenu(string.(1:3), pagesize=1, warn=false) -@test simulate_input(3, radio_menu, :down, :down, :down, :down, :enter) +@test simulate_input(radio_menu, :down, :down, :down, :down, :enter) == 3 diff --git a/stdlib/REPL/test/TerminalMenus/multiselect_menu.jl b/stdlib/REPL/test/TerminalMenus/multiselect_menu.jl index d625554c813b0..d6568fb6b04c1 100644 --- a/stdlib/REPL/test/TerminalMenus/multiselect_menu.jl +++ b/stdlib/REPL/test/TerminalMenus/multiselect_menu.jl @@ -52,6 +52,6 @@ end # Test SDTIN multi_menu = MultiSelectMenu(string.(1:10), charset=:ascii) -@test simulate_input(Set([1,2]), multi_menu, :enter, :down, :enter, 'd') +@test simulate_input(multi_menu, :enter, :down, :enter, 'd') == Set([1,2]) multi_menu = MultiSelectMenu(["single option"], charset=:ascii) -@test simulate_input(Set([1]), multi_menu, :up, :up, :down, :enter, 'd') +@test simulate_input(multi_menu, :up, :up, :down, :enter, 'd') == Set([1]) diff --git a/stdlib/REPL/test/TerminalMenus/multiselect_with_skip_menu.jl b/stdlib/REPL/test/TerminalMenus/multiselect_with_skip_menu.jl index 84f259ad7642c..609b168c2ddba 100644 --- a/stdlib/REPL/test/TerminalMenus/multiselect_with_skip_menu.jl +++ b/stdlib/REPL/test/TerminalMenus/multiselect_with_skip_menu.jl @@ -121,10 +121,10 @@ menu = MultiSelectWithSkipMenu(string.(1:5), selected=[2, 3]) buf = IOBuffer() TerminalMenus.printmenu(buf, menu, 1; init=true) @test occursin("2 items selected", String(take!(buf))) -@test simulate_input(Set([2, 3, 4]), menu, 'n', :enter, 'd') +@test simulate_input(menu, 'n', :enter, 'd') == Set([2, 3, 4]) buf = IOBuffer() TerminalMenus.printmenu(buf, menu, 1; init=true) @test occursin("3 items selected", String(take!(buf))) menu = MultiSelectWithSkipMenu(string.(1:5), selected=[2, 3]) -@test simulate_input(Set([2]), menu, 'P', :enter, 'd', cursor=5) +@test simulate_input(menu, 'P', :enter, 'd', cursor=5) == Set([2]) diff --git a/stdlib/REPL/test/TerminalMenus/radio_menu.jl b/stdlib/REPL/test/TerminalMenus/radio_menu.jl index 696be1324a8e3..5ca6422717425 100644 --- a/stdlib/REPL/test/TerminalMenus/radio_menu.jl +++ b/stdlib/REPL/test/TerminalMenus/radio_menu.jl @@ -45,10 +45,10 @@ end # Test using stdin radio_menu = RadioMenu(string.(1:10); charset=:ascii) -@test simulate_input(3, radio_menu, :down, :down, :enter) +@test simulate_input(radio_menu, :down, :down, :enter) == 3 radio_menu = RadioMenu(["single option"], charset=:ascii) -@test simulate_input(1, radio_menu, :up, :up, :down, :up, :enter) +@test simulate_input(radio_menu, :up, :up, :down, :up, :enter) == 1 radio_menu = RadioMenu(string.(1:3), pagesize=1, charset=:ascii) -@test simulate_input(3, radio_menu, :down, :down, :down, :down, :enter) +@test simulate_input(radio_menu, :down, :down, :down, :down, :enter) == 3 radio_menu = RadioMenu(["apple", "banana", "cherry"]; keybindings=collect('a':'c'), charset=:ascii) -@test simulate_input(2, radio_menu, 'b') +@test simulate_input(radio_menu, 'b') == 2 diff --git a/stdlib/REPL/test/TerminalMenus/runtests.jl b/stdlib/REPL/test/TerminalMenus/runtests.jl index 62a91cc0a1256..c594958a36670 100644 --- a/stdlib/REPL/test/TerminalMenus/runtests.jl +++ b/stdlib/REPL/test/TerminalMenus/runtests.jl @@ -4,21 +4,22 @@ import REPL using REPL.TerminalMenus using Test -function simulate_input(expected, menu::TerminalMenus.AbstractMenu, keys...; - kwargs...) +function simulate_input(menu::TerminalMenus.AbstractMenu, keys...; kwargs...) keydict = Dict(:up => "\e[A", :down => "\e[B", :enter => "\r") + new_stdin = Base.BufferStream() for key in keys if isa(key, Symbol) - write(stdin.buffer, keydict[key]) + write(new_stdin, keydict[key]) else - write(stdin.buffer, "$key") + write(new_stdin, "$key") end end + TerminalMenus.terminal.in_stream = new_stdin - request(menu; suppress_output=true, kwargs...) == expected + return request(menu; suppress_output=true, kwargs...) end include("radio_menu.jl") From 2cf85b615ef3a3264b670e0ef2bff8a64532a0e8 Mon Sep 17 00:00:00 2001 From: pchintalapudi <34727397+pchintalapudi@users.noreply.github.com> Date: Tue, 24 May 2022 23:12:07 +0200 Subject: [PATCH 67/68] Fix use-after-free bugs in debuginfo (#45016) Co-authored-by: Dilum Aluthge --- src/Makefile | 8 +- src/codegen.cpp | 3 - src/debug-registry.h | 183 ++++++++++++ src/debuginfo.cpp | 670 +++++++++++++++++++++---------------------- src/jitlayers.h | 7 + src/processor.h | 5 + 6 files changed, 529 insertions(+), 347 deletions(-) create mode 100644 src/debug-registry.h diff --git a/src/Makefile b/src/Makefile index b7235597fd08c..e0ab9568fd242 100644 --- a/src/Makefile +++ b/src/Makefile @@ -279,12 +279,12 @@ $(BUILDDIR)/julia_flisp.boot: $(addprefix $(SRCDIR)/,jlfrontend.scm flisp/aliase # additional dependency links $(BUILDDIR)/codegen-stubs.o $(BUILDDIR)/codegen-stubs.dbg.obj: $(SRCDIR)/intrinsics.h -$(BUILDDIR)/aotcompile.o $(BUILDDIR)/aotcompile.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/codegen_shared.h +$(BUILDDIR)/aotcompile.o $(BUILDDIR)/aotcompile.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/codegen_shared.h $(SRCDIR)/debug-registry.h $(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SRCDIR)/flisp/*.h $(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/iddict.c $(SRCDIR)/builtin_proto.h $(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,\ - intrinsics.cpp jitlayers.h intrinsics.h codegen_shared.h cgutils.cpp ccall.cpp abi_*.cpp processor.h builtin_proto.h) -$(BUILDDIR)/debuginfo.o $(BUILDDIR)/debuginfo.dbg.obj: $(addprefix $(SRCDIR)/,debuginfo.h processor.h) + intrinsics.cpp jitlayers.h debug-registry.h intrinsics.h codegen_shared.h cgutils.cpp ccall.cpp abi_*.cpp processor.h builtin_proto.h) +$(BUILDDIR)/debuginfo.o $(BUILDDIR)/debuginfo.dbg.obj: $(addprefix $(SRCDIR)/,debuginfo.h processor.h jitlayers.h debug-registry.h) $(BUILDDIR)/disasm.o $(BUILDDIR)/disasm.dbg.obj: $(SRCDIR)/debuginfo.h $(SRCDIR)/processor.h $(BUILDDIR)/dump.o $(BUILDDIR)/dump.dbg.obj: $(addprefix $(SRCDIR)/,common_symbols1.inc common_symbols2.inc builtin_proto.h serialize.h) $(BUILDDIR)/gc-debug.o $(BUILDDIR)/gc-debug.dbg.obj: $(SRCDIR)/gc.h @@ -292,7 +292,7 @@ $(BUILDDIR)/gc-pages.o $(BUILDDIR)/gc-pages.dbg.obj: $(SRCDIR)/gc.h $(BUILDDIR)/gc.o $(BUILDDIR)/gc.dbg.obj: $(SRCDIR)/gc.h $(SRCDIR)/gc-alloc-profiler.h $(BUILDDIR)/init.o $(BUILDDIR)/init.dbg.obj: $(SRCDIR)/builtin_proto.h $(BUILDDIR)/interpreter.o $(BUILDDIR)/interpreter.dbg.obj: $(SRCDIR)/builtin_proto.h -$(BUILDDIR)/jitlayers.o $(BUILDDIR)/jitlayers.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/codegen_shared.h +$(BUILDDIR)/jitlayers.o $(BUILDDIR)/jitlayers.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/codegen_shared.h $(SRCDIR)/debug-registry.h $(BUILDDIR)/jltypes.o $(BUILDDIR)/jltypes.dbg.obj: $(SRCDIR)/builtin_proto.h $(build_shlibdir)/libllvmcalltest.$(SHLIB_EXT): $(SRCDIR)/codegen_shared.h $(BUILDDIR)/julia_version.h $(BUILDDIR)/llvm-alloc-helpers.o $(BUILDDIR)/llvm-alloc-helpers.dbg.obj: $(SRCDIR)/codegen_shared.h $(SRCDIR)/llvm-pass-helpers.h $(SRCDIR)/llvm-alloc-helpers.h diff --git a/src/codegen.cpp b/src/codegen.cpp index cccb4b12cfc1d..517577df0f324 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -8155,8 +8155,6 @@ char jl_using_oprofile_jitevents = 0; // Non-zero if running under OProfile char jl_using_perf_jitevents = 0; #endif -void jl_init_debuginfo(void); - extern "C" void jl_init_llvm(void) { builtin_func_map = @@ -8199,7 +8197,6 @@ extern "C" void jl_init_llvm(void) jl_default_debug_info_kind = (int) DICompileUnit::DebugEmissionKind::FullDebug; imaging_mode = jl_options.image_codegen || (jl_generating_output() && !jl_options.incremental); jl_default_cgparams.generic_context = jl_nothing; - jl_init_debuginfo(); InitializeNativeTarget(); InitializeNativeTargetAsmPrinter(); diff --git a/src/debug-registry.h b/src/debug-registry.h new file mode 100644 index 0000000000000..dfdb09ca84519 --- /dev/null +++ b/src/debug-registry.h @@ -0,0 +1,183 @@ +#include +#include +#include + +#include "julia_internal.h" +#include "processor.h" + +#include +#include +#include + +typedef struct { + const llvm::object::ObjectFile *obj; + llvm::DIContext *ctx; + int64_t slide; +} objfileentry_t; + + +// Central registry for resolving function addresses to `jl_method_instance_t`s and +// originating `ObjectFile`s (for the DWARF debug info). +// +// A global singleton instance is notified by the JIT whenever a new object is emitted, +// and later queried by the various function info APIs. We also use the chance to handle +// some platform-specific unwind info registration (which is unrelated to the query +// functionality). +class JITDebugInfoRegistry +{ +public: + template + struct Locked { + + template + struct Lock { + std::unique_lock lock; + CResourceT &resource; + + Lock(std::mutex &mutex, CResourceT &resource) JL_NOTSAFEPOINT : lock(mutex), resource(resource) {} + Lock(Lock &&) JL_NOTSAFEPOINT = default; + Lock &operator=(Lock &&) JL_NOTSAFEPOINT = default; + + CResourceT &operator*() JL_NOTSAFEPOINT { + return resource; + } + + const CResourceT &operator*() const JL_NOTSAFEPOINT { + return resource; + } + + CResourceT *operator->() JL_NOTSAFEPOINT { + return &**this; + } + + const CResourceT *operator->() const JL_NOTSAFEPOINT { + return &**this; + } + + operator const CResourceT &() const JL_NOTSAFEPOINT { + return resource; + } + + ~Lock() JL_NOTSAFEPOINT = default; + }; + private: + + mutable std::mutex mutex; + ResourceT resource; + public: + typedef Lock LockT; + typedef Lock ConstLockT; + + Locked(ResourceT resource = ResourceT()) JL_NOTSAFEPOINT : mutex(), resource(std::move(resource)) {} + + LockT operator*() JL_NOTSAFEPOINT { + return LockT(mutex, resource); + } + + ConstLockT operator*() const JL_NOTSAFEPOINT { + return ConstLockT(mutex, resource); + } + + ~Locked() JL_NOTSAFEPOINT = default; + }; + + template + struct jl_pthread_key_t { + static_assert(std::is_trivially_default_constructible::value, "Invalid datatype for pthread key!"); + static_assert(std::is_trivially_destructible::value, "Expected datatype to be trivially destructible!"); + static_assert(sizeof(datatype) == sizeof(void*), "Expected datatype to be like a void*!"); + pthread_key_t key; + + void init() JL_NOTSAFEPOINT { + if (pthread_key_create(&key, NULL)) + jl_error("fatal: pthread_key_create failed"); + } + + operator datatype() JL_NOTSAFEPOINT { + return reinterpret_cast(pthread_getspecific(key)); + } + + jl_pthread_key_t &operator=(datatype val) JL_NOTSAFEPOINT { + pthread_setspecific(key, reinterpret_cast(val)); + return *this; + } + + void destroy() JL_NOTSAFEPOINT { + pthread_key_delete(key); + } + }; + + struct sysimg_info_t { + uint64_t jl_sysimage_base; + jl_sysimg_fptrs_t sysimg_fptrs; + jl_method_instance_t **sysimg_fvars_linfo; + size_t sysimg_fvars_n; + }; + + struct libc_frames_t { +#if defined(_OS_DARWIN_) && defined(LLVM_SHLIB) + std::atomic libc_register_frame_{nullptr}; + std::atomic libc_deregister_frame_{nullptr}; + + void libc_register_frame(const char *Entry) JL_NOTSAFEPOINT; + + void libc_deregister_frame(const char *Entry) JL_NOTSAFEPOINT; +#endif + }; +private: + + struct ObjectInfo { + const llvm::object::ObjectFile *object = nullptr; + size_t SectionSize = 0; + ptrdiff_t slide = 0; + llvm::object::SectionRef Section{}; + llvm::DIContext *context = nullptr; + }; + + template + using rev_map = std::map>; + + typedef rev_map objectmap_t; + typedef rev_map objfilemap_t; + + objectmap_t objectmap{}; + rev_map> linfomap{}; + + // Maintain a mapping of unrealized function names -> linfo objects + // so that when we see it get emitted, we can add a link back to the linfo + // that it came from (providing name, type signature, file info, etc.) + Locked> codeinst_in_flight{}; + + Locked sysimg_info{}; + + Locked objfilemap{}; + + static std::string mangle(llvm::StringRef Name, const llvm::DataLayout &DL) JL_NOTSAFEPOINT; + +public: + + JITDebugInfoRegistry() JL_NOTSAFEPOINT; + ~JITDebugInfoRegistry() JL_NOTSAFEPOINT = default; + + // Any function that acquires this lock must be either a unmanaged thread + // or in the GC safe region and must NOT allocate anything through the GC + // while holding this lock. + // Certain functions in this file might be called from an unmanaged thread + // and cannot have any interaction with the julia runtime + // They also may be re-entrant, and operating while threads are paused, so we + // separately manage the re-entrant count behavior for safety across platforms + // Note that we cannot safely upgrade read->write + uv_rwlock_t debuginfo_asyncsafe{}; + jl_pthread_key_t debuginfo_asyncsafe_held{}; + libc_frames_t libc_frames{}; + + void add_code_in_flight(llvm::StringRef name, jl_code_instance_t *codeinst, const llvm::DataLayout &DL) JL_NOTSAFEPOINT; + jl_method_instance_t *lookupLinfo(size_t pointer) JL_NOTSAFEPOINT; + void registerJITObject(const llvm::object::ObjectFile &Object, + std::function getLoadAddress, + std::function lookupWriteAddress) JL_NOTSAFEPOINT; + objectmap_t& getObjectMap() JL_NOTSAFEPOINT; + void set_sysimg_info(sysimg_info_t info) JL_NOTSAFEPOINT; + Locked::ConstLockT get_sysimg_info() const JL_NOTSAFEPOINT; + Locked::LockT get_objfile_map() JL_NOTSAFEPOINT; +}; diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 0e246160a3c16..76a203eead0db 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -34,47 +34,129 @@ using namespace llvm; #include #include #include +#include #include "julia_assert.h" #ifdef _OS_DARWIN_ #include #endif -typedef object::SymbolRef SymRef; +#include "jitlayers.h" -// Any function that acquires this lock must be either a unmanaged thread -// or in the GC safe region and must NOT allocate anything through the GC -// while holding this lock. -// Certain functions in this file might be called from an unmanaged thread -// and cannot have any interaction with the julia runtime -// They also may be re-entrant, and operating while threads are paused, so we -// separately manage the re-entrant count behavior for safety across platforms -// Note that we cannot safely upgrade read->write -static uv_rwlock_t debuginfo_asyncsafe; -static pthread_key_t debuginfo_asyncsafe_held; +static JITDebugInfoRegistry &getJITDebugRegistry() JL_NOTSAFEPOINT { + return jl_ExecutionEngine->getDebugInfoRegistry(); +} + +struct debug_link_info { + StringRef filename; + uint32_t crc32; +}; + +extern "C" JL_DLLEXPORT void jl_lock_profile_impl(void) JL_NOTSAFEPOINT; +extern "C" JL_DLLEXPORT void jl_unlock_profile_impl(void) JL_NOTSAFEPOINT; + +template +static void jl_profile_atomic(T f); + +#if (defined(_OS_LINUX_) || defined(_OS_FREEBSD_) || (defined(_OS_DARWIN_) && defined(LLVM_SHLIB))) +extern "C" void __register_frame(void*); +extern "C" void __deregister_frame(void*); + +template +static void processFDEs(const char *EHFrameAddr, size_t EHFrameSize, callback f) +{ + const char *P = EHFrameAddr; + const char *End = P + EHFrameSize; + do { + const char *Entry = P; + P += 4; + assert(P <= End); + uint32_t Length = *(const uint32_t*)Entry; + // Length == 0: Terminator + if (Length == 0) + break; + assert(P + Length <= End); + uint32_t Offset = *(const uint32_t*)P; + // Offset == 0: CIE + if (Offset != 0) + f(Entry); + P += Length; + } while (P != End); +} +#endif + +std::string JITDebugInfoRegistry::mangle(StringRef Name, const DataLayout &DL) JL_NOTSAFEPOINT +{ + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, Name, DL); + } + return MangledName; +} + +void JITDebugInfoRegistry::add_code_in_flight(StringRef name, jl_code_instance_t *codeinst, const DataLayout &DL) JL_NOTSAFEPOINT { + (**codeinst_in_flight)[mangle(name, DL)] = codeinst; +} + +jl_method_instance_t *JITDebugInfoRegistry::lookupLinfo(size_t pointer) JL_NOTSAFEPOINT +{ + jl_lock_profile_impl(); + auto region = linfomap.lower_bound(pointer); + jl_method_instance_t *linfo = NULL; + if (region != linfomap.end() && pointer < region->first + region->second.first) + linfo = region->second.second; + jl_unlock_profile_impl(); + return linfo; +} -void jl_init_debuginfo(void) +//Protected by debuginfo_asyncsafe +JITDebugInfoRegistry::objectmap_t & +JITDebugInfoRegistry::getObjectMap() JL_NOTSAFEPOINT { + return objectmap; +} + +void JITDebugInfoRegistry::set_sysimg_info(sysimg_info_t info) JL_NOTSAFEPOINT { + (**this->sysimg_info) = info; +} + +JITDebugInfoRegistry::Locked::ConstLockT +JITDebugInfoRegistry::get_sysimg_info() const JL_NOTSAFEPOINT { + return *this->sysimg_info; +} + +JITDebugInfoRegistry::Locked::LockT +JITDebugInfoRegistry::get_objfile_map() JL_NOTSAFEPOINT { + return *this->objfilemap; +} + +JITDebugInfoRegistry::JITDebugInfoRegistry() JL_NOTSAFEPOINT { uv_rwlock_init(&debuginfo_asyncsafe); - if (pthread_key_create(&debuginfo_asyncsafe_held, NULL)) - jl_error("fatal: pthread_key_create failed"); + debuginfo_asyncsafe_held.init(); } -extern "C" JL_DLLEXPORT void jl_lock_profile_impl(void) +struct unw_table_entry +{ + int32_t start_ip_offset; + int32_t fde_offset; +}; + +extern "C" JL_DLLEXPORT void jl_lock_profile_impl(void) JL_NOTSAFEPOINT { - uintptr_t held = (uintptr_t)pthread_getspecific(debuginfo_asyncsafe_held); + uintptr_t held = getJITDebugRegistry().debuginfo_asyncsafe_held; if (held++ == 0) - uv_rwlock_rdlock(&debuginfo_asyncsafe); - pthread_setspecific(debuginfo_asyncsafe_held, (void*)held); + uv_rwlock_rdlock(&getJITDebugRegistry().debuginfo_asyncsafe); + getJITDebugRegistry().debuginfo_asyncsafe_held = held; } -extern "C" JL_DLLEXPORT void jl_unlock_profile_impl(void) +extern "C" JL_DLLEXPORT void jl_unlock_profile_impl(void) JL_NOTSAFEPOINT { - uintptr_t held = (uintptr_t)pthread_getspecific(debuginfo_asyncsafe_held); + uintptr_t held = getJITDebugRegistry().debuginfo_asyncsafe_held; assert(held); if (--held == 0) - uv_rwlock_rdunlock(&debuginfo_asyncsafe); - pthread_setspecific(debuginfo_asyncsafe_held, (void*)held); + uv_rwlock_rdunlock(&getJITDebugRegistry().debuginfo_asyncsafe); + getJITDebugRegistry().debuginfo_asyncsafe_held = held; } // some actions aren't signal (especially profiler) safe so we acquire a lock @@ -82,8 +164,8 @@ extern "C" JL_DLLEXPORT void jl_unlock_profile_impl(void) template static void jl_profile_atomic(T f) { - assert(0 == (uintptr_t)pthread_getspecific(debuginfo_asyncsafe_held)); - uv_rwlock_wrlock(&debuginfo_asyncsafe); + assert(0 == getJITDebugRegistry().debuginfo_asyncsafe_held); + uv_rwlock_wrlock(&getJITDebugRegistry().debuginfo_asyncsafe); #ifndef _OS_WINDOWS_ sigset_t sset; sigset_t oset; @@ -94,36 +176,14 @@ static void jl_profile_atomic(T f) #ifndef _OS_WINDOWS_ pthread_sigmask(SIG_SETMASK, &oset, NULL); #endif - uv_rwlock_wrunlock(&debuginfo_asyncsafe); + uv_rwlock_wrunlock(&getJITDebugRegistry().debuginfo_asyncsafe); } // --- storing and accessing source location metadata --- - -struct ObjectInfo { - const object::ObjectFile *object; - size_t SectionSize; - ptrdiff_t slide; - object::SectionRef Section; - DIContext *context; -}; - -// Maintain a mapping of unrealized function names -> linfo objects -// so that when we see it get emitted, we can add a link back to the linfo -// that it came from (providing name, type signature, file info, etc.) -static StringMap codeinst_in_flight; -static std::string mangle(StringRef Name, const DataLayout &DL) -{ - std::string MangledName; - { - raw_string_ostream MangledNameStream(MangledName); - Mangler::getNameWithPrefix(MangledNameStream, Name, DL); - } - return MangledName; -} void jl_add_code_in_flight(StringRef name, jl_code_instance_t *codeinst, const DataLayout &DL) { - codeinst_in_flight[mangle(name, DL)] = codeinst; + getJITDebugRegistry().add_code_in_flight(name, codeinst, DL); } @@ -181,218 +241,184 @@ static void create_PRUNTIME_FUNCTION(uint8_t *Code, size_t Size, StringRef fnnam } #endif -struct revcomp { - bool operator() (const size_t& lhs, const size_t& rhs) const - { return lhs>rhs; } -}; - - -// Central registry for resolving function addresses to `jl_method_instance_t`s and -// originating `ObjectFile`s (for the DWARF debug info). -// -// A global singleton instance is notified by the JIT whenever a new object is emitted, -// and later queried by the various function info APIs. We also use the chance to handle -// some platform-specific unwind info registration (which is unrelated to the query -// functionality). -class JITObjectRegistry +void JITDebugInfoRegistry::registerJITObject(const object::ObjectFile &Object, + std::function getLoadAddress, + std::function lookupWriteAddress) { - std::map objectmap; - std::map, revcomp> linfomap; + jl_ptls_t ptls = jl_current_task->ptls; + // This function modify codeinst->fptr in GC safe region. + // This should be fine since the GC won't scan this field. + int8_t gc_state = jl_gc_safe_enter(ptls); -public: - jl_method_instance_t *lookupLinfo(size_t pointer) JL_NOTSAFEPOINT - { - jl_lock_profile_impl(); - auto region = linfomap.lower_bound(pointer); - jl_method_instance_t *linfo = NULL; - if (region != linfomap.end() && pointer < region->first + region->second.first) - linfo = region->second.second; - jl_unlock_profile_impl(); - return linfo; - } - - void registerJITObject(const object::ObjectFile &Object, - std::function getLoadAddress, - std::function lookupWriteAddress) - { - jl_ptls_t ptls = jl_current_task->ptls; - // This function modify codeinst->fptr in GC safe region. - // This should be fine since the GC won't scan this field. - int8_t gc_state = jl_gc_safe_enter(ptls); - - object::section_iterator EndSection = Object.section_end(); + object::section_iterator EndSection = Object.section_end(); #ifdef _CPU_ARM_ - // ARM does not have/use .eh_frame - uint64_t arm_exidx_addr = 0; - size_t arm_exidx_len = 0; - uint64_t arm_text_addr = 0; - size_t arm_text_len = 0; - for (auto §ion: Object.sections()) { - bool istext = false; - if (section.isText()) { - istext = true; - } - else { - auto sName = section.getName(); - if (!sName) - continue; - if (sName.get() != ".ARM.exidx") { - continue; - } + // ARM does not have/use .eh_frame + uint64_t arm_exidx_addr = 0; + size_t arm_exidx_len = 0; + uint64_t arm_text_addr = 0; + size_t arm_text_len = 0; + for (auto §ion: Object.sections()) { + bool istext = false; + if (section.isText()) { + istext = true; + } + else { + auto sName = section.getName(); + if (!sName) + continue; + if (sName.get() != ".ARM.exidx") { + continue; } - uint64_t loadaddr = getLoadAddress(section.getName().get()); - size_t seclen = section.getSize(); - if (istext) { - arm_text_addr = loadaddr; - arm_text_len = seclen; - if (!arm_exidx_addr) { - continue; - } + } + uint64_t loadaddr = getLoadAddress(section.getName().get()); + size_t seclen = section.getSize(); + if (istext) { + arm_text_addr = loadaddr; + arm_text_len = seclen; + if (!arm_exidx_addr) { + continue; } - else { - arm_exidx_addr = loadaddr; - arm_exidx_len = seclen; - if (!arm_text_addr) { - continue; - } + } + else { + arm_exidx_addr = loadaddr; + arm_exidx_len = seclen; + if (!arm_text_addr) { + continue; } - unw_dyn_info_t *di = new unw_dyn_info_t; - di->gp = 0; - di->format = UNW_INFO_FORMAT_ARM_EXIDX; - di->start_ip = (uintptr_t)arm_text_addr; - di->end_ip = (uintptr_t)(arm_text_addr + arm_text_len); - di->u.rti.name_ptr = 0; - di->u.rti.table_data = arm_exidx_addr; - di->u.rti.table_len = arm_exidx_len; - jl_profile_atomic([&]() { - _U_dyn_register(di); - }); - break; } + unw_dyn_info_t *di = new unw_dyn_info_t; + di->gp = 0; + di->format = UNW_INFO_FORMAT_ARM_EXIDX; + di->start_ip = (uintptr_t)arm_text_addr; + di->end_ip = (uintptr_t)(arm_text_addr + arm_text_len); + di->u.rti.name_ptr = 0; + di->u.rti.table_data = arm_exidx_addr; + di->u.rti.table_len = arm_exidx_len; + jl_profile_atomic([&]() { + _U_dyn_register(di); + }); + break; + } #endif #if defined(_OS_WINDOWS_) - uint64_t SectionAddrCheck = 0; - uint64_t SectionLoadCheck = 0; (void)SectionLoadCheck; - uint64_t SectionWriteCheck = 0; (void)SectionWriteCheck; - uint8_t *UnwindData = NULL; + uint64_t SectionAddrCheck = 0; + uint64_t SectionLoadCheck = 0; (void)SectionLoadCheck; + uint64_t SectionWriteCheck = 0; (void)SectionWriteCheck; + uint8_t *UnwindData = NULL; #if defined(_CPU_X86_64_) - uint8_t *catchjmp = NULL; - for (const object::SymbolRef &sym_iter : Object.symbols()) { - StringRef sName = cantFail(sym_iter.getName()); - if (sName.equals("__UnwindData") || sName.equals("__catchjmp")) { - uint64_t Addr = cantFail(sym_iter.getAddress()); - auto Section = cantFail(sym_iter.getSection()); - assert(Section != EndSection && Section->isText()); - uint64_t SectionAddr = Section->getAddress(); - StringRef secName = cantFail(Section->getName()); - uint64_t SectionLoadAddr = getLoadAddress(secName); - assert(SectionLoadAddr); - if (SectionAddrCheck) // assert that all of the Sections are at the same location - assert(SectionAddrCheck == SectionAddr && - SectionLoadCheck == SectionLoadAddr); - SectionAddrCheck = SectionAddr; - SectionLoadCheck = SectionLoadAddr; - SectionWriteCheck = SectionLoadAddr; - if (lookupWriteAddress) - SectionWriteCheck = (uintptr_t)lookupWriteAddress((void*)SectionLoadAddr); - Addr += SectionWriteCheck - SectionLoadCheck; - if (sName.equals("__UnwindData")) { - UnwindData = (uint8_t*)Addr; - } - else if (sName.equals("__catchjmp")) { - catchjmp = (uint8_t*)Addr; - } - } - } - assert(catchjmp); - assert(UnwindData); - assert(SectionAddrCheck); - assert(SectionLoadCheck); - assert(!memcmp(catchjmp, "\0\0\0\0\0\0\0\0\0\0\0\0", 12) && - !memcmp(UnwindData, "\0\0\0\0\0\0\0\0\0\0\0\0", 12)); - catchjmp[0] = 0x48; - catchjmp[1] = 0xb8; // mov RAX, QWORD PTR [&__julia_personality] - *(uint64_t*)(&catchjmp[2]) = (uint64_t)&__julia_personality; - catchjmp[10] = 0xff; - catchjmp[11] = 0xe0; // jmp RAX - UnwindData[0] = 0x09; // version info, UNW_FLAG_EHANDLER - UnwindData[1] = 4; // size of prolog (bytes) - UnwindData[2] = 2; // count of unwind codes (slots) - UnwindData[3] = 0x05; // frame register (rbp) = rsp - UnwindData[4] = 4; // second instruction - UnwindData[5] = 0x03; // mov RBP, RSP - UnwindData[6] = 1; // first instruction - UnwindData[7] = 0x50; // push RBP - *(DWORD*)&UnwindData[8] = (DWORD)(catchjmp - (uint8_t*)SectionWriteCheck); // relative location of catchjmp - UnwindData -= SectionWriteCheck - SectionLoadCheck; -#endif // defined(_OS_X86_64_) -#endif // defined(_OS_WINDOWS_) - - auto symbols = object::computeSymbolSizes(Object); - bool first = true; - for (const auto &sym_size : symbols) { - const object::SymbolRef &sym_iter = sym_size.first; - object::SymbolRef::Type SymbolType = cantFail(sym_iter.getType()); - if (SymbolType != object::SymbolRef::ST_Function) continue; + uint8_t *catchjmp = NULL; + for (const object::SymbolRef &sym_iter : Object.symbols()) { + StringRef sName = cantFail(sym_iter.getName()); + if (sName.equals("__UnwindData") || sName.equals("__catchjmp")) { uint64_t Addr = cantFail(sym_iter.getAddress()); auto Section = cantFail(sym_iter.getSection()); - if (Section == EndSection) continue; - if (!Section->isText()) continue; + assert(Section != EndSection && Section->isText()); uint64_t SectionAddr = Section->getAddress(); StringRef secName = cantFail(Section->getName()); uint64_t SectionLoadAddr = getLoadAddress(secName); - Addr -= SectionAddr - SectionLoadAddr; - StringRef sName = cantFail(sym_iter.getName()); - uint64_t SectionSize = Section->getSize(); - size_t Size = sym_size.second; -#if defined(_OS_WINDOWS_) - if (SectionAddrCheck) + assert(SectionLoadAddr); + if (SectionAddrCheck) // assert that all of the Sections are at the same location assert(SectionAddrCheck == SectionAddr && - SectionLoadCheck == SectionLoadAddr); + SectionLoadCheck == SectionLoadAddr); SectionAddrCheck = SectionAddr; SectionLoadCheck = SectionLoadAddr; - create_PRUNTIME_FUNCTION( - (uint8_t*)(uintptr_t)Addr, (size_t)Size, sName, - (uint8_t*)(uintptr_t)SectionLoadAddr, (size_t)SectionSize, UnwindData); + SectionWriteCheck = SectionLoadAddr; + if (lookupWriteAddress) + SectionWriteCheck = (uintptr_t)lookupWriteAddress((void*)SectionLoadAddr); + Addr += SectionWriteCheck - SectionLoadCheck; + if (sName.equals("__UnwindData")) { + UnwindData = (uint8_t*)Addr; + } + else if (sName.equals("__catchjmp")) { + catchjmp = (uint8_t*)Addr; + } + } + } + assert(catchjmp); + assert(UnwindData); + assert(SectionAddrCheck); + assert(SectionLoadCheck); + assert(!memcmp(catchjmp, "\0\0\0\0\0\0\0\0\0\0\0\0", 12) && + !memcmp(UnwindData, "\0\0\0\0\0\0\0\0\0\0\0\0", 12)); + catchjmp[0] = 0x48; + catchjmp[1] = 0xb8; // mov RAX, QWORD PTR [&__julia_personality] + *(uint64_t*)(&catchjmp[2]) = (uint64_t)&__julia_personality; + catchjmp[10] = 0xff; + catchjmp[11] = 0xe0; // jmp RAX + UnwindData[0] = 0x09; // version info, UNW_FLAG_EHANDLER + UnwindData[1] = 4; // size of prolog (bytes) + UnwindData[2] = 2; // count of unwind codes (slots) + UnwindData[3] = 0x05; // frame register (rbp) = rsp + UnwindData[4] = 4; // second instruction + UnwindData[5] = 0x03; // mov RBP, RSP + UnwindData[6] = 1; // first instruction + UnwindData[7] = 0x50; // push RBP + *(DWORD*)&UnwindData[8] = (DWORD)(catchjmp - (uint8_t*)SectionWriteCheck); // relative location of catchjmp + UnwindData -= SectionWriteCheck - SectionLoadCheck; +#endif // defined(_OS_X86_64_) +#endif // defined(_OS_WINDOWS_) + + auto symbols = object::computeSymbolSizes(Object); + bool first = true; + for (const auto &sym_size : symbols) { + const object::SymbolRef &sym_iter = sym_size.first; + object::SymbolRef::Type SymbolType = cantFail(sym_iter.getType()); + if (SymbolType != object::SymbolRef::ST_Function) continue; + uint64_t Addr = cantFail(sym_iter.getAddress()); + auto Section = cantFail(sym_iter.getSection()); + if (Section == EndSection) continue; + if (!Section->isText()) continue; + uint64_t SectionAddr = Section->getAddress(); + StringRef secName = cantFail(Section->getName()); + uint64_t SectionLoadAddr = getLoadAddress(secName); + Addr -= SectionAddr - SectionLoadAddr; + StringRef sName = cantFail(sym_iter.getName()); + uint64_t SectionSize = Section->getSize(); + size_t Size = sym_size.second; +#if defined(_OS_WINDOWS_) + if (SectionAddrCheck) + assert(SectionAddrCheck == SectionAddr && + SectionLoadCheck == SectionLoadAddr); + SectionAddrCheck = SectionAddr; + SectionLoadCheck = SectionLoadAddr; + create_PRUNTIME_FUNCTION( + (uint8_t*)(uintptr_t)Addr, (size_t)Size, sName, + (uint8_t*)(uintptr_t)SectionLoadAddr, (size_t)SectionSize, UnwindData); #endif + jl_code_instance_t *codeinst = NULL; + { + auto lock = *this->codeinst_in_flight; + auto &codeinst_in_flight = *lock; StringMap::iterator codeinst_it = codeinst_in_flight.find(sName); - jl_code_instance_t *codeinst = NULL; if (codeinst_it != codeinst_in_flight.end()) { codeinst = codeinst_it->second; codeinst_in_flight.erase(codeinst_it); } - jl_profile_atomic([&]() { - if (codeinst) - linfomap[Addr] = std::make_pair(Size, codeinst->def); - if (first) { - ObjectInfo tmp = {&Object, - (size_t)SectionSize, - (ptrdiff_t)(SectionAddr - SectionLoadAddr), - *Section, - nullptr, - }; - objectmap[SectionLoadAddr] = tmp; - first = false; - } - }); } - jl_gc_safe_leave(ptls, gc_state); - } - - std::map& getObjectMap() JL_NOTSAFEPOINT - { - return objectmap; + jl_profile_atomic([&]() { + if (codeinst) + linfomap[Addr] = std::make_pair(Size, codeinst->def); + if (first) { + objectmap[SectionLoadAddr] = {&Object, + (size_t)SectionSize, + (ptrdiff_t)(SectionAddr - SectionLoadAddr), + *Section, + nullptr, + }; + first = false; + } + }); } -}; + jl_gc_safe_leave(ptls, gc_state); +} -static JITObjectRegistry jl_jit_object_registry; void jl_register_jit_object(const object::ObjectFile &Object, std::function getLoadAddress, std::function lookupWriteAddress) { - jl_jit_object_registry.registerJITObject(Object, getLoadAddress, lookupWriteAddress); + getJITDebugRegistry().registerJITObject(Object, getLoadAddress, lookupWriteAddress); } // TODO: convert the safe names from aotcomile.cpp:makeSafeName back into symbols @@ -462,10 +488,10 @@ static int lookup_pointer( // DWARFContext/DWARFUnit update some internal tables during these queries, so // a lock is needed. - assert(0 == (uintptr_t)pthread_getspecific(debuginfo_asyncsafe_held)); - uv_rwlock_wrlock(&debuginfo_asyncsafe); + assert(0 == getJITDebugRegistry().debuginfo_asyncsafe_held); + uv_rwlock_wrlock(&getJITDebugRegistry().debuginfo_asyncsafe); auto inlineInfo = context->getInliningInfoForAddress(makeAddress(Section, pointer + slide), infoSpec); - uv_rwlock_wrunlock(&debuginfo_asyncsafe); + uv_rwlock_wrunlock(&getJITDebugRegistry().debuginfo_asyncsafe); int fromC = (*frames)[0].fromC; int n_frames = inlineInfo.getNumberOfFrames(); @@ -488,9 +514,9 @@ static int lookup_pointer( info = inlineInfo.getFrame(i); } else { - uv_rwlock_wrlock(&debuginfo_asyncsafe); + uv_rwlock_wrlock(&getJITDebugRegistry().debuginfo_asyncsafe); info = context->getLineInfoForAddress(makeAddress(Section, pointer + slide), infoSpec); - uv_rwlock_wrunlock(&debuginfo_asyncsafe); + uv_rwlock_wrunlock(&getJITDebugRegistry().debuginfo_asyncsafe); } jl_frame_t *frame = &(*frames)[i]; @@ -534,13 +560,37 @@ static int lookup_pointer( #ifndef _OS_WINDOWS_ #include #endif -typedef struct { - const llvm::object::ObjectFile *obj; - DIContext *ctx; - int64_t slide; -} objfileentry_t; -typedef std::map obfiletype; -static obfiletype objfilemap; + + + +#if defined(_OS_DARWIN_) && defined(LLVM_SHLIB) + +void JITDebugInfoRegistry::libc_frames_t::libc_register_frame(const char *Entry) { + auto libc_register_frame_ = jl_atomic_load_relaxed(&this->libc_register_frame_); + if (!libc_register_frame_) { + libc_register_frame_ = (void(*)(void*))dlsym(RTLD_NEXT, "__register_frame"); + jl_atomic_store_relaxed(&this->libc_register_frame_, libc_register_frame_); + } + assert(libc_register_frame_); + jl_profile_atomic([&]() { + libc_register_frame_(const_cast(Entry)); + __register_frame(const_cast(Entry)); + }); +} + +void JITDebugInfoRegistry::libc_frames_t::libc_deregister_frame(const char *Entry) { + auto libc_deregister_frame_ = jl_atomic_load_relaxed(&this->libc_deregister_frame_); + if (!libc_deregister_frame_) { + libc_deregister_frame_ = (void(*)(void*))dlsym(RTLD_NEXT, "__deregister_frame"); + jl_atomic_store_relaxed(&this->libc_deregister_frame_, libc_deregister_frame_); + } + assert(libc_deregister_frame_); + jl_profile_atomic([&]() { + libc_deregister_frame_(const_cast(Entry)); + __deregister_frame(const_cast(Entry)); + }); +} +#endif static bool getObjUUID(llvm::object::MachOObjectFile *obj, uint8_t uuid[16]) JL_NOTSAFEPOINT { @@ -553,11 +603,6 @@ static bool getObjUUID(llvm::object::MachOObjectFile *obj, uint8_t uuid[16]) JL_ } return false; } - -struct debug_link_info { - StringRef filename; - uint32_t crc32; -}; static debug_link_info getDebuglink(const object::ObjectFile &Obj) JL_NOTSAFEPOINT { debug_link_info info = {}; @@ -668,19 +713,11 @@ openDebugInfo(StringRef debuginfopath, const debug_link_info &info) std::move(error_splitobj.get()), std::move(SplitFile.get())); } - -static uint64_t jl_sysimage_base; -static jl_sysimg_fptrs_t sysimg_fptrs; -static jl_method_instance_t **sysimg_fvars_linfo; -static size_t sysimg_fvars_n; extern "C" JL_DLLEXPORT void jl_register_fptrs_impl(uint64_t sysimage_base, const jl_sysimg_fptrs_t *fptrs, jl_method_instance_t **linfos, size_t n) { - jl_sysimage_base = (uintptr_t)sysimage_base; - sysimg_fptrs = *fptrs; - sysimg_fvars_linfo = linfos; - sysimg_fvars_n = n; + getJITDebugRegistry().set_sysimg_info({(uintptr_t) sysimage_base, *fptrs, linfos, n}); } template @@ -695,7 +732,7 @@ static void get_function_name_and_base(llvm::object::SectionRef Section, size_t void **saddr, char **name, bool untrusted_dladdr) JL_NOTSAFEPOINT { // Assume we only need base address for sysimg for now - if (!insysimage || !sysimg_fptrs.base) + if (!insysimage || !getJITDebugRegistry().get_sysimg_info()->sysimg_fptrs.base) saddr = nullptr; bool needs_saddr = saddr && (!*saddr || untrusted_dladdr); bool needs_name = name && (!*name || untrusted_dladdr); @@ -721,7 +758,7 @@ static void get_function_name_and_base(llvm::object::SectionRef Section, size_t } if (Section.getObject() && (needs_saddr || needs_name)) { size_t distance = (size_t)-1; - SymRef sym_found; + object::SymbolRef sym_found; for (auto sym : Section.getObject()->symbols()) { if (!Section.containsSymbol(sym)) continue; @@ -796,7 +833,7 @@ static void get_function_name_and_base(llvm::object::SectionRef Section, size_t #endif } -static objfileentry_t &find_object_file(uint64_t fbase, StringRef fname) JL_NOTSAFEPOINT +static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSAFEPOINT { int isdarwin = 0, islinux = 0, iswindows = 0; #if defined(_OS_DARWIN_) @@ -809,12 +846,11 @@ static objfileentry_t &find_object_file(uint64_t fbase, StringRef fname) JL_NOTS (void)iswindows; // GOAL: Read debuginfo from file - // TODO: need read/write lock here for objfilemap synchronization - obfiletype::iterator it = objfilemap.find(fbase); - if (it != objfilemap.end()) + objfileentry_t entry{nullptr, nullptr, 0}; + auto success = getJITDebugRegistry().get_objfile_map()->emplace(fbase, entry); + if (!success.second) // Return cached value - return it->second; - auto &entry = objfilemap[fbase]; // default initialized + return success.first->second; // GOAL: Assign errorobj StringRef objpath; @@ -984,8 +1020,9 @@ static objfileentry_t &find_object_file(uint64_t fbase, StringRef fname) JL_NOTS auto binary = errorobj->takeBinary(); binary.first.release(); binary.second.release(); - // update cache entry = {debugobj, context, slide}; + // update cache + (*getJITDebugRegistry().get_objfile_map())[fbase] = entry; } else { // TODO: report the error instead of silently consuming it? @@ -1043,7 +1080,7 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * if (fname.empty()) // empirically, LoadedImageName might be missing fname = ModuleInfo.ImageName; DWORD64 fbase = ModuleInfo.BaseOfImage; - bool insysimage = (fbase == jl_sysimage_base); + bool insysimage = (fbase == getJITDebugRegistry().get_sysimg_info()->jl_sysimage_base); if (isSysImg) *isSysImg = insysimage; if (onlySysImg && !insysimage) @@ -1083,7 +1120,7 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * fbase = (uintptr_t)dlinfo.dli_fbase; #endif StringRef fname; - bool insysimage = (fbase == jl_sysimage_base); + bool insysimage = (fbase == getJITDebugRegistry().get_sysimg_info()->jl_sysimage_base); if (saddr && !(insysimage && untrusted_dladdr)) *saddr = dlinfo.dli_saddr; if (isSysImg) @@ -1098,7 +1135,7 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * jl_copy_str(filename, dlinfo.dli_fname); fname = dlinfo.dli_fname; #endif // ifdef _OS_WINDOWS_ - auto &entry = find_object_file(fbase, fname); + auto entry = find_object_file(fbase, fname); *slide = entry.slide; *context = entry.ctx; if (entry.obj) @@ -1139,20 +1176,23 @@ static int jl_getDylibFunctionInfo(jl_frame_t **frames, size_t pointer, int skip return 1; } frame0->fromC = !isSysImg; - if (isSysImg && sysimg_fptrs.base && saddr) { - intptr_t diff = (uintptr_t)saddr - (uintptr_t)sysimg_fptrs.base; - for (size_t i = 0; i < sysimg_fptrs.nclones; i++) { - if (diff == sysimg_fptrs.clone_offsets[i]) { - uint32_t idx = sysimg_fptrs.clone_idxs[i] & jl_sysimg_val_mask; - if (idx < sysimg_fvars_n) // items after this were cloned but not referenced directly by a method (such as our ccall PLT thunks) - frame0->linfo = sysimg_fvars_linfo[idx]; - break; + { + auto sysimg_locked = getJITDebugRegistry().get_sysimg_info(); + if (isSysImg && sysimg_locked->sysimg_fptrs.base && saddr) { + intptr_t diff = (uintptr_t)saddr - (uintptr_t)sysimg_locked->sysimg_fptrs.base; + for (size_t i = 0; i < sysimg_locked->sysimg_fptrs.nclones; i++) { + if (diff == sysimg_locked->sysimg_fptrs.clone_offsets[i]) { + uint32_t idx = sysimg_locked->sysimg_fptrs.clone_idxs[i] & jl_sysimg_val_mask; + if (idx < sysimg_locked->sysimg_fvars_n) // items after this were cloned but not referenced directly by a method (such as our ccall PLT thunks) + frame0->linfo = sysimg_locked->sysimg_fvars_linfo[idx]; + break; + } } - } - for (size_t i = 0; i < sysimg_fvars_n; i++) { - if (diff == sysimg_fptrs.offsets[i]) { - frame0->linfo = sysimg_fvars_linfo[i]; - break; + for (size_t i = 0; i < sysimg_locked->sysimg_fvars_n; i++) { + if (diff == sysimg_locked->sysimg_fptrs.offsets[i]) { + frame0->linfo = sysimg_locked->sysimg_fvars_linfo[i]; + break; + } } } } @@ -1163,13 +1203,13 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, object::SectionRef *Section, llvm::DIContext **context) JL_NOTSAFEPOINT { int found = 0; - assert(0 == (uintptr_t)pthread_getspecific(debuginfo_asyncsafe_held)); - uv_rwlock_wrlock(&debuginfo_asyncsafe); - std::map &objmap = jl_jit_object_registry.getObjectMap(); - std::map::iterator fit = objmap.lower_bound(fptr); - + assert(0 == getJITDebugRegistry().debuginfo_asyncsafe_held); + uv_rwlock_wrlock(&getJITDebugRegistry().debuginfo_asyncsafe); if (symsize) *symsize = 0; + + auto &objmap = getJITDebugRegistry().getObjectMap(); + auto fit = objmap.lower_bound(fptr); if (fit != objmap.end() && fptr < fit->first + fit->second.SectionSize) { *slide = fit->second.slide; *Section = fit->second.Section; @@ -1180,7 +1220,7 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, } found = 1; } - uv_rwlock_wrunlock(&debuginfo_asyncsafe); + uv_rwlock_wrunlock(&getJITDebugRegistry().debuginfo_asyncsafe); return found; } @@ -1199,7 +1239,7 @@ extern "C" JL_DLLEXPORT int jl_getFunctionInfo_impl(jl_frame_t **frames_out, siz int64_t slide; uint64_t symsize; if (jl_DI_for_fptr(pointer, &symsize, &slide, &Section, &context)) { - frames[0].linfo = jl_jit_object_registry.lookupLinfo(pointer); + frames[0].linfo = getJITDebugRegistry().lookupLinfo(pointer); int nf = lookup_pointer(Section, context, frames_out, pointer, slide, true, noInline); return nf; } @@ -1208,36 +1248,9 @@ extern "C" JL_DLLEXPORT int jl_getFunctionInfo_impl(jl_frame_t **frames_out, siz extern "C" jl_method_instance_t *jl_gdblookuplinfo(void *p) JL_NOTSAFEPOINT { - return jl_jit_object_registry.lookupLinfo((size_t)p); + return getJITDebugRegistry().lookupLinfo((size_t)p); } -#if (defined(_OS_LINUX_) || defined(_OS_FREEBSD_) || (defined(_OS_DARWIN_) && defined(LLVM_SHLIB))) -extern "C" void __register_frame(void*); -extern "C" void __deregister_frame(void*); - -template -static void processFDEs(const char *EHFrameAddr, size_t EHFrameSize, callback f) -{ - const char *P = EHFrameAddr; - const char *End = P + EHFrameSize; - do { - const char *Entry = P; - P += 4; - assert(P <= End); - uint32_t Length = *(const uint32_t*)Entry; - // Length == 0: Terminator - if (Length == 0) - break; - assert(P + Length <= End); - uint32_t Offset = *(const uint32_t*)P; - // Offset == 0: CIE - if (Offset != 0) - f(Entry); - P += Length; - } while (P != End); -} -#endif - #if defined(_OS_DARWIN_) && defined(LLVM_SHLIB) /* @@ -1248,37 +1261,20 @@ static void processFDEs(const char *EHFrameAddr, size_t EHFrameSize, callback f) * ourselves to ensure the right one gets picked. */ -static void (*libc_register_frame)(void*) = NULL; -static void (*libc_deregister_frame)(void*) = NULL; - // This implementation handles frame registration for local targets. void register_eh_frames(uint8_t *Addr, size_t Size) { // On OS X OS X __register_frame takes a single FDE as an argument. // See http://lists.cs.uiuc.edu/pipermail/llvmdev/2013-April/061768.html processFDEs((char*)Addr, Size, [](const char *Entry) { - if (!libc_register_frame) { - libc_register_frame = (void(*)(void*))dlsym(RTLD_NEXT, "__register_frame"); - } - assert(libc_register_frame); - jl_profile_atomic([&]() { - libc_register_frame(const_cast(Entry)); - __register_frame(const_cast(Entry)); - }); + getJITDebugRegistry().libc_frames.libc_register_frame(Entry); }); } void deregister_eh_frames(uint8_t *Addr, size_t Size) { processFDEs((char*)Addr, Size, [](const char *Entry) { - if (!libc_deregister_frame) { - libc_deregister_frame = (void(*)(void*))dlsym(RTLD_NEXT, "__deregister_frame"); - } - assert(libc_deregister_frame); - jl_profile_atomic([&]() { - libc_deregister_frame(const_cast(Entry)); - __deregister_frame(const_cast(Entry)); - }); + getJITDebugRegistry().libc_frames.libc_deregister_frame(Entry); }); } @@ -1287,12 +1283,6 @@ void deregister_eh_frames(uint8_t *Addr, size_t Size) !defined(_CPU_ARM_) // ARM does not have/use .eh_frame, so we handle this elsewhere #include -struct unw_table_entry -{ - int32_t start_ip_offset; - int32_t fde_offset; -}; - // Skip over an arbitrary long LEB128 encoding. // Return the pointer to the first unprocessed byte. static const uint8_t *consume_leb128(const uint8_t *Addr, const uint8_t *End) @@ -1630,8 +1620,8 @@ uint64_t jl_getUnwindInfo_impl(uint64_t dwAddr) { // Might be called from unmanaged thread jl_lock_profile_impl(); - std::map &objmap = jl_jit_object_registry.getObjectMap(); - std::map::iterator it = objmap.lower_bound(dwAddr); + auto &objmap = getJITDebugRegistry().getObjectMap(); + auto it = objmap.lower_bound(dwAddr); uint64_t ipstart = 0; // ip of the start of the section (if found) if (it != objmap.end() && dwAddr < it->first + it->second.SectionSize) { ipstart = (uint64_t)(uintptr_t)(*it).first; diff --git a/src/jitlayers.h b/src/jitlayers.h index ba3f81fa66997..af47102173a7b 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -11,6 +11,7 @@ #include #include "julia_assert.h" +#include "debug-registry.h" // As of LLVM 13, there are two runtime JIT linker implementations, the older // RuntimeDyld (used via orc::RTDyldObjectLinkingLayer) and the newer JITLink @@ -214,6 +215,10 @@ class JuliaOJIT { const DataLayout& getDataLayout() const; const Triple& getTargetTriple() const; size_t getTotalBytes() const; + + JITDebugInfoRegistry &getDebugInfoRegistry() JL_NOTSAFEPOINT { + return DebugRegistry; + } private: std::string getMangledName(StringRef Name); std::string getMangledName(const GlobalValue *GV); @@ -236,6 +241,8 @@ class JuliaOJIT { orc::JITDylib &GlobalJD; orc::JITDylib &JD; + JITDebugInfoRegistry DebugRegistry; + #ifndef JL_USE_JITLINK std::shared_ptr MemMgr; #endif diff --git a/src/processor.h b/src/processor.h index 1d385cfc80b98..f3b571cf9b937 100644 --- a/src/processor.h +++ b/src/processor.h @@ -1,5 +1,8 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license +#ifndef JL_PROCESSOR_H +#define JL_PROCESSOR_H + #include "support/dtypes.h" #include "julia.h" @@ -215,3 +218,5 @@ extern "C" JL_DLLEXPORT std::vector jl_get_llvm_clone_targets( std::string jl_get_cpu_name_llvm(void); std::string jl_get_cpu_features_llvm(void); #endif + +#endif From 8bca2f4b3d92b1d4fc234ff516b7ba9a80c6f915 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Wed, 25 May 2022 16:52:23 +0900 Subject: [PATCH 68/68] fix #45440, improve the robustness of concrete-evaled callsite inlining --- base/compiler/ssair/inlining.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index 5e3ad4febe60f..f5fb200c03c98 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -735,20 +735,20 @@ end function compileable_specialization(et::Union{EdgeTracker, Nothing}, match::MethodMatch, effects::Effects) mi = specialize_method(match; compilesig=true) - mi !== nothing && et !== nothing && push!(et, mi::MethodInstance) mi === nothing && return nothing + et !== nothing && push!(et, mi) return InvokeCase(mi, effects) end function compileable_specialization(et::Union{EdgeTracker, Nothing}, linfo::MethodInstance, effects::Effects) mi = specialize_method(linfo.def::Method, linfo.specTypes, linfo.sparam_vals; compilesig=true) - mi !== nothing && et !== nothing && push!(et, mi::MethodInstance) mi === nothing && return nothing + et !== nothing && push!(et, mi) return InvokeCase(mi, effects) end -function compileable_specialization(et::Union{EdgeTracker, Nothing}, (; linfo)::InferenceResult, effects::Effects) - return compileable_specialization(et, linfo, effects) +function compileable_specialization(et::Union{EdgeTracker, Nothing}, result::InferenceResult, effects::Effects) + return compileable_specialization(et, result.linfo, effects) end function resolve_todo(todo::InliningTodo, state::InliningState, flag::UInt8) @@ -1257,7 +1257,11 @@ function handle_const_call!( result = results[j] if isa(result, ConstResult) case = const_result_item(result, state) - push!(cases, InliningCase(result.mi.specTypes, case)) + if case === nothing + fully_covered = false + else + push!(cases, InliningCase(result.mi.specTypes, case)) + end elseif isa(result, InferenceResult) fully_covered &= handle_inf_result!(result, argtypes, flag, state, cases) else