Skip to content

add logic to prefer loading modules that are already loaded #55908

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 4, 2024
Merged

add logic to prefer loading modules that are already loaded #55908

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion base/Base.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -648,7 +648,7 @@ function __init__()
empty!(explicit_loaded_modules)
empty!(loaded_precompiles) # If we load a packageimage when building the image this might not be empty
for (mod, key) in module_keys
loaded_precompiles[key => module_build_id(mod)] = mod
push!(get!(Vector{Module}, loaded_precompiles, key), mod)
end
if haskey(ENV, "JULIA_MAX_NUM_PRECOMPILE_FILES")
MAX_NUM_PRECOMPILE_FILES[] = parse(Int, ENV["JULIA_MAX_NUM_PRECOMPILE_FILES"])
Expand Down
184 changes: 103 additions & 81 deletions base/loading.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1231,7 +1231,7 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No
dep = depmods[i]
dep isa Module && continue
_, depkey, depbuild_id = dep::Tuple{String, PkgId, UInt128}
dep = loaded_precompiles[depkey => depbuild_id]
dep = something(maybe_loaded_precompile(depkey, depbuild_id))
@assert PkgId(dep) == depkey && module_build_id(dep) === depbuild_id
depmods[i] = dep
end
Expand Down Expand Up @@ -1337,6 +1337,7 @@ end

function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String)
# This function is also used by PkgCacheInspector.jl
assert_havelock(require_lock)
restored = sv[1]::Vector{Any}
for M in restored
M = M::Module
Expand All @@ -1345,7 +1346,7 @@ function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String)
end
if parentmodule(M) === M
push!(loaded_modules_order, M)
loaded_precompiles[pkg => module_build_id(M)] = M
push!(get!(Vector{Module}, loaded_precompiles, pkg), M)
end
end

Expand Down Expand Up @@ -1945,90 +1946,102 @@ end
assert_havelock(require_lock)
paths = find_all_in_cache_path(pkg, DEPOT_PATH)
newdeps = PkgId[]
for path_to_try in paths::Vector{String}
staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try; reasons, stalecheck)
if staledeps === true
continue
end
try
staledeps, ocachefile, newbuild_id = staledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
# finish checking staledeps module graph
for i in eachindex(staledeps)
dep = staledeps[i]
dep isa Module && continue
modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128}
modpaths = find_all_in_cache_path(modkey, DEPOT_PATH)
for modpath_to_try in modpaths
modstaledeps = stale_cachefile(modkey, modbuild_id, modpath, modpath_to_try; stalecheck)
if modstaledeps === true
continue
end
modstaledeps, modocachepath, _ = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
staledeps[i] = (modpath, modkey, modbuild_id, modpath_to_try, modstaledeps, modocachepath)
@goto check_next_dep
try_build_ids = UInt128[build_id]
if build_id == UInt128(0)
let loaded = get(loaded_precompiles, pkg, nothing)
if loaded !== nothing
for mod in loaded # try these in reverse original load order to see if one is already valid
pushfirst!(try_build_ids, module_build_id(mod))
end
@debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache."
@goto check_next_path
@label check_next_dep
end
M = get(loaded_precompiles, pkg => newbuild_id, nothing)
if isa(M, Module)
stalecheck && register_root_module(M)
return M
end
if stalecheck
try
touch(path_to_try) # update timestamp of precompilation file
catch ex # file might be read-only and then we fail to update timestamp, which is fine
ex isa IOError || rethrow()
end
end
end
for build_id in try_build_ids
for path_to_try in paths::Vector{String}
staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try; reasons, stalecheck)
if staledeps === true
continue
end
# finish loading module graph into staledeps
# TODO: call all start_loading calls (in reverse order) before calling any _include_from_serialized, since start_loading will drop the loading lock
for i in eachindex(staledeps)
dep = staledeps[i]
dep isa Module && continue
modpath, modkey, modbuild_id, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, UInt128, String, Vector{Any}, Union{Nothing, String}}
dep = start_loading(modkey, modbuild_id, stalecheck)
while true
if dep isa Module
if PkgId(dep) == modkey && module_build_id(dep) === modbuild_id
break
else
@debug "Rejecting cache file $path_to_try because module $modkey got loaded at a different version than expected."
@goto check_next_path
try
staledeps, ocachefile, newbuild_id = staledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
# finish checking staledeps module graph
for i in eachindex(staledeps)
dep = staledeps[i]
dep isa Module && continue
modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128}
modpaths = find_all_in_cache_path(modkey, DEPOT_PATH)
for modpath_to_try in modpaths
modstaledeps = stale_cachefile(modkey, modbuild_id, modpath, modpath_to_try; stalecheck)
if modstaledeps === true
continue
end
modstaledeps, modocachepath, _ = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
staledeps[i] = (modpath, modkey, modbuild_id, modpath_to_try, modstaledeps, modocachepath)
@goto check_next_dep
end
@debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache."
@goto check_next_path
@label check_next_dep
end
M = maybe_loaded_precompile(pkg, newbuild_id)
if isa(M, Module)
stalecheck && register_root_module(M)
return M
end
if stalecheck
try
touch(path_to_try) # update timestamp of precompilation file
catch ex # file might be read-only and then we fail to update timestamp, which is fine
ex isa IOError || rethrow()
end
if dep === nothing
try
set_pkgorigin_version_path(modkey, modpath)
dep = _include_from_serialized(modkey, modcachepath, modocachepath, modstaledeps; register = stalecheck)
finally
end_loading(modkey, dep)
end
# finish loading module graph into staledeps
# TODO: call all start_loading calls (in reverse order) before calling any _include_from_serialized, since start_loading will drop the loading lock
for i in eachindex(staledeps)
dep = staledeps[i]
dep isa Module && continue
modpath, modkey, modbuild_id, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, UInt128, String, Vector{Any}, Union{Nothing, String}}
dep = start_loading(modkey, modbuild_id, stalecheck)
while true
if dep isa Module
if PkgId(dep) == modkey && module_build_id(dep) === modbuild_id
break
else
@debug "Rejecting cache file $path_to_try because module $modkey got loaded at a different version than expected."
@goto check_next_path
end
end
if !isa(dep, Module)
@debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep
@goto check_next_path
else
push!(newdeps, modkey)
if dep === nothing
try
set_pkgorigin_version_path(modkey, modpath)
dep = _include_from_serialized(modkey, modcachepath, modocachepath, modstaledeps; register = stalecheck)
finally
end_loading(modkey, dep)
end
if !isa(dep, Module)
@debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep
@goto check_next_path
else
push!(newdeps, modkey)
end
end
end
staledeps[i] = dep
end
staledeps[i] = dep
end
restored = get(loaded_precompiles, pkg => newbuild_id, nothing)
if !isa(restored, Module)
restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps; register = stalecheck)
end
isa(restored, Module) && return restored
@debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored
@label check_next_path
finally
for modkey in newdeps
insert_extension_triggers(modkey)
stalecheck && run_package_callbacks(modkey)
restored = maybe_loaded_precompile(pkg, newbuild_id)
if !isa(restored, Module)
restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps; register = stalecheck)
end
isa(restored, Module) && return restored
@debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored
@label check_next_path
finally
for modkey in newdeps
insert_extension_triggers(modkey)
stalecheck && run_package_callbacks(modkey)
end
empty!(newdeps)
end
empty!(newdeps)
end
end
return nothing
Expand All @@ -2047,7 +2060,7 @@ function start_loading(modkey::PkgId, build_id::UInt128, stalecheck::Bool)
loaded = stalecheck ? maybe_root_module(modkey) : nothing
loaded isa Module && return loaded
if build_id != UInt128(0)
loaded = get(loaded_precompiles, modkey => build_id, nothing)
loaded = maybe_loaded_precompile(modkey, build_id)
loaded isa Module && return loaded
end
loading = get(package_locks, modkey, nothing)
Expand Down Expand Up @@ -2377,12 +2390,21 @@ const pkgorigins = Dict{PkgId,PkgOrigin}()

const explicit_loaded_modules = Dict{PkgId,Module}() # Emptied on Julia start
const loaded_modules = Dict{PkgId,Module}() # available to be explicitly loaded
const loaded_precompiles = Dict{Pair{PkgId,UInt128},Module}() # extended (complete) list of modules, available to be loaded
const loaded_precompiles = Dict{PkgId,Vector{Module}}() # extended (complete) list of modules, available to be loaded
const loaded_modules_order = Vector{Module}()
const module_keys = IdDict{Module,PkgId}() # the reverse of loaded_modules

root_module_key(m::Module) = @lock require_lock module_keys[m]

function maybe_loaded_precompile(key::PkgId, buildid::UInt128)
assert_havelock(require_lock)
mods = get(loaded_precompiles, key, nothing)
mods === nothing && return
for mod in mods
module_build_id(mod) == buildid && return mod
end
end

function module_build_id(m::Module)
hi, lo = ccall(:jl_module_build_id, NTuple{2,UInt64}, (Any,), m)
return (UInt128(hi) << 64) | lo
Expand All @@ -2403,7 +2425,7 @@ end
end
end
end
haskey(loaded_precompiles, key => module_build_id(m)) || push!(loaded_modules_order, m)
maybe_loaded_precompile(key, module_build_id(m)) === nothing && push!(loaded_modules_order, m)
loaded_modules[key] = m
explicit_loaded_modules[key] = m
module_keys[m] = key
Expand Down Expand Up @@ -3789,8 +3811,8 @@ end
for i in 1:ndeps
req_key, req_build_id = required_modules[i]
# Check if module is already loaded
if !stalecheck && haskey(loaded_precompiles, req_key => req_build_id)
M = loaded_precompiles[req_key => req_build_id]
M = stalecheck ? nothing : maybe_loaded_precompile(req_key, req_build_id)
if M !== nothing
@assert PkgId(M) == req_key && module_build_id(M) === req_build_id
depmods[i] = M
elseif root_module_exists(req_key)
Expand Down
36 changes: 34 additions & 2 deletions test/loading.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

original_depot_path = copy(Base.DEPOT_PATH)

using Test

# Tests for @__LINE__ inside and outside of macros
# NOTE: the __LINE__ numbers for these first couple tests are significant, so
# adding any lines here will make those tests fail
@test (@__LINE__) == 8

macro macro_caller_lineno()
Expand Down Expand Up @@ -33,6 +33,9 @@ end
@test @nested_LINE_expansion() == ((@__LINE__() - 4, @__LINE__() - 12), @__LINE__())
@test @nested_LINE_expansion2() == ((@__LINE__() - 5, @__LINE__() - 9), @__LINE__())

original_depot_path = copy(Base.DEPOT_PATH)
include("precompile_utils.jl")

loaded_files = String[]
push!(Base.include_callbacks, (mod::Module, fn::String) -> push!(loaded_files, fn))
include("test_sourcepath.jl")
Expand Down Expand Up @@ -1603,3 +1606,32 @@ end
copy!(LOAD_PATH, old_load_path)
end
end

@testset "require_stdlib loading duplication" begin
depot_path = mktempdir()
oldBase64 = nothing
try
push!(empty!(DEPOT_PATH), depot_path)
Base64_key = Base.PkgId(Base.UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"), "Base64")
oldBase64 = Base.unreference_module(Base64_key)
cc = Base.compilecache(Base64_key)
@test Base.isprecompiled(Base64_key, cachepaths=String[cc[1]])
empty!(DEPOT_PATH)
Base.require_stdlib(Base64_key)
push!(DEPOT_PATH, depot_path)
append!(DEPOT_PATH, original_depot_path)
oldloaded = @lock(Base.require_lock, length(get(Base.loaded_precompiles, Base64_key, Module[])))
Base.require(Base64_key)
@test @lock(Base.require_lock, length(get(Base.loaded_precompiles, Base64_key, Module[]))) == oldloaded
Base.unreference_module(Base64_key)
empty!(DEPOT_PATH)
push!(DEPOT_PATH, depot_path)
Base.require(Base64_key)
@test @lock(Base.require_lock, length(get(Base.loaded_precompiles, Base64_key, Module[]))) == oldloaded + 1
Base.unreference_module(Base64_key)
finally
oldBase64 === nothing || Base.register_root_module(oldBase64)
copy!(DEPOT_PATH, original_depot_path)
rm(depot_path, force=true, recursive=true)
end
end
Loading