From ea1a0d2512be72a4568348e6b5fdfbbe687436c3 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 23 Feb 2024 14:59:20 -0500 Subject: [PATCH] when loading code for internal purposes, load stdlib files directly(#53326) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This bypasses DEPOT_PATH, LOAD_PATH, and stale checks when loading known stdlib code for known purposes from known locations, specifically to avoid the problem that I cannot fix my tools because I have used my tools to break my tools. This helps avoid the situation that the user can break important Pkg, REPL, Socket or similar features simply because they chose to remove `@stdlib` from their environment. For example, if you make an edit to REPL, then this will trigger recompilation and load the edited version: ``` $ ./julia -e 'using REPL' -iq ┌ Info: Precompiling REPL [3fa0cd96-eef1-5676-8a61-b3b8758bbffb] (cache misses: include_dependency fsize change (2), invalid header (10), mismatched flags (1)) └ @ Base loading.jl:2643 julia> ``` But this will load the version that shipped with Julia, regardless of the state of the cache or the source code (unless you delete the cache files): ``` $ ./julia -iq julia> ``` Fixes #53365 --- base/Base.jl | 3 +- base/client.jl | 7 +- base/loading.jl | 311 +++++++++++------- base/stream.jl | 4 +- base/util.jl | 2 +- .../InteractiveUtils/src/InteractiveUtils.jl | 4 +- stdlib/LinearAlgebra/src/LinearAlgebra.jl | 2 +- stdlib/REPL/src/REPL.jl | 22 +- test/precompile.jl | 6 +- 9 files changed, 210 insertions(+), 151 deletions(-) diff --git a/base/Base.jl b/base/Base.jl index c729ab9ca707e..e02b132aae6e4 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -629,8 +629,7 @@ function profile_printing_listener(cond::Base.AsyncCondition) profile = nothing try while _trywait(cond) - # this call to require is mostly legal, only because Profile has no dependencies and is usually in LOAD_PATH - profile = @something(profile, require(PkgId(UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"), "Profile")))::Module + profile = @something(profile, require_stdlib(PkgId(UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"), "Profile")))::Module invokelatest(profile.peek_report[]) if Base.get_bool_env("JULIA_PROFILE_PEEK_HEAP_SNAPSHOT", false) === true println(stderr, "Saving heap snapshot...") diff --git a/base/client.jl b/base/client.jl index 201792c786b51..31a3016489dcd 100644 --- a/base/client.jl +++ b/base/client.jl @@ -260,7 +260,7 @@ function exec_options(opts) # Load Distributed module only if any of the Distributed options have been specified. distributed_mode = (opts.worker == 1) || (opts.nprocs > 0) || (opts.machine_file != C_NULL) if distributed_mode - let Distributed = require(PkgId(UUID((0x8ba89e20_285c_5b6f, 0x9357_94700520ee1b)), "Distributed")) + let Distributed = require_stdlib(PkgId(UUID((0x8ba89e20_285c_5b6f, 0x9357_94700520ee1b)), "Distributed")) Core.eval(MainInclude, :(const Distributed = $Distributed)) Core.eval(Main, :(using Base.MainInclude.Distributed)) end @@ -386,7 +386,8 @@ function load_InteractiveUtils(mod::Module=Main) # load interactive-only libraries if !isdefined(MainInclude, :InteractiveUtils) try - let InteractiveUtils = require(PkgId(UUID(0xb77e0a4c_d291_57a0_90e8_8db25a27a240), "InteractiveUtils")) + # TODO: we have to use require_stdlib here because it is a dependency of REPL, but we would sort of prefer not to + let InteractiveUtils = require_stdlib(PkgId(UUID(0xb77e0a4c_d291_57a0_90e8_8db25a27a240), "InteractiveUtils")) Core.eval(MainInclude, :(const InteractiveUtils = $InteractiveUtils)) end catch ex @@ -401,7 +402,7 @@ end function load_REPL() # load interactive-only libraries try - return Base.require(PkgId(UUID(0x3fa0cd96_eef1_5676_8a61_b3b8758bbffb), "REPL")) + return Base.require_stdlib(PkgId(UUID(0x3fa0cd96_eef1_5676_8a61_b3b8758bbffb), "REPL")) catch ex @warn "Failed to import REPL" exception=(ex, catch_backtrace()) end diff --git a/base/loading.jl b/base/loading.jl index 3c7bb28cfdd17..2be97237238dd 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -256,9 +256,9 @@ struct LoadingCache env_project_file::Dict{String, Union{Bool, String}} project_file_manifest_path::Dict{String, Union{Nothing, String}} require_parsed::Set{String} - identified_where::Dict{Tuple{PkgId, String}, Union{Nothing, Tuple{PkgId, Union{Nothing, String}}}} - identified::Dict{String, Union{Nothing, Tuple{PkgId, Union{Nothing, String}}}} - located::Dict{Tuple{PkgId, Union{String, Nothing}}, Union{Tuple{Union{String, Nothing}, Union{String, Nothing}}, Nothing}} + identified_where::Dict{Tuple{PkgId, String}, Union{Nothing, Tuple{PkgId, String}}} + identified::Dict{String, Union{Nothing, Tuple{PkgId, String}}} + located::Dict{Tuple{PkgId, Union{String, Nothing}}, Union{Tuple{String, String}, Nothing}} end const LOADING_CACHE = Ref{Union{LoadingCache, Nothing}}(nothing) LoadingCache() = LoadingCache(load_path(), Dict(), Dict(), Dict(), Set(), Dict(), Dict(), Dict()) @@ -298,7 +298,7 @@ end ## package identification: determine unique identity of package to be loaded ## # Used by Pkg but not used in loading itself -function find_package(arg) +function find_package(arg) # ::Union{Nothing,String} pkgenv = identify_package_env(arg) pkgenv === nothing && return nothing pkg, env = pkgenv @@ -307,21 +307,21 @@ end """ Base.identify_package_env(name::String)::Union{Tuple{PkgId, String}, Nothing} - Base.identify_package_env(where::Union{Module,PkgId}, name::String)::Union{Tuple{PkgId, String} Nothing} + Base.identify_package_env(where::Union{Module,PkgId}, name::String)::Union{Tuple{PkgId, Union{String, Nothing}}, Nothing} Same as [`Base.identify_package`](@ref) except that the path to the environment where the package is identified -is also returned. +is also returned, except when the identity is not identified. """ identify_package_env(where::Module, name::String) = identify_package_env(PkgId(where), name) function identify_package_env(where::PkgId, name::String) cache = LOADING_CACHE[] if cache !== nothing - pkg_env = get(cache.identified_where, (where, name), nothing) - pkg_env === nothing || return pkg_env + pkg_env = get(cache.identified_where, (where, name), missing) + pkg_env === missing || return pkg_env end pkg_env = nothing if where.name === name - pkg_env = where, nothing + return (where, nothing) elseif where.uuid === nothing pkg_env = identify_package_env(name) # ignore `where` else @@ -342,8 +342,8 @@ end function identify_package_env(name::String) cache = LOADING_CACHE[] if cache !== nothing - pkg_env = get(cache.identified, name, nothing) - pkg_env === nothing || return pkg_env + pkg_env = get(cache.identified, name, missing) + pkg_env === missing || return pkg_env end pkg_env = nothing for env in load_path() @@ -390,17 +390,16 @@ identify_package(where::Module, name::String) = _nothing_or_first(identify_packa identify_package(where::PkgId, name::String) = _nothing_or_first(identify_package_env(where, name)) identify_package(name::String) = _nothing_or_first(identify_package_env(name)) -function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing) +function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing)::Union{Nothing,Tuple{String,String}} cache = LOADING_CACHE[] if cache !== nothing - pathenv = get(cache.located, (pkg, stopenv), nothing) - pathenv === nothing || return pathenv + pathenv = get(cache.located, (pkg, stopenv), missing) + pathenv === missing || return pathenv end path = nothing env′ = nothing if pkg.uuid === nothing for env in load_path() - env′ = env # look for the toplevel pkg `pkg.name` in this entry found = project_deps_get(env, pkg.name) if found !== nothing @@ -410,6 +409,7 @@ function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing) # return the path the entry point for the code, if it could be found # otherwise, signal failure path = implicit_manifest_uuid_path(env, pkg) + env′ = env @goto done end end @@ -419,7 +419,6 @@ function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing) end else for env in load_path() - env′ = env path = manifest_uuid_path(env, pkg) # missing is used as a sentinel to stop looking further down in envs if path === missing @@ -428,6 +427,7 @@ function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing) end if path !== nothing path = entry_path(path, pkg.name) + env′ = env @goto done end if !(loading_extension || precompiling_extension) @@ -439,14 +439,16 @@ function locate_package_env(pkg::PkgId, stopenv::Union{String, Nothing}=nothing) mbypath = manifest_uuid_path(Sys.STDLIB, pkg) if mbypath isa String path = entry_path(mbypath, pkg.name) + env′ = Sys.STDLIB @goto done end end @label done if cache !== nothing - cache.located[(pkg, stopenv)] = path, env′ + cache.located[(pkg, stopenv)] = path === nothing ? nothing : (path, something(env′)) end - return path, env′ + path === nothing && return nothing + return path, something(env′) end """ @@ -1083,11 +1085,12 @@ end # want it to pick up caches that already exist for other optimization levels const ignore_compiled_cache = PkgId[] -function find_all_in_cache_path(pkg::PkgId) +function find_all_in_cache_path(pkg::PkgId, DEPOT_PATH::typeof(DEPOT_PATH)=DEPOT_PATH) paths = String[] pkg in ignore_compiled_cache && return paths entrypath, entryfile = cache_file_entry(pkg) - for path in joinpath.(DEPOT_PATH, entrypath) + for path in DEPOT_PATH + path = joinpath(path, entrypath) isdir(path) || continue for file in readdir(path, sort = false) # no sort given we sort later if !((pkg.uuid === nothing && file == entryfile * ".ji") || @@ -1142,6 +1145,8 @@ cachefile_from_ocachefile(cachefile) = string(chopsuffix(cachefile, ".$(Libc.Lib # use an Int counter so that nested @time_imports calls all remain open const TIMING_IMPORTS = Threads.Atomic{Int}(0) +# loads a precompile cache file, ignoring stale_cachefile tests +# assuming all depmods are already loaded and everything is valid # these return either the array of modules loaded from the path / content given # or an Exception that describes why it couldn't be loaded # and it reconnects the Base.Docs.META @@ -1169,6 +1174,15 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No t_comp_before = cumulative_compile_time_ns() end + for i in 1:length(depmods) + dep = depmods[i] + dep isa Module && continue + _, depkey, depbuild_id = dep::Tuple{String, PkgId, UInt128} + @assert root_module_exists(depkey) + dep = root_module(depkey) + depmods[i] = dep + end + if ocachepath !== nothing @debug "Loading object cache file $ocachepath for $(repr("text/plain", pkg))" sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any, Cint, Cstring, Cint), ocachepath, depmods, false, pkg.name, ignore_native) @@ -1314,9 +1328,6 @@ function insert_extension_triggers(pkg::PkgId) path_env_loc = locate_package_env(pkg) path_env_loc === nothing && return path, env_loc = path_env_loc - if path === nothing || env_loc === nothing - return - end insert_extension_triggers(env_loc, pkg) end @@ -1637,7 +1648,7 @@ function isprecompiled(pkg::PkgId; cachepaths::Vector{String}=Base.find_all_in_cache_path(pkg), sourcepath::Union{String,Nothing}=Base.locate_package(pkg), flags::CacheFlags=CacheFlags()) - isnothing(sourcepath) && error("Cannot locate source for $(repr(pkg))") + isnothing(sourcepath) && error("Cannot locate source for $(repr("text/plain", pkg))") for path_to_try in cachepaths staledeps = stale_cachefile(sourcepath, path_to_try, ignore_loaded = true, requested_flags=flags) if staledeps === true @@ -1674,10 +1685,9 @@ function isprecompiled(pkg::PkgId; return false end -# loads a precompile cache file, after checking stale_cachefile tests +# search for a precompile cache file to load, after some various checks function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt128) assert_havelock(require_lock) - loaded = nothing if root_module_exists(modkey) loaded = root_module(modkey) else @@ -1687,7 +1697,7 @@ function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt128) modpath = locate_package(modkey) modpath === nothing && return nothing set_pkgorigin_version_path(modkey, String(modpath)) - loaded = _require_search_from_serialized(modkey, String(modpath), build_id) + loaded = _require_search_from_serialized(modkey, String(modpath), build_id, true) finally end_loading(modkey, loaded) end @@ -1697,46 +1707,10 @@ function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt128) end end end - if !(loaded isa Module) || PkgId(loaded) != modkey - return ErrorException("Required dependency $modkey failed to load from a cache file.") + if loaded isa Module && PkgId(loaded) == modkey && module_build_id(loaded) === build_id + return loaded end - return loaded -end - -# loads a precompile cache file, ignoring stale_cachefile tests -# assuming all depmods are already loaded and everything is valid -function _tryrequire_from_serialized(modkey::PkgId, path::String, ocachepath::Union{Nothing, String}, sourcepath::String, depmods::Vector{Any}) - assert_havelock(require_lock) - loaded = nothing - if root_module_exists(modkey) - loaded = root_module(modkey) - else - loaded = start_loading(modkey) - if loaded === nothing - try - for i in 1:length(depmods) - dep = depmods[i] - dep isa Module && continue - _, depkey, depbuild_id = dep::Tuple{String, PkgId, UInt128} - @assert root_module_exists(depkey) - dep = root_module(depkey) - depmods[i] = dep - end - set_pkgorigin_version_path(modkey, sourcepath) - loaded = _include_from_serialized(modkey, path, ocachepath, depmods) - finally - end_loading(modkey, loaded) - end - if loaded isa Module - insert_extension_triggers(modkey) - run_package_callbacks(modkey) - end - end - end - if !(loaded isa Module) || PkgId(loaded) != modkey - return ErrorException("Required dependency $modkey failed to load from a cache file.") - end - return loaded + return ErrorException("Required dependency $modkey failed to load from a cache file.") end # returns whether the package is tracked in coverage or malloc tracking based on @@ -1766,7 +1740,7 @@ function pkg_tracked(includes) end # loads a precompile cache file, ignoring stale_cachefile tests -# load the best available (non-stale) version of all dependent modules first +# load all dependent modules first function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}) assert_havelock(require_lock) local depmodnames @@ -1808,56 +1782,92 @@ end # returns `nothing` if require found a precompile cache for this sourcepath, but couldn't load it # returns the set of modules restored if the cache load succeeded -@constprop :none function _require_search_from_serialized(pkg::PkgId, sourcepath::String, build_id::UInt128; reasons=nothing) +@constprop :none function _require_search_from_serialized(pkg::PkgId, sourcepath::String, build_id::UInt128, stalecheck::Bool; reasons=nothing, DEPOT_PATH::typeof(DEPOT_PATH)=DEPOT_PATH) assert_havelock(require_lock) - paths = find_all_in_cache_path(pkg) + paths = find_all_in_cache_path(pkg, DEPOT_PATH) + newdeps = PkgId[] for path_to_try in paths::Vector{String} - staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try; reasons) + staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try; reasons, stalecheck) if staledeps === true continue end - staledeps, ocachefile = staledeps::Tuple{Vector{Any}, Union{Nothing, String}} - # finish checking staledeps module graph - for i in 1:length(staledeps) - dep = staledeps[i] - dep isa Module && continue - modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128} - modpaths = find_all_in_cache_path(modkey) - for modpath_to_try in modpaths - modstaledeps = stale_cachefile(modkey, modbuild_id, modpath, modpath_to_try) - if modstaledeps === true - continue - end - modstaledeps, modocachepath = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}} - staledeps[i] = (modpath, modkey, modpath_to_try, modstaledeps, modocachepath) - @goto check_next_dep - end - @debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache." - @goto check_next_path - @label check_next_dep - end try - touch(path_to_try) # update timestamp of precompilation file - catch ex # file might be read-only and then we fail to update timestamp, which is fine - ex isa IOError || rethrow() - end - # finish loading module graph into staledeps - for i in 1:length(staledeps) - dep = staledeps[i] - dep isa Module && continue - modpath, modkey, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, String, Vector{Any}, Union{Nothing, String}} - dep = _tryrequire_from_serialized(modkey, modcachepath, modocachepath, modpath, modstaledeps) - if !isa(dep, Module) - @debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep + staledeps, ocachefile = staledeps::Tuple{Vector{Any}, Union{Nothing, String}} + # finish checking staledeps module graph + for i in 1:length(staledeps) + dep = staledeps[i] + dep isa Module && continue + modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128} + modpaths = find_all_in_cache_path(modkey, DEPOT_PATH) + for modpath_to_try in modpaths + modstaledeps = stale_cachefile(modkey, modbuild_id, modpath, modpath_to_try; stalecheck) + if modstaledeps === true + continue + end + modstaledeps, modocachepath = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}} + staledeps[i] = (modpath, modkey, modbuild_id, modpath_to_try, modstaledeps, modocachepath) + @goto check_next_dep + end + @debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache." @goto check_next_path + @label check_next_dep + end + if stalecheck + try + touch(path_to_try) # update timestamp of precompilation file + catch ex # file might be read-only and then we fail to update timestamp, which is fine + ex isa IOError || rethrow() + end end - staledeps[i] = dep + # finish loading module graph into staledeps + for i in 1:length(staledeps) + dep = staledeps[i] + dep isa Module && continue + modpath, modkey, modbuild_id, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, UInt128, String, Vector{Any}, Union{Nothing, String}} + dep = nothing + if root_module_exists(modkey) + dep = root_module(modkey) + end + while true + if dep isa Module + if PkgId(dep) == modkey && module_build_id(dep) === modbuild_id + break + else + if stalecheck + @debug "Rejecting cache file $path_to_try because module $modkey is already loaded and incompatible." + @goto check_next_path + end + end + end + dep = start_loading(modkey) + if dep === nothing + try + set_pkgorigin_version_path(modkey, modpath) + dep = _include_from_serialized(modkey, modcachepath, modocachepath, modstaledeps) + finally + end_loading(modkey, dep) + end + if !isa(dep, Module) + @debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep + @goto check_next_path + else + push!(newdeps, modkey) + end + end + end + staledeps[i] = dep + end + restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps) + isa(restored, Module) && return restored + @debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored + @label check_next_path + finally + for modkey in newdeps + insert_extension_triggers(modkey) + run_package_callbacks(modkey) + end + empty!(newdeps) end - restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps) - isa(restored, Module) && return restored - @debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored - continue - @label check_next_path end return nothing end @@ -2241,14 +2251,12 @@ function unreference_module(key::PkgId) end # whoever takes the package_locks[pkg] must call this function immediately -function set_pkgorigin_version_path(pkg::PkgId, path::Union{String,Nothing}) +function set_pkgorigin_version_path(pkg::PkgId, path::String) assert_havelock(require_lock) pkgorigin = get!(PkgOrigin, pkgorigins, pkg) - if path !== nothing - # Pkg needs access to the version of packages in the sysimage. - if generating_output(#=incremental=#false) - pkgorigin.version = get_pkgversion_from_path(joinpath(dirname(path), "..")) - end + # Pkg needs access to the version of packages in the sysimage. + if generating_output(#=incremental=#false) + pkgorigin.version = get_pkgversion_from_path(joinpath(dirname(path), "..")) end pkgorigin.path = path nothing @@ -2281,7 +2289,7 @@ function _require(pkg::PkgId, env=nothing) # attempt to load the module file via the precompile cache locations if JLOptions().use_compiled_modules != 0 @label load_from_cache - m = _require_search_from_serialized(pkg, path, UInt128(0); reasons) + m = _require_search_from_serialized(pkg, path, UInt128(0), true; reasons) if m isa Module return m end @@ -2319,7 +2327,7 @@ function _require(pkg::PkgId, env=nothing) # spawn off a new incremental pre-compile task for recursive `require` calls cachefile_or_module = maybe_cachefile_lock(pkg, path) do # double-check now that we have lock - m = _require_search_from_serialized(pkg, path, UInt128(0)) + m = _require_search_from_serialized(pkg, path, UInt128(0), true) m isa Module && return m compilecache(pkg, path; reasons) end @@ -2376,10 +2384,10 @@ function _require(pkg::PkgId, env=nothing) return loaded end -# Only used from test/precompile.jl -function _require_from_serialized(uuidkey::PkgId, path::String, ocachepath::Union{String, Nothing}) +# load a serialized file directly +function _require_from_serialized(uuidkey::PkgId, path::String, ocachepath::Union{String, Nothing}, sourcepath::String) @lock require_lock begin - set_pkgorigin_version_path(uuidkey, nothing) + set_pkgorigin_version_path(uuidkey, sourcepath) newm = _tryrequire_from_serialized(uuidkey, path, ocachepath) newm isa Module || throw(newm) insert_extension_triggers(uuidkey) @@ -2389,6 +2397,58 @@ function _require_from_serialized(uuidkey::PkgId, path::String, ocachepath::Unio end end +# load a serialized file directly from append_bundled_depot_path for uuidkey without stalechecks +function require_stdlib(uuidkey::PkgId, ext::Union{Nothing, String}=nothing) + @lock require_lock begin + if root_module_exists(uuidkey) + return loaded_modules[uuidkey] + end + # first since this is a stdlib, try to look there directly first + env = Sys.STDLIB + #sourcepath = "" + if ext === nothing + sourcepath = normpath(env, uuidkey.name, "src", uuidkey.name * ".jl") + else + sourcepath = find_ext_path(normpath(joinpath(env, uuidkey.name)), ext) + uuidkey = PkgId(uuid5(uuidkey.uuid, ext), ext) + end + #mbypath = manifest_uuid_path(env, uuidkey) + #if mbypath isa String + # sourcepath = entry_path(mbypath, uuidkey.name) + #else + # # if the user deleted the stdlib folder, we next try using their environment + # sourcepath = locate_package_env(uuidkey) + # if sourcepath !== nothing + # sourcepath, env = sourcepath + # end + #end + #if sourcepath === nothing + # throw(ArgumentError(""" + # Package $(repr("text/plain", uuidkey)) is required but does not seem to be installed. + # """)) + #end + set_pkgorigin_version_path(uuidkey, sourcepath) + depot_path = append_bundled_depot_path!(empty(DEPOT_PATH)) + newm = start_loading(uuidkey) + newm === nothing || return newm + try + newm = _require_search_from_serialized(uuidkey, sourcepath, UInt128(0), false; DEPOT_PATH=depot_path) + finally + end_loading(uuidkey, newm) + end + if newm isa Module + # After successfully loading, notify downstream consumers + insert_extension_triggers(env, uuidkey) + run_package_callbacks(uuidkey) + else + # if the user deleted their bundled depot, next try to load it completely normally + newm = _require(uuidkey) + end + return newm + end +end + + # relative-path load @@ -3393,7 +3453,9 @@ list_reasons(::Nothing) = "" return stale_cachefile(PkgId(""), UInt128(0), modpath, cachefile; ignore_loaded, requested_flags, reasons) end @constprop :none function stale_cachefile(modkey::PkgId, build_id::UInt128, modpath::String, cachefile::String; - ignore_loaded::Bool = false, requested_flags::CacheFlags=CacheFlags(), reasons::Union{Dict{String,Int},Nothing}=nothing) + ignore_loaded::Bool=false, requested_flags::CacheFlags=CacheFlags(), + reasons::Union{Dict{String,Int},Nothing}=nothing, stalecheck::Bool=true) + # XXX: this function appears to dl all of the file validation, not just those checks related to stale io = open(cachefile, "r") try checksum = isvalid_cache_header(io) @@ -3468,7 +3530,7 @@ end M = root_module(req_key) if PkgId(M) == req_key && module_build_id(M) === req_build_id depmods[i] = M - elseif ignore_loaded + elseif ignore_loaded || !stalecheck # Used by Pkg.precompile given that there it's ok to precompile different versions of loaded packages @goto locate_branch else @@ -3478,7 +3540,7 @@ end end else @label locate_branch - path = locate_package(req_key) + path = locate_package(req_key) # TODO: add env and/or skip this when stalecheck is false if path === nothing @debug "Rejecting cache file $cachefile because dependency $req_key not found." record_reason(reasons, "dep missing source") @@ -3491,13 +3553,12 @@ end # check if this file is going to provide one of our concrete dependencies # or if it provides a version that conflicts with our concrete dependencies # or neither - skip_check = false for (req_key, req_build_id) in _concrete_dependencies build_id = get(modules, req_key, UInt64(0)) if build_id !== UInt64(0) build_id |= UInt128(checksum) << 64 if build_id === req_build_id - skip_check = true + stalecheck = false break end @debug "Rejecting cache file $cachefile because it provides the wrong build_id (got $((UUID(build_id)))) for $req_key (want $(UUID(req_build_id)))" @@ -3507,7 +3568,7 @@ end end # now check if this file's content hash has changed relative to its source files - if !skip_check + if stalecheck if !samefile(includes[1].filename, modpath) && !samefile(fixup_stdlib_path(includes[1].filename), modpath) @debug "Rejecting cache file $cachefile because it is for file $(includes[1].filename) not file $modpath" record_reason(reasons, "wrong source") diff --git a/base/stream.jl b/base/stream.jl index 3de61181e978d..3264b8f153677 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -304,7 +304,7 @@ function init_stdio(handle::Ptr{Cvoid}) elseif t == UV_TTY io = TTY(handle, StatusOpen) elseif t == UV_TCP - Sockets = require(PkgId(UUID((0x6462fe0b_24de_5631, 0x8697_dd941f90decc)), "Sockets")) + Sockets = require_stdlib(PkgId(UUID((0x6462fe0b_24de_5631, 0x8697_dd941f90decc)), "Sockets")) io = Sockets.TCPSocket(handle, StatusOpen) elseif t == UV_NAMED_PIPE io = PipeEndpoint(handle, StatusOpen) @@ -341,7 +341,7 @@ function open(h::OS_HANDLE) elseif t == UV_TTY io = TTY(h) elseif t == UV_TCP - Sockets = require(PkgId(UUID((0x6462fe0b_24de_5631, 0x8697_dd941f90decc)), "Sockets")) + Sockets = require_stdlib(PkgId(UUID((0x6462fe0b_24de_5631, 0x8697_dd941f90decc)), "Sockets")) io = Sockets.TCPSocket(h) elseif t == UV_NAMED_PIPE io = PipeEndpoint(h) diff --git a/base/util.jl b/base/util.jl index a26a874c7a461..847d89e895af4 100644 --- a/base/util.jl +++ b/base/util.jl @@ -695,7 +695,7 @@ function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS / 2), catch buf = PipeBuffer() original_load_path = copy(Base.LOAD_PATH); empty!(Base.LOAD_PATH); pushfirst!(Base.LOAD_PATH, "@stdlib") - let InteractiveUtils = Base.require(Base, :InteractiveUtils) + let InteractiveUtils = Base.require_stdlib(Base, :InteractiveUtils) @invokelatest InteractiveUtils.versioninfo(buf) end empty!(Base.LOAD_PATH); append!(Base.LOAD_PATH, original_load_path) diff --git a/stdlib/InteractiveUtils/src/InteractiveUtils.jl b/stdlib/InteractiveUtils/src/InteractiveUtils.jl index 7391a0d6b2f00..519acfc8a2f33 100644 --- a/stdlib/InteractiveUtils/src/InteractiveUtils.jl +++ b/stdlib/InteractiveUtils/src/InteractiveUtils.jl @@ -338,7 +338,7 @@ export peakflops function peakflops(n::Integer=4096; eltype::DataType=Float64, ntrials::Integer=3, parallel::Bool=false) # Base.depwarn("`peakflops` has moved to the LinearAlgebra module, " * # "add `using LinearAlgebra` to your imports.", :peakflops) - let LinearAlgebra = Base.require(Base.PkgId( + let LinearAlgebra = Base.require_stdlib(Base.PkgId( Base.UUID((0x37e2e46d_f89d_539d,0xb4ee_838fcccc9c8e)), "LinearAlgebra")) return LinearAlgebra.peakflops(n, eltype=eltype, ntrials=ntrials, parallel=parallel) end @@ -353,7 +353,7 @@ function report_bug(kind) if Base.locate_package(BugReportingId) === nothing @info "Package `BugReporting` not found - attempting temporary installation" # Create a temporary environment and add BugReporting - let Pkg = Base.require(Base.PkgId( + let Pkg = Base.require_stdlib(Base.PkgId( Base.UUID((0x44cfe95a_1eb2_52ea,0xb672_e2afdf69b78f)), "Pkg")) mktempdir() do tmp old_load_path = copy(LOAD_PATH) diff --git a/stdlib/LinearAlgebra/src/LinearAlgebra.jl b/stdlib/LinearAlgebra/src/LinearAlgebra.jl index 793775992fdea..acacf4b20cfc7 100644 --- a/stdlib/LinearAlgebra/src/LinearAlgebra.jl +++ b/stdlib/LinearAlgebra/src/LinearAlgebra.jl @@ -691,7 +691,7 @@ function peakflops(n::Integer=4096; eltype::DataType=Float64, ntrials::Integer=3 end if parallel - let Distributed = Base.require(Base.PkgId( + let Distributed = Base.require_stdlib(Base.PkgId( Base.UUID((0x8ba89e20_285c_5b6f, 0x9357_94700520ee1b)), "Distributed")) nworkers = @invokelatest Distributed.nworkers() results = @invokelatest Distributed.pmap(peakflops, fill(n, nworkers)) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index f39aed51cebaf..840e6daaf1909 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -1188,25 +1188,23 @@ function setup_interface( ']' => function (s::MIState,o...) if isempty(s) || position(LineEdit.buffer(s)) == 0 pkgid = Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg") - if Base.locate_package(pkgid) !== nothing # Only try load Pkg if we can find it - Pkg = Base.require(pkgid) - REPLExt = Base.get_extension(Pkg, :REPLExt) - # Pkg should have loaded its REPL mode by now, let's find it so we can transition to it. - pkg_mode = nothing + REPLExt = Base.require_stdlib(pkgid, "REPLExt") + pkg_mode = nothing + if REPLExt isa Module && isdefined(REPLExt, :PkgCompletionProvider) for mode in repl.interface.modes if mode isa LineEdit.Prompt && mode.complete isa REPLExt.PkgCompletionProvider pkg_mode = mode break end end - # TODO: Cache the `pkg_mode`? - if pkg_mode !== nothing - buf = copy(LineEdit.buffer(s)) - transition(s, pkg_mode) do - LineEdit.state(s, pkg_mode).input_buffer = buf - end - return + end + # TODO: Cache the `pkg_mode`? + if pkg_mode !== nothing + buf = copy(LineEdit.buffer(s)) + transition(s, pkg_mode) do + LineEdit.state(s, pkg_mode).input_buffer = buf end + return end end edit_insert(s, ']') diff --git a/test/precompile.jl b/test/precompile.jl index 9a035997ce9f0..ce7e9e77217b0 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -407,11 +407,11 @@ precompile_test_harness(false) do dir else ocachefile = nothing end - # use _require_from_serialized to ensure that the test fails if - # the module doesn't reload from the image: + # use _require_from_serialized to ensure that the test fails if + # the module doesn't reload from the image: @test_warn "@ccallable was already defined for this method name" begin @test_logs (:warn, "Replacing module `$Foo_module`") begin - m = Base._require_from_serialized(Base.PkgId(Foo), cachefile, ocachefile) + m = Base._require_from_serialized(Base.PkgId(Foo), cachefile, ocachefile, Foo_file) @test isa(m, Module) end end