From eebacab497dcf9e049293434384690ea904e3e7c Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 21 Oct 2022 14:18:24 -0400 Subject: [PATCH 1/5] fix #46778, precompile() for abstract but compileable signatures --- src/gf.c | 125 ++++++++++++++++++++++++++++++++++++--------- test/precompile.jl | 7 +++ 2 files changed, 107 insertions(+), 25 deletions(-) diff --git a/src/gf.c b/src/gf.c index 1f896921d45f5..668072172a38a 100644 --- a/src/gf.c +++ b/src/gf.c @@ -2255,6 +2255,39 @@ JL_DLLEXPORT jl_value_t *jl_normalize_to_compilable_sig(jl_methtable_t *mt, jl_t return is_compileable ? (jl_value_t*)tt : jl_nothing; } +// return a MethodInstance for a compileable method_match +jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *match, size_t world, size_t min_valid, size_t max_valid, int mt_cache) +{ + jl_method_t *m = match->method; + jl_svec_t *env = match->sparams; + jl_tupletype_t *ti = match->spec_types; + jl_method_instance_t *mi = NULL; + if (jl_is_datatype(ti)) { + jl_methtable_t *mt = jl_method_get_table(m); + if ((jl_value_t*)mt != jl_nothing) { + // get the specialization without caching it + if (mt_cache && ((jl_datatype_t*)ti)->isdispatchtuple) { + // Since we also use this presence in the cache + // to trigger compilation when producing `.ji` files, + // inject it there now if we think it will be + // used via dispatch later (e.g. because it was hinted via a call to `precompile`) + JL_LOCK(&mt->writelock); + mi = cache_method(mt, &mt->cache, (jl_value_t*)mt, ti, m, world, min_valid, max_valid, env); + JL_UNLOCK(&mt->writelock); + } + else { + jl_value_t *tt = jl_normalize_to_compilable_sig(mt, ti, env, m); + JL_GC_PUSH1(&tt); + if (tt != jl_nothing) { + mi = jl_specializations_get_linfo(m, (jl_value_t*)tt, env); + } + JL_GC_POP(); + } + } + } + return mi; +} + // compile-time method lookup jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES_ROOT, size_t world, size_t *min_valid, size_t *max_valid, int mt_cache) { @@ -2274,36 +2307,78 @@ jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES *max_valid = max_valid2; if (matches == jl_false || jl_array_len(matches) != 1 || ambig) return NULL; - jl_value_t *tt = NULL; - JL_GC_PUSH2(&matches, &tt); + JL_GC_PUSH1(&matches); jl_method_match_t *match = (jl_method_match_t*)jl_array_ptr_ref(matches, 0); - jl_method_t *m = match->method; - jl_svec_t *env = match->sparams; - jl_tupletype_t *ti = match->spec_types; - jl_method_instance_t *nf = NULL; - if (jl_is_datatype(ti)) { - jl_methtable_t *mt = jl_method_get_table(m); - if ((jl_value_t*)mt != jl_nothing) { - // get the specialization without caching it - if (mt_cache && ((jl_datatype_t*)ti)->isdispatchtuple) { - // Since we also use this presence in the cache - // to trigger compilation when producing `.ji` files, - // inject it there now if we think it will be - // used via dispatch later (e.g. because it was hinted via a call to `precompile`) - JL_LOCK(&mt->writelock); - nf = cache_method(mt, &mt->cache, (jl_value_t*)mt, ti, m, world, min_valid2, max_valid2, env); - JL_UNLOCK(&mt->writelock); - } - else { - tt = jl_normalize_to_compilable_sig(mt, ti, env, m); - if (tt != jl_nothing) { - nf = jl_specializations_get_linfo(m, (jl_value_t*)tt, env); + jl_method_instance_t *mi = jl_method_match_to_mi(match, world, min_valid2, max_valid2, mt_cache); + JL_GC_POP(); + return mi; +} + +// Get a MethodInstance for a precompile() call. This uses a special kind of lookup that +// tries to find a method for which the requested signature is compileable. +jl_method_instance_t *jl_get_compile_hint_specialization(jl_tupletype_t *types JL_PROPAGATES_ROOT, size_t world, size_t *min_valid, size_t *max_valid, int mt_cache) +{ + if (jl_has_free_typevars((jl_value_t*)types)) + return NULL; // don't poison the cache due to a malformed query + if (!jl_has_concrete_subtype((jl_value_t*)types)) + return NULL; + + size_t min_valid2 = 1; + size_t max_valid2 = ~(size_t)0; + int ambig = 0; + jl_value_t *matches = jl_matching_methods(types, jl_nothing, -1, 0, world, &min_valid2, &max_valid2, &ambig); + if (*min_valid < min_valid2) + *min_valid = min_valid2; + if (*max_valid > max_valid2) + *max_valid = max_valid2; + size_t i, n = jl_array_len(matches); + if (n == 0) + return NULL; + JL_GC_PUSH1(&matches); + jl_method_match_t *match = NULL; + if (n == 1) { + match = (jl_method_match_t*)jl_array_ptr_ref(matches, 0); + } + else { + // first, select methods for which `types` is compileable + size_t count = 0; + for (i = 0; i < n; i++) { + jl_method_match_t *match1 = (jl_method_match_t*)jl_array_ptr_ref(matches, i); + if (jl_isa_compileable_sig(types, match1->method)) + jl_array_ptr_set(matches, count++, (jl_value_t*)match1); + } + jl_array_del_end((jl_array_t*)matches, n - count); + n = count; + // now remove methods that are more specific than others in the list. + // this is because the intent of precompiling e.g. f(::DataType) is to + // compile that exact method if it exists, and not lots of f(::Type{X}) methods + int exclude; + count = 0; + for (i = 0; i < n; i++) { + jl_method_match_t *match1 = (jl_method_match_t*)jl_array_ptr_ref(matches, i); + exclude = 0; + for (size_t j = n-1; j > i; j--) { // more general methods maybe more likely to be at end + jl_method_match_t *match2 = (jl_method_match_t*)jl_array_ptr_ref(matches, j); + if (jl_type_morespecific(match1->method->sig, match2->method->sig)) { + exclude = 1; + break; } } + if (!exclude) + jl_array_ptr_set(matches, count++, (jl_value_t*)match1); + if (count > 1) + break; } + // at this point if there are 0 matches left we found nothing, or if there are + // more than one the request is ambiguous and we ignore it. + if (count == 1) + match = (jl_method_match_t*)jl_array_ptr_ref(matches, 0); } + jl_method_instance_t *mi = NULL; + if (match != NULL) + mi = jl_method_match_to_mi(match, world, min_valid2, max_valid2, mt_cache); JL_GC_POP(); - return nf; + return mi; } static void _generate_from_hint(jl_method_instance_t *mi, size_t world) @@ -2370,7 +2445,7 @@ JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types) size_t world = jl_atomic_load_acquire(&jl_world_counter); size_t min_valid = 0; size_t max_valid = ~(size_t)0; - jl_method_instance_t *mi = jl_get_specialization1(types, world, &min_valid, &max_valid, 1); + jl_method_instance_t *mi = jl_get_compile_hint_specialization(types, world, &min_valid, &max_valid, 1); if (mi == NULL) return 0; JL_GC_PROMISE_ROOTED(mi); diff --git a/test/precompile.jl b/test/precompile.jl index 098d1ffbba231..5b49ad4a3b31a 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1556,3 +1556,10 @@ end empty!(Base.DEPOT_PATH) append!(Base.DEPOT_PATH, original_depot_path) + +@testset "issue 46778" begin + f46778(::Any, ::Type{Int}) = 1 + f46778(::Any, ::DataType) = 2 + @test precompile(Tuple{typeof(f46778), Int, DataType}) + @test which(f46778, Tuple{Any,DataType}).specializations[1].cache.invoke != C_NULL +end From 8b5e4bac87ad0be2e06f53fa22c1d44bc8d9bb00 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Wed, 22 Sep 2021 13:10:59 +0200 Subject: [PATCH 2/5] refactor the code loading of packages --- base/Base.jl | 10 +- base/codeloading2.jl | 425 ++++++++++++++++++++++++++ base/deprecated.jl | 66 +++++ base/initdefs.jl | 2 - base/loading.jl | 691 +++++++------------------------------------ test/loading.jl | 6 +- 6 files changed, 607 insertions(+), 593 deletions(-) create mode 100644 base/codeloading2.jl diff --git a/base/Base.jl b/base/Base.jl index 29a6f9ed4366d..f8009e787b190 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -413,16 +413,17 @@ include("initdefs.jl") # worker threads include("threadcall.jl") -# code loading include("uuid.jl") -include("pkgid.jl") -include("toml_parser.jl") -include("loading.jl") # misc useful functions & macros include("timing.jl") include("util.jl") +# code loading +include("pkgid.jl") +include("toml_parser.jl") +include("loading.jl") + include("asyncmap.jl") # deprecated functions @@ -519,6 +520,7 @@ function __init__() init_depot_path() init_load_path() init_active_project() + init_stdlib_path_env() append!(empty!(_sysimage_modules), keys(loaded_modules)) if haskey(ENV, "JULIA_MAX_NUM_PRECOMPILE_FILES") MAX_NUM_PRECOMPILE_FILES[] = parse(Int, ENV["JULIA_MAX_NUM_PRECOMPILE_FILES"]) diff --git a/base/codeloading2.jl b/base/codeloading2.jl new file mode 100644 index 0000000000000..7f1be15e28618 --- /dev/null +++ b/base/codeloading2.jl @@ -0,0 +1,425 @@ + +######################### +# Implicit environments # +######################### + +struct ImplicitEnvPkg + path::String # The entry point of the package relative to the implicit environment path + uuid::Union{Nothing, UUID} + deps::Union{Nothing, Vector{PkgId}} # Eventual deps of project file of package (in case it has a Project.toml file) +end + +# An implicit environment (or package directory) is a folder in the LOAD_PATH without a project file. +# A package X exists in a package directory if the directory contains one of the following "entry point" files: +mutable struct ImplicitEnv + path::String # mutable so that we can replace the cached STDLIB implicit env with the runtime path + const pkgs::Dict{String, ImplicitEnvPkg} +end + +function _ImplicitEnv(envpath::String) + envpath = abspath(envpath) + pkgs = Dict{String, ImplicitEnvPkg}() + for path in readdir(envpath; join=true) + dir, ext = splitext(path) + name = basename(dir) + pkg = nothing + # Package defined by a single file X.jl: + if ext == ".jl" && isfile_casesensitive(path) + pkg = ImplicitEnvPkg(path, nothing, nothing) + # Package defined by a folder X/src/X.jl or X.jl/src/X.jl: + elseif isaccessibledir(path) + entry_point = joinpath(path, "src", name * ".jl") + if isfile_casesensitive(entry_point) + # Does the package have a project file? + project_file = nothing + for proj in project_names + maybe_project_file = joinpath(dir, proj) + isfile_casesensitive(maybe_project_file) && (project_file = maybe_project_file) + end + # It did have a project file: + if project_file !== nothing + project_d = parsed_toml(project_file) + uuid = project_file_uuid(project_d, project_file) + deps = PkgId[] + # Get the explicit deps, these are the only deps that can be loaded inside the package: + for (name, uuid) in get(Dict{String, Any}, project_d, "deps") + uuid = uuid::String + push!(deps, PkgId(UUID(uuid), name)) + end + pkg = ImplicitEnvPkg(relpath(entry_point, envpath), uuid, deps) + # No project file: no uuid and no explicit deps: + else + pkg = ImplicitEnvPkg(relpath(entry_point, envpath), nothing, nothing) + end + end + end + if pkg !== nothing + pkgs[name] = pkg + end + # TODO: It is possible to both have e.g. a `X.jl` file and a `X/src/X.jl` package which is a name collison. + # warn about that? + end + return ImplicitEnv(envpath, pkgs) +end + +# The stdlib environment is an implicit environemnt. +# It is also constant so we might as well just cache it. +const STDLIB_ENVIRONMENT = _ImplicitEnv(Sys.STDLIB) +init_stdlib_path_env() = STDLIB_ENVIRONMENT.path = Sys.STDLIB + +function ImplicitEnv(envpath::String) + if envpath == Sys.STDLIB + return STDLIB_ENVIRONMENT + else + return _ImplicitEnv(envpath) + end +end + +function _identify_package(env::ImplicitEnv, name::String)::Union{Nothing, PkgId} + pkg = get(env.pkgs, name, nothing) + pkg === nothing && return nothing + return PkgId(pkg.uuid, name) +end + +function _locate_package(env::ImplicitEnv, pkg::PkgId)::Union{Nothing, String} + pkg′ = get(env.pkgs, pkg.name, nothing) + pkg′ === nothing && return nothing + # We also need to check that an eventual uuid in `pkg` matches the one in the env: + return pkg′.uuid !== pkg.uuid ? nothing : joinpath(env.path, pkg′.path) +end + + +######################### +# Explicit environments # +######################### + +# An explicit environment is a folder with a `Project.toml` file and (most often) +# a `Manifest.toml` file. The `Project.toml` file describes what can be loaded at +# top-level and the Manifest.toml describes what packages can be loaded in other packages +# as well as how the path is looked up for a package +struct ExplicitEnv + path::String + project_deps::Dict{String, UUID} # [deps] in Project.toml + project_extras::Dict{String, UUID} # [extras] in Project.toml + deps::Dict{UUID, Dict{String, UUID}} # all dependencies in Manifest.toml + lookup_strategy::Dict{UUID, Union{ + SHA1, # `git-tree-sha1` entry + String, # `path` entry + Nothing, # stdlib (no `path` nor `git-tree-sha1`) + Missing}} # not present in the manifest + prefs::Union{Nothing, Dict{String, Any}} + local_prefs::Union{Nothing, Dict{String, Any}} +end + +function ExplicitEnv(envpath::String) + envpath = abspath(envpath) + project_d = parsed_toml(envpath) + + project_deps = Dict{String, UUID}() + # Collect all direct dependencies of the project + for (name, uuid) in get(Dict{String, Any}, project_d, "deps")::Dict{String, Any} + project_deps[name] = UUID(uuid::String) + end + + project_extras = Dict{String, UUID}() + # Collect all "extras" dependencies of the project + for (name, uuid) in get(Dict{String, Any}, project_d, "extras")::Dict{String, Any} + project_extras[name] = UUID(uuid::String) + end + + # This project might be a package, in that case, that is also a "dependency" + # of the project. + name = get(project_d, "name", nothing)::Union{String, Nothing} + pkg_uuid = UUID(project_file_uuid(project_d, envpath)) + if name !== nothing + project_deps[name] = pkg_uuid + end + + manifest = project_file_manifest_path(envpath) + manifest_d = manifest === nothing ? Dict{String, Any}() : parsed_toml(manifest) + + # Dependencies in a manifest can either be stored compressed (when name is unique among all packages) + # in which case it is a `Vector{String}` or expanded where it is a `name => uuid` mapping. + deps = Dict{UUID, Union{Vector{String}, Dict{String, UUID}}}() + sizehint!(deps, length(manifest_d)) + name_to_uuid = Dict{String, UUID}() + sizehint!(name_to_uuid, length(manifest_d)) + lookup_strategy = Dict{UUID, Union{SHA1, String, Nothing, Missing}}() + sizehint!(lookup_strategy, length(manifest_d)) + + for (name, pkg_infos) in get_deps(manifest_d) + pkg_infos = pkg_infos::Vector{Any} + for pkg_info in pkg_infos + m_uuid = UUID(pkg_info["uuid"]::String) + + # If we have multiple packages with the same name we will overwrite things here + # but that is fine since we will only use the information in here for packages + # with unique names + name_to_uuid[name] = m_uuid + deps_pkg = get(Vector{String}, pkg_info, "deps")::Union{Vector{String}, Dict{String, Any}} + # Compressed format with unique names: + if deps_pkg isa Vector{String} + deps[m_uuid] = deps_pkg + # Exapanded format: + else + deps[m_uuid] = Dict{String, UUID}(name_dep => UUID(dep_uuid::String) for (name_dep, dep_uuid) in deps_pkg) + end + + # Determine strategy to find package + lookup_strat = begin + if (path = get(pkg_info, "path", nothing)::Union{String, Nothing}) !== nothing + path + elseif (git_tree_sha_str = get(pkg_info, "git-tree-sha1", nothing)::Union{String, Nothing}) !== nothing + SHA1(git_tree_sha_str) + else + nothing + end + end + lookup_strategy[m_uuid] = lookup_strat + end + end + + # No matter if the deps were stored compressed or not in the manifest, + # we internally store them expanded + deps_expanded = Dict{UUID, Dict{String, UUID}}() + sizehint!(deps_expanded, length(deps)) + + if name !== nothing + deps_expanded[pkg_uuid] = project_deps + # N.b `path` entries in the Project file is currently not understood by Pkg + path = get(project_d, "path", nothing) + entry_point = path !== nothing ? path : dirname(envpath) + lookup_strategy[pkg_uuid] = entry_point + end + + for (pkg, deps) in deps + # dependencies was already expanded so use it directly: + if deps isa Dict{String,UUID} + deps_expanded[pkg] = deps + # find the (unique) UUID associated with the name + else + deps_pkg = Dict{String, UUID}() + sizehint!(deps_pkg, length(deps)) + for dep in deps + deps_pkg[dep] = name_to_uuid[dep] + end + deps_expanded[pkg] = deps_pkg + end + end + + # Everything that does not yet have a lookup_strategy is missing from the manifest + for (name, uuid) in project_deps + get!(lookup_strategy, uuid, missing) + end + + # Preferences: + prefs = get(project_d, "preferences", nothing) + + # `(Julia)LocalPreferences.toml` + project_dir = dirname(envpath) + local_prefs = nothing + for name in preferences_names + toml_path = joinpath(project_dir, name) + if isfile(toml_path) + local_prefs = parsed_toml(toml_path) + break + end + end + + return ExplicitEnv(envpath, project_deps, project_extras, deps_expanded, lookup_strategy, prefs, local_prefs) +end + +# Marker to return when we should have been able to load a package but failed. +# At that point, we should not keep searching for the package in other environments +const STOP = :stop + +function _identify_package(env::ExplicitEnv, where::PkgId, name::String)::Union{Nothing, PkgId, Symbol} + where.name == name && return where # Loading X inside X + where.uuid === nothing && return identify_package(env, name) + where_pkg = get(env.deps, where.uuid, nothing) + where_pkg === nothing && return nothing # `where` is not in current env + uuid = get(where_pkg, name, nothing) + uuid === nothing && return STOP # we found `where` but not allowed to load `name` in it. + return PkgId(uuid, name) +end + +function _identify_package(env::ExplicitEnv, name::String)::Union{Nothing, PkgId} + uuid = get(env.project_deps, name, nothing) + uuid === nothing && return nothing + return PkgId(uuid, name) +end + +const STDLIBS = Set(readdir(Sys.STDLIB::String)) + +function _locate_package(env::ExplicitEnv, pkg::PkgId)::Union{Nothing, String, Symbol} + pkg.uuid === nothing && return nothing + haskey(env.lookup_strategy, pkg.uuid) || return nothing + + lookup_strategy = env.lookup_strategy[pkg.uuid] + + # Not found in manifest: + if lookup_strategy isa Missing + return STOP + # Stdlib: + elseif lookup_strategy isa Nothing + # @assert pkg.name in STDLIBS + # Check that UUID is matching? + stdlib_path = joinpath(Sys.STDLIB::String, pkg.name, "src", pkg.name * ".jl") + return isfile_casesensitive(stdlib_path) ? stdlib_path : nothing + # Path: + elseif lookup_strategy isa String + # `path` in a manifest are defined relative the project path + path = normpath(dirname(env.path), lookup_strategy) + if isaccessibledir(path) + path = joinpath(path, "src", pkg.name * ".jl") + return isfile_casesensitive(path) ? path : STOP + elseif isfile(path) + return path + else + return STOP + end + # Versioned + elseif lookup_strategy isa SHA1 + hash = lookup_strategy + # Try find it in a depot + for slug in (version_slug(pkg.uuid, hash), version_slug(pkg.uuid, hash, 4)) + for depot in DEPOT_PATH + path = joinpath(depot, "packages", pkg.name, slug, "src", pkg.name * ".jl") + isfile(path) && return path + end + end + return STOP + else + error("unhandled lookup strategy") + end +end + + +#################### +# EnvironmentStack # +#################### + +# An environment stack is the stack of environments formed via load_path() (the expanded LOAD_PATH) +struct EnvironmentStack + load_path::Vector{String} + envs::Vector{Union{ImplicitEnv, ExplicitEnv}} +end + +function EnvironmentStack(environments = load_path()) + envs = Union{ImplicitEnv, ExplicitEnv}[] + for env in environments + if isfile(env) + push!(envs, ExplicitEnv(env)) + elseif isaccessibledir(env) + push!(envs, ImplicitEnv(env)) + end + end + return EnvironmentStack(environments, envs) +end + +function identify_package(envstack::EnvironmentStack, where::PkgId, name::String)::Union{Nothing, PkgId, Symbol} + where.name == name && return where # X loaded in X + where.uuid === nothing && return identify_package(envstack, name) + for env in envstack.envs + if env isa ExplicitEnv + pkg = _identify_package(env, where, name) + if pkg === STOP + return nothing + elseif pkg isa PkgId + return pkg + end + # keep looking + else + where_pkg = get(env.pkgs, where.name, nothing) + # Found a package with the correct name in the implicit environment: + if where_pkg !== nothing + # But wrong uuid, keep looking: + where_pkg.uuid == where.uuid || continue + for env in envstack.envs + maybe_pkg = _identify_package(env, name) + if maybe_pkg !== nothing + if where_pkg.deps === nothing || maybe_pkg in where_pkg.deps + return maybe_pkg + end + end + end + # Could not find a valid package with the correct name that could be + # loaded in `where`. + return nothing + end + end + end + return nothing +end + +function identify_package(envstack::EnvironmentStack, name::String)::Union{Nothing, PkgId} + for env in envstack.envs + pkg = _identify_package(env, name) + pkg !== nothing && return pkg + end + return nothing +end + +function locate_package(envstack::EnvironmentStack, pkg::PkgId)::Union{Nothing, String} + for env in envstack.envs + path = _locate_package(env, pkg) + path === STOP && return nothing + path !== nothing && return path + end + return nothing +end + +""" + Base.identify_package(name::String)::Union{PkgId, Nothing} + Base.identify_package(where::Union{Module,PkgId}, name::String)::Union{PkgId, Nothing} + +Identify the package by its name from the current environment stack, returning +its `PkgId`, or `nothing` if it cannot be found. + +If only the `name` argument is provided, it searches each environment in the +stack and its named direct dependencies. + +There `where` argument provides the context from where to search for the +package: in this case it first checks if the name matches the context itself, +otherwise it searches all recursive dependencies (from the resolved manifest of +each environment) until it locates the context `where`, and from there +identifies the dependency with with the corresponding name. + +```julia-repl +julia> Base.identify_package("Pkg") # Pkg is a dependency of the default environment +Pkg [44cfe95a-1eb2-52ea-b672-e2afdf69b78f] + +julia> using LinearAlgebra + +julia> Base.identify_package(LinearAlgebra, "Pkg") # Pkg is not a dependency of LinearAlgebra +``` +""" +identify_package(where::PkgId, name::String) = identify_package(EnvironmentStack(), where, name) +identify_package(name::String) = identify_package(EnvironmentStack(), name) + +identify_package(where::Module, name::String) = identify_package(PkgId(where), name) + + +""" + Base.locate_package(pkg::PkgId)::Union{String, Nothing} + +The path to the entry-point file for the package corresponding to the identifier +`pkg`, or `nothing` if not found. See also [`identify_package`](@ref). + +```julia-repl +julia> pkg = Base.identify_package("Pkg") +Pkg [44cfe95a-1eb2-52ea-b672-e2afdf69b78f] + +julia> Base.locate_package(pkg) +"/path/to/julia/stdlib/v$(VERSION.major).$(VERSION.minor)/Pkg/src/Pkg.jl" +``` +""" +locate_package(pkg::PkgId) = locate_package(EnvironmentStack(), pkg) + +# Used by Pkg but not used in loading itself +function find_package(envstack::EnvironmentStack, arg) + pkg = identify_package(envstack, arg) + pkg === nothing && return nothing + return locate_package(envstack, pkg) +end +find_package(arg) = find_package(EnvironmentStack(), arg) diff --git a/base/deprecated.jl b/base/deprecated.jl index 6953cd600cacd..a312b00399a41 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -363,4 +363,70 @@ end end end +# Used by Prefences.jl +function env_project_file(env::String)::Union{Bool,String} + if isdir(env) + for proj in project_names + maybe_project_file = joinpath(env, proj) + if isfile_casesensitive(maybe_project_file) + project_file = maybe_project_file + break + end + end + project_file =true + elseif basename(env) in project_names && isfile_casesensitive(env) + project_file = env + else + project_file = false + end + return project_file +end + +function get_uuid_name(project_toml::String, uuid::UUID) + project = parsed_toml(project_toml) + return get_uuid_name(project, uuid) +end + +# Used by Prefences.jl +function get_uuid_name(project::Dict{String, Any}, uuid::UUID) + uuid_p = get(project, "uuid", nothing)::Union{Nothing, String} + name = get(project, "name", nothing)::Union{Nothing, String} + if name !== nothing && uuid_p !== nothing && UUID(uuid_p) == uuid + return name + end + deps = get(project, "deps", nothing)::Union{Nothing, Dict{String, Any}} + if deps !== nothing + for (k, v) in deps + if uuid == UUID(v::String) + return k + end + end + end + for subkey in ("deps", "extras") + subsection = get(project, subkey, nothing)::Union{Nothing, Dict{String, Any}} + if subsection !== nothing + for (k, v) in subsection + if uuid == UUID(v::String) + return k + end + end + end + end + return nothing +end + +# used by Profile.jl +# find project file's top-level UUID entry (or nothing) +function project_file_name_uuid(project_file::String, name::String)::PkgId + d = parsed_toml(project_file) + uuid′ = get(d, "uuid", nothing)::Union{String, Nothing} + uuid = uuid′ === nothing ? dummy_uuid(project_file) : UUID(uuid′) + name = get(d, "name", name)::String + return PkgId(uuid, name) +end + +# Used by Pkg.jl +struct LoadingCache end +LOADING_CACHE = Ref{Union{LoadingCache, Nothing}}(nothing) + # END 1.9 deprecations diff --git a/base/initdefs.jl b/base/initdefs.jl index 1988e56c6eb1d..4c2831020278c 100644 --- a/base/initdefs.jl +++ b/base/initdefs.jl @@ -335,8 +335,6 @@ Return the fully expanded value of [`LOAD_PATH`](@ref) that is searched for proj packages. """ function load_path() - cache = LOADING_CACHE[] - cache !== nothing && return cache.load_path paths = String[] for env in LOAD_PATH path = load_path_expand(env) diff --git a/base/loading.jl b/base/loading.jl index 1a933b274b7de..50a1349484bd5 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -160,11 +160,6 @@ const ns_dummy_uuid = UUID("fe0723d6-3a44-4c41-8065-ee0f42c8ceab") function dummy_uuid(project_file::String) @lock require_lock begin - cache = LOADING_CACHE[] - if cache !== nothing - uuid = get(cache.dummy_uuid, project_file, nothing) - uuid === nothing || return uuid - end project_path = try realpath(project_file) catch ex @@ -172,9 +167,6 @@ function dummy_uuid(project_file::String) project_file end uuid = uuid5(ns_dummy_uuid, project_path) - if cache !== nothing - cache.dummy_uuid[project_file] = uuid - end return uuid end end @@ -250,17 +242,6 @@ function get_updated_dict(p::TOML.Parser, f::CachedTOMLDict) return f.d end -struct LoadingCache - load_path::Vector{String} - dummy_uuid::Dict{String, UUID} - env_project_file::Dict{String, Union{Bool, String}} - project_file_manifest_path::Dict{String, Union{Nothing, String}} - require_parsed::Set{String} -end -const LOADING_CACHE = Ref{Union{LoadingCache, Nothing}}(nothing) -LoadingCache() = LoadingCache(load_path(), Dict(), Dict(), Dict(), Set()) - - struct TOMLCache p::TOML.Parser d::Dict{String, CachedTOMLDict} @@ -270,142 +251,74 @@ const TOML_CACHE = TOMLCache(TOML.Parser(), Dict{String, Dict{String, Any}}()) parsed_toml(project_file::AbstractString) = parsed_toml(project_file, TOML_CACHE, require_lock) function parsed_toml(project_file::AbstractString, toml_cache::TOMLCache, toml_lock::ReentrantLock) lock(toml_lock) do - cache = LOADING_CACHE[] - dd = if !haskey(toml_cache.d, project_file) + if !haskey(toml_cache.d, project_file) d = CachedTOMLDict(toml_cache.p, project_file) toml_cache.d[project_file] = d - d.d + return d.d else d = toml_cache.d[project_file] - # We are in a require call and have already parsed this TOML file - # assume that it is unchanged to avoid hitting disk - if cache !== nothing && project_file in cache.require_parsed - d.d - else - get_updated_dict(toml_cache.p, d) - end + return get_updated_dict(toml_cache.p, d) end - if cache !== nothing - push!(cache.require_parsed, project_file) - end - return dd end end -## package identification: determine unique identity of package to be loaded ## -# Used by Pkg but not used in loading itself -function find_package(arg) - pkgenv = identify_package_env(arg) - pkgenv === nothing && return nothing - pkg, env = pkgenv - return locate_package(pkg, env) +const project_names = ("JuliaProject.toml", "Project.toml") +const manifest_names = ("JuliaManifest.toml", "Manifest.toml") +const preferences_names = ("JuliaLocalPreferences.toml", "LocalPreferences.toml") + +function locate_project_file(env::String) + for proj in project_names + project_file = joinpath(env, proj) + if isfile_casesensitive(project_file) + return project_file + end + end + return true end -""" - Base.identify_package_env(name::String)::Union{Tuple{PkgId, String}, Nothing} - Base.identify_package_env(where::Union{Module,PkgId}, name::String)::Union{Tuple{PkgId, String} Nothing} +function project_file_uuid(d::Dict, project_file::String)::UUID + uuid′ = get(d, "uuid", nothing)::Union{String, Nothing} + return uuid′ === nothing ? dummy_uuid(project_file) : UUID(uuid′) +end -Same as [`Base.identify_package`](@ref) except that the path to the environment where the package is identified -is also returned. -""" -identify_package_env(where::Module, name::String) = identify_package_env(PkgId(where), name) -function identify_package_env(where::PkgId, name::String) - where.name === name && return where, nothing - where.uuid === nothing && return identify_package_env(name) # ignore `where` - for env in load_path() - pkgid = manifest_deps_get(env, where, name) - pkgid === nothing && continue # not found--keep looking - pkgid.uuid === nothing || return pkgid, env # found in explicit environment--use it - return nothing # found in implicit environment--return "not found" +function project_file_manifest_path(project_file::String)::Union{Nothing,String} + dir = dirname(project_file) + for mfst in manifest_names + manifest_file = joinpath(dir, mfst) + isfile_casesensitive(manifest_file) && return manifest_file end return nothing end -function identify_package_env(name::String) - for env in load_path() - uuid = project_deps_get(env, name) - uuid === nothing || return uuid, env # found--return it +function is_v1_format_manifest(raw_manifest::Dict) + if haskey(raw_manifest, "manifest_format") + mf = raw_manifest["manifest_format"] + if mf isa Dict && haskey(mf, "uuid") + # the off-chance where an old format manifest has a dep called "manifest_format" + return true + end + return false + else + return true end - return nothing end -_nothing_or_first(x) = x === nothing ? nothing : first(x) - -""" - Base.identify_package(name::String)::Union{PkgId, Nothing} - Base.identify_package(where::Union{Module,PkgId}, name::String)::Union{PkgId, Nothing} - -Identify the package by its name from the current environment stack, returning -its `PkgId`, or `nothing` if it cannot be found. - -If only the `name` argument is provided, it searches each environment in the -stack and its named direct dependencies. - -There `where` argument provides the context from where to search for the -package: in this case it first checks if the name matches the context itself, -otherwise it searches all recursive dependencies (from the resolved manifest of -each environment) until it locates the context `where`, and from there -identifies the dependency with with the corresponding name. - -```julia-repl -julia> Base.identify_package("Pkg") # Pkg is a dependency of the default environment -Pkg [44cfe95a-1eb2-52ea-b672-e2afdf69b78f] - -julia> using LinearAlgebra - -julia> Base.identify_package(LinearAlgebra, "Pkg") # Pkg is not a dependency of LinearAlgebra -``` -""" -identify_package(where::Module, name::String) = _nothing_or_first(identify_package_env(where, name)) -identify_package(where::PkgId, name::String) = _nothing_or_first(identify_package_env(where, name)) -identify_package(name::String) = _nothing_or_first(identify_package_env(name)) +# returns a deps list for both old and new manifest formats +function get_deps(raw_manifest::Dict) + if is_v1_format_manifest(raw_manifest) + return raw_manifest + else + # if the manifest has no deps, there won't be a `deps` field + return get(Dict{String, Any}, raw_manifest, "deps") + end +end -""" - Base.locate_package(pkg::PkgId)::Union{String, Nothing} +include("codeloading2.jl") -The path to the entry-point file for the package corresponding to the identifier -`pkg`, or `nothing` if not found. See also [`identify_package`](@ref). -```julia-repl -julia> pkg = Base.identify_package("Pkg") -Pkg [44cfe95a-1eb2-52ea-b672-e2afdf69b78f] +## other code loading functionality ## -julia> Base.locate_package(pkg) -"/path/to/julia/stdlib/v$(VERSION.major).$(VERSION.minor)/Pkg/src/Pkg.jl" -``` -""" -function locate_package(pkg::PkgId, stopenv::Union{String, Nothing}=nothing)::Union{Nothing,String} - if pkg.uuid === nothing - for env in load_path() - # look for the toplevel pkg `pkg.name` in this entry - found = project_deps_get(env, pkg.name) - if found !== nothing - @assert found.name == pkg.name - if found.uuid === nothing - # pkg.name is present in this directory or project file, - # return the path the entry point for the code, if it could be found - # otherwise, signal failure - return implicit_manifest_uuid_path(env, pkg) - end - end - stopenv == env && return nothing - end - else - for env in load_path() - path = manifest_uuid_path(env, pkg) - # missing is used as a sentinel to stop looking further down in envs - path === missing && return nothing - path === nothing || return entry_path(path, pkg.name) - stopenv == env && break - end - # Allow loading of stdlibs if the name/uuid are given - # e.g. if they have been explicitly added to the project/manifest - path = manifest_uuid_path(Sys.STDLIB, pkg) - path isa String && return entry_path(path, pkg.name) - end - return nothing -end """ pathof(m::Module) @@ -499,352 +412,6 @@ function pkgversion(m::Module) end end -## generic project & manifest API ## - -const project_names = ("JuliaProject.toml", "Project.toml") -const manifest_names = ("JuliaManifest.toml", "Manifest.toml") -const preferences_names = ("JuliaLocalPreferences.toml", "LocalPreferences.toml") - -function locate_project_file(env::String) - for proj in project_names - project_file = joinpath(env, proj) - if isfile_casesensitive(project_file) - return project_file - end - end - return true -end - -# classify the LOAD_PATH entry to be one of: -# - `false`: nonexistent / nothing to see here -# - `true`: `env` is an implicit environment -# - `path`: the path of an explicit project file -function env_project_file(env::String)::Union{Bool,String} - @lock require_lock begin - cache = LOADING_CACHE[] - if cache !== nothing - project_file = get(cache.env_project_file, env, nothing) - project_file === nothing || return project_file - end - if isdir(env) - project_file = locate_project_file(env) - elseif basename(env) in project_names && isfile_casesensitive(env) - project_file = env - else - project_file = false - end - if cache !== nothing - cache.env_project_file[env] = project_file - end - return project_file - end -end - -function project_deps_get(env::String, name::String)::Union{Nothing,PkgId} - project_file = env_project_file(env) - if project_file isa String - pkg_uuid = explicit_project_deps_get(project_file, name) - pkg_uuid === nothing || return PkgId(pkg_uuid, name) - elseif project_file - return implicit_project_deps_get(env, name) - end - return nothing -end - -function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Nothing,PkgId} - uuid = where.uuid - @assert uuid !== nothing - project_file = env_project_file(env) - if project_file isa String - # first check if `where` names the Project itself - proj = project_file_name_uuid(project_file, where.name) - if proj == where - # if `where` matches the project, use [deps] section as manifest, and stop searching - pkg_uuid = explicit_project_deps_get(project_file, name) - return PkgId(pkg_uuid, name) - end - # look for manifest file and `where` stanza - return explicit_manifest_deps_get(project_file, uuid, name) - elseif project_file - # if env names a directory, search it - return implicit_manifest_deps_get(env, where, name) - end - return nothing -end - -function manifest_uuid_path(env::String, pkg::PkgId)::Union{Nothing,String,Missing} - project_file = env_project_file(env) - if project_file isa String - proj = project_file_name_uuid(project_file, pkg.name) - if proj == pkg - # if `pkg` matches the project, return the project itself - return project_file_path(project_file, pkg.name) - end - # look for manifest file and `where` stanza - return explicit_manifest_uuid_path(project_file, pkg) - elseif project_file - # if env names a directory, search it - return implicit_manifest_uuid_path(env, pkg) - end - return nothing -end - -# find project file's top-level UUID entry (or nothing) -function project_file_name_uuid(project_file::String, name::String)::PkgId - d = parsed_toml(project_file) - uuid′ = get(d, "uuid", nothing)::Union{String, Nothing} - uuid = uuid′ === nothing ? dummy_uuid(project_file) : UUID(uuid′) - name = get(d, "name", name)::String - return PkgId(uuid, name) -end - -function project_file_path(project_file::String, name::String) - d = parsed_toml(project_file) - joinpath(dirname(project_file), get(d, "path", "")::String) -end - -# find project file's corresponding manifest file -function project_file_manifest_path(project_file::String)::Union{Nothing,String} - @lock require_lock begin - cache = LOADING_CACHE[] - if cache !== nothing - manifest_path = get(cache.project_file_manifest_path, project_file, missing) - manifest_path === missing || return manifest_path - end - dir = abspath(dirname(project_file)) - d = parsed_toml(project_file) - explicit_manifest = get(d, "manifest", nothing)::Union{String, Nothing} - manifest_path = nothing - if explicit_manifest !== nothing - manifest_file = normpath(joinpath(dir, explicit_manifest)) - if isfile_casesensitive(manifest_file) - manifest_path = manifest_file - end - end - if manifest_path === nothing - for mfst in manifest_names - manifest_file = joinpath(dir, mfst) - if isfile_casesensitive(manifest_file) - manifest_path = manifest_file - break - end - end - end - if cache !== nothing - cache.project_file_manifest_path[project_file] = manifest_path - end - return manifest_path - end -end - -# given a directory (implicit env from LOAD_PATH) and a name, -# check if it is an implicit package -function entry_point_and_project_file_inside(dir::String, name::String)::Union{Tuple{Nothing,Nothing},Tuple{String,Nothing},Tuple{String,String}} - path = normpath(joinpath(dir, "src", "$name.jl")) - isfile_casesensitive(path) || return nothing, nothing - for proj in project_names - project_file = normpath(joinpath(dir, proj)) - isfile_casesensitive(project_file) || continue - return path, project_file - end - return path, nothing -end - -# given a project directory (implicit env from LOAD_PATH) and a name, -# find an entry point for `name`, and see if it has an associated project file -function entry_point_and_project_file(dir::String, name::String)::Union{Tuple{Nothing,Nothing},Tuple{String,Nothing},Tuple{String,String}} - path = normpath(joinpath(dir, "$name.jl")) - isfile_casesensitive(path) && return path, nothing - dir = joinpath(dir, name) - path, project_file = entry_point_and_project_file_inside(dir, name) - path === nothing || return path, project_file - dir = dir * ".jl" - path, project_file = entry_point_and_project_file_inside(dir, name) - path === nothing || return path, project_file - return nothing, nothing -end - -# given a path and a name, return the entry point -function entry_path(path::String, name::String)::Union{Nothing,String} - isfile_casesensitive(path) && return normpath(path) - path = normpath(joinpath(path, "src", "$name.jl")) - isfile_casesensitive(path) && return path - return nothing # source not found -end - -## explicit project & manifest API ## - -# find project file root or deps `name => uuid` mapping -# return `nothing` if `name` is not found -function explicit_project_deps_get(project_file::String, name::String)::Union{Nothing,UUID} - d = parsed_toml(project_file) - root_uuid = dummy_uuid(project_file) - if get(d, "name", nothing)::Union{String, Nothing} === name - uuid = get(d, "uuid", nothing)::Union{String, Nothing} - return uuid === nothing ? root_uuid : UUID(uuid) - end - deps = get(d, "deps", nothing)::Union{Dict{String, Any}, Nothing} - if deps !== nothing - uuid = get(deps, name, nothing)::Union{String, Nothing} - uuid === nothing || return UUID(uuid) - end - return nothing -end - -function is_v1_format_manifest(raw_manifest::Dict) - if haskey(raw_manifest, "manifest_format") - mf = raw_manifest["manifest_format"] - if mf isa Dict && haskey(mf, "uuid") - # the off-chance where an old format manifest has a dep called "manifest_format" - return true - end - return false - else - return true - end -end - -# returns a deps list for both old and new manifest formats -function get_deps(raw_manifest::Dict) - if is_v1_format_manifest(raw_manifest) - return raw_manifest - else - # if the manifest has no deps, there won't be a `deps` field - return get(Dict{String, Any}, raw_manifest, "deps")::Dict{String, Any} - end -end - -# find `where` stanza and return the PkgId for `name` -# return `nothing` if it did not find `where` (indicating caller should continue searching) -function explicit_manifest_deps_get(project_file::String, where::UUID, name::String)::Union{Nothing,PkgId} - manifest_file = project_file_manifest_path(project_file) - manifest_file === nothing && return nothing # manifest not found--keep searching LOAD_PATH - d = get_deps(parsed_toml(manifest_file)) - found_where = false - found_name = false - for (dep_name, entries) in d - entries::Vector{Any} - for entry in entries - entry = entry::Dict{String, Any} - uuid = get(entry, "uuid", nothing)::Union{String, Nothing} - uuid === nothing && continue - if UUID(uuid) === where - found_where = true - # deps is either a list of names (deps = ["DepA", "DepB"]) or - # a table of entries (deps = {"DepA" = "6ea...", "DepB" = "55d..."} - deps = get(entry, "deps", nothing)::Union{Vector{String}, Dict{String, Any}, Nothing} - deps === nothing && continue - if deps isa Vector{String} - found_name = name in deps - break - else - deps = deps::Dict{String, Any} - for (dep, uuid) in deps - uuid::String - if dep === name - return PkgId(UUID(uuid), name) - end - end - end - end - end - end - found_where || return nothing - found_name || return PkgId(name) - # Only reach here if deps was not a dict which mean we have a unique name for the dep - name_deps = get(d, name, nothing)::Union{Nothing, Vector{Any}} - if name_deps === nothing || length(name_deps) != 1 - error("expected a single entry for $(repr(name)) in $(repr(project_file))") - end - entry = first(name_deps::Vector{Any})::Dict{String, Any} - uuid = get(entry, "uuid", nothing)::Union{String, Nothing} - uuid === nothing && return nothing - return PkgId(UUID(uuid), name) -end - -# find `uuid` stanza, return the corresponding path -function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{Nothing,String,Missing} - manifest_file = project_file_manifest_path(project_file) - manifest_file === nothing && return nothing # no manifest, skip env - - d = get_deps(parsed_toml(manifest_file)) - entries = get(d, pkg.name, nothing)::Union{Nothing, Vector{Any}} - entries === nothing && return nothing # TODO: allow name to mismatch? - for entry in entries - entry = entry::Dict{String, Any} - uuid = get(entry, "uuid", nothing)::Union{Nothing, String} - uuid === nothing && continue - if UUID(uuid) === pkg.uuid - return explicit_manifest_entry_path(manifest_file, pkg, entry) - end - end - return nothing -end - -function explicit_manifest_entry_path(manifest_file::String, pkg::PkgId, entry::Dict{String,Any}) - path = get(entry, "path", nothing)::Union{Nothing, String} - if path !== nothing - path = normpath(abspath(dirname(manifest_file), path)) - return path - end - hash = get(entry, "git-tree-sha1", nothing)::Union{Nothing, String} - hash === nothing && return nothing - hash = SHA1(hash) - # Keep the 4 since it used to be the default - uuid = pkg.uuid::UUID # checked within `explicit_manifest_uuid_path` - for slug in (version_slug(uuid, hash), version_slug(uuid, hash, 4)) - for depot in DEPOT_PATH - path = joinpath(depot, "packages", pkg.name, slug) - ispath(path) && return abspath(path) - end - end - # no depot contains the package, return missing to stop looking - return missing -end - -## implicit project & manifest API ## - -# look for an entry point for `name` from a top-level package (no environment) -# otherwise return `nothing` to indicate the caller should keep searching -function implicit_project_deps_get(dir::String, name::String)::Union{Nothing,PkgId} - path, project_file = entry_point_and_project_file(dir, name) - if project_file === nothing - path === nothing && return nothing - return PkgId(name) - end - proj = project_file_name_uuid(project_file, name) - proj.name == name || return nothing - return proj -end - -# look for an entry-point for `name`, check that UUID matches -# if there's a project file, look up `name` in its deps and return that -# otherwise return `nothing` to indicate the caller should keep searching -function implicit_manifest_deps_get(dir::String, where::PkgId, name::String)::Union{Nothing,PkgId} - @assert where.uuid !== nothing - project_file = entry_point_and_project_file(dir, where.name)[2] - project_file === nothing && return nothing # a project file is mandatory for a package with a uuid - proj = project_file_name_uuid(project_file, where.name) - proj == where || return nothing # verify that this is the correct project file - # this is the correct project, so stop searching here - pkg_uuid = explicit_project_deps_get(project_file, name) - return PkgId(pkg_uuid, name) -end - -# look for an entry-point for `pkg` and return its path if UUID matches -function implicit_manifest_uuid_path(dir::String, pkg::PkgId)::Union{Nothing,String} - path, project_file = entry_point_and_project_file(dir, pkg.name) - if project_file === nothing - pkg.uuid === nothing || return nothing - return path - end - proj = project_file_name_uuid(project_file, pkg.name) - proj == pkg || return nothing - return path -end - -## other code loading functionality ## - function find_source_file(path::AbstractString) (isabspath(path) || isfile(path)) && return path base_path = joinpath(Sys.BINDIR, DATAROOTDIR, "julia", "base", path) @@ -861,7 +428,7 @@ function find_all_in_cache_path(pkg::PkgId) paths = String[] entrypath, entryfile = cache_file_entry(pkg) for path in joinpath.(DEPOT_PATH, entrypath) - isdir(path) || continue + isaccessibledir(path) || continue for file in readdir(path, sort = false) # no sort given we sort later if !((pkg.uuid === nothing && file == entryfile * ".ji") || (pkg.uuid !== nothing && startswith(file, entryfile * "_"))) @@ -1236,47 +803,42 @@ For more details regarding code loading, see the manual sections on [modules](@r """ function require(into::Module, mod::Symbol) @lock require_lock begin - LOADING_CACHE[] = LoadingCache() - try - uuidkey_env = identify_package_env(into, String(mod)) - # Core.println("require($(PkgId(into)), $mod) -> $uuidkey from env \"$env\"") - if uuidkey_env === nothing - where = PkgId(into) - if where.uuid === nothing - hint, dots = begin - if isdefined(into, mod) && getfield(into, mod) isa Module - true, "." - elseif isdefined(parentmodule(into), mod) && getfield(parentmodule(into), mod) isa Module - true, ".." - else - false, "" - end + uuidkey = identify_package(into, String(mod)) + # Core.println("require($(PkgId(into)), $mod) -> $uuidkey from env \"$env\"") + if uuidkey === nothing + where = PkgId(into) + if where.uuid === nothing + hint, dots = begin + if isdefined(into, mod) && getfield(into, mod) isa Module + true, "." + elseif isdefined(parentmodule(into), mod) && getfield(parentmodule(into), mod) isa Module + true, ".." + else + false, "" end - hint_message = hint ? ", maybe you meant `import/using $(dots)$(mod)`" : "" - start_sentence = hint ? "Otherwise, run" : "Run" - throw(ArgumentError(""" - Package $mod not found in current path$hint_message. - - $start_sentence `import Pkg; Pkg.add($(repr(String(mod))))` to install the $mod package.""")) - else - throw(ArgumentError(""" - Package $(where.name) does not have $mod in its dependencies: - - You may have a partially installed environment. Try `Pkg.instantiate()` - to ensure all packages in the environment are installed. - - Or, if you have $(where.name) checked out for development and have - added $mod as a dependency but haven't updated your primary - environment's manifest file, try `Pkg.resolve()`. - - Otherwise you may need to report an issue with $(where.name)""")) end + hint_message = hint ? ", maybe you meant `import/using $(dots)$(mod)`" : "" + start_sentence = hint ? "Otherwise, run" : "Run" + throw(ArgumentError(""" + Package $mod not found in current path$hint_message. + - $start_sentence `import Pkg; Pkg.add($(repr(String(mod))))` to install the $mod package.""")) + else + throw(ArgumentError(""" + Package $(where.name) does not have $mod in its dependencies: + - You may have a partially installed environment. Try `Pkg.instantiate()` + to ensure all packages in the environment are installed. + - Or, if you have $(where.name) checked out for development and have + added $mod as a dependency but haven't updated your primary + environment's manifest file, try `Pkg.resolve()`. + - Otherwise you may need to report an issue with $(where.name)""")) end - uuidkey, env = uuidkey_env - if _track_dependencies[] - push!(_require_dependencies, (into, binpack(uuidkey), 0.0)) - end - return _require_prelocked(uuidkey, env) - finally - LOADING_CACHE[] = nothing + full_warning_showed[] = true end + if _track_dependencies[] + push!(_require_dependencies, (into, binpack(uuidkey), 0.0)) end + return _require_prelocked(uuidkey) + end # @lock end mutable struct PkgOrigin @@ -1289,10 +851,10 @@ const pkgorigins = Dict{PkgId,PkgOrigin}() require(uuidkey::PkgId) = @lock require_lock _require_prelocked(uuidkey) -function _require_prelocked(uuidkey::PkgId, env=nothing) +function _require_prelocked(uuidkey::PkgId) assert_havelock(require_lock) if !root_module_exists(uuidkey) - newm = _require(uuidkey, env) + newm = _require(uuidkey) if newm === nothing error("package `$(uuidkey.name)` did not define the expected \ module `$(uuidkey.name)`, check for typos in package module name") @@ -1380,7 +942,7 @@ function set_pkgorigin_version_path(pkg::PkgId, path::Union{String,Nothing}) end # Returns `nothing` or the new(ish) module -function _require(pkg::PkgId, env=nothing) +function _require(pkg::PkgId) assert_havelock(require_lock) # handle recursive calls to require loading = get(package_locks, pkg, false) @@ -1395,7 +957,7 @@ function _require(pkg::PkgId, env=nothing) try toplevel_load[] = false # perform the search operation to select the module file require intends to load - path = locate_package(pkg, env) + path = locate_package(pkg) if path === nothing throw(ArgumentError(""" Package $pkg is required but does not seem to be installed: @@ -1726,7 +1288,7 @@ end function compilecache_path(pkg::PkgId, prefs_hash::UInt64)::String entrypath, entryfile = cache_file_entry(pkg) cachepath = joinpath(DEPOT_PATH[1], entrypath) - isdir(cachepath) || mkpath(cachepath) + isaccessibledir(cachepath) || mkpath(cachepath) if pkg.uuid === nothing abspath(cachepath, entryfile) * ".ji" else @@ -2005,40 +1567,6 @@ function srctext_files(f::IO, srctextpos::Int64) return files end -# Test to see if this UUID is mentioned in this `Project.toml`; either as -# the top-level UUID (e.g. that of the project itself), as a dependency, -# or as an extra for Preferences. -function get_uuid_name(project::Dict{String, Any}, uuid::UUID) - uuid_p = get(project, "uuid", nothing)::Union{Nothing, String} - name = get(project, "name", nothing)::Union{Nothing, String} - if name !== nothing && uuid_p !== nothing && UUID(uuid_p) == uuid - return name - end - deps = get(project, "deps", nothing)::Union{Nothing, Dict{String, Any}} - if deps !== nothing - for (k, v) in deps - if uuid == UUID(v::String) - return k - end - end - end - for subkey in ("deps", "extras") - subsection = get(project, subkey, nothing)::Union{Nothing, Dict{String, Any}} - if subsection !== nothing - for (k, v) in subsection - if uuid == UUID(v::String) - return k - end - end - end - end - return nothing -end - -function get_uuid_name(project_toml::String, uuid::UUID) - project = parsed_toml(project_toml) - return get_uuid_name(project, uuid) -end # If we've asked for a specific UUID, this function will extract the prefs # for that particular UUID. Otherwise, it returns all preferences. @@ -2050,38 +1578,38 @@ function filter_preferences(prefs::Dict{String, Any}, pkg_name) end end -function collect_preferences(project_toml::String, uuid::Union{UUID,Nothing}) +############### +# Preferences # +############### + +function collect_preferences(env::ExplicitEnv, uuid::Union{UUID,Nothing}) # We'll return a list of dicts to be merged dicts = Dict{String, Any}[] - project = parsed_toml(project_toml) + # If we've been given a UUID, map that to the name of the package as + # recorded in the preferences section. If we can't find that mapping, + # exit out, as it means there's no way preferences can be set for that + # UUID, as we only allow actual dependencies to have preferences set. pkg_name = nothing if uuid !== nothing - # If we've been given a UUID, map that to the name of the package as - # recorded in the preferences section. If we can't find that mapping, - # exit out, as it means there's no way preferences can be set for that - # UUID, as we only allow actual dependencies to have preferences set. - pkg_name = get_uuid_name(project, uuid) + for (name, uuid′) in env.project_deps + uuid == uuid′ && (pkg_name = name) + end if pkg_name === nothing - return dicts + for (name, uuid′) in env.project_extras + uuid == uuid′ && (pkg_name = name) + end end + pkg_name === nothing && return dicts end - # Look first inside of `Project.toml` to see we have preferences embedded within there - proj_preferences = get(Dict{String, Any}, project, "preferences")::Dict{String, Any} - push!(dicts, filter_preferences(proj_preferences, pkg_name)) - - # Next, look for `(Julia)LocalPreferences.toml` files next to this `Project.toml` - project_dir = dirname(project_toml) - for name in preferences_names - toml_path = joinpath(project_dir, name) - if isfile(toml_path) - prefs = parsed_toml(toml_path) - push!(dicts, filter_preferences(prefs, pkg_name)) + # Look first to see if we have preferences embedded within there + if env.prefs !== nothing + push!(dicts, filter_preferences(env.prefs, pkg_name)) + end - # If we find `JuliaLocalPreferences.toml`, don't look for `LocalPreferences.toml` - break - end + if env.local_prefs !== nothing + push!(dicts, filter_preferences(env.local_prefs, pkg_name)) end return dicts @@ -2118,16 +1646,13 @@ function recursive_prefs_merge(base::Dict{String, Any}, overrides::Dict{String, return new_base end -function get_preferences(uuid::Union{UUID,Nothing} = nothing) +function get_preferences(uuid::Union{UUID,Nothing} = nothing, envstack=EnvironmentStack()) merged_prefs = Dict{String,Any}() - for env in reverse(load_path()) - project_toml = env_project_file(env) - if !isa(project_toml, String) - continue - end + for env in reverse(envstack.envs) + env isa ExplicitEnv || continue # Collect all dictionaries from the current point in the load path, then merge them in - dicts = collect_preferences(project_toml, uuid) + dicts = collect_preferences(env, uuid) merged_prefs = recursive_prefs_merge(merged_prefs, dicts...) end return merged_prefs diff --git a/test/loading.jl b/test/loading.jl index d057f0b3c3702..cca18a4fb68b0 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -214,8 +214,6 @@ end @test root.uuid == root_uuid @test this == nothing @test that == nothing - - @test Base.get_uuid_name(project_file, this_uuid) == "This" finally copy!(LOAD_PATH, old_load_path) end @@ -982,8 +980,8 @@ end write(joinpath(tmp, "Env1", "Manifest.toml"), """ """) # Package in current env not present in manifest - pkg, env = Base.identify_package_env("Baz") - @test Base.locate_package(pkg, env) === nothing + pkg = Base.identify_package("Baz") + @test Base.locate_package(pkg) === nothing finally copy!(LOAD_PATH, old_load_path) copy!(DEPOT_PATH, old_depot_path) From 9be5978b7b8dfcdd804c46277df1a4fec491d536 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Mon, 12 Sep 2022 15:48:08 +0200 Subject: [PATCH 3/5] pass the environment stack to precompile workers via serialization --- base/codeloading2.jl | 14 ++++++++- base/loading.jl | 30 +++++++++++++++++-- contrib/generate_precompile.jl | 25 ++++++++++++++++ .../md5 | 1 + .../sha512 | 1 + stdlib/Pkg.version | 2 +- stdlib/Serialization/src/Serialization.jl | 6 ++-- 7 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 deps/checksums/Pkg-45003c2e00a50d0594b927782f8342739860e3da.tar.gz/md5 create mode 100644 deps/checksums/Pkg-45003c2e00a50d0594b927782f8342739860e3da.tar.gz/sha512 diff --git a/base/codeloading2.jl b/base/codeloading2.jl index 7f1be15e28618..577327aeb62ff 100644 --- a/base/codeloading2.jl +++ b/base/codeloading2.jl @@ -305,7 +305,19 @@ struct EnvironmentStack envs::Vector{Union{ImplicitEnv, ExplicitEnv}} end -function EnvironmentStack(environments = load_path()) + +# Caching + +const CACHED_ENV_STACK = Ref{Union{EnvironmentStack, Nothing}}(nothing) + +function EnvironmentStack(environments = nothing) + if CACHED_ENV_STACK[] !== nothing + return CACHED_ENV_STACK[] + end + # Avoid looking up `load_path` until it is really needed. + if environments === nothing + environments = load_path() + end envs = Union{ImplicitEnv, ExplicitEnv}[] for env in environments if isfile(env) diff --git a/base/loading.jl b/base/loading.jl index 50a1349484bd5..1bb0365129801 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -159,7 +159,6 @@ end const ns_dummy_uuid = UUID("fe0723d6-3a44-4c41-8065-ee0f42c8ceab") function dummy_uuid(project_file::String) - @lock require_lock begin project_path = try realpath(project_file) catch ex @@ -168,7 +167,6 @@ function dummy_uuid(project_file::String) end uuid = uuid5(ns_dummy_uuid, project_path) return uuid - end end ## package path slugs: turning UUID + SHA1 into a pair of 4-byte "slugs" ## @@ -256,6 +254,11 @@ function parsed_toml(project_file::AbstractString, toml_cache::TOMLCache, toml_l toml_cache.d[project_file] = d return d.d else + # We are in a require call and have already parsed this TOML file + # assume that it is unchanged to avoid hitting disk + if CACHED_ENV_STACK[] !== nothing + return d.d + end d = toml_cache.d[project_file] return get_updated_dict(toml_cache.p, d) end @@ -803,6 +806,9 @@ For more details regarding code loading, see the manual sections on [modules](@r """ function require(into::Module, mod::Symbol) @lock require_lock begin + no_initial_cache = CACHED_ENV_STACK[] === nothing + try + no_initial_cache && (CACHED_ENV_STACK[] = EnvironmentStack()) uuidkey = identify_package(into, String(mod)) # Core.println("require($(PkgId(into)), $mod) -> $uuidkey from env \"$env\"") if uuidkey === nothing @@ -838,6 +844,9 @@ function require(into::Module, mod::Symbol) push!(_require_dependencies, (into, binpack(uuidkey), 0.0)) end return _require_prelocked(uuidkey) + finally + no_initial_cache && (CACHED_ENV_STACK[] = nothing) + end # try end # @lock end @@ -1238,6 +1247,7 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto end const PRECOMPILE_TRACE_COMPILE = Ref{String}() +const ENV_SERIALIZATION_FILE = Ref{Union{Nothing,String}}(nothing) function create_expr_cache(pkg::PkgId, input::String, output::String, concrete_deps::typeof(_concrete_dependencies), internal_stderr::IO = stderr, internal_stdout::IO = stdout) @nospecialize internal_stderr internal_stdout rm(output, force=true) # Remove file if it exists @@ -1272,6 +1282,22 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, concrete_d stderr = internal_stderr, stdout = internal_stdout), "w", stdout) # write data over stdin to avoid the (unlikely) case of exceeding max command line size + serialization_pkgid = PkgId(UUID("9e88b42a-f829-5b0c-bbe9-9e923198166b"), "Serialization") + if haskey(loaded_modules, serialization_pkgid) + local ftmp + if ENV_SERIALIZATION_FILE[] === nothing || !isfile(ENV_SERIALIZATION_FILE[]) + Serialization = loaded_modules[serialization_pkgid] + ftmp, iotmp = mktemp() + Serialization.serialize(iotmp, EnvironmentStack()) + close(iotmp) + else + ftmp = ENV_SERIALIZATION_FILE[] + end + write(io.in, """ + Base.ENV_SERIALIZATION_FILE[] = $(repr(ftmp)) + Base.CACHED_ENV_STACK[] = (Base.loaded_modules[Base.PkgId(Base.UUID("9e88b42a-f829-5b0c-bbe9-9e923198166b"), "Serialization")].deserialize(IOBuffer(read($(repr(ftmp)))))) + """) + end write(io.in, """ Base.include_package_for_output($(pkg_str(pkg)), $(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)), $(repr(load_path)), $deps, $(repr(source_path(nothing)))) diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 295b24d22e1c7..0a08d7f69aca8 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -186,6 +186,31 @@ if Libdl !== nothing """ end +Serialization = get(Base.loaded_modules, + Base.PkgId(Base.UUID("9e88b42a-f829-5b0c-bbe9-9e923198166b"), "Serialization"), + nothing) +if Serialization !== nothing + # Deserializing environment + hardcoded_precompile_statements *= """ + precompile(Tuple{typeof(Serialization.deserialize), String}) + precompile(Tuple{typeof(Serialization.deserialize), Serialization.Serializer{Base.IOStream}, Type{Base.EnvironmentStack}}) + precompile(Tuple{typeof(Serialization.deserialize_fillarray!), Array{String, 1}, Serialization.Serializer{Base.IOStream}}) + precompile(Tuple{typeof(Serialization.deserialize), Serialization.Serializer{Base.IOStream}, Type{Union}}) + precompile(Tuple{typeof(Serialization.deserialize_fillarray!), Array{Union{Base.ExplicitEnv, Base.ImplicitEnv}, 1}, Serialization.Serializer{Base.IOStream}}) + precompile(Tuple{typeof(Serialization.deserialize), Serialization.Serializer{Base.IOStream}, Type{Base.ExplicitEnv}}) + precompile(Tuple{typeof(Serialization.deserialize), Serialization.Serializer{Base.IOStream}, Type{Base.Dict{String, Base.UUID}}}) + precompile(Tuple{typeof(Serialization.deserialize), Serialization.Serializer{Base.IOStream}, Type{Base.UUID}}) + precompile(Tuple{typeof(Serialization.deserialize), Serialization.Serializer{Base.IOStream}, Type{Base.Dict{Base.UUID, Base.Dict{String, Base.UUID}}}}) + precompile(Tuple{typeof(Serialization.deserialize), Serialization.Serializer{Base.IOStream}, Type{Base.Dict{Base.UUID, Union{Base.Missing, Nothing, Base.SHA1, String}}}}) + precompile(Tuple{typeof(Serialization.deserialize), Serialization.Serializer{Base.IOStream}, Type{Base.SHA1}}) + precompile(Tuple{typeof(Serialization.deserialize), Serialization.Serializer{Base.IOStream}, Type{Base.ImplicitEnv}}) + precompile(Tuple{typeof(Serialization.deserialize), Serialization.Serializer{Base.IOStream}, Type{Base.Dict{String, Base.ImplicitEnvPkg}}}) + precompile(Tuple{typeof(Serialization.deserialize), Serialization.Serializer{Base.IOStream}, Type{Base.ImplicitEnvPkg}}) + precompile(Tuple{typeof(Serialization.deserialize_fillarray!), Array{Base.PkgId, 1}, Serialization.Serializer{Base.IOStream}}) + precompile(Tuple{typeof(Serialization.deserialize), Serialization.Serializer{Base.IOStream}, Type{Base.PkgId}}) + """ +end + Test = get(Base.loaded_modules, Base.PkgId(Base.UUID("8dfed614-e22c-5e08-85e1-65c5234f0b40"), "Test"), nothing) diff --git a/deps/checksums/Pkg-45003c2e00a50d0594b927782f8342739860e3da.tar.gz/md5 b/deps/checksums/Pkg-45003c2e00a50d0594b927782f8342739860e3da.tar.gz/md5 new file mode 100644 index 0000000000000..edb60b88f7068 --- /dev/null +++ b/deps/checksums/Pkg-45003c2e00a50d0594b927782f8342739860e3da.tar.gz/md5 @@ -0,0 +1 @@ +da299f922804dae87cefd024e9c96866 diff --git a/deps/checksums/Pkg-45003c2e00a50d0594b927782f8342739860e3da.tar.gz/sha512 b/deps/checksums/Pkg-45003c2e00a50d0594b927782f8342739860e3da.tar.gz/sha512 new file mode 100644 index 0000000000000..907c65a38ec19 --- /dev/null +++ b/deps/checksums/Pkg-45003c2e00a50d0594b927782f8342739860e3da.tar.gz/sha512 @@ -0,0 +1 @@ +fa9edd15a7e4e754ece0ae7f7a5b3dda744340211e0857e4f59d5050c86fe73ff938cb6afe6588caaa0945dedcd0f0373a92e38fc47c2673dbec02df4ce40251 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index faff813896433..3969844b75daa 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = ed6a5497e46ed541b2718c404c0f468b7f92263a +PKG_SHA1 = 45003c2e00a50d0594b927782f8342739860e3da PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 diff --git a/stdlib/Serialization/src/Serialization.jl b/stdlib/Serialization/src/Serialization.jl index 0ad24addd9028..be3b5d78e7879 100644 --- a/stdlib/Serialization/src/Serialization.jl +++ b/stdlib/Serialization/src/Serialization.jl @@ -1474,7 +1474,7 @@ function deserialize_string(s::AbstractSerializer, len::Int) end # default DataType deserializer -function deserialize(s::AbstractSerializer, t::DataType) +function deserialize(s::AbstractSerializer, ::Type{T}) where {T} nf = length(t.types) if isprimitivetype(t) return read(s.io, t) @@ -1489,7 +1489,7 @@ function deserialize(s::AbstractSerializer, t::DataType) end return x elseif nf == 0 - return ccall(:jl_new_struct_uninit, Any, (Any,), t) + return ccall(:jl_new_struct_uninit, Any, (Any,), T) else na = nf vflds = Vector{Any}(undef, nf) @@ -1502,7 +1502,7 @@ function deserialize(s::AbstractSerializer, t::DataType) na >= i && (na = i - 1) # rest of tail must be undefined values end end - return ccall(:jl_new_structv, Any, (Any, Ptr{Any}, UInt32), t, vflds, na) + return ccall(:jl_new_structv, Any, (Any, Ptr{Any}, UInt32), T, vflds, na) end end From 47c3edf5ee58d8e0a1d51386792ec6314c2515bf Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Wed, 9 Nov 2022 13:49:52 +0100 Subject: [PATCH 4/5] fixup! pass the environment stack to precompile workers via serialization --- stdlib/Serialization/src/Serialization.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stdlib/Serialization/src/Serialization.jl b/stdlib/Serialization/src/Serialization.jl index be3b5d78e7879..0ad24addd9028 100644 --- a/stdlib/Serialization/src/Serialization.jl +++ b/stdlib/Serialization/src/Serialization.jl @@ -1474,7 +1474,7 @@ function deserialize_string(s::AbstractSerializer, len::Int) end # default DataType deserializer -function deserialize(s::AbstractSerializer, ::Type{T}) where {T} +function deserialize(s::AbstractSerializer, t::DataType) nf = length(t.types) if isprimitivetype(t) return read(s.io, t) @@ -1489,7 +1489,7 @@ function deserialize(s::AbstractSerializer, ::Type{T}) where {T} end return x elseif nf == 0 - return ccall(:jl_new_struct_uninit, Any, (Any,), T) + return ccall(:jl_new_struct_uninit, Any, (Any,), t) else na = nf vflds = Vector{Any}(undef, nf) @@ -1502,7 +1502,7 @@ function deserialize(s::AbstractSerializer, ::Type{T}) where {T} na >= i && (na = i - 1) # rest of tail must be undefined values end end - return ccall(:jl_new_structv, Any, (Any, Ptr{Any}, UInt32), T, vflds, na) + return ccall(:jl_new_structv, Any, (Any, Ptr{Any}, UInt32), t, vflds, na) end end From bb7c98eefd688794858a353b62a4a8d82204e226 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Mon, 19 Feb 2024 19:00:04 +0100 Subject: [PATCH 5/5] some updates --- base/codeloading2.jl | 212 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 167 insertions(+), 45 deletions(-) diff --git a/base/codeloading2.jl b/base/codeloading2.jl index 577327aeb62ff..4e797d83db9ee 100644 --- a/base/codeloading2.jl +++ b/base/codeloading2.jl @@ -1,3 +1,6 @@ +using Base: PkgId, UUID, SHA1, parsed_toml, project_file_name_uuid, project_names, + project_file_manifest_path, get_deps, preferences_names, isaccessibledir, isfile_casesensitive +using UUIDs ######################### # Implicit environments # @@ -39,7 +42,7 @@ function _ImplicitEnv(envpath::String) # It did have a project file: if project_file !== nothing project_d = parsed_toml(project_file) - uuid = project_file_uuid(project_d, project_file) + uuid = project_file_name_uuid(envpath, "").uuid deps = PkgId[] # Get the explicit deps, these are the only deps that can be loaded inside the package: for (name, uuid) in get(Dict{String, Any}, project_d, "deps") @@ -99,40 +102,98 @@ end # as well as how the path is looked up for a package struct ExplicitEnv path::String - project_deps::Dict{String, UUID} # [deps] in Project.toml - project_extras::Dict{String, UUID} # [extras] in Project.toml - deps::Dict{UUID, Dict{String, UUID}} # all dependencies in Manifest.toml + project_deps::Vector{UUID} # [deps] in Project.toml + project_weakdeps::Vector{UUID} # [weakdeps] in Project.toml + project_extras::Vector{UUID} # [extras] in Project.toml + project_extensions::Dict{String, Vector{UUID}} # [exts] in Project.toml + deps::Dict{UUID, Vector{UUID}} # all dependencies in Manifest.toml + weakdeps::Dict{UUID, Vector{UUID}} # all weak dependencies in Manifest.toml + extensions::Dict{UUID, Dict{String, Vector{UUID}}} + # Lookup name for a UUID + names::Dict{UUID, String} lookup_strategy::Dict{UUID, Union{ SHA1, # `git-tree-sha1` entry String, # `path` entry Nothing, # stdlib (no `path` nor `git-tree-sha1`) Missing}} # not present in the manifest - prefs::Union{Nothing, Dict{String, Any}} - local_prefs::Union{Nothing, Dict{String, Any}} + #prefs::Union{Nothing, Dict{String, Any}} + #local_prefs::Union{Nothing, Dict{String, Any}} end +#= + +[[deps.PGFPlotsX]] +deps = ["ArgCheck", "Dates", "DefaultApplication", "DocStringExtensions", "MacroTools", "OrderedCollections", "Parameters", "Requires", "Tables"] +path = "../../../../../../../Users/kristoffercarlsson/JuliaPkgs/PGFPlotsX.jl" +uuid = "8314cec4-20b6-5062-9cdb-752b83310925" +version = "1.6.1" + + [deps.PGFPlotsX.extensions] + ColorsExt = "Colors" + ContourExt = "Contour" + MeasurementsExt = "Measurements" + StatsBaseExt = "StatsBase" + + [deps.PGFPlotsX.weakdeps] + Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" + Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" + Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" + StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +=# + + function ExplicitEnv(envpath::String) envpath = abspath(envpath) project_d = parsed_toml(envpath) - project_deps = Dict{String, UUID}() + # TODO: Perhaps verify that two packages with the same UUID do not have different names? + names = Dict{UUID, String}() + project_uuid_to_name = Dict{String, UUID}() + + project_deps = UUID[] + project_weakdeps = UUID[] + project_extras = UUID[] + # Collect all direct dependencies of the project - for (name, uuid) in get(Dict{String, Any}, project_d, "deps")::Dict{String, Any} - project_deps[name] = UUID(uuid::String) + for key in ["deps", "weakdeps", "extras"] + for (name, _uuid) in get(Dict{String, Any}, project_d, key)::Dict{String, Any} + v = key == "deps" ? project_deps : + key == "weakdeps" ? project_weakdeps : + key == "extras" ? project_extras : + error() + uuid = UUID(_uuid) + push!(v, uuid) + names[UUID(uuid)] = name + project_uuid_to_name[name] = UUID(uuid) + end end - project_extras = Dict{String, UUID}() - # Collect all "extras" dependencies of the project - for (name, uuid) in get(Dict{String, Any}, project_d, "extras")::Dict{String, Any} - project_extras[name] = UUID(uuid::String) + project_extensions = Dict{String, Vector{UUID}}() + # Collect all extensions of the project + for (name, triggers::Union{String, Vector{String}}) in get(Dict{String, Any}, project_d, "extensions")::Dict{String, Any} + if triggers isa String + triggers = [triggers] + end + uuids = UUID[] + for trigger in triggers + uuid = get(project_uuid_to_name, trigger, nothing) + if uuid === nothing + error("Trigger $trigger for extension $name not found in project") + end + push!(uuids, uuid) + end + project_extensions[name] = uuids end # This project might be a package, in that case, that is also a "dependency" # of the project. - name = get(project_d, "name", nothing)::Union{String, Nothing} - pkg_uuid = UUID(project_file_uuid(project_d, envpath)) - if name !== nothing - project_deps[name] = pkg_uuid + proj_name = get(project_d, "name", nothing)::Union{String, Nothing} + _proj_uuid = get(project_d, "uuid", nothing)::Union{String, Nothing} + proj_uuid = _proj_uuid === nothing ? nothing : UUID(_proj_uuid) + if proj_name !== nothing + # TODO: Error on missing uuid? + push!(project_deps, UUID(proj_uuid)) + names[UUID(proj_uuid)] = proj_name end manifest = project_file_manifest_path(envpath) @@ -140,11 +201,16 @@ function ExplicitEnv(envpath::String) # Dependencies in a manifest can either be stored compressed (when name is unique among all packages) # in which case it is a `Vector{String}` or expanded where it is a `name => uuid` mapping. - deps = Dict{UUID, Union{Vector{String}, Dict{String, UUID}}}() - sizehint!(deps, length(manifest_d)) + deps = Dict{UUID, Union{Vector{String}, Vector{UUID}}}() + weakdeps = Dict{UUID, Union{Vector{String}, Vector{UUID}}}() + extensions = Dict{UUID, Dict{String, Vector{String}}}() name_to_uuid = Dict{String, UUID}() - sizehint!(name_to_uuid, length(manifest_d)) lookup_strategy = Dict{UUID, Union{SHA1, String, Nothing, Missing}}() + + sizehint!(deps, length(manifest_d)) + sizehint!(weakdeps, length(manifest_d)) + sizehint!(extensions, length(manifest_d)) + sizehint!(name_to_uuid, length(manifest_d)) sizehint!(lookup_strategy, length(manifest_d)) for (name, pkg_infos) in get_deps(manifest_d) @@ -155,15 +221,40 @@ function ExplicitEnv(envpath::String) # If we have multiple packages with the same name we will overwrite things here # but that is fine since we will only use the information in here for packages # with unique names + names[m_uuid] = name name_to_uuid[name] = m_uuid - deps_pkg = get(Vector{String}, pkg_info, "deps")::Union{Vector{String}, Dict{String, Any}} - # Compressed format with unique names: - if deps_pkg isa Vector{String} - deps[m_uuid] = deps_pkg - # Exapanded format: - else - deps[m_uuid] = Dict{String, UUID}(name_dep => UUID(dep_uuid::String) for (name_dep, dep_uuid) in deps_pkg) + + for key in ["deps", "weakdeps"] + deps_pkg = get(Vector{String}, pkg_info, key)::Union{Vector{String}, Dict{String, Any}} + d = key == "deps" ? deps : + key == "weakdeps" ? weakdeps : + error() + + # Compressed format with unique names: + if deps_pkg isa Vector{String} + d[m_uuid] = deps_pkg + # Expanded format: + else + uuids = UUID[] + for (name_dep, _dep_uuid::String) in deps_pkg + dep_uuid = UUID(_dep_uuid) + push!(uuids, dep_uuid) + names[dep_uuid] = name_dep + end + d[m_uuid] = uuids + end + end + + # Extensions + deps_pkg = get(Dict{String, Any}, pkg_info, "extensions")::Dict{String, Any} + for (ext, triggers) in deps_pkg + triggers = triggers::Union{String, Vector{String}} + if triggers isa String + triggers = [triggers] + end + deps_pkg[ext] = triggers end + extensions[m_uuid] = deps_pkg # Determine strategy to find package lookup_strat = begin @@ -181,37 +272,65 @@ function ExplicitEnv(envpath::String) # No matter if the deps were stored compressed or not in the manifest, # we internally store them expanded - deps_expanded = Dict{UUID, Dict{String, UUID}}() + deps_expanded = Dict{UUID, Vector{UUID}}() + weakdeps_expanded = Dict{UUID, Vector{UUID}}() + extensions_expanded = Dict{UUID, Dict{String, Vector{UUID}}}() sizehint!(deps_expanded, length(deps)) + sizehint!(weakdeps_expanded, length(deps)) + sizehint!(extensions_expanded, length(deps)) - if name !== nothing - deps_expanded[pkg_uuid] = project_deps - # N.b `path` entries in the Project file is currently not understood by Pkg + if proj_name !== nothing + deps_expanded[proj_uuid] = project_deps path = get(project_d, "path", nothing) entry_point = path !== nothing ? path : dirname(envpath) - lookup_strategy[pkg_uuid] = entry_point + lookup_strategy[proj_uuid] = entry_point end - for (pkg, deps) in deps - # dependencies was already expanded so use it directly: - if deps isa Dict{String,UUID} - deps_expanded[pkg] = deps - # find the (unique) UUID associated with the name - else - deps_pkg = Dict{String, UUID}() - sizehint!(deps_pkg, length(deps)) - for dep in deps - deps_pkg[dep] = name_to_uuid[dep] + for key in ["deps", "weakdeps"] + d = key == "deps" ? deps : + key == "weakdeps" ? weakdeps : + error() + d_expanded = key == "deps" ? deps_expanded : + key == "weakdeps" ? weakdeps_expanded : + error() + for (pkg, deps) in d + # dependencies was already expanded so use it directly: + if deps isa Vector{UUID} + d_expanded[pkg] = deps + for dep in deps + name_to_uuid[names[dep]] = dep + end + # find the (unique) UUID associated with the name + else + deps_pkg = UUID[] + sizehint!(deps_pkg, length(deps)) + for dep in deps + push!(deps_pkg, name_to_uuid[dep]) + end + d_expanded[pkg] = deps_pkg + end + end + end + + for (pkg, exts) in extensions + exts_expanded = Dict{String, Vector{UUID}}() + for (ext, triggers) in exts + triggers_expanded = UUID[] + sizehint!(triggers_expanded, length(triggers)) + for trigger in triggers + push!(triggers_expanded, name_to_uuid[trigger]) end - deps_expanded[pkg] = deps_pkg + exts_expanded[ext] = triggers_expanded end + extensions_expanded[pkg] = exts_expanded end # Everything that does not yet have a lookup_strategy is missing from the manifest - for (name, uuid) in project_deps + for uuid in project_deps get!(lookup_strategy, uuid, missing) end + #= # Preferences: prefs = get(project_d, "preferences", nothing) @@ -225,8 +344,11 @@ function ExplicitEnv(envpath::String) break end end + =# - return ExplicitEnv(envpath, project_deps, project_extras, deps_expanded, lookup_strategy, prefs, local_prefs) + return ExplicitEnv(envpath, project_deps, project_weakdeps, project_extras, + project_extensions, deps_expanded, weakdeps_expanded, extensions_expanded, + names, lookup_strategy, #=prefs, local_prefs=#) end # Marker to return when we should have been able to load a package but failed.