From e0954b97fb6fc89791fa28a0aeaa17eb883b37a4 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 7 Jul 2022 16:57:01 +0200 Subject: [PATCH 01/26] add support for glue packages --- docs/make.jl | 1 + docs/src/creating-packages.md | 75 ++++++++++++ src/API.jl | 9 +- src/Operations.jl | 112 +++++++++++++----- src/Pkg.jl | 4 +- src/Registry/registry_instance.jl | 69 ++++++++++- src/Resolve/graphtype.jl | 8 +- src/Types.jl | 8 +- src/manifest.jl | 68 +++++++---- src/project.jl | 30 +++-- test/gluedeps.jl | 21 ++++ test/resolve_utils.jl | 2 +- test/runtests.jl | 1 + .../HasDepWithGluePkgs.jl/Manifest.toml | 57 +++++++++ .../HasDepWithGluePkgs.jl/Project.toml | 17 +++ .../src/HasDepWithGluePkgs.jl | 15 +++ .../HasDepWithGluePkgs.jl/test/runtests.jl | 5 + .../HasGluePkgs.jl/Manifest.toml | 10 ++ .../HasGluePkgs.jl/Project.toml | 22 ++++ .../HasGluePkgs.jl/glue/GlueOffsetArrays.jl | 13 ++ .../HasGluePkgs.jl/src/HasGluePkgs.jl | 11 ++ .../HasGluePkgs.jl/test/runtests.jl | 8 ++ 22 files changed, 485 insertions(+), 81 deletions(-) create mode 100644 test/gluedeps.jl create mode 100644 test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml create mode 100644 test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Project.toml create mode 100644 test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/src/HasDepWithGluePkgs.jl create mode 100644 test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/test/runtests.jl create mode 100644 test/test_packages/GluePkgExamples/HasGluePkgs.jl/Manifest.toml create mode 100644 test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml create mode 100644 test/test_packages/GluePkgExamples/HasGluePkgs.jl/glue/GlueOffsetArrays.jl create mode 100644 test/test_packages/GluePkgExamples/HasGluePkgs.jl/src/HasGluePkgs.jl create mode 100644 test/test_packages/GluePkgExamples/HasGluePkgs.jl/test/runtests.jl diff --git a/docs/make.jl b/docs/make.jl index 24b0c4244c..163cb9e7bd 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -35,6 +35,7 @@ makedocs( "managing-packages.md", "environments.md", "creating-packages.md", + "gluedeps.md", "compatibility.md", "registries.md", "artifacts.md", diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index ebf86517b4..df45a97957 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -245,6 +245,81 @@ using Test Every dependency should in general have a compatibility constraint on it. This is an important topic so there is a separate chapter about it: [Compatibility](@ref Compatibility). +## Conditional loading of code in packages (Glue packages) + +!!! note + This is a somewhat advanced section which can be skipped for people new to Julia and Julia packages. + +It is sometimes desirable to be able to extend some functionality of a package without having to +unconditionally take on the cost (in terms of e.g. load time) of adding an extra dependency. +A *glue package* is a file that gets automatically loaded when some other set of packages are +loaded into the Julia session. + +A useful application of glue packages could be for a plotting package that should be able to plot +objects from a wide variety of different Julia packages. +Adding all those different Julia packages as dependencies +could be expensive since they would end up getting loaded even if they were never used. +Instead, these code required to plot objects for specific packages can be put into separate files +(glue packages) which is only loaded when + +Below is an example of how the code can be structured for a use case as outlined above. + + `Project.toml`: + ```toml +name = "Plotting" +version = "0.1.0" +uuid = "..." + +[deps] +Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" + +[gluedeps] +Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" + +[gluepkgs] +# name of glue package to the left +# glue dependencies required to load the glue pkg to the right +# use a list for multiple glue dependencies +GlueContour = "Contour" + +[compat] # compat can also be given on glue dependencies +Colors = "0.12.8" +Contour = "0.6.2" +``` + +`src/Plotting.jl`: +```julia +module Plotting +using Colors + +function plot(x::Colors.Color) + # Some functionality for plotting a color here +end + +end # module +``` + +`glue/GlueContour.jl` (can also be in `glue/GlueContour/GlueContour.jl`): +```julia +module GlueContour + +using Plotting, Contour + +function Plotting.plot(c::Contour.ContourCollection) + # Some functionality for plotting a contour here +end + +end # module +``` + +A user that depends on `Plotting` will not pay the code of the "glue code" inside the `GlueContour` module. +It is only when the `Contour` package actually gets loaded that the `GlueCountour` glue package will get loaded +and provide the new functionality. + +Compatibility can be set on glue dependencies just like normal dependencies. + +A glue package will only be loaded if the glue dependencies are loaded from the same environment or environments higher in the environment stack than the package itself. + ## Package naming guidelines Package names should be sensible to most Julia users, *even to those who are not domain experts*. diff --git a/src/API.jl b/src/API.jl index a365b1c72a..850e36a559 100644 --- a/src/API.jl +++ b/src/API.jl @@ -154,7 +154,7 @@ for f in (:develop, :add, :rm, :up, :pin, :free, :test, :build, :status, :why) pkgs = deepcopy(pkgs) # don't mutate input foreach(handle_package_input!, pkgs) ret = $f(ctx, pkgs; kwargs...) - $(f in (:add, :up, :pin, :free, :build)) && Pkg._auto_precompile(ctx) + $(f in (:add, :up, :pin, :free, :build)) && Pkg._auto_precompile(ctx) # rm does too, but it's handled differently $(f in (:up, :pin, :free, :rm)) && Pkg._auto_gc(ctx) return ret end @@ -301,9 +301,11 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode=PKGMODE_PROJECT, all_p ensure_resolved(ctx, ctx.env.manifest, pkgs) Operations.rm(ctx, pkgs; mode) + return end + function append_all_pkgs!(pkgs, ctx, mode) if mode == PKGMODE_PROJECT || mode == PKGMODE_COMBINED for (name::String, uuid::UUID) in ctx.env.project.deps @@ -1163,6 +1165,8 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: isempty(depsmap) && pkgerror("No direct dependencies found matching $(repr(pkgs))") end + target = string(isempty(pkgs) ? "project" : join(pkgs, ", "), "...") + pkg_queue = Base.PkgId[] failed_deps = Dict{Base.PkgId, String}() skipped_deps = Base.PkgId[] @@ -1309,7 +1313,7 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: iob = IOBuffer() name = is_direct_dep ? pkg.name : string(color_string(pkg.name, :light_black)) !fancyprint && lock(print_lock) do - isempty(pkg_queue) && printpkgstyle(io, :Precompiling, "environment...") + isempty(pkg_queue) && printpkgstyle(io, :Precompiling, target) end push!(pkg_queue, pkg) started[pkg] = true @@ -1613,6 +1617,7 @@ function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode= if compat diff && pkgerror("Compat status has no `diff` mode") outdated && pkgerror("Compat status has no `outdated` mode") + Operations.print_compat(ctx, pkgs; io) else Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated) diff --git a/src/Operations.jl b/src/Operations.jl index 53178462d8..c835accf39 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -142,6 +142,15 @@ function update_manifest!(env::EnvCache, pkgs::Vector{PackageSpec}, deps_map, ju else entry.deps = deps_map[pkg.uuid] end + v = joinpath(source_path(env.project_file, pkg), "Project.toml") + if isfile(v) + p = Types.read_project(v) + entry.gluedeps = p.gluedeps + entry.gluepkgs = p.gluepkgs + for (name, _) in p.gluedeps + delete!(entry.deps, name) + end + end env.manifest[pkg.uuid] = entry end prune_manifest(env) @@ -198,24 +207,29 @@ function reset_all_compat!(proj::Project) return nothing end -function collect_project!(pkg::PackageSpec, path::String, - deps_map::Dict{UUID,Vector{PackageSpec}}) - deps_map[pkg.uuid] = PackageSpec[] +function collect_project(pkg::PackageSpec, path::String) + deps = PackageSpec[] + glues = Set{UUID}() project_file = projectfile_path(path; strict=true) if project_file === nothing pkgerror("could not find project file for package $(err_rep(pkg)) at `$path`") end project = read_package(project_file) - julia_compat = get_compat(project, "julia") #= # TODO, this should either error or be quiet + julia_compat = get_compat(project, "julia") if julia_compat !== nothing && !(VERSION in julia_compat) println(io, "julia version requirement for package $(err_rep(pkg)) not satisfied") end =# for (name, uuid) in project.deps vspec = get_compat(project, name) - push!(deps_map[pkg.uuid], PackageSpec(name, uuid, vspec)) + push!(deps, PackageSpec(name, uuid, vspec)) + end + for (name, uuid) in project.gluedeps + vspec = get_compat(project, name) + push!(deps, PackageSpec(name, uuid, vspec)) + push!(glues, uuid) end if project.version !== nothing pkg.version = project.version @@ -223,7 +237,7 @@ function collect_project!(pkg::PackageSpec, path::String, # @warn("project file for $(pkg.name) is missing a `version` entry") pkg.version = VersionNumber(0) end - return + return deps, glues end is_tracking_path(pkg) = pkg.path !== nothing @@ -258,9 +272,12 @@ end function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UUID, String}) deps_map = Dict{UUID,Vector{PackageSpec}}() + glue_map = Dict{UUID,Set{UUID}}() if env.pkg !== nothing pkg = env.pkg - collect_project!(pkg, dirname(env.project_file), deps_map) + deps, glues = collect_project(pkg, dirname(env.project_file)) + deps_map[pkg.uuid] = deps + glue_map[pkg.uuid] = glues names[pkg.uuid] = pkg.name end for pkg in pkgs @@ -268,7 +285,9 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU if !isdir(path) pkgerror("expected package $(err_rep(pkg)) to exist at path `$path`") end - collect_project!(pkg, path, deps_map) + deps, glues = collect_project(pkg, path) + deps_map[pkg.uuid] = deps + glue_map[pkg.uuid] = glues end fixed = Dict{UUID,Resolve.Fixed}() @@ -285,7 +304,7 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU idx = findfirst(pkg -> pkg.uuid == uuid, pkgs) fix_pkg = pkgs[idx] end - fixed[uuid] = Resolve.Fixed(fix_pkg.version, q) + fixed[uuid] = Resolve.Fixed(fix_pkg.version, q, glue_map[uuid]) end return fixed end @@ -407,6 +426,7 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} # pkg -> version -> (dependency => compat): all_compat = Dict{UUID,Dict{VersionNumber,Dict{UUID,VersionSpec}}}() + glue_compat = Dict{UUID,Dict{VersionNumber,Set{UUID}}}() for (fp, fx) in fixed all_compat[fp] = Dict(fx.version => Dict{UUID,VersionSpec}()) @@ -418,7 +438,8 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} for uuid in unseen push!(seen, uuid) uuid in keys(fixed) && continue - all_compat_u = get_or_make!(all_compat, uuid) + all_compat_u = get_or_make!(all_compat, uuid) + glue_compat_u = get_or_make!(glue_compat, uuid) uuid_is_stdlib = false stdlib_name = "" @@ -446,35 +467,56 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} push!(uuids, other_uuid) all_compat_u_vr[other_uuid] = VersionSpec() end + + if !isempty(proj.gluedeps) + glue_all_compat_u_vr = get_or_make!(glue_compat_u, v) + for (_, other_uuid) in proj.gluedeps + push!(uuids, other_uuid) + all_compat_u_vr[other_uuid] = VersionSpec() + push!(glue_all_compat_u_vr, other_uuid) + end + end else for reg in registries pkg = get(reg, uuid, nothing) pkg === nothing && continue info = Registry.registry_info(pkg) - for (v, compat_info) in Registry.compat_info(info) - # Filter yanked and if we are in offline mode also downloaded packages - # TODO, pull this into a function - Registry.isyanked(info, v) && continue - if Pkg.OFFLINE_MODE[] - pkg_spec = PackageSpec(name=pkg.name, uuid=pkg.uuid, version=v, tree_hash=Registry.treehash(info, v)) - is_package_downloaded(env.project_file, pkg_spec) || continue - end - # Skip package version that are not the same as external packages in sysimage - if PKGORIGIN_HAVE_VERSION && RESPECT_SYSIMAGE_VERSIONS[] && julia_version == VERSION - pkgid = Base.PkgId(uuid, pkg.name) - if Base.in_sysimage(pkgid) - pkgorigin = get(Base.pkgorigins, pkgid, nothing) - if pkgorigin !== nothing && pkgorigin.version !== nothing - if v != pkgorigin.version - continue + function add_compat!(d, cinfo) + for (v, compat_info) in cinfo + # Filter yanked and if we are in offline mode also downloaded packages + # TODO, pull this into a function + Registry.isyanked(info, v) && continue + if Pkg.OFFLINE_MODE[] + pkg_spec = PackageSpec(name=pkg.name, uuid=pkg.uuid, version=v, tree_hash=Registry.treehash(info, v)) + is_package_downloaded(env.project_file, pkg_spec) || continue + end + + # Skip package version that are not the same as external packages in sysimage + if PKGORIGIN_HAVE_VERSION && RESPECT_SYSIMAGE_VERSIONS[] && julia_version == VERSION + pkgid = Base.PkgId(uuid, pkg.name) + if Base.in_sysimage(pkgid) + pkgorigin = get(Base.pkgorigins, pkgid, nothing) + if pkgorigin !== nothing && pkgorigin.version !== nothing + if v != pkgorigin.version + continue + end end end end + dv = get_or_make!(d, v) + merge!(dv, compat_info) + union!(uuids, keys(compat_info)) + end + end + add_compat!(all_compat_u, Registry.compat_info(info)) + glue_compat_info = Registry.glue_compat_info(info) + if glue_compat_info !== nothing + add_compat!(all_compat_u, glue_compat_info) + # Version to Set + for (v, compat_info) in glue_compat_info + glue_compat_u[v] = keys(compat_info) end - - all_compat_u[v] = compat_info - union!(uuids, keys(compat_info)) end end end @@ -493,7 +535,7 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} end end - return Resolve.Graph(all_compat, uuid_to_name, reqs, fixed, false, julia_version), + return Resolve.Graph(all_compat, glue_compat, uuid_to_name, reqs, fixed, false, julia_version), all_compat end @@ -1735,9 +1777,13 @@ function gen_target_project(ctx::Context, pkg::PackageSpec, source_path::String, test_project.deps = source_env.project.deps # collect test dependencies for name in get(source_env.project.targets, target, String[]) - uuid = get(source_env.project.extras, name, nothing) + uuid = nothing + for list in [source_env.project.extras, source_env.project.gluedeps] + uuid = get(list, name, nothing) + uuid === nothing || break + end if uuid === nothing - pkgerror("`$name` declared as a `$target` dependency, but no such entry in `extras`") + pkgerror("`$name` declared as a `$target` dependency, but no such entry in `extras` or `gluedeps`") end test_project.deps[name] = uuid end @@ -2083,6 +2129,7 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie lpadding = 2 package_statuses = PackageStatusData[] + installed_cache = Dict{Base.PkgId, Bool}() for (uuid, old, new) in xs if Types.is_project_uuid(env, uuid) continue @@ -2116,6 +2163,7 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie no_packages_upgradable &= (!changed || !pkg_upgradable) no_visible_packages_heldback &= (!changed || !pkg_heldback) no_packages_heldback &= !pkg_heldback + push!(package_statuses, PackageStatusData(uuid, old, new, pkg_downloaded, pkg_upgradable, pkg_heldback, cinfo, changed)) end diff --git a/src/Pkg.jl b/src/Pkg.jl index 87adf6d7fc..8d166f4a13 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -735,9 +735,9 @@ end # Precompilation # ################## -function _auto_precompile(ctx::Types.Context; warn_loaded = true, already_instantiated = false) +function _auto_precompile(ctx::Types.Context, pkgs::Vector{String}=String[]; warn_loaded = true, already_instantiated = false) if Base.JLOptions().use_compiled_modules == 1 && get_bool_env("JULIA_PKG_PRECOMPILE_AUTO"; default="true") - Pkg.precompile(ctx; internal_call=true, warn_loaded = warn_loaded, already_instantiated = already_instantiated) + Pkg.precompile(ctx, pkgs; internal_call=true, warn_loaded = warn_loaded, already_instantiated = already_instantiated) end end diff --git a/src/Registry/registry_instance.jl b/src/Registry/registry_instance.jl index 8b59dec914..44825f7014 100644 --- a/src/Registry/registry_instance.jl +++ b/src/Registry/registry_instance.jl @@ -36,8 +36,9 @@ custom_isfile(in_memory_registry::Union{Dict, Nothing}, folder::AbstractString, git_tree_sha1::Base.SHA1 yanked::Bool @lazy uncompressed_compat::Union{Dict{UUID, VersionSpec}} + @lazy glue_uncompressed_compat::Union{Dict{UUID, VersionSpec}} end -VersionInfo(git_tree_sha1::Base.SHA1, yanked::Bool) = VersionInfo(git_tree_sha1, yanked, uninit) +VersionInfo(git_tree_sha1::Base.SHA1, yanked::Bool) = VersionInfo(git_tree_sha1, yanked, uninit, uninit) # This is the information that exists in e.g. General/A/ACME struct PkgInfo @@ -53,6 +54,12 @@ struct PkgInfo # Deps.toml deps::Dict{VersionRange, Dict{String, UUID}} + + # WeakCompat.toml + glue_compat::Dict{VersionRange, Dict{String, VersionSpec}} + + # GlueDeps.toml + glue_deps::Dict{VersionRange, Dict{String, UUID}} end isyanked(pkg::PkgInfo, v::VersionNumber) = pkg.version_info[v].yanked @@ -100,8 +107,8 @@ function initialize_uncompressed!(pkg::PkgInfo, versions = keys(pkg.version_info sort!(versions) - uncompressed_compat = uncompress(pkg.compat, versions) - uncompressed_deps = uncompress(pkg.deps, versions) + uncompressed_compat = uncompress(pkg.compat, versions) + uncompressed_deps = uncompress(pkg.deps, versions) for v in versions vinfo = pkg.version_info[v] @@ -119,11 +126,43 @@ function initialize_uncompressed!(pkg::PkgInfo, versions = keys(pkg.version_info return pkg end +function initialize_glue_uncompressed!(pkg::PkgInfo, versions = keys(pkg.version_info)) + # Only valid to call this with existing versions of the package + # Remove all versions we have already uncompressed + versions = filter!(v -> !isinit(pkg.version_info[v], :glue_uncompressed_compat), collect(versions)) + + sort!(versions) + + glue_uncompressed_compat = uncompress(pkg.glue_compat, versions) + glue_uncompressed_deps = uncompress(pkg.glue_deps, versions) + + for v in versions + vinfo = pkg.version_info[v] + glue_compat = Dict{UUID, VersionSpec}() + glue_uncompressed_deps_v = glue_uncompressed_deps[v] + glue_uncompressed_compat_v = glue_uncompressed_compat[v] + for (pkg, uuid) in glue_uncompressed_deps_v + vspec = get(glue_uncompressed_compat_v, pkg, nothing) + glue_compat[uuid] = vspec === nothing ? VersionSpec() : vspec + end + @init! vinfo.glue_uncompressed_compat = glue_compat + end + return pkg +end + function compat_info(pkg::PkgInfo) initialize_uncompressed!(pkg) return Dict(v => info.uncompressed_compat for (v, info) in pkg.version_info) end +function glue_compat_info(pkg::PkgInfo) + if isempty(pkg.glue_deps) + return nothing + end + initialize_glue_uncompressed!(pkg) + return Dict(v => info.glue_uncompressed_compat for (v, info) in pkg.version_info) +end + @lazy struct PkgEntry # Registry.toml: path::String @@ -181,7 +220,29 @@ function init_package_info!(pkg::PkgEntry) # All packages depend on julia deps[VersionRange()] = Dict("julia" => JULIA_UUID) - @init! pkg.info = PkgInfo(repo, subdir, version_info, compat, deps) + # WeakCompat.toml + glue_compat_data_toml = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "WeakCompat.toml")) ? + parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "WeakCompat.toml")) : Dict{String, Any}() + glue_compat_data_toml = convert(Dict{String, Dict{String, Union{String, Vector{String}}}}, glue_compat_data_toml) + glue_compat = Dict{VersionRange, Dict{String, VersionSpec}}() + for (v, data) in glue_compat_data_toml + vr = VersionRange(v) + d = Dict{String, VersionSpec}(dep => VersionSpec(vr_dep) for (dep, vr_dep) in data) + glue_compat[vr] = d + end + + # GlueDeps.toml + glue_deps_data_toml = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "GlueDeps.toml")) ? + parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "GlueDeps.toml")) : Dict{String, Any}() + glue_deps_data_toml = convert(Dict{String, Dict{String, String}}, glue_deps_data_toml) + glue_deps = Dict{VersionRange, Dict{String, UUID}}() + for (v, data) in glue_deps_data_toml + vr = VersionRange(v) + d = Dict{String, UUID}(dep => UUID(uuid) for (dep, uuid) in data) + glue_deps[vr] = d + end + + @init! pkg.info = PkgInfo(repo, subdir, version_info, compat, deps, glue_compat, glue_deps) return pkg.info end diff --git a/src/Resolve/graphtype.jl b/src/Resolve/graphtype.jl index ecd1d92669..94d2d1882b 100644 --- a/src/Resolve/graphtype.jl +++ b/src/Resolve/graphtype.jl @@ -236,13 +236,12 @@ mutable struct Graph function Graph( compat::Dict{UUID,Dict{VersionNumber,Dict{UUID,VersionSpec}}}, + compat_glue::Dict{UUID,Dict{VersionNumber,Set{UUID}}}, uuid_to_name::Dict{UUID,String}, reqs::Requires, fixed::Dict{UUID,Fixed}, verbose::Bool = false, julia_version::Union{VersionNumber,Nothing} = VERSION - ; - compat_weak::Dict{UUID,Dict{VersionNumber,Set{UUID}}} = Dict{UUID,Dict{VersionNumber,Set{UUID}}}(), ) # Tell the resolver about julia itself @@ -256,7 +255,6 @@ mutable struct Graph data = GraphData(compat, uuid_to_name, verbose) pkgs, np, spp, pdict, pvers, vdict, rlog = data.pkgs, data.np, data.spp, data.pdict, data.pvers, data.vdict, data.rlog - extended_deps = let spp = spp # Due to https://github.com/JuliaLang/julia/issues/15276 [Vector{Dict{Int,BitVector}}(undef, spp[p0]-1) for p0 = 1:np] end @@ -276,14 +274,14 @@ mutable struct Graph # Translate the requirements into bit masks # Hot code, measure performance before changing req_msk = Dict{Int,BitVector}() - maybe_weak = haskey(compat_weak, uuid0) && haskey(compat_weak[uuid0], vn) + maybe_weak = haskey(compat_glue, uuid0) && haskey(compat_glue[uuid0], vn) for (p1, vs) in req pv = pvers[p1] req_msk_p1 = BitVector(undef, spp[p1]) @inbounds for i in 1:spp[p1] - 1 req_msk_p1[i] = pv[i] ∈ vs end - weak = maybe_weak && (pkgs[p1] ∈ compat_weak[uuid0][vn]) + weak = maybe_weak && (pkgs[p1] ∈ compat_glue[uuid0][vn]) req_msk_p1[end] = weak req_msk[p1] = req_msk_p1 end diff --git a/src/Types.jl b/src/Types.jl index 2146149399..2e7ff22ff0 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -239,6 +239,8 @@ Base.@kwdef mutable struct Project manifest::Union{String, Nothing} = nothing # Sections deps::Dict{String,UUID} = Dict{String,UUID}() + gluedeps::Dict{String,UUID} = Dict{String,UUID}() + gluepkgs::Dict{String,Union{Vector{String}, String}} = Dict{String,String}() extras::Dict{String,UUID} = Dict{String,UUID}() targets::Dict{String,Vector{String}} = Dict{String,Vector{String}}() compat::Dict{String,Compat} = Dict{String,Compat}() @@ -262,6 +264,8 @@ Base.@kwdef mutable struct PackageEntry repo::GitRepo = GitRepo() tree_hash::Union{Nothing,SHA1} = nothing deps::Dict{String,UUID} = Dict{String,UUID}() + gluedeps::Dict{String,UUID} = Dict{String,UUID}() + gluepkgs::Dict{String,Union{Vector{String}, String}} = Dict{String,String}() uuid::Union{Nothing, UUID} = nothing other::Union{Dict,Nothing} = nothing end @@ -272,9 +276,11 @@ Base.:(==)(t1::PackageEntry, t2::PackageEntry) = t1.name == t2.name && t1.repo == t2.repo && t1.tree_hash == t2.tree_hash && t1.deps == t2.deps && + t1.gluedeps == t2.gluedeps && + t1.gluepkgs == t2.gluepkgs && t1.uuid == t2.uuid # omits `other` -Base.hash(x::PackageEntry, h::UInt) = foldr(hash, [x.name, x.version, x.path, x.pinned, x.repo, x.tree_hash, x.deps, x.uuid], init=h) # omits `other` +Base.hash(x::PackageEntry, h::UInt) = foldr(hash, [x.name, x.version, x.path, x.pinned, x.repo, x.tree_hash, x.deps, x.gluedeps,x.gluepkgs, x.uuid], init=h) # omits `other` Base.@kwdef mutable struct Manifest julia_version::Union{Nothing,VersionNumber} = nothing # only set to VERSION when resolving diff --git a/src/manifest.jl b/src/manifest.jl index 69e20e25d3..79521957a9 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -83,19 +83,22 @@ struct Stage1 uuid::UUID entry::PackageEntry deps::Union{Vector{String}, Dict{String,UUID}} + gluedeps::Union{Vector{String}, Dict{String,UUID}} end -normalize_deps(name, uuid, deps, manifest) = deps -function normalize_deps(name, uuid, deps::Vector{String}, manifest::Dict{String,Vector{Stage1}}) +normalize_deps(name, uuid, deps, manifest; isglue=false) = deps +function normalize_deps(name, uuid, deps::Vector{String}, manifest::Dict{String,Vector{Stage1}}; isglue=false) if length(deps) != length(unique(deps)) pkgerror("Duplicate entry in `$name=$uuid`'s `deps` field.") end final = Dict{String,UUID}() for dep in deps infos = get(manifest, dep, nothing) - if infos === nothing - pkgerror("`$name=$uuid` depends on `$dep`, ", - "but no such entry exists in the manifest.") + if !isglue + if infos === nothing + pkgerror("`$name=$uuid` depends on `$dep`, ", + "but no such entry exists in the manifest.") + end end # should have used dict format instead of vector format length(infos) == 1 || pkgerror("Invalid manifest format. ", @@ -110,21 +113,30 @@ function validate_manifest(julia_version::Union{Nothing,VersionNumber}, manifest for (name, infos) in stage1, info in infos info.entry.deps = normalize_deps(name, info.uuid, info.deps, stage1) end + for (name, infos) in stage1, info in infos + info.entry.gluedeps = normalize_deps(name, info.uuid, info.gluedeps, stage1; isglue=true) + end # invariant: all dependencies are now normalized to Dict{String,UUID} deps = Dict{UUID, PackageEntry}() for (name, infos) in stage1, info in infos deps[info.uuid] = info.entry end # now just verify the graph structure - for (entry_uuid, entry) in deps, (name, uuid) in entry.deps - dep_entry = get(deps, uuid, nothing) - if dep_entry === nothing - pkgerror("`$(entry.name)=$(entry_uuid)` depends on `$name=$uuid`, ", - "but no such entry exists in the manifest.") - end - if dep_entry.name != name - pkgerror("`$(entry.name)=$(entry_uuid)` depends on `$name=$uuid`, ", - "but entry with UUID `$uuid` has name `$(dep_entry.name)`.") + for (entry_uuid, entry) in deps + for (deptype, isglue) in [(entry.deps, false), (entry.gluedeps, true)] + for (name, uuid) in deptype + dep_entry = get(deps, uuid, nothing) + if !isglue + if dep_entry === nothing + pkgerror("`$(entry.name)=$(entry_uuid)` depends on `$name=$uuid`, ", + "but no such entry exists in the manifest.") + end + if dep_entry.name != name + pkgerror("`$(entry.name)=$(entry_uuid)` depends on `$name=$uuid`, ", + "but entry with UUID `$uuid` has name `$(dep_entry.name)`.") + end + end + end end end return Manifest(; julia_version, manifest_format, deps, other) @@ -147,6 +159,7 @@ function Manifest(raw::Dict, f_or_io::Union{String, IO})::Manifest entry.name = name uuid = nothing deps = nothing + gluedeps = nothing try entry.pinned = read_pinned(get(info, "pinned", nothing)) uuid = read_field("uuid", nothing, info, safe_uuid)::UUID @@ -158,13 +171,15 @@ function Manifest(raw::Dict, f_or_io::Union{String, IO})::Manifest entry.tree_hash = read_field("git-tree-sha1", nothing, info, safe_SHA1) entry.uuid = uuid deps = read_deps(get(info::Dict, "deps", nothing)) + gluedeps = read_deps(get(info::Dict, "gluedeps", nothing)) + entry.gluepkgs = get(Dict{String, String}, info::Dict, "gluepkgs") catch # TODO: Should probably not unconditionally log something @debug "Could not parse manifest entry for `$name`" f_or_io rethrow() end entry.other = info::Union{Dict,Nothing} - stage1[name] = push!(get(stage1, name, Stage1[]), Stage1(uuid, entry, deps)) + stage1[name] = push!(get(stage1, name, Stage1[]), Stage1(uuid, entry, deps, gluedeps)) end # by this point, all the fields of the `PackageEntry`s have been type casted # but we have *not* verified the _graph_ structure of the manifest @@ -257,18 +272,25 @@ function destructure(manifest::Manifest)::Dict entry!(new_entry, "repo-url", repo_source) entry!(new_entry, "repo-rev", entry.repo.rev) entry!(new_entry, "repo-subdir", entry.repo.subdir) - if isempty(entry.deps) - delete!(new_entry, "deps") - else - if all(dep -> unique_name[first(dep)], entry.deps) - new_entry["deps"] = sort(collect(keys(entry.deps))) + for (deptype, depname) in [(entry.deps, "deps"), (entry.gluedeps, "gluedeps")] + if isempty(deptype) + delete!(new_entry, depname) else - new_entry["deps"] = Dict{String,String}() - for (name, uuid) in entry.deps - new_entry["deps"][name] = string(uuid) + if all(dep -> haskey(unique_name, first(dep)), deptype) && all(dep -> unique_name[first(dep)], deptype) + new_entry[depname] = sort(collect(keys(deptype))) + else + new_entry[depname] = Dict{String,String}() + for (name, uuid) in deptype + new_entry[depname][name] = string(uuid) + end end end end + + # TODO: Write this inline + if !isempty(entry.gluepkgs) + entry!(new_entry, "gluepkgs", entry.gluepkgs) + end if manifest.manifest_format.major == 1 push!(get!(raw, entry.name, Dict{String,Any}[]), new_entry) elseif manifest.manifest_format.major == 2 diff --git a/src/project.jl b/src/project.jl index dfb0de7210..4930e82d4e 100644 --- a/src/project.jl +++ b/src/project.jl @@ -2,7 +2,7 @@ # UTILS # ######### listed_deps(project::Project) = - append!(collect(keys(project.deps)), collect(keys(project.extras))) + append!(collect(keys(project.deps)), collect(keys(project.extras)), collect(keys(project.gluedeps))) ########### # READING # @@ -73,21 +73,26 @@ end read_project_compat(raw, project::Project) = pkgerror("Expected `compat` section to be a key-value list") -function validate(project::Project) +function validate(project::Project; file=nothing) # deps + location_string = file === nothing ? "" : " at $(repr(file))." dep_uuids = collect(values(project.deps)) if length(dep_uuids) != length(unique(dep_uuids)) - pkgerror("Two different dependencies can not have the same uuid") + pkgerror("Two different dependencies can not have the same uuid" * location_string) + end + glue_dep_uuids = collect(values(project.gluedeps)) + if length(glue_dep_uuids) != length(unique(glue_dep_uuids)) + pkgerror("Two different glue dependencies can not have the same uuid" * location_string) end # extras extra_uuids = collect(values(project.extras)) if length(extra_uuids) != length(unique(extra_uuids)) - pkgerror("Two different `extra` dependencies can not have the same uuid") + pkgerror("Two different `extra` dependencies can not have the same uuid" * location_string) end - dep_names = keys(project.deps) # TODO decide what to do in when `add`ing a dep that is already in `extras` # also, reintroduce test files for this #= + dep_names = keys(project.deps) for (name, uuid) in project.extras name in dep_names && pkgerror("name `$name` is listed in both `deps` and `extras`") uuid in dep_uuids && pkgerror("uuid `$uuid` is listed in both `deps` and `extras`") @@ -100,18 +105,18 @@ function validate(project::Project) pkgerror("A dependency was named twice in target `$target`") end dep in listed || pkgerror(""" - Dependency `$dep` in target `$target` not listed in `deps` or `extras` section. - """) + Dependency `$dep` in target `$target` not listed in `deps`, `gluedeps` or `extras` section + """ * location_string) end # compat for (name, version) in project.compat name == "julia" && continue name in listed || - pkgerror("Compat `$name` not listed in `deps` or `extras` section.") + pkgerror("Compat `$name` not listed in `deps`, `gluedeps` or `extras` section" * location_string) end end -function Project(raw::Dict) +function Project(raw::Dict; file=nothing) project = Project() project.other = raw project.name = get(raw, "name", nothing)::Union{String, Nothing} @@ -119,10 +124,12 @@ function Project(raw::Dict) project.uuid = read_project_uuid(get(raw, "uuid", nothing)) project.version = read_project_version(get(raw, "version", nothing)) project.deps = read_project_deps(get(raw, "deps", nothing), "deps") + project.gluedeps = read_project_deps(get(raw, "gluedeps", nothing), "gluedeps") + project.gluepkgs = get(Dict{String, String}, raw, "gluepkgs") project.extras = read_project_deps(get(raw, "extras", nothing), "extras") project.compat = read_project_compat(get(raw, "compat", nothing), project) project.targets = read_project_targets(get(raw, "targets", nothing), project) - validate(project) + validate(project; file) return project end @@ -139,7 +146,7 @@ function read_project(f_or_io::Union{String, IO}) end rethrow() end - return Project(raw) + return Project(raw; file= f_or_io isa IO ? nothing : f_or_io) end @@ -168,6 +175,7 @@ function destructure(project::Project)::Dict entry!("version", project.version) entry!("manifest", project.manifest) entry!("deps", project.deps) + entry!("gluedeps", project.gluedeps) entry!("extras", project.extras) entry!("compat", Dict(name => x.str for (name, x) in project.compat)) entry!("targets", project.targets) diff --git a/test/gluedeps.jl b/test/gluedeps.jl new file mode 100644 index 0000000000..3e251c402b --- /dev/null +++ b/test/gluedeps.jl @@ -0,0 +1,21 @@ +using .Utils +using Test + +@testset "weak deps" begin + isolate(loaded_depot=true) do + Pkg.activate(; temp=true) + Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasGluePkgs.jl")) + Pkg.test("HasGluePkgs") + end + isolate(loaded_depot=true) do + Pkg.activate(; temp=true) + Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasDepWithGluePkgs.jl")) + Pkg.test("HasDepWithGluePkgs") + end + + isolate(loaded_depot=true) do + Pkg.activate(; temp=true) + Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasGluePkgs.jl")) + @test_throws Pkg.Resolve.ResolverError Pkg.add(; name = "OffsetArrays", version = "0.9.0") + end +end diff --git a/test/resolve_utils.jl b/test/resolve_utils.jl index 8763949e6c..981cb638b7 100644 --- a/test/resolve_utils.jl +++ b/test/resolve_utils.jl @@ -81,7 +81,7 @@ function graph_from_data(deps_data) end end end - return Graph(all_compat, uuid_to_name, Requires(), fixed, VERBOSE; compat_weak=all_compat_w) + return Graph(all_compat, all_compat_w, uuid_to_name, Requires(), fixed, VERBOSE) end function reqs_from_data(reqs_data, graph::Graph) reqs = Dict{UUID,VersionSpec}() diff --git a/test/runtests.jl b/test/runtests.jl index 7e3fedc819..a75bced3a6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -66,6 +66,7 @@ Logging.with_logger(hide_logs ? Logging.NullLogger() : Logging.current_logger()) "api.jl", "registry.jl", "subdir.jl", + "gluedeps.jl", "artifacts.jl", "binaryplatforms.jl", "platformengines.jl", diff --git a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml new file mode 100644 index 0000000000..4627434d62 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml @@ -0,0 +1,57 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.10.0-DEV" +manifest_format = "2.0" +project_hash = "1af04d62b804224fdf802daf4a4b96e213299d30" + +[[deps.Adapt]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "195c5505521008abea5aee4f96930717958eac6f" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "3.4.0" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.0.1+0" + +[[deps.Example]] +git-tree-sha1 = "46e44e869b4d90b96bd8ed1fdcf32244fddfb6cc" +uuid = "7876af07-990d-54b4-ab0e-23690620f79a" +version = "0.5.3" + +[[deps.HasGluePkgs]] +deps = ["Example"] +path = "../HasGluePkgs.jl" +uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" +version = "0.1.0" +gluedeps = ["OffsetArrays"] + + [deps.HasGluePkgs.gluepkgs] + GlueOffsetArrays = "OffsetArrays" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.OffsetArrays]] +deps = ["Adapt"] +git-tree-sha1 = "f71d8950b724e9ff6110fc948dff5a329f901d64" +uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +version = "1.12.8" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.21+0" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.2.0+0" diff --git a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Project.toml b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Project.toml new file mode 100644 index 0000000000..2036b85044 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Project.toml @@ -0,0 +1,17 @@ +name = "HasDepWithGluePkgs" +uuid = "d4ef3d4a-8e22-4710-85d8-c6cf2eb9efca" +version = "0.1.0" + +[deps] +HasGluePkgs = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" + +[compat] +HasGluePkgs = "0.1" +OffsetArrays = "1.12" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/src/HasDepWithGluePkgs.jl b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/src/HasDepWithGluePkgs.jl new file mode 100644 index 0000000000..66a109b9a8 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/src/HasDepWithGluePkgs.jl @@ -0,0 +1,15 @@ +module HasDepWithGluePkgs + +using HasGluePkgs +using OffsetArrays: OffsetArray +# Loading OffsetArrays makes the glue module "GlueOffsetArrays" to load + +function do_something() + # @info "First do something with the basic array support in B" + HasGluePkgs.foo(rand(Float64, 2)) + + # @info "Now do something with extended OffsetArray support in B" + HasGluePkgs.foo(OffsetArray(rand(Float64, 2), 0:1)) +end + +end # module diff --git a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/test/runtests.jl b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/test/runtests.jl new file mode 100644 index 0000000000..318c70c3f5 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/test/runtests.jl @@ -0,0 +1,5 @@ +using HasDepWithGluePkgs +using Test + +@test HasDepWithGluePkgs.HasGluePkgs.offsetarrays_loaded +HasDepWithGluePkgs.do_something() diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Manifest.toml b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Manifest.toml new file mode 100644 index 0000000000..f9461df371 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Manifest.toml @@ -0,0 +1,10 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.10.0-DEV" +manifest_format = "2.0" +project_hash = "549efd0c32255f4b8717f50f49778e183bcc0e36" + +[[deps.Example]] +git-tree-sha1 = "46e44e869b4d90b96bd8ed1fdcf32244fddfb6cc" +uuid = "7876af07-990d-54b4-ab0e-23690620f79a" +version = "0.5.3" diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml new file mode 100644 index 0000000000..d3b8260a17 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml @@ -0,0 +1,22 @@ +name = "HasGluePkgs" +uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" +version = "0.1.0" + +[deps] +Example = "7876af07-990d-54b4-ab0e-23690620f79a" + +[gluedeps] +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" + +[gluepkgs] +GlueOffsetArrays = "OffsetArrays" + +[compat] +Example = "0.5" +OffsetArrays = "1.12" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["OffsetArrays", "Test"] diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/glue/GlueOffsetArrays.jl b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/glue/GlueOffsetArrays.jl new file mode 100644 index 0000000000..897a7b8188 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/glue/GlueOffsetArrays.jl @@ -0,0 +1,13 @@ +module GlueOffsetArrays + +using HasGluePkgs, OffsetArrays + +function foo(::OffsetArray) + return 2 +end + +function __init__() + HasGluePkgs.offsetarrays_loaded = true +end + +end diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/src/HasGluePkgs.jl b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/src/HasGluePkgs.jl new file mode 100644 index 0000000000..212512cd73 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/src/HasGluePkgs.jl @@ -0,0 +1,11 @@ +module HasGluePkgs + +using Example + +function foo(::AbstractArray) + return 1 +end + +offsetarrays_loaded = false + +end # module diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/test/runtests.jl b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/test/runtests.jl new file mode 100644 index 0000000000..8042719f35 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/test/runtests.jl @@ -0,0 +1,8 @@ +using HasGluePkgs +using Test + +@test !HasGluePkgs.offsetarrays_loaded + +using OffsetArrays + +@test HasGluePkgs.offsetarrays_loaded From 2d6adbb90c67fef652f63aaf464054335132746e Mon Sep 17 00:00:00 2001 From: Ian Date: Thu, 24 Nov 2022 23:19:58 -0500 Subject: [PATCH 02/26] add some general explanation comments --- src/API.jl | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/API.jl b/src/API.jl index 850e36a559..09150fb4f5 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1098,6 +1098,7 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: man = ctx.env.manifest deps_pair_or_nothing = Iterators.map(man) do dep + # make a flat map of each dep and its deps pkg = Base.PkgId(first(dep), last(dep).name) Base.in_sysimage(pkg) && return nothing deps = [Base.PkgId(last(x), first(x)) for x in last(dep).deps] @@ -1105,6 +1106,7 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: end depsmap = Dict{Base.PkgId, Vector{Base.PkgId}}(Iterators.filter(!isnothing, deps_pair_or_nothing)) #flat map of each dep and its deps + # if the active environment is a package, add that ctx_env_pkg = ctx.env.pkg if ctx_env_pkg !== nothing && isfile( joinpath( dirname(ctx.env.project_file), "src", "$(ctx_env_pkg.name).jl") ) depsmap[Base.PkgId(ctx_env_pkg.uuid, ctx_env_pkg.name)] = [ @@ -1114,8 +1116,10 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: push!(direct_deps, Base.PkgId(ctx_env_pkg.uuid, ctx_env_pkg.name)) end + # return early if no deps isempty(depsmap) && return + # initialize signalling started = Dict{Base.PkgId,Bool}() was_processed = Dict{Base.PkgId,Base.Event}() was_recompiled = Dict{Base.PkgId,Bool}() @@ -1126,9 +1130,12 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: was_recompiled[pkgid] = false push!(pkg_specs, get_or_make_pkgspec(pkg_specs, ctx, pkgid.uuid)) end + + # remove packages that are suspended because they errored before + # note that when `Pkg.precompile` is manually called, all suspended packages are unsuspended precomp_prune_suspended!(pkg_specs) - # guarding against circular deps + # find and guard against circular deps circular_deps = Base.PkgId[] function in_deps(_pkgs, deps, dmap) isempty(deps) && return false @@ -1145,8 +1152,8 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: @warn """Circular dependency detected. Precompilation will be skipped for:\n $(join(string.(circular_deps), "\n "))""" end + # if a list of packages is given, restrict to dependencies of given packages if !isempty(pkgs) - # if a list of packages is given, restrict to dependencies of given packages function collect_all_deps(depsmap, dep, alldeps=Base.PkgId[]) append!(alldeps, depsmap[dep]) for _dep in depsmap[dep] @@ -1164,7 +1171,6 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: filter!(d->in(first(d), keep), depsmap) isempty(depsmap) && pkgerror("No direct dependencies found matching $(repr(pkgs))") end - target = string(isempty(pkgs) ? "project" : join(pkgs, ", "), "...") pkg_queue = Base.PkgId[] @@ -1203,7 +1209,8 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: end end - t_print = @async begin # fancy print loop + ## fancy print loop + t_print = @async begin try wait(first_started) (isempty(pkg_queue) || interrupted_or_done.set) && return @@ -1278,7 +1285,8 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: end tasks = Task[] Base.LOADING_CACHE[] = Base.LoadingCache() - for (pkg, deps) in depsmap # precompilation loop + ## precompilation loop + for (pkg, deps) in depsmap paths = Base.find_all_in_cache_path(pkg) sourcepath = Base.locate_package(pkg) if sourcepath === nothing From 83ad6550baaf78601fe32c6ae099917b27f63df4 Mon Sep 17 00:00:00 2001 From: Ian Date: Thu, 24 Nov 2022 23:20:44 -0500 Subject: [PATCH 03/26] precompile glue packages too --- src/API.jl | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/API.jl b/src/API.jl index 09150fb4f5..aea35a0f82 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1096,15 +1096,29 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: for (name, uuid) in ctx.env.project.deps if !Base.in_sysimage(Base.PkgId(uuid, name)) ] - man = ctx.env.manifest - deps_pair_or_nothing = Iterators.map(man) do dep # make a flat map of each dep and its deps + depsmap = Dict{Base.PkgId, Vector{Base.PkgId}}() + pkg_specs = PackageSpec[] + for dep in ctx.env.manifest pkg = Base.PkgId(first(dep), last(dep).name) - Base.in_sysimage(pkg) && return nothing + Base.in_sysimage(pkg) && continue deps = [Base.PkgId(last(x), first(x)) for x in last(dep).deps] - return pkg => filter!(!Base.in_sysimage, deps) + depsmap[pkg] = filter!(!Base.in_sysimage, deps) + # add any glue packages + for (gluepkg_name, gluedep_names) in last(dep).gluepkgs + gluepkg_deps = copy(deps) # depends on the deps of the parent package + push!(gluepkg_deps, pkg) # depends on parent package + gluedep_names = gluedep_names isa String ? String[gluedep_names] : gluedep_names + for gluedep_name in gluedep_names + gluedep = Base.identify_package(gluedep_name) + !isnothing(gluedep) && push!(gluepkg_deps, gluedep) + end + gluepkg_uuid = Base.uuid5(pkg.uuid, gluepkg_name) + gluepkg = Base.PkgId(gluepkg_uuid, gluepkg_name) + push!(pkg_specs, PackageSpec(uuid = gluepkg_uuid, name = gluepkg_name)) # create this here as the name cannot be looked up easily later via the uuid + depsmap[gluepkg] = filter!(!Base.in_sysimage, gluepkg_deps) + end end - depsmap = Dict{Base.PkgId, Vector{Base.PkgId}}(Iterators.filter(!isnothing, deps_pair_or_nothing)) #flat map of each dep and its deps # if the active environment is a package, add that ctx_env_pkg = ctx.env.pkg @@ -1123,7 +1137,6 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: started = Dict{Base.PkgId,Bool}() was_processed = Dict{Base.PkgId,Base.Event}() was_recompiled = Dict{Base.PkgId,Bool}() - pkg_specs = PackageSpec[] for pkgid in keys(depsmap) started[pkgid] = false was_processed[pkgid] = Base.Event() From 1a697362337028d2420b8a607731044b6eaf6b19 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 7 Jul 2022 16:57:01 +0200 Subject: [PATCH 04/26] add support for glue packages --- docs/make.jl | 1 + docs/src/creating-packages.md | 75 ++++++++++++ src/API.jl | 9 +- src/Operations.jl | 114 +++++++++++++----- src/Pkg.jl | 4 +- src/Registry/registry_instance.jl | 69 ++++++++++- src/Resolve/graphtype.jl | 8 +- src/Types.jl | 8 +- src/manifest.jl | 68 +++++++---- src/project.jl | 30 +++-- test/gluedeps.jl | 21 ++++ test/resolve_utils.jl | 2 +- test/runtests.jl | 1 + .../HasDepWithGluePkgs.jl/Manifest.toml | 57 +++++++++ .../HasDepWithGluePkgs.jl/Project.toml | 17 +++ .../src/HasDepWithGluePkgs.jl | 15 +++ .../HasDepWithGluePkgs.jl/test/runtests.jl | 5 + .../HasGluePkgs.jl/Manifest.toml | 10 ++ .../HasGluePkgs.jl/Project.toml | 22 ++++ .../HasGluePkgs.jl/glue/GlueOffsetArrays.jl | 13 ++ .../HasGluePkgs.jl/src/HasGluePkgs.jl | 11 ++ .../HasGluePkgs.jl/test/runtests.jl | 8 ++ 22 files changed, 486 insertions(+), 82 deletions(-) create mode 100644 test/gluedeps.jl create mode 100644 test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml create mode 100644 test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Project.toml create mode 100644 test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/src/HasDepWithGluePkgs.jl create mode 100644 test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/test/runtests.jl create mode 100644 test/test_packages/GluePkgExamples/HasGluePkgs.jl/Manifest.toml create mode 100644 test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml create mode 100644 test/test_packages/GluePkgExamples/HasGluePkgs.jl/glue/GlueOffsetArrays.jl create mode 100644 test/test_packages/GluePkgExamples/HasGluePkgs.jl/src/HasGluePkgs.jl create mode 100644 test/test_packages/GluePkgExamples/HasGluePkgs.jl/test/runtests.jl diff --git a/docs/make.jl b/docs/make.jl index 24b0c4244c..163cb9e7bd 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -35,6 +35,7 @@ makedocs( "managing-packages.md", "environments.md", "creating-packages.md", + "gluedeps.md", "compatibility.md", "registries.md", "artifacts.md", diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index ebf86517b4..df45a97957 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -245,6 +245,81 @@ using Test Every dependency should in general have a compatibility constraint on it. This is an important topic so there is a separate chapter about it: [Compatibility](@ref Compatibility). +## Conditional loading of code in packages (Glue packages) + +!!! note + This is a somewhat advanced section which can be skipped for people new to Julia and Julia packages. + +It is sometimes desirable to be able to extend some functionality of a package without having to +unconditionally take on the cost (in terms of e.g. load time) of adding an extra dependency. +A *glue package* is a file that gets automatically loaded when some other set of packages are +loaded into the Julia session. + +A useful application of glue packages could be for a plotting package that should be able to plot +objects from a wide variety of different Julia packages. +Adding all those different Julia packages as dependencies +could be expensive since they would end up getting loaded even if they were never used. +Instead, these code required to plot objects for specific packages can be put into separate files +(glue packages) which is only loaded when + +Below is an example of how the code can be structured for a use case as outlined above. + + `Project.toml`: + ```toml +name = "Plotting" +version = "0.1.0" +uuid = "..." + +[deps] +Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" + +[gluedeps] +Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" + +[gluepkgs] +# name of glue package to the left +# glue dependencies required to load the glue pkg to the right +# use a list for multiple glue dependencies +GlueContour = "Contour" + +[compat] # compat can also be given on glue dependencies +Colors = "0.12.8" +Contour = "0.6.2" +``` + +`src/Plotting.jl`: +```julia +module Plotting +using Colors + +function plot(x::Colors.Color) + # Some functionality for plotting a color here +end + +end # module +``` + +`glue/GlueContour.jl` (can also be in `glue/GlueContour/GlueContour.jl`): +```julia +module GlueContour + +using Plotting, Contour + +function Plotting.plot(c::Contour.ContourCollection) + # Some functionality for plotting a contour here +end + +end # module +``` + +A user that depends on `Plotting` will not pay the code of the "glue code" inside the `GlueContour` module. +It is only when the `Contour` package actually gets loaded that the `GlueCountour` glue package will get loaded +and provide the new functionality. + +Compatibility can be set on glue dependencies just like normal dependencies. + +A glue package will only be loaded if the glue dependencies are loaded from the same environment or environments higher in the environment stack than the package itself. + ## Package naming guidelines Package names should be sensible to most Julia users, *even to those who are not domain experts*. diff --git a/src/API.jl b/src/API.jl index a365b1c72a..850e36a559 100644 --- a/src/API.jl +++ b/src/API.jl @@ -154,7 +154,7 @@ for f in (:develop, :add, :rm, :up, :pin, :free, :test, :build, :status, :why) pkgs = deepcopy(pkgs) # don't mutate input foreach(handle_package_input!, pkgs) ret = $f(ctx, pkgs; kwargs...) - $(f in (:add, :up, :pin, :free, :build)) && Pkg._auto_precompile(ctx) + $(f in (:add, :up, :pin, :free, :build)) && Pkg._auto_precompile(ctx) # rm does too, but it's handled differently $(f in (:up, :pin, :free, :rm)) && Pkg._auto_gc(ctx) return ret end @@ -301,9 +301,11 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode=PKGMODE_PROJECT, all_p ensure_resolved(ctx, ctx.env.manifest, pkgs) Operations.rm(ctx, pkgs; mode) + return end + function append_all_pkgs!(pkgs, ctx, mode) if mode == PKGMODE_PROJECT || mode == PKGMODE_COMBINED for (name::String, uuid::UUID) in ctx.env.project.deps @@ -1163,6 +1165,8 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: isempty(depsmap) && pkgerror("No direct dependencies found matching $(repr(pkgs))") end + target = string(isempty(pkgs) ? "project" : join(pkgs, ", "), "...") + pkg_queue = Base.PkgId[] failed_deps = Dict{Base.PkgId, String}() skipped_deps = Base.PkgId[] @@ -1309,7 +1313,7 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: iob = IOBuffer() name = is_direct_dep ? pkg.name : string(color_string(pkg.name, :light_black)) !fancyprint && lock(print_lock) do - isempty(pkg_queue) && printpkgstyle(io, :Precompiling, "environment...") + isempty(pkg_queue) && printpkgstyle(io, :Precompiling, target) end push!(pkg_queue, pkg) started[pkg] = true @@ -1613,6 +1617,7 @@ function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode= if compat diff && pkgerror("Compat status has no `diff` mode") outdated && pkgerror("Compat status has no `outdated` mode") + Operations.print_compat(ctx, pkgs; io) else Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated) diff --git a/src/Operations.jl b/src/Operations.jl index 53178462d8..dfa1148693 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -142,6 +142,15 @@ function update_manifest!(env::EnvCache, pkgs::Vector{PackageSpec}, deps_map, ju else entry.deps = deps_map[pkg.uuid] end + v = joinpath(source_path(env.project_file, pkg), "Project.toml") + if isfile(v) + p = Types.read_project(v) + entry.gluedeps = p.gluedeps + entry.gluepkgs = p.gluepkgs + for (name, _) in p.gluedeps + delete!(entry.deps, name) + end + end env.manifest[pkg.uuid] = entry end prune_manifest(env) @@ -198,24 +207,29 @@ function reset_all_compat!(proj::Project) return nothing end -function collect_project!(pkg::PackageSpec, path::String, - deps_map::Dict{UUID,Vector{PackageSpec}}) - deps_map[pkg.uuid] = PackageSpec[] +function collect_project(pkg::PackageSpec, path::String) + deps = PackageSpec[] + glues = Set{UUID}() project_file = projectfile_path(path; strict=true) if project_file === nothing pkgerror("could not find project file for package $(err_rep(pkg)) at `$path`") end project = read_package(project_file) - julia_compat = get_compat(project, "julia") #= # TODO, this should either error or be quiet + julia_compat = get_compat(project, "julia") if julia_compat !== nothing && !(VERSION in julia_compat) println(io, "julia version requirement for package $(err_rep(pkg)) not satisfied") end =# for (name, uuid) in project.deps vspec = get_compat(project, name) - push!(deps_map[pkg.uuid], PackageSpec(name, uuid, vspec)) + push!(deps, PackageSpec(name, uuid, vspec)) + end + for (name, uuid) in project.gluedeps + vspec = get_compat(project, name) + push!(deps, PackageSpec(name, uuid, vspec)) + push!(glues, uuid) end if project.version !== nothing pkg.version = project.version @@ -223,7 +237,7 @@ function collect_project!(pkg::PackageSpec, path::String, # @warn("project file for $(pkg.name) is missing a `version` entry") pkg.version = VersionNumber(0) end - return + return deps, glues end is_tracking_path(pkg) = pkg.path !== nothing @@ -258,9 +272,12 @@ end function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UUID, String}) deps_map = Dict{UUID,Vector{PackageSpec}}() + glue_map = Dict{UUID,Set{UUID}}() if env.pkg !== nothing pkg = env.pkg - collect_project!(pkg, dirname(env.project_file), deps_map) + deps, glues = collect_project(pkg, dirname(env.project_file)) + deps_map[pkg.uuid] = deps + glue_map[pkg.uuid] = glues names[pkg.uuid] = pkg.name end for pkg in pkgs @@ -268,7 +285,9 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU if !isdir(path) pkgerror("expected package $(err_rep(pkg)) to exist at path `$path`") end - collect_project!(pkg, path, deps_map) + deps, glues = collect_project(pkg, path) + deps_map[pkg.uuid] = deps + glue_map[pkg.uuid] = glues end fixed = Dict{UUID,Resolve.Fixed}() @@ -285,7 +304,7 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU idx = findfirst(pkg -> pkg.uuid == uuid, pkgs) fix_pkg = pkgs[idx] end - fixed[uuid] = Resolve.Fixed(fix_pkg.version, q) + fixed[uuid] = Resolve.Fixed(fix_pkg.version, q, glue_map[uuid]) end return fixed end @@ -407,6 +426,7 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} # pkg -> version -> (dependency => compat): all_compat = Dict{UUID,Dict{VersionNumber,Dict{UUID,VersionSpec}}}() + glue_compat = Dict{UUID,Dict{VersionNumber,Set{UUID}}}() for (fp, fx) in fixed all_compat[fp] = Dict(fx.version => Dict{UUID,VersionSpec}()) @@ -418,7 +438,8 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} for uuid in unseen push!(seen, uuid) uuid in keys(fixed) && continue - all_compat_u = get_or_make!(all_compat, uuid) + all_compat_u = get_or_make!(all_compat, uuid) + glue_compat_u = get_or_make!(glue_compat, uuid) uuid_is_stdlib = false stdlib_name = "" @@ -446,35 +467,56 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} push!(uuids, other_uuid) all_compat_u_vr[other_uuid] = VersionSpec() end + + if !isempty(proj.gluedeps) + glue_all_compat_u_vr = get_or_make!(glue_compat_u, v) + for (_, other_uuid) in proj.gluedeps + push!(uuids, other_uuid) + all_compat_u_vr[other_uuid] = VersionSpec() + push!(glue_all_compat_u_vr, other_uuid) + end + end else for reg in registries pkg = get(reg, uuid, nothing) pkg === nothing && continue info = Registry.registry_info(pkg) - for (v, compat_info) in Registry.compat_info(info) - # Filter yanked and if we are in offline mode also downloaded packages - # TODO, pull this into a function - Registry.isyanked(info, v) && continue - if Pkg.OFFLINE_MODE[] - pkg_spec = PackageSpec(name=pkg.name, uuid=pkg.uuid, version=v, tree_hash=Registry.treehash(info, v)) - is_package_downloaded(env.project_file, pkg_spec) || continue - end - # Skip package version that are not the same as external packages in sysimage - if PKGORIGIN_HAVE_VERSION && RESPECT_SYSIMAGE_VERSIONS[] && julia_version == VERSION - pkgid = Base.PkgId(uuid, pkg.name) - if Base.in_sysimage(pkgid) - pkgorigin = get(Base.pkgorigins, pkgid, nothing) - if pkgorigin !== nothing && pkgorigin.version !== nothing - if v != pkgorigin.version - continue + function add_compat!(d, cinfo) + for (v, compat_info) in cinfo + # Filter yanked and if we are in offline mode also downloaded packages + # TODO, pull this into a function + Registry.isyanked(info, v) && continue + if Pkg.OFFLINE_MODE[] + pkg_spec = PackageSpec(name=pkg.name, uuid=pkg.uuid, version=v, tree_hash=Registry.treehash(info, v)) + is_package_downloaded(env.project_file, pkg_spec) || continue + end + + # Skip package version that are not the same as external packages in sysimage + if PKGORIGIN_HAVE_VERSION && RESPECT_SYSIMAGE_VERSIONS[] && julia_version == VERSION + pkgid = Base.PkgId(uuid, pkg.name) + if Base.in_sysimage(pkgid) + pkgorigin = get(Base.pkgorigins, pkgid, nothing) + if pkgorigin !== nothing && pkgorigin.version !== nothing + if v != pkgorigin.version + continue + end end end end + dv = get_or_make!(d, v) + merge!(dv, compat_info) + union!(uuids, keys(compat_info)) + end + end + add_compat!(all_compat_u, Registry.compat_info(info)) + glue_compat_info = Registry.glue_compat_info(info) + if glue_compat_info !== nothing + add_compat!(all_compat_u, glue_compat_info) + # Version to Set + for (v, compat_info) in glue_compat_info + glue_compat_u[v] = keys(compat_info) end - - all_compat_u[v] = compat_info - union!(uuids, keys(compat_info)) end end end @@ -493,7 +535,7 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} end end - return Resolve.Graph(all_compat, uuid_to_name, reqs, fixed, false, julia_version), + return Resolve.Graph(all_compat, glue_compat, uuid_to_name, reqs, fixed, false, julia_version), all_compat end @@ -1127,7 +1169,7 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode::PackageMode) # only declare `compat` for remaining direct or `extra` dependencies # `julia` is always an implicit direct dependency filter!(ctx.env.project.compat) do (name, _) - name == "julia" || name in keys(ctx.env.project.deps) || name in keys(ctx.env.project.extras) + name == "julia" || name in keys(ctx.env.project.deps) || name in keys(ctx.env.project.extras) || name in keys(ctx.env.project.gluedeps) end deps_names = union(keys(ctx.env.project.deps), keys(ctx.env.project.extras)) filter!(ctx.env.project.targets) do (target, deps) @@ -1735,9 +1777,13 @@ function gen_target_project(ctx::Context, pkg::PackageSpec, source_path::String, test_project.deps = source_env.project.deps # collect test dependencies for name in get(source_env.project.targets, target, String[]) - uuid = get(source_env.project.extras, name, nothing) + uuid = nothing + for list in [source_env.project.extras, source_env.project.gluedeps] + uuid = get(list, name, nothing) + uuid === nothing || break + end if uuid === nothing - pkgerror("`$name` declared as a `$target` dependency, but no such entry in `extras`") + pkgerror("`$name` declared as a `$target` dependency, but no such entry in `extras` or `gluedeps`") end test_project.deps[name] = uuid end @@ -2083,6 +2129,7 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie lpadding = 2 package_statuses = PackageStatusData[] + installed_cache = Dict{Base.PkgId, Bool}() for (uuid, old, new) in xs if Types.is_project_uuid(env, uuid) continue @@ -2116,6 +2163,7 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie no_packages_upgradable &= (!changed || !pkg_upgradable) no_visible_packages_heldback &= (!changed || !pkg_heldback) no_packages_heldback &= !pkg_heldback + push!(package_statuses, PackageStatusData(uuid, old, new, pkg_downloaded, pkg_upgradable, pkg_heldback, cinfo, changed)) end diff --git a/src/Pkg.jl b/src/Pkg.jl index 87adf6d7fc..8d166f4a13 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -735,9 +735,9 @@ end # Precompilation # ################## -function _auto_precompile(ctx::Types.Context; warn_loaded = true, already_instantiated = false) +function _auto_precompile(ctx::Types.Context, pkgs::Vector{String}=String[]; warn_loaded = true, already_instantiated = false) if Base.JLOptions().use_compiled_modules == 1 && get_bool_env("JULIA_PKG_PRECOMPILE_AUTO"; default="true") - Pkg.precompile(ctx; internal_call=true, warn_loaded = warn_loaded, already_instantiated = already_instantiated) + Pkg.precompile(ctx, pkgs; internal_call=true, warn_loaded = warn_loaded, already_instantiated = already_instantiated) end end diff --git a/src/Registry/registry_instance.jl b/src/Registry/registry_instance.jl index 8b59dec914..44825f7014 100644 --- a/src/Registry/registry_instance.jl +++ b/src/Registry/registry_instance.jl @@ -36,8 +36,9 @@ custom_isfile(in_memory_registry::Union{Dict, Nothing}, folder::AbstractString, git_tree_sha1::Base.SHA1 yanked::Bool @lazy uncompressed_compat::Union{Dict{UUID, VersionSpec}} + @lazy glue_uncompressed_compat::Union{Dict{UUID, VersionSpec}} end -VersionInfo(git_tree_sha1::Base.SHA1, yanked::Bool) = VersionInfo(git_tree_sha1, yanked, uninit) +VersionInfo(git_tree_sha1::Base.SHA1, yanked::Bool) = VersionInfo(git_tree_sha1, yanked, uninit, uninit) # This is the information that exists in e.g. General/A/ACME struct PkgInfo @@ -53,6 +54,12 @@ struct PkgInfo # Deps.toml deps::Dict{VersionRange, Dict{String, UUID}} + + # WeakCompat.toml + glue_compat::Dict{VersionRange, Dict{String, VersionSpec}} + + # GlueDeps.toml + glue_deps::Dict{VersionRange, Dict{String, UUID}} end isyanked(pkg::PkgInfo, v::VersionNumber) = pkg.version_info[v].yanked @@ -100,8 +107,8 @@ function initialize_uncompressed!(pkg::PkgInfo, versions = keys(pkg.version_info sort!(versions) - uncompressed_compat = uncompress(pkg.compat, versions) - uncompressed_deps = uncompress(pkg.deps, versions) + uncompressed_compat = uncompress(pkg.compat, versions) + uncompressed_deps = uncompress(pkg.deps, versions) for v in versions vinfo = pkg.version_info[v] @@ -119,11 +126,43 @@ function initialize_uncompressed!(pkg::PkgInfo, versions = keys(pkg.version_info return pkg end +function initialize_glue_uncompressed!(pkg::PkgInfo, versions = keys(pkg.version_info)) + # Only valid to call this with existing versions of the package + # Remove all versions we have already uncompressed + versions = filter!(v -> !isinit(pkg.version_info[v], :glue_uncompressed_compat), collect(versions)) + + sort!(versions) + + glue_uncompressed_compat = uncompress(pkg.glue_compat, versions) + glue_uncompressed_deps = uncompress(pkg.glue_deps, versions) + + for v in versions + vinfo = pkg.version_info[v] + glue_compat = Dict{UUID, VersionSpec}() + glue_uncompressed_deps_v = glue_uncompressed_deps[v] + glue_uncompressed_compat_v = glue_uncompressed_compat[v] + for (pkg, uuid) in glue_uncompressed_deps_v + vspec = get(glue_uncompressed_compat_v, pkg, nothing) + glue_compat[uuid] = vspec === nothing ? VersionSpec() : vspec + end + @init! vinfo.glue_uncompressed_compat = glue_compat + end + return pkg +end + function compat_info(pkg::PkgInfo) initialize_uncompressed!(pkg) return Dict(v => info.uncompressed_compat for (v, info) in pkg.version_info) end +function glue_compat_info(pkg::PkgInfo) + if isempty(pkg.glue_deps) + return nothing + end + initialize_glue_uncompressed!(pkg) + return Dict(v => info.glue_uncompressed_compat for (v, info) in pkg.version_info) +end + @lazy struct PkgEntry # Registry.toml: path::String @@ -181,7 +220,29 @@ function init_package_info!(pkg::PkgEntry) # All packages depend on julia deps[VersionRange()] = Dict("julia" => JULIA_UUID) - @init! pkg.info = PkgInfo(repo, subdir, version_info, compat, deps) + # WeakCompat.toml + glue_compat_data_toml = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "WeakCompat.toml")) ? + parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "WeakCompat.toml")) : Dict{String, Any}() + glue_compat_data_toml = convert(Dict{String, Dict{String, Union{String, Vector{String}}}}, glue_compat_data_toml) + glue_compat = Dict{VersionRange, Dict{String, VersionSpec}}() + for (v, data) in glue_compat_data_toml + vr = VersionRange(v) + d = Dict{String, VersionSpec}(dep => VersionSpec(vr_dep) for (dep, vr_dep) in data) + glue_compat[vr] = d + end + + # GlueDeps.toml + glue_deps_data_toml = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "GlueDeps.toml")) ? + parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "GlueDeps.toml")) : Dict{String, Any}() + glue_deps_data_toml = convert(Dict{String, Dict{String, String}}, glue_deps_data_toml) + glue_deps = Dict{VersionRange, Dict{String, UUID}}() + for (v, data) in glue_deps_data_toml + vr = VersionRange(v) + d = Dict{String, UUID}(dep => UUID(uuid) for (dep, uuid) in data) + glue_deps[vr] = d + end + + @init! pkg.info = PkgInfo(repo, subdir, version_info, compat, deps, glue_compat, glue_deps) return pkg.info end diff --git a/src/Resolve/graphtype.jl b/src/Resolve/graphtype.jl index ecd1d92669..94d2d1882b 100644 --- a/src/Resolve/graphtype.jl +++ b/src/Resolve/graphtype.jl @@ -236,13 +236,12 @@ mutable struct Graph function Graph( compat::Dict{UUID,Dict{VersionNumber,Dict{UUID,VersionSpec}}}, + compat_glue::Dict{UUID,Dict{VersionNumber,Set{UUID}}}, uuid_to_name::Dict{UUID,String}, reqs::Requires, fixed::Dict{UUID,Fixed}, verbose::Bool = false, julia_version::Union{VersionNumber,Nothing} = VERSION - ; - compat_weak::Dict{UUID,Dict{VersionNumber,Set{UUID}}} = Dict{UUID,Dict{VersionNumber,Set{UUID}}}(), ) # Tell the resolver about julia itself @@ -256,7 +255,6 @@ mutable struct Graph data = GraphData(compat, uuid_to_name, verbose) pkgs, np, spp, pdict, pvers, vdict, rlog = data.pkgs, data.np, data.spp, data.pdict, data.pvers, data.vdict, data.rlog - extended_deps = let spp = spp # Due to https://github.com/JuliaLang/julia/issues/15276 [Vector{Dict{Int,BitVector}}(undef, spp[p0]-1) for p0 = 1:np] end @@ -276,14 +274,14 @@ mutable struct Graph # Translate the requirements into bit masks # Hot code, measure performance before changing req_msk = Dict{Int,BitVector}() - maybe_weak = haskey(compat_weak, uuid0) && haskey(compat_weak[uuid0], vn) + maybe_weak = haskey(compat_glue, uuid0) && haskey(compat_glue[uuid0], vn) for (p1, vs) in req pv = pvers[p1] req_msk_p1 = BitVector(undef, spp[p1]) @inbounds for i in 1:spp[p1] - 1 req_msk_p1[i] = pv[i] ∈ vs end - weak = maybe_weak && (pkgs[p1] ∈ compat_weak[uuid0][vn]) + weak = maybe_weak && (pkgs[p1] ∈ compat_glue[uuid0][vn]) req_msk_p1[end] = weak req_msk[p1] = req_msk_p1 end diff --git a/src/Types.jl b/src/Types.jl index 2146149399..2e7ff22ff0 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -239,6 +239,8 @@ Base.@kwdef mutable struct Project manifest::Union{String, Nothing} = nothing # Sections deps::Dict{String,UUID} = Dict{String,UUID}() + gluedeps::Dict{String,UUID} = Dict{String,UUID}() + gluepkgs::Dict{String,Union{Vector{String}, String}} = Dict{String,String}() extras::Dict{String,UUID} = Dict{String,UUID}() targets::Dict{String,Vector{String}} = Dict{String,Vector{String}}() compat::Dict{String,Compat} = Dict{String,Compat}() @@ -262,6 +264,8 @@ Base.@kwdef mutable struct PackageEntry repo::GitRepo = GitRepo() tree_hash::Union{Nothing,SHA1} = nothing deps::Dict{String,UUID} = Dict{String,UUID}() + gluedeps::Dict{String,UUID} = Dict{String,UUID}() + gluepkgs::Dict{String,Union{Vector{String}, String}} = Dict{String,String}() uuid::Union{Nothing, UUID} = nothing other::Union{Dict,Nothing} = nothing end @@ -272,9 +276,11 @@ Base.:(==)(t1::PackageEntry, t2::PackageEntry) = t1.name == t2.name && t1.repo == t2.repo && t1.tree_hash == t2.tree_hash && t1.deps == t2.deps && + t1.gluedeps == t2.gluedeps && + t1.gluepkgs == t2.gluepkgs && t1.uuid == t2.uuid # omits `other` -Base.hash(x::PackageEntry, h::UInt) = foldr(hash, [x.name, x.version, x.path, x.pinned, x.repo, x.tree_hash, x.deps, x.uuid], init=h) # omits `other` +Base.hash(x::PackageEntry, h::UInt) = foldr(hash, [x.name, x.version, x.path, x.pinned, x.repo, x.tree_hash, x.deps, x.gluedeps,x.gluepkgs, x.uuid], init=h) # omits `other` Base.@kwdef mutable struct Manifest julia_version::Union{Nothing,VersionNumber} = nothing # only set to VERSION when resolving diff --git a/src/manifest.jl b/src/manifest.jl index 69e20e25d3..79521957a9 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -83,19 +83,22 @@ struct Stage1 uuid::UUID entry::PackageEntry deps::Union{Vector{String}, Dict{String,UUID}} + gluedeps::Union{Vector{String}, Dict{String,UUID}} end -normalize_deps(name, uuid, deps, manifest) = deps -function normalize_deps(name, uuid, deps::Vector{String}, manifest::Dict{String,Vector{Stage1}}) +normalize_deps(name, uuid, deps, manifest; isglue=false) = deps +function normalize_deps(name, uuid, deps::Vector{String}, manifest::Dict{String,Vector{Stage1}}; isglue=false) if length(deps) != length(unique(deps)) pkgerror("Duplicate entry in `$name=$uuid`'s `deps` field.") end final = Dict{String,UUID}() for dep in deps infos = get(manifest, dep, nothing) - if infos === nothing - pkgerror("`$name=$uuid` depends on `$dep`, ", - "but no such entry exists in the manifest.") + if !isglue + if infos === nothing + pkgerror("`$name=$uuid` depends on `$dep`, ", + "but no such entry exists in the manifest.") + end end # should have used dict format instead of vector format length(infos) == 1 || pkgerror("Invalid manifest format. ", @@ -110,21 +113,30 @@ function validate_manifest(julia_version::Union{Nothing,VersionNumber}, manifest for (name, infos) in stage1, info in infos info.entry.deps = normalize_deps(name, info.uuid, info.deps, stage1) end + for (name, infos) in stage1, info in infos + info.entry.gluedeps = normalize_deps(name, info.uuid, info.gluedeps, stage1; isglue=true) + end # invariant: all dependencies are now normalized to Dict{String,UUID} deps = Dict{UUID, PackageEntry}() for (name, infos) in stage1, info in infos deps[info.uuid] = info.entry end # now just verify the graph structure - for (entry_uuid, entry) in deps, (name, uuid) in entry.deps - dep_entry = get(deps, uuid, nothing) - if dep_entry === nothing - pkgerror("`$(entry.name)=$(entry_uuid)` depends on `$name=$uuid`, ", - "but no such entry exists in the manifest.") - end - if dep_entry.name != name - pkgerror("`$(entry.name)=$(entry_uuid)` depends on `$name=$uuid`, ", - "but entry with UUID `$uuid` has name `$(dep_entry.name)`.") + for (entry_uuid, entry) in deps + for (deptype, isglue) in [(entry.deps, false), (entry.gluedeps, true)] + for (name, uuid) in deptype + dep_entry = get(deps, uuid, nothing) + if !isglue + if dep_entry === nothing + pkgerror("`$(entry.name)=$(entry_uuid)` depends on `$name=$uuid`, ", + "but no such entry exists in the manifest.") + end + if dep_entry.name != name + pkgerror("`$(entry.name)=$(entry_uuid)` depends on `$name=$uuid`, ", + "but entry with UUID `$uuid` has name `$(dep_entry.name)`.") + end + end + end end end return Manifest(; julia_version, manifest_format, deps, other) @@ -147,6 +159,7 @@ function Manifest(raw::Dict, f_or_io::Union{String, IO})::Manifest entry.name = name uuid = nothing deps = nothing + gluedeps = nothing try entry.pinned = read_pinned(get(info, "pinned", nothing)) uuid = read_field("uuid", nothing, info, safe_uuid)::UUID @@ -158,13 +171,15 @@ function Manifest(raw::Dict, f_or_io::Union{String, IO})::Manifest entry.tree_hash = read_field("git-tree-sha1", nothing, info, safe_SHA1) entry.uuid = uuid deps = read_deps(get(info::Dict, "deps", nothing)) + gluedeps = read_deps(get(info::Dict, "gluedeps", nothing)) + entry.gluepkgs = get(Dict{String, String}, info::Dict, "gluepkgs") catch # TODO: Should probably not unconditionally log something @debug "Could not parse manifest entry for `$name`" f_or_io rethrow() end entry.other = info::Union{Dict,Nothing} - stage1[name] = push!(get(stage1, name, Stage1[]), Stage1(uuid, entry, deps)) + stage1[name] = push!(get(stage1, name, Stage1[]), Stage1(uuid, entry, deps, gluedeps)) end # by this point, all the fields of the `PackageEntry`s have been type casted # but we have *not* verified the _graph_ structure of the manifest @@ -257,18 +272,25 @@ function destructure(manifest::Manifest)::Dict entry!(new_entry, "repo-url", repo_source) entry!(new_entry, "repo-rev", entry.repo.rev) entry!(new_entry, "repo-subdir", entry.repo.subdir) - if isempty(entry.deps) - delete!(new_entry, "deps") - else - if all(dep -> unique_name[first(dep)], entry.deps) - new_entry["deps"] = sort(collect(keys(entry.deps))) + for (deptype, depname) in [(entry.deps, "deps"), (entry.gluedeps, "gluedeps")] + if isempty(deptype) + delete!(new_entry, depname) else - new_entry["deps"] = Dict{String,String}() - for (name, uuid) in entry.deps - new_entry["deps"][name] = string(uuid) + if all(dep -> haskey(unique_name, first(dep)), deptype) && all(dep -> unique_name[first(dep)], deptype) + new_entry[depname] = sort(collect(keys(deptype))) + else + new_entry[depname] = Dict{String,String}() + for (name, uuid) in deptype + new_entry[depname][name] = string(uuid) + end end end end + + # TODO: Write this inline + if !isempty(entry.gluepkgs) + entry!(new_entry, "gluepkgs", entry.gluepkgs) + end if manifest.manifest_format.major == 1 push!(get!(raw, entry.name, Dict{String,Any}[]), new_entry) elseif manifest.manifest_format.major == 2 diff --git a/src/project.jl b/src/project.jl index dfb0de7210..4930e82d4e 100644 --- a/src/project.jl +++ b/src/project.jl @@ -2,7 +2,7 @@ # UTILS # ######### listed_deps(project::Project) = - append!(collect(keys(project.deps)), collect(keys(project.extras))) + append!(collect(keys(project.deps)), collect(keys(project.extras)), collect(keys(project.gluedeps))) ########### # READING # @@ -73,21 +73,26 @@ end read_project_compat(raw, project::Project) = pkgerror("Expected `compat` section to be a key-value list") -function validate(project::Project) +function validate(project::Project; file=nothing) # deps + location_string = file === nothing ? "" : " at $(repr(file))." dep_uuids = collect(values(project.deps)) if length(dep_uuids) != length(unique(dep_uuids)) - pkgerror("Two different dependencies can not have the same uuid") + pkgerror("Two different dependencies can not have the same uuid" * location_string) + end + glue_dep_uuids = collect(values(project.gluedeps)) + if length(glue_dep_uuids) != length(unique(glue_dep_uuids)) + pkgerror("Two different glue dependencies can not have the same uuid" * location_string) end # extras extra_uuids = collect(values(project.extras)) if length(extra_uuids) != length(unique(extra_uuids)) - pkgerror("Two different `extra` dependencies can not have the same uuid") + pkgerror("Two different `extra` dependencies can not have the same uuid" * location_string) end - dep_names = keys(project.deps) # TODO decide what to do in when `add`ing a dep that is already in `extras` # also, reintroduce test files for this #= + dep_names = keys(project.deps) for (name, uuid) in project.extras name in dep_names && pkgerror("name `$name` is listed in both `deps` and `extras`") uuid in dep_uuids && pkgerror("uuid `$uuid` is listed in both `deps` and `extras`") @@ -100,18 +105,18 @@ function validate(project::Project) pkgerror("A dependency was named twice in target `$target`") end dep in listed || pkgerror(""" - Dependency `$dep` in target `$target` not listed in `deps` or `extras` section. - """) + Dependency `$dep` in target `$target` not listed in `deps`, `gluedeps` or `extras` section + """ * location_string) end # compat for (name, version) in project.compat name == "julia" && continue name in listed || - pkgerror("Compat `$name` not listed in `deps` or `extras` section.") + pkgerror("Compat `$name` not listed in `deps`, `gluedeps` or `extras` section" * location_string) end end -function Project(raw::Dict) +function Project(raw::Dict; file=nothing) project = Project() project.other = raw project.name = get(raw, "name", nothing)::Union{String, Nothing} @@ -119,10 +124,12 @@ function Project(raw::Dict) project.uuid = read_project_uuid(get(raw, "uuid", nothing)) project.version = read_project_version(get(raw, "version", nothing)) project.deps = read_project_deps(get(raw, "deps", nothing), "deps") + project.gluedeps = read_project_deps(get(raw, "gluedeps", nothing), "gluedeps") + project.gluepkgs = get(Dict{String, String}, raw, "gluepkgs") project.extras = read_project_deps(get(raw, "extras", nothing), "extras") project.compat = read_project_compat(get(raw, "compat", nothing), project) project.targets = read_project_targets(get(raw, "targets", nothing), project) - validate(project) + validate(project; file) return project end @@ -139,7 +146,7 @@ function read_project(f_or_io::Union{String, IO}) end rethrow() end - return Project(raw) + return Project(raw; file= f_or_io isa IO ? nothing : f_or_io) end @@ -168,6 +175,7 @@ function destructure(project::Project)::Dict entry!("version", project.version) entry!("manifest", project.manifest) entry!("deps", project.deps) + entry!("gluedeps", project.gluedeps) entry!("extras", project.extras) entry!("compat", Dict(name => x.str for (name, x) in project.compat)) entry!("targets", project.targets) diff --git a/test/gluedeps.jl b/test/gluedeps.jl new file mode 100644 index 0000000000..3e251c402b --- /dev/null +++ b/test/gluedeps.jl @@ -0,0 +1,21 @@ +using .Utils +using Test + +@testset "weak deps" begin + isolate(loaded_depot=true) do + Pkg.activate(; temp=true) + Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasGluePkgs.jl")) + Pkg.test("HasGluePkgs") + end + isolate(loaded_depot=true) do + Pkg.activate(; temp=true) + Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasDepWithGluePkgs.jl")) + Pkg.test("HasDepWithGluePkgs") + end + + isolate(loaded_depot=true) do + Pkg.activate(; temp=true) + Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasGluePkgs.jl")) + @test_throws Pkg.Resolve.ResolverError Pkg.add(; name = "OffsetArrays", version = "0.9.0") + end +end diff --git a/test/resolve_utils.jl b/test/resolve_utils.jl index 8763949e6c..981cb638b7 100644 --- a/test/resolve_utils.jl +++ b/test/resolve_utils.jl @@ -81,7 +81,7 @@ function graph_from_data(deps_data) end end end - return Graph(all_compat, uuid_to_name, Requires(), fixed, VERBOSE; compat_weak=all_compat_w) + return Graph(all_compat, all_compat_w, uuid_to_name, Requires(), fixed, VERBOSE) end function reqs_from_data(reqs_data, graph::Graph) reqs = Dict{UUID,VersionSpec}() diff --git a/test/runtests.jl b/test/runtests.jl index 7e3fedc819..a75bced3a6 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -66,6 +66,7 @@ Logging.with_logger(hide_logs ? Logging.NullLogger() : Logging.current_logger()) "api.jl", "registry.jl", "subdir.jl", + "gluedeps.jl", "artifacts.jl", "binaryplatforms.jl", "platformengines.jl", diff --git a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml new file mode 100644 index 0000000000..4627434d62 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml @@ -0,0 +1,57 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.10.0-DEV" +manifest_format = "2.0" +project_hash = "1af04d62b804224fdf802daf4a4b96e213299d30" + +[[deps.Adapt]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "195c5505521008abea5aee4f96930717958eac6f" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "3.4.0" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.0.1+0" + +[[deps.Example]] +git-tree-sha1 = "46e44e869b4d90b96bd8ed1fdcf32244fddfb6cc" +uuid = "7876af07-990d-54b4-ab0e-23690620f79a" +version = "0.5.3" + +[[deps.HasGluePkgs]] +deps = ["Example"] +path = "../HasGluePkgs.jl" +uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" +version = "0.1.0" +gluedeps = ["OffsetArrays"] + + [deps.HasGluePkgs.gluepkgs] + GlueOffsetArrays = "OffsetArrays" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.OffsetArrays]] +deps = ["Adapt"] +git-tree-sha1 = "f71d8950b724e9ff6110fc948dff5a329f901d64" +uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +version = "1.12.8" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.21+0" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.2.0+0" diff --git a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Project.toml b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Project.toml new file mode 100644 index 0000000000..2036b85044 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Project.toml @@ -0,0 +1,17 @@ +name = "HasDepWithGluePkgs" +uuid = "d4ef3d4a-8e22-4710-85d8-c6cf2eb9efca" +version = "0.1.0" + +[deps] +HasGluePkgs = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" + +[compat] +HasGluePkgs = "0.1" +OffsetArrays = "1.12" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/src/HasDepWithGluePkgs.jl b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/src/HasDepWithGluePkgs.jl new file mode 100644 index 0000000000..66a109b9a8 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/src/HasDepWithGluePkgs.jl @@ -0,0 +1,15 @@ +module HasDepWithGluePkgs + +using HasGluePkgs +using OffsetArrays: OffsetArray +# Loading OffsetArrays makes the glue module "GlueOffsetArrays" to load + +function do_something() + # @info "First do something with the basic array support in B" + HasGluePkgs.foo(rand(Float64, 2)) + + # @info "Now do something with extended OffsetArray support in B" + HasGluePkgs.foo(OffsetArray(rand(Float64, 2), 0:1)) +end + +end # module diff --git a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/test/runtests.jl b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/test/runtests.jl new file mode 100644 index 0000000000..318c70c3f5 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/test/runtests.jl @@ -0,0 +1,5 @@ +using HasDepWithGluePkgs +using Test + +@test HasDepWithGluePkgs.HasGluePkgs.offsetarrays_loaded +HasDepWithGluePkgs.do_something() diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Manifest.toml b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Manifest.toml new file mode 100644 index 0000000000..f9461df371 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Manifest.toml @@ -0,0 +1,10 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.10.0-DEV" +manifest_format = "2.0" +project_hash = "549efd0c32255f4b8717f50f49778e183bcc0e36" + +[[deps.Example]] +git-tree-sha1 = "46e44e869b4d90b96bd8ed1fdcf32244fddfb6cc" +uuid = "7876af07-990d-54b4-ab0e-23690620f79a" +version = "0.5.3" diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml new file mode 100644 index 0000000000..d3b8260a17 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml @@ -0,0 +1,22 @@ +name = "HasGluePkgs" +uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" +version = "0.1.0" + +[deps] +Example = "7876af07-990d-54b4-ab0e-23690620f79a" + +[gluedeps] +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" + +[gluepkgs] +GlueOffsetArrays = "OffsetArrays" + +[compat] +Example = "0.5" +OffsetArrays = "1.12" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["OffsetArrays", "Test"] diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/glue/GlueOffsetArrays.jl b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/glue/GlueOffsetArrays.jl new file mode 100644 index 0000000000..897a7b8188 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/glue/GlueOffsetArrays.jl @@ -0,0 +1,13 @@ +module GlueOffsetArrays + +using HasGluePkgs, OffsetArrays + +function foo(::OffsetArray) + return 2 +end + +function __init__() + HasGluePkgs.offsetarrays_loaded = true +end + +end diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/src/HasGluePkgs.jl b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/src/HasGluePkgs.jl new file mode 100644 index 0000000000..212512cd73 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/src/HasGluePkgs.jl @@ -0,0 +1,11 @@ +module HasGluePkgs + +using Example + +function foo(::AbstractArray) + return 1 +end + +offsetarrays_loaded = false + +end # module diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/test/runtests.jl b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/test/runtests.jl new file mode 100644 index 0000000000..8042719f35 --- /dev/null +++ b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/test/runtests.jl @@ -0,0 +1,8 @@ +using HasGluePkgs +using Test + +@test !HasGluePkgs.offsetarrays_loaded + +using OffsetArrays + +@test HasGluePkgs.offsetarrays_loaded From 041e9f015183b6f9572c3759b6bbd3d4ce8d93f4 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Fri, 25 Nov 2022 09:36:35 +0100 Subject: [PATCH 05/26] fix order of sections in Project file --- src/project.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/project.jl b/src/project.jl index 4930e82d4e..12c0f76c0a 100644 --- a/src/project.jl +++ b/src/project.jl @@ -182,7 +182,7 @@ function destructure(project::Project)::Dict return raw end -_project_key_order = ["name", "uuid", "keywords", "license", "desc", "deps", "compat"] +_project_key_order = ["name", "uuid", "keywords", "license", "desc", "deps", "gluedeps", "gluepkgs", "compat"] project_key_order(key::String) = something(findfirst(x -> x == key, _project_key_order), length(_project_key_order) + 1) From f2376f8314636c7f9956020f4b0017247653b77e Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Fri, 25 Nov 2022 09:51:15 +0100 Subject: [PATCH 06/26] tweak printing of glue pkgs in precompilation --- src/API.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/API.jl b/src/API.jl index 29dd681763..53a7206987 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1096,6 +1096,7 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: for (name, uuid) in ctx.env.project.deps if !Base.in_sysimage(Base.PkgId(uuid, name)) ] + gluepkgs = Dict{Base.PkgId, String}() # gluepkg -> parent # make a flat map of each dep and its deps depsmap = Dict{Base.PkgId, Vector{Base.PkgId}}() pkg_specs = PackageSpec[] @@ -1117,6 +1118,7 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: gluepkg = Base.PkgId(gluepkg_uuid, gluepkg_name) push!(pkg_specs, PackageSpec(uuid = gluepkg_uuid, name = gluepkg_name)) # create this here as the name cannot be looked up easily later via the uuid depsmap[gluepkg] = filter!(!Base.in_sysimage, gluepkg_deps) + gluepkgs[gluepkg] = pkg.name end end @@ -1260,7 +1262,8 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: final_loop || print(iostr, sprint(io -> show_progress(io, bar; termwidth = displaysize(ctx.io)[2]); context=io), "\n") for dep in pkg_queue_show loaded = warn_loaded && haskey(Base.loaded_modules, dep) - name = dep in direct_deps ? dep.name : string(color_string(dep.name, :light_black)) + _name = haskey(gluepkgs, dep) ? string(gluepkgs[dep], " → ", dep.name) : dep.name + name = dep in direct_deps ? _name : string(color_string(_name, :light_black)) if dep in precomperr_deps print(iostr, color_string(" ? ", Base.warn_color()), name, "\n") elseif haskey(failed_deps, dep) @@ -1334,7 +1337,8 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: Base.acquire(parallel_limiter) is_direct_dep = pkg in direct_deps iob = IOBuffer() - name = is_direct_dep ? pkg.name : string(color_string(pkg.name, :light_black)) + _name = haskey(gluepkgs, pkg) ? string(gluepkgs[pkg], " → ", pkg.name) : pkg.name + name = is_direct_dep ? _name : string(color_string(_name, :light_black)) !fancyprint && lock(print_lock) do isempty(pkg_queue) && printpkgstyle(io, :Precompiling, target) end From 6eb01e15893b0d9f33eea43bbe08583bf5d305c6 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Fri, 25 Nov 2022 10:30:37 +0100 Subject: [PATCH 07/26] add docs for backwards compat --- docs/src/creating-packages.md | 52 +++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index df45a97957..969e201f37 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -253,7 +253,8 @@ This is an important topic so there is a separate chapter about it: [Compatibili It is sometimes desirable to be able to extend some functionality of a package without having to unconditionally take on the cost (in terms of e.g. load time) of adding an extra dependency. A *glue package* is a file that gets automatically loaded when some other set of packages are -loaded into the Julia session. +loaded into the Julia session. This is very similar to functionality that the external package +Requires.jl used to provide, but which is now aviable directly through Julia. A useful application of glue packages could be for a plotting package that should be able to plot objects from a wide variety of different Julia packages. @@ -270,9 +271,6 @@ name = "Plotting" version = "0.1.0" uuid = "..." -[deps] -Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" - [gluedeps] Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" @@ -283,17 +281,15 @@ Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" GlueContour = "Contour" [compat] # compat can also be given on glue dependencies -Colors = "0.12.8" Contour = "0.6.2" ``` `src/Plotting.jl`: ```julia module Plotting -using Colors -function plot(x::Colors.Color) - # Some functionality for plotting a color here +function plot(x::Vector) + # Some functionality for plotting a vector here end end # module @@ -312,14 +308,48 @@ end end # module ``` -A user that depends on `Plotting` will not pay the code of the "glue code" inside the `GlueContour` module. -It is only when the `Contour` package actually gets loaded that the `GlueCountour` glue package will get loaded +A user that depends on `Plotting` will not pay the cost of the "glue code" inside the `GlueContour` module. +It is only when the `Contour` package actually gets loaded that the `GlueCountour` glue package is loaded and provide the new functionality. -Compatibility can be set on glue dependencies just like normal dependencies. +Compatibility can be set on glue dependencies just like normal dependencies and they can be used in the `test` +target to make them available when testing. A glue package will only be loaded if the glue dependencies are loaded from the same environment or environments higher in the environment stack than the package itself. + +!!! compat + In order to be compatible with earlier versions of Julia, some extra steps have to be taken. + These are: Duplicate the packages under `[gluedeps]` into `[extras]`. This is an unfortunate + duplication but without doing this the project verifier under older Julia versions will complain (error). + +### Backwards compatibility with Requires.jl + +Since glue packages and Requires.jl are solving the same problem, +it is useful to know how to use Requires.jl on older Julia versions while seamlessly +transitioning into using glue packages on newer Julia versions that support it. +This is done by making the following changes (using the example above): + +- Add the following to the package file. This makes it so that Requires.jl loads and inserts the + callback only when glue packages are not supported + ```julia + # This symbol is only defined on Julia versions that support glue packages + if isdefined(Base, :get_gluepkg) + using Requires + function __init__() + @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../glue/GlueContour.jl) + end + end + ``` + +- Do the following change in the glue packages for loading the glue dependency: + ```julia + isdefined(Base, :get_gluepkg) ? (using Contour) : (using ..Contour) + ``` + +The package should now work with Requires.jl on Julia versions before glue packages were introduced +and with glue packages afterward. + ## Package naming guidelines Package names should be sensible to most Julia users, *even to those who are not domain experts*. From b591e39048d6510ac3154e5978b7283efc1c079a Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Fri, 25 Nov 2022 12:10:53 +0100 Subject: [PATCH 08/26] add a status option for glue --- src/API.jl | 8 +-- src/Operations.jl | 74 +++++++++++++++++++++++++--- src/Pkg.jl | 5 +- src/REPLMode/command_declarations.jl | 6 +++ test/gluedeps.jl | 1 + 5 files changed, 83 insertions(+), 11 deletions(-) diff --git a/src/API.jl b/src/API.jl index 53a7206987..61409d3b9b 100644 --- a/src/API.jl +++ b/src/API.jl @@ -154,7 +154,7 @@ for f in (:develop, :add, :rm, :up, :pin, :free, :test, :build, :status, :why) pkgs = deepcopy(pkgs) # don't mutate input foreach(handle_package_input!, pkgs) ret = $f(ctx, pkgs; kwargs...) - $(f in (:add, :up, :pin, :free, :build)) && Pkg._auto_precompile(ctx) # rm does too, but it's handled differently + $(f in (:add, :up, :pin, :free, :build)) && Pkg._auto_precompile(ctx) $(f in (:up, :pin, :free, :rm)) && Pkg._auto_gc(ctx) return ret end @@ -1640,14 +1640,14 @@ end @deprecate status(mode::PackageMode) status(mode=mode) -function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=PKGMODE_PROJECT, outdated::Bool=false, compat::Bool=false, io::IO=stdout_f(), kwargs...) +function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=PKGMODE_PROJECT, outdated::Bool=false, compat::Bool=false, glue::Bool=false, io::IO=stdout_f(), kwargs...) if compat diff && pkgerror("Compat status has no `diff` mode") outdated && pkgerror("Compat status has no `outdated` mode") - + weak && pkgerror("Compat status has no `weak` mode") Operations.print_compat(ctx, pkgs; io) else - Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated) + Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated, glue) end return nothing end diff --git a/src/Operations.jl b/src/Operations.jl index dfa1148693..b22e677cea 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -2080,6 +2080,35 @@ function is_package_downloaded(project_file::String, pkg::PackageSpec; platform= return true end +function status_glue_info(pkg::PackageSpec, env::EnvCache) + manifest = env.manifest + manifest_info = get(manifest, pkg.uuid, nothing) + manifest_info === nothing && return nothing + gluedepses = manifest_info.gluedeps + gluepkgs = manifest_info.gluepkgs + if !isempty(gluedepses) && !isempty(gluepkgs) + v = GlueInfo[] + for (gluepkg, gluedeps) in gluepkgs + gluedeps isa String && (gluedeps = String[gluedeps]) + gluepkg_loaded = (Base.get_gluepkg(Base.PkgId(pkg.uuid, pkg.name), Symbol(gluepkg)) !== nothing) + # Check if deps are loaded + gluedeps_info= Tuple{String, Bool}[] + for gluedep in gluedeps + uuid = gluedepses[gluedep] + loaded = haskey(Base.loaded_modules, Base.PkgId(uuid, gluedep)) + push!(gluedeps_info, (gluedep, loaded)) + end + push!(v, GlueInfo((gluepkg, gluepkg_loaded), gluedeps_info)) + end + return v + end + return nothing +end + +struct GlueInfo + gluepkg::Tuple{String, Bool} # name, loaded + gluedeps::Vector{Tuple{String, Bool}} # name, loaded +end struct PackageStatusData uuid::UUID old::Union{Nothing, PackageSpec} @@ -2089,10 +2118,11 @@ struct PackageStatusData heldback::Bool compat_data::Union{Nothing, Tuple{Vector{String}, VersionNumber, VersionNumber}} changed::Bool + glueinfo::Union{Nothing, Vector{GlueInfo}} end function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registries::Vector{Registry.RegistryInstance}, header::Symbol, - uuids::Vector, names::Vector; manifest=true, diff=false, ignore_indent::Bool, outdated::Bool, io::IO, + uuids::Vector, names::Vector; manifest=true, diff=false, ignore_indent::Bool, outdated::Bool, glue::Bool, io::IO, mode::PackageMode, hidden_upgrades_info::Bool, show_usagetips::Bool=true) not_installed_indicator = sprint((io, args) -> printstyled(io, args...; color=Base.error_color()), "→", context=io) upgradable_indicator = sprint((io, args) -> printstyled(io, args...; color=:green), "⌃", context=io) @@ -2129,7 +2159,6 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie lpadding = 2 package_statuses = PackageStatusData[] - installed_cache = Dict{Base.PkgId, Bool}() for (uuid, old, new) in xs if Types.is_project_uuid(env, uuid) continue @@ -2137,6 +2166,7 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie latest_version = true # Outdated info cinfo = nothing + glue_info = nothing if !isnothing(new) && !is_stdlib(new.uuid) cinfo = status_compat_info(new, env, registries) if cinfo !== nothing @@ -2148,6 +2178,17 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie continue end + if !isnothing(new) && !is_stdlib(new.uuid) + glue_info = status_glue_info(new, env) + end + + if glue && glue_info == nothing + continue + end + + + # TODO: Show glue deps for project as well + pkg_downloaded = !is_instantiated(new) || is_package_downloaded(env.project_file, new) new_ver_avail = !latest_version && !Operations.is_tracking_repo(new) && !Operations.is_tracking_path(new) @@ -2164,7 +2205,7 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie no_visible_packages_heldback &= (!changed || !pkg_heldback) no_packages_heldback &= !pkg_heldback - push!(package_statuses, PackageStatusData(uuid, old, new, pkg_downloaded, pkg_upgradable, pkg_heldback, cinfo, changed)) + push!(package_statuses, PackageStatusData(uuid, old, new, pkg_downloaded, pkg_upgradable, pkg_heldback, cinfo, changed, glue_info)) end for pkg in package_statuses @@ -2209,6 +2250,27 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie printstyled(io, pkg_str; color=Base.warn_color()) end end + + if glue && !diff && pkg.glueinfo !== nothing + println(io) + for (i, glue) in enumerate(pkg.glueinfo) + sym = i == length(pkg.glueinfo) ? '└' : '├' + function print_glue_entry(io, (name, installed)) + color = installed ? :light_green : :light_black + printstyled(io, name, ;color) + end + print(io, " ", sym, "─ ") + print_glue_entry(io, glue.gluepkg) + + print(io, " [") + join(io,sprint.(print_glue_entry, glue.gluedeps; context=io), ", ") + print(io, "]") + if i != length(pkg.glueinfo) + println(io) + end + end + end + println(io) end @@ -2263,7 +2325,7 @@ end function status(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}=PackageSpec[]; header=nothing, mode::PackageMode=PKGMODE_PROJECT, git_diff::Bool=false, env_diff=nothing, ignore_indent=true, - io::IO, outdated::Bool=false, hidden_upgrades_info::Bool=false, show_usagetips::Bool=true) + io::IO, outdated::Bool=false, glue::Bool=false, hidden_upgrades_info::Bool=false, show_usagetips::Bool=true) io == Base.devnull && return # if a package, print header if header === nothing && env.pkg !== nothing @@ -2290,10 +2352,10 @@ function status(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pk diff = old_env !== nothing header = something(header, diff ? :Diff : :Status) if mode == PKGMODE_PROJECT || mode == PKGMODE_COMBINED - print_status(env, old_env, registries, header, filter_uuids, filter_names; manifest=false, diff, ignore_indent, io, outdated, mode, hidden_upgrades_info, show_usagetips) + print_status(env, old_env, registries, header, filter_uuids, filter_names; manifest=false, diff, ignore_indent, io, outdated, glue, mode, hidden_upgrades_info, show_usagetips) end if mode == PKGMODE_MANIFEST || mode == PKGMODE_COMBINED - print_status(env, old_env, registries, header, filter_uuids, filter_names; diff, ignore_indent, io, outdated, mode, hidden_upgrades_info, show_usagetips) + print_status(env, old_env, registries, header, filter_uuids, filter_names; diff, ignore_indent, io, outdated, glue, mode, hidden_upgrades_info, show_usagetips) end if is_manifest_current(env) === false tip = show_usagetips ? " It is recommended to `Pkg.resolve()` or consider `Pkg.update()` if necessary." : "" diff --git a/src/Pkg.jl b/src/Pkg.jl index 8d166f4a13..76eebe7732 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -419,7 +419,7 @@ from packages that are tracking a path. const resolve = API.resolve """ - Pkg.status([pkgs...]; outdated::Bool=false, mode::PackageMode=PKGMODE_PROJECT, diff::Bool=false, compat::Bool=false, io::IO=stdout) + Pkg.status([pkgs...]; outdated::Bool=false, mode::PackageMode=PKGMODE_PROJECT, diff::Bool=false, compat::Bool=false, glue::Bool=false io::IO=stdout) Print out the status of the project/manifest. @@ -450,6 +450,9 @@ that are in the project (explicitly added). If `mode` is `PKGMODE_MANIFEST`, print status also about those in the manifest (recursive dependencies). If there are any packages listed as arguments, the output will be limited to those packages. +Setting `glue=true` will show dependencies with glue packages and what glue dependencies +of those that are currently loaded. + Setting `diff=true` will, if the environment is in a git repository, limit the output to the difference as compared to the last git commit. diff --git a/src/REPLMode/command_declarations.jl b/src/REPLMode/command_declarations.jl index 1c29bd6437..e3209b068e 100644 --- a/src/REPLMode/command_declarations.jl +++ b/src/REPLMode/command_declarations.jl @@ -376,6 +376,7 @@ PSA[:name => "status", PSA[:name => "diff", :short_name => "d", :api => :diff => true], PSA[:name => "outdated", :short_name => "o", :api => :outdated => true], PSA[:name => "compat", :short_name => "c", :api => :compat => true], + PSA[:name => "glue", :short_name => "g", :api => :glue => true], ], :completions => complete_installed_packages, :description => "summarize contents of and changes to environment", @@ -383,6 +384,8 @@ PSA[:name => "status", [st|status] [-d|--diff] [-o|--outdated] [pkgs...] [st|status] [-d|--diff] [-o|--outdated] [-p|--project] [pkgs...] [st|status] [-d|--diff] [-o|--outdated] [-m|--manifest] [pkgs...] + [st|status] [-d|--diff] [-g|--glue] [-p|--project] [pkgs...] + [st|status] [-d|--diff] [-g|--glue] [-m|--manifest] [pkgs...] [st|status] [-c|--compat] [pkgs...] Show the status of the current environment. Packages marked with `⌃` have new @@ -391,6 +394,9 @@ new versions available, but cannot be installed due to compatibility constraints. To see why use `pkg> status --outdated` which shows any packages that are not at their latest version and if any packages are holding them back. +Use `pkg> status --glue` to show dependencies with glue packages and what glue dependencies +of those that are currently loaded. + In `--project` mode (default), the status of the project file is summarized. In `--manifest` mode the output also includes the recursive dependencies of added packages given in the manifest. If there are any packages listed as arguments the output will be limited to those packages. diff --git a/test/gluedeps.jl b/test/gluedeps.jl index 3e251c402b..aaa2ae16dc 100644 --- a/test/gluedeps.jl +++ b/test/gluedeps.jl @@ -11,6 +11,7 @@ using Test Pkg.activate(; temp=true) Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasDepWithGluePkgs.jl")) Pkg.test("HasDepWithGluePkgs") + Pkg.status(; glue=true, io=IOBuffer()) # TODO: Test output end isolate(loaded_depot=true) do From e23c49e9e6cd6083c711d60dce0fea11f84843a2 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Fri, 25 Nov 2022 15:57:45 +0100 Subject: [PATCH 09/26] fixup precompile --- src/API.jl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/API.jl b/src/API.jl index 61409d3b9b..6b381909ca 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1106,14 +1106,22 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: deps = [Base.PkgId(last(x), first(x)) for x in last(dep).deps] depsmap[pkg] = filter!(!Base.in_sysimage, deps) # add any glue packages + gluedeps = last(dep).gluedeps for (gluepkg_name, gluedep_names) in last(dep).gluepkgs - gluepkg_deps = copy(deps) # depends on the deps of the parent package + gluepkg_deps = Base.PkgId[] # depends on the deps of the parent package push!(gluepkg_deps, pkg) # depends on parent package + all_gluedeps_available = true gluedep_names = gluedep_names isa String ? String[gluedep_names] : gluedep_names for gluedep_name in gluedep_names - gluedep = Base.identify_package(gluedep_name) - !isnothing(gluedep) && push!(gluepkg_deps, gluedep) + gluedep_uuid = gluedeps[gluedep_name] + if gluedep_uuid in keys(ctx.env.manifest.deps) + push!(gluepkg_deps, Base.PkgId(gluedep_uuid, gluedep_name)) + else + all_gluedeps_available = false + break + end end + all_gluedeps_available || continue gluepkg_uuid = Base.uuid5(pkg.uuid, gluepkg_name) gluepkg = Base.PkgId(gluepkg_uuid, gluepkg_name) push!(pkg_specs, PackageSpec(uuid = gluepkg_uuid, name = gluepkg_name)) # create this here as the name cannot be looked up easily later via the uuid From 941998b65eb32a4b141de19f46af7622faacbc85 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Fri, 25 Nov 2022 16:50:08 +0100 Subject: [PATCH 10/26] add tests for glue packages retrieved from a registry --- src/Operations.jl | 24 ++++++++++++++----- test/gluedeps.jl | 12 ++++++++++ .../H/HasDepWithGluePkgs/Compat.toml | 3 +++ .../H/HasDepWithGluePkgs/Deps.toml | 3 +++ .../H/HasDepWithGluePkgs/Package.toml | 4 ++++ .../H/HasDepWithGluePkgs/Versions.toml | 2 ++ .../GlueRegistry/H/HasGluePkgs/Compat.toml | 2 ++ .../GlueRegistry/H/HasGluePkgs/Deps.toml | 2 ++ .../H/HasGluePkgs/GlueCompat.toml | 3 +++ .../GlueRegistry/H/HasGluePkgs/GlueDeps.toml | 3 +++ .../GlueRegistry/H/HasGluePkgs/Package.toml | 4 ++++ .../GlueRegistry/H/HasGluePkgs/Versions.toml | 2 ++ .../GlueRegistry/Registry.toml | 6 +++++ 13 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Compat.toml create mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Deps.toml create mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Package.toml create mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Versions.toml create mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Compat.toml create mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Deps.toml create mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/GlueCompat.toml create mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/GlueDeps.toml create mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Package.toml create mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Versions.toml create mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/Registry.toml diff --git a/src/Operations.jl b/src/Operations.jl index b22e677cea..ce68499019 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -142,7 +142,19 @@ function update_manifest!(env::EnvCache, pkgs::Vector{PackageSpec}, deps_map, ju else entry.deps = deps_map[pkg.uuid] end + env.manifest[pkg.uuid] = entry + end + prune_manifest(env) + record_project_hash(env) +end + +# This has to be done after the packages have been downloaded +# since we need access to the Projet file to read the information +# about glue packages +function fixup_glue!(env, pkgs) + for pkg in pkgs v = joinpath(source_path(env.project_file, pkg), "Project.toml") + entry = env.manifest[pkg.uuid] if isfile(v) p = Types.read_project(v) entry.gluedeps = p.gluedeps @@ -151,13 +163,9 @@ function update_manifest!(env::EnvCache, pkgs::Vector{PackageSpec}, deps_map, ju delete!(entry.deps, name) end end - env.manifest[pkg.uuid] = entry end - prune_manifest(env) - record_project_hash(env) end - #################### # Registry Loading # #################### @@ -1314,7 +1322,8 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}(); pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version) update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new_apply = download_source(ctx) - + fixup_glue!(ctx.env, pkgs) + # After downloading resolutionary packages, search for (Julia)Artifacts.toml files # and ensure they are all downloaded and unpacked as well: download_artifacts(ctx.env, platform=platform, julia_version=ctx.julia_version, io=ctx.io) @@ -1336,6 +1345,7 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}, new_git::Set{UUID}; pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version) update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new_apply = download_source(ctx) + fixup_glue!(ctx.env, pkgs) download_artifacts(ctx.env; platform=platform, julia_version=ctx.julia_version, io=ctx.io) write_env(ctx.env) # write env before building show_update(ctx.env, ctx.registries; io=ctx.io) @@ -1463,6 +1473,7 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}, level::UpgradeLevel; end update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new_apply = download_source(ctx) + fixup_glue!(ctx.env, pkgs) download_artifacts(ctx.env, julia_version=ctx.julia_version, io=ctx.io) write_env(ctx.env; skip_writing_project) # write env before building show_update(ctx.env, ctx.registries; io=ctx.io, hidden_upgrades_info = true) @@ -1503,8 +1514,8 @@ function pin(ctx::Context, pkgs::Vector{PackageSpec}) pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, PRESERVE_TIERED, ctx.julia_version) update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) - new = download_source(ctx) + fixup_glue!(ctx.env, pkgs) download_artifacts(ctx.env; julia_version=ctx.julia_version, io=ctx.io) write_env(ctx.env) # write env before building show_update(ctx.env, ctx.registries; io=ctx.io) @@ -1546,6 +1557,7 @@ function free(ctx::Context, pkgs::Vector{PackageSpec}; err_if_free=true) update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new = download_source(ctx) + fixup_glue!(ctx.env, pkgs) download_artifacts(ctx.env, io=ctx.io) write_env(ctx.env) # write env before building show_update(ctx.env, ctx.registries; io=ctx.io) diff --git a/test/gluedeps.jl b/test/gluedeps.jl index aaa2ae16dc..8775551524 100644 --- a/test/gluedeps.jl +++ b/test/gluedeps.jl @@ -19,4 +19,16 @@ using Test Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasGluePkgs.jl")) @test_throws Pkg.Resolve.ResolverError Pkg.add(; name = "OffsetArrays", version = "0.9.0") end + + isolate(loaded_depot=false) do + depot = mktempdir(); empty!(DEPOT_PATH); push!(DEPOT_PATH, depot) + Pkg.activate(; temp=true) + Pkg.Registry.add(Pkg.RegistrySpec(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "GlueRegistry"))) + Pkg.Registry.add("General") + Pkg.add("HasGluePkgs") + Pkg.test("HasGluePkgs") + Pkg.add("HasDepWithGluePkgs") + Pkg.test("HasDepWithGluePkgs") + @test_throws Pkg.Resolve.ResolverError Pkg.add(; name = "OffsetArrays", version = "0.9.0") + end end diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Compat.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Compat.toml new file mode 100644 index 0000000000..bebba50133 --- /dev/null +++ b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Compat.toml @@ -0,0 +1,3 @@ +[0] +HasGluePkgs = "0.1" +OffsetArrays = "1.12.0-1" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Deps.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Deps.toml new file mode 100644 index 0000000000..6949deaaa8 --- /dev/null +++ b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Deps.toml @@ -0,0 +1,3 @@ +[0] +HasGluePkgs = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Package.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Package.toml new file mode 100644 index 0000000000..cfc83472eb --- /dev/null +++ b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Package.toml @@ -0,0 +1,4 @@ +name = "HasDepWithGluePkgs" +uuid = "d4ef3d4a-8e22-4710-85d8-c6cf2eb9efca" +repo = "https://github.com/KristofferC/GluePkgExamples.jl.git" +subdir = "HasDepWithGluePkgs.jl" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Versions.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Versions.toml new file mode 100644 index 0000000000..e3c08bf782 --- /dev/null +++ b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Versions.toml @@ -0,0 +1,2 @@ +["0.1.0"] +git-tree-sha1 = "c5337abfa2dc37e5b9dc967a93ffa05cfd642840" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Compat.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Compat.toml new file mode 100644 index 0000000000..6acf004c98 --- /dev/null +++ b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Compat.toml @@ -0,0 +1,2 @@ +[0] +Example = "0.5" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Deps.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Deps.toml new file mode 100644 index 0000000000..4a437a62fc --- /dev/null +++ b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Deps.toml @@ -0,0 +1,2 @@ +[0] +Example = "7876af07-990d-54b4-ab0e-23690620f79a" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/GlueCompat.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/GlueCompat.toml new file mode 100644 index 0000000000..7b6d607bfd --- /dev/null +++ b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/GlueCompat.toml @@ -0,0 +1,3 @@ +[0] +CSV = "0.10" +OffsetArrays = "1.12.0-1" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/GlueDeps.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/GlueDeps.toml new file mode 100644 index 0000000000..f6d210c744 --- /dev/null +++ b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/GlueDeps.toml @@ -0,0 +1,3 @@ +[0] +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Package.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Package.toml new file mode 100644 index 0000000000..8c4873fb49 --- /dev/null +++ b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Package.toml @@ -0,0 +1,4 @@ +name = "HasGluePkgs" +uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" +repo = "https://github.com/KristofferC/GluePkgExamples.jl.git" +subdir = "HasGluePkgs.jl" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Versions.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Versions.toml new file mode 100644 index 0000000000..41f9f13b02 --- /dev/null +++ b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Versions.toml @@ -0,0 +1,2 @@ +["0.1.0"] +git-tree-sha1 = "a854ae1bd07517256b369164c77d5f3d88451b3d" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/Registry.toml b/test/test_packages/GluePkgExamples/GlueRegistry/Registry.toml new file mode 100644 index 0000000000..32f7335209 --- /dev/null +++ b/test/test_packages/GluePkgExamples/GlueRegistry/Registry.toml @@ -0,0 +1,6 @@ +name = "GlueRegistry" +uuid = "c22930c5-f744-4ce3-9117-e610c805c832" + +[packages] +4d3288b3-3afc-4bb6-85f3-489fffe514c8 = { name = "HasGluePkgs", path = "H/HasGluePkgs" } +d4ef3d4a-8e22-4710-85d8-c6cf2eb9efca = { name = "HasDepWithGluePkgs", path = "H/HasDepWithGluePkgs" } From 2bfd2ab4e014660f077c7d7594dabeaa1aa92b72 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Fri, 25 Nov 2022 16:51:13 +0100 Subject: [PATCH 11/26] add small note --- docs/src/creating-packages.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index 969e201f37..db1f2ab3b1 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -297,7 +297,7 @@ end # module `glue/GlueContour.jl` (can also be in `glue/GlueContour/GlueContour.jl`): ```julia -module GlueContour +module GlueContour # Should be same name as the file (just like a normal package) using Plotting, Contour From abc9e210216b682825a04e78331a504d48d498d3 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 25 Nov 2022 19:14:43 +0100 Subject: [PATCH 12/26] Update docs/src/creating-packages.md Co-authored-by: Ian Butterworth --- docs/src/creating-packages.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index db1f2ab3b1..487d1d8f51 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -254,7 +254,7 @@ It is sometimes desirable to be able to extend some functionality of a package w unconditionally take on the cost (in terms of e.g. load time) of adding an extra dependency. A *glue package* is a file that gets automatically loaded when some other set of packages are loaded into the Julia session. This is very similar to functionality that the external package -Requires.jl used to provide, but which is now aviable directly through Julia. +Requires.jl used to provide, but which is now available directly through Julia. A useful application of glue packages could be for a plotting package that should be able to plot objects from a wide variety of different Julia packages. From 92bed4c1b0bb3005e08ab76a9d69b8799e077aa3 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 25 Nov 2022 19:15:18 +0100 Subject: [PATCH 13/26] Update src/API.jl Co-authored-by: Ian Butterworth --- src/API.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/API.jl b/src/API.jl index 6b381909ca..04dce688db 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1108,7 +1108,7 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: # add any glue packages gluedeps = last(dep).gluedeps for (gluepkg_name, gluedep_names) in last(dep).gluepkgs - gluepkg_deps = Base.PkgId[] # depends on the deps of the parent package + gluepkg_deps = Base.PkgId[] push!(gluepkg_deps, pkg) # depends on parent package all_gluedeps_available = true gluedep_names = gluedep_names isa String ? String[gluedep_names] : gluedep_names From 49cd28a1d497b78f58fe4e264c259409f08bfd76 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 25 Nov 2022 19:16:31 +0100 Subject: [PATCH 14/26] Update src/API.jl Co-authored-by: Ian Butterworth --- src/API.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/API.jl b/src/API.jl index 04dce688db..529e0d7776 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1196,8 +1196,6 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: end target = string(isempty(pkgs) ? "project" : join(pkgs, ", "), "...") - target = string(isempty(pkgs) ? "project" : join(pkgs, ", "), "...") - pkg_queue = Base.PkgId[] failed_deps = Dict{Base.PkgId, String}() skipped_deps = Base.PkgId[] From a9e5ffa0878eea8ef6d2e5eca779783787d0550f Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Fri, 25 Nov 2022 21:46:33 +0100 Subject: [PATCH 15/26] fixup fixup_glue --- src/Operations.jl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index ce68499019..ae045479dc 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -154,13 +154,15 @@ end function fixup_glue!(env, pkgs) for pkg in pkgs v = joinpath(source_path(env.project_file, pkg), "Project.toml") - entry = env.manifest[pkg.uuid] - if isfile(v) - p = Types.read_project(v) - entry.gluedeps = p.gluedeps - entry.gluepkgs = p.gluepkgs - for (name, _) in p.gluedeps - delete!(entry.deps, name) + if haskey(env.manifest, pkg.uuid) + entry = env.manifest[pkg.uuid] + if isfile(v) + p = Types.read_project(v) + entry.gluedeps = p.gluedeps + entry.gluepkgs = p.gluepkgs + for (name, _) in p.gluedeps + delete!(entry.deps, name) + end end end end From c52bf2763f4a0729956f6448f1563b5f8fb47076 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Fri, 25 Nov 2022 22:05:26 +0100 Subject: [PATCH 16/26] weak -> glue --- src/API.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/API.jl b/src/API.jl index 529e0d7776..1d266014bd 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1650,7 +1650,7 @@ function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode= if compat diff && pkgerror("Compat status has no `diff` mode") outdated && pkgerror("Compat status has no `outdated` mode") - weak && pkgerror("Compat status has no `weak` mode") + glue && pkgerror("Compat status has no `glue` mode") Operations.print_compat(ctx, pkgs; io) else Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated, glue) From f1a47ed18bd3a306e318143befe9466a5f5fd571 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Sun, 27 Nov 2022 13:08:22 +0100 Subject: [PATCH 17/26] fix tests passing when dep warnings are errors (as they are on Julia CI) --- test/gluedeps.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/gluedeps.jl b/test/gluedeps.jl index 8775551524..cd75132106 100644 --- a/test/gluedeps.jl +++ b/test/gluedeps.jl @@ -5,13 +5,17 @@ using Test isolate(loaded_depot=true) do Pkg.activate(; temp=true) Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasGluePkgs.jl")) - Pkg.test("HasGluePkgs") + Pkg.test("HasGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn end isolate(loaded_depot=true) do Pkg.activate(; temp=true) Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasDepWithGluePkgs.jl")) - Pkg.test("HasDepWithGluePkgs") - Pkg.status(; glue=true, io=IOBuffer()) # TODO: Test output + Pkg.test("HasDepWithGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn + io = IOBuffer() + Pkg.status(; glue=true, mode=Pkg.PKGMODE_MANIFEST, io) + # TODO: Test output when glue deps are loaded etc. + str = String(take!(io)) + @test contains(str, "└─ GlueOffsetArrays [OffsetArrays]" ) end isolate(loaded_depot=true) do @@ -26,9 +30,9 @@ using Test Pkg.Registry.add(Pkg.RegistrySpec(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "GlueRegistry"))) Pkg.Registry.add("General") Pkg.add("HasGluePkgs") - Pkg.test("HasGluePkgs") + Pkg.test("HasGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn Pkg.add("HasDepWithGluePkgs") - Pkg.test("HasDepWithGluePkgs") + Pkg.test("HasDepWithGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn @test_throws Pkg.Resolve.ResolverError Pkg.add(; name = "OffsetArrays", version = "0.9.0") end end From 169e1283db9be9a28508651c7db9593ef24be440 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Tue, 29 Nov 2022 16:35:06 +0100 Subject: [PATCH 18/26] gluedeps -> weakdeps --- docs/make.jl | 1 - docs/src/creating-packages.md | 51 ++++++++++++++----- src/API.jl | 4 +- src/Operations.jl | 44 ++++++++-------- src/Registry/registry_instance.jl | 48 ++++++++--------- src/Types.jl | 8 +-- src/manifest.jl | 14 ++--- src/project.jl | 14 ++--- test/gluepkgs.jl | 38 ++++++++++++++ test/runtests.jl | 2 +- .../H/HasDepWithGluePkgs/Compat.toml | 4 +- .../H/HasDepWithGluePkgs/Versions.toml | 4 +- .../GlueRegistry/H/HasGluePkgs/Versions.toml | 4 +- .../{GlueCompat.toml => WeakCompat.toml} | 0 .../{GlueDeps.toml => WeakDeps.toml} | 0 .../HasDepWithGluePkgs.jl/Manifest.toml | 2 +- .../HasGluePkgs.jl/Project.toml | 2 +- 17 files changed, 152 insertions(+), 88 deletions(-) create mode 100644 test/gluepkgs.jl rename test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/{GlueCompat.toml => WeakCompat.toml} (100%) rename test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/{GlueDeps.toml => WeakDeps.toml} (100%) diff --git a/docs/make.jl b/docs/make.jl index 163cb9e7bd..24b0c4244c 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -35,7 +35,6 @@ makedocs( "managing-packages.md", "environments.md", "creating-packages.md", - "gluedeps.md", "compatibility.md", "registries.md", "artifacts.md", diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index 487d1d8f51..cc9af0dc57 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -245,14 +245,32 @@ using Test Every dependency should in general have a compatibility constraint on it. This is an important topic so there is a separate chapter about it: [Compatibility](@ref Compatibility). +## Weak dependencies + +!!! note + This is a somewhat advanced usage of Pkg which can be skipped for people new to Julia and Julia packages. + +A weak dependency is a dependency that will not automatically install when the package is installed but +you can still control what versions of that package is allowed to install by setting compatibility on it. +These are listed in the project file under the `[weakdeps]` section: + +```toml +[weakdeps] +SomePackage = "b3785f31-9d33-4cdf-bc73-f646780f1739" + +[compat] +SomePackage = "1.2" +``` + +The current usage of this is almost solely limited to "glue packages" which is described in the next section. ## Conditional loading of code in packages (Glue packages) !!! note - This is a somewhat advanced section which can be skipped for people new to Julia and Julia packages. + This is a somewhat advanced usage of Pkg which can be skipped for people new to Julia and Julia packages. It is sometimes desirable to be able to extend some functionality of a package without having to -unconditionally take on the cost (in terms of e.g. load time) of adding an extra dependency. -A *glue package* is a file that gets automatically loaded when some other set of packages are +unconditionally take on the cost (in terms of e.g. load time) of adding a full dependency on that package. +A *glue package* is something resembling a package that gets automatically loaded when *some other set of packages* are loaded into the Julia session. This is very similar to functionality that the external package Requires.jl used to provide, but which is now available directly through Julia. @@ -260,10 +278,11 @@ A useful application of glue packages could be for a plotting package that shoul objects from a wide variety of different Julia packages. Adding all those different Julia packages as dependencies could be expensive since they would end up getting loaded even if they were never used. -Instead, these code required to plot objects for specific packages can be put into separate files -(glue packages) which is only loaded when +Instead, the code required to plot objects for specific packages can be put into separate files +(glue packages) and these are loaded only when the packages that defines the type we want to plot +are loaded. -Below is an example of how the code can be structured for a use case as outlined above. +Below is an example of how the code can be structured for a use case outlined above: `Project.toml`: ```toml @@ -271,7 +290,7 @@ name = "Plotting" version = "0.1.0" uuid = "..." -[gluedeps] +[weakdeps] Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" [gluepkgs] @@ -280,7 +299,7 @@ Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" # use a list for multiple glue dependencies GlueContour = "Contour" -[compat] # compat can also be given on glue dependencies +[compat] Contour = "0.6.2" ``` @@ -310,17 +329,23 @@ end # module A user that depends on `Plotting` will not pay the cost of the "glue code" inside the `GlueContour` module. It is only when the `Contour` package actually gets loaded that the `GlueCountour` glue package is loaded -and provide the new functionality. +and provides the new functionality. -Compatibility can be set on glue dependencies just like normal dependencies and they can be used in the `test` -target to make them available when testing. +If one considers `GlueContour` as a completely separate package, it could be argued that defining `Plotting.plot(c::Contour.ContourCollection)` is +type piracy since `GlueContour` does not "own" neigher the method `Plotting.plot` nor the type `Contour.ContourCollection`. +However, for glue packages, it is ok to assume that the glue package owns the methdos in its parent package. +In fact, this type of "type piracies" is one of the most standard use cases for glue packages. A glue package will only be loaded if the glue dependencies are loaded from the same environment or environments higher in the environment stack than the package itself. +!!! note + If you use a manifest generated by a Julia version that does not know about glue packages with a Julia version that does + know about them, the glue packages will not load. This is because the manifest lacks some information that tells Julia + when it should load these packages. So make sure you use a manifest generated at least the Julia version you are using. !!! compat In order to be compatible with earlier versions of Julia, some extra steps have to be taken. - These are: Duplicate the packages under `[gluedeps]` into `[extras]`. This is an unfortunate + These are: Duplicate the packages under `[weakdeps]` into `[extras]`. This is an unfortunate duplication but without doing this the project verifier under older Julia versions will complain (error). ### Backwards compatibility with Requires.jl @@ -334,7 +359,7 @@ This is done by making the following changes (using the example above): callback only when glue packages are not supported ```julia # This symbol is only defined on Julia versions that support glue packages - if isdefined(Base, :get_gluepkg) + if !isdefined(Base, :get_gluepkg) using Requires function __init__() @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../glue/GlueContour.jl) diff --git a/src/API.jl b/src/API.jl index 1d266014bd..3d303023d7 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1106,14 +1106,14 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: deps = [Base.PkgId(last(x), first(x)) for x in last(dep).deps] depsmap[pkg] = filter!(!Base.in_sysimage, deps) # add any glue packages - gluedeps = last(dep).gluedeps + weakdeps = last(dep).weakdeps for (gluepkg_name, gluedep_names) in last(dep).gluepkgs gluepkg_deps = Base.PkgId[] push!(gluepkg_deps, pkg) # depends on parent package all_gluedeps_available = true gluedep_names = gluedep_names isa String ? String[gluedep_names] : gluedep_names for gluedep_name in gluedep_names - gluedep_uuid = gluedeps[gluedep_name] + gluedep_uuid = weakdeps[gluedep_name] if gluedep_uuid in keys(ctx.env.manifest.deps) push!(gluepkg_deps, Base.PkgId(gluedep_uuid, gluedep_name)) else diff --git a/src/Operations.jl b/src/Operations.jl index ae045479dc..74b28e0d1a 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -158,9 +158,9 @@ function fixup_glue!(env, pkgs) entry = env.manifest[pkg.uuid] if isfile(v) p = Types.read_project(v) - entry.gluedeps = p.gluedeps + entry.weakdeps = p.weakdeps entry.gluepkgs = p.gluepkgs - for (name, _) in p.gluedeps + for (name, _) in p.weakdeps delete!(entry.deps, name) end end @@ -236,7 +236,7 @@ function collect_project(pkg::PackageSpec, path::String) vspec = get_compat(project, name) push!(deps, PackageSpec(name, uuid, vspec)) end - for (name, uuid) in project.gluedeps + for (name, uuid) in project.weakdeps vspec = get_compat(project, name) push!(deps, PackageSpec(name, uuid, vspec)) push!(glues, uuid) @@ -436,7 +436,7 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} # pkg -> version -> (dependency => compat): all_compat = Dict{UUID,Dict{VersionNumber,Dict{UUID,VersionSpec}}}() - glue_compat = Dict{UUID,Dict{VersionNumber,Set{UUID}}}() + weak_compat = Dict{UUID,Dict{VersionNumber,Set{UUID}}}() for (fp, fx) in fixed all_compat[fp] = Dict(fx.version => Dict{UUID,VersionSpec}()) @@ -449,7 +449,7 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} push!(seen, uuid) uuid in keys(fixed) && continue all_compat_u = get_or_make!(all_compat, uuid) - glue_compat_u = get_or_make!(glue_compat, uuid) + weak_compat_u = get_or_make!(weak_compat, uuid) uuid_is_stdlib = false stdlib_name = "" @@ -478,9 +478,9 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} all_compat_u_vr[other_uuid] = VersionSpec() end - if !isempty(proj.gluedeps) - glue_all_compat_u_vr = get_or_make!(glue_compat_u, v) - for (_, other_uuid) in proj.gluedeps + if !isempty(proj.weakdeps) + glue_all_compat_u_vr = get_or_make!(weak_compat_u, v) + for (_, other_uuid) in proj.weakdeps push!(uuids, other_uuid) all_compat_u_vr[other_uuid] = VersionSpec() push!(glue_all_compat_u_vr, other_uuid) @@ -520,12 +520,12 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} end end add_compat!(all_compat_u, Registry.compat_info(info)) - glue_compat_info = Registry.glue_compat_info(info) - if glue_compat_info !== nothing - add_compat!(all_compat_u, glue_compat_info) + weak_compat_info = Registry.weak_compat_info(info) + if weak_compat_info !== nothing + add_compat!(all_compat_u, weak_compat_info) # Version to Set - for (v, compat_info) in glue_compat_info - glue_compat_u[v] = keys(compat_info) + for (v, compat_info) in weak_compat_info + weak_compat_u[v] = keys(compat_info) end end end @@ -545,7 +545,7 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} end end - return Resolve.Graph(all_compat, glue_compat, uuid_to_name, reqs, fixed, false, julia_version), + return Resolve.Graph(all_compat, weak_compat, uuid_to_name, reqs, fixed, false, julia_version), all_compat end @@ -1179,7 +1179,7 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode::PackageMode) # only declare `compat` for remaining direct or `extra` dependencies # `julia` is always an implicit direct dependency filter!(ctx.env.project.compat) do (name, _) - name == "julia" || name in keys(ctx.env.project.deps) || name in keys(ctx.env.project.extras) || name in keys(ctx.env.project.gluedeps) + name == "julia" || name in keys(ctx.env.project.deps) || name in keys(ctx.env.project.extras) || name in keys(ctx.env.project.weakdeps) end deps_names = union(keys(ctx.env.project.deps), keys(ctx.env.project.extras)) filter!(ctx.env.project.targets) do (target, deps) @@ -1792,12 +1792,12 @@ function gen_target_project(ctx::Context, pkg::PackageSpec, source_path::String, # collect test dependencies for name in get(source_env.project.targets, target, String[]) uuid = nothing - for list in [source_env.project.extras, source_env.project.gluedeps] + for list in [source_env.project.extras, source_env.project.weakdeps] uuid = get(list, name, nothing) uuid === nothing || break end if uuid === nothing - pkgerror("`$name` declared as a `$target` dependency, but no such entry in `extras` or `gluedeps`") + pkgerror("`$name` declared as a `$target` dependency, but no such entry in `extras` or `weakdeps`") end test_project.deps[name] = uuid end @@ -2098,9 +2098,9 @@ function status_glue_info(pkg::PackageSpec, env::EnvCache) manifest = env.manifest manifest_info = get(manifest, pkg.uuid, nothing) manifest_info === nothing && return nothing - gluedepses = manifest_info.gluedeps + weakdepses = manifest_info.weakdeps gluepkgs = manifest_info.gluepkgs - if !isempty(gluedepses) && !isempty(gluepkgs) + if !isempty(weakdepses) && !isempty(gluepkgs) v = GlueInfo[] for (gluepkg, gluedeps) in gluepkgs gluedeps isa String && (gluedeps = String[gluedeps]) @@ -2108,7 +2108,7 @@ function status_glue_info(pkg::PackageSpec, env::EnvCache) # Check if deps are loaded gluedeps_info= Tuple{String, Bool}[] for gluedep in gluedeps - uuid = gluedepses[gluedep] + uuid = weakdepses[gluedep] loaded = haskey(Base.loaded_modules, Base.PkgId(uuid, gluedep)) push!(gluedeps_info, (gluedep, loaded)) end @@ -2121,7 +2121,7 @@ end struct GlueInfo gluepkg::Tuple{String, Bool} # name, loaded - gluedeps::Vector{Tuple{String, Bool}} # name, loaded + weakdeps::Vector{Tuple{String, Bool}} # name, loaded end struct PackageStatusData uuid::UUID @@ -2277,7 +2277,7 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie print_glue_entry(io, glue.gluepkg) print(io, " [") - join(io,sprint.(print_glue_entry, glue.gluedeps; context=io), ", ") + join(io,sprint.(print_glue_entry, glue.weakdeps; context=io), ", ") print(io, "]") if i != length(pkg.glueinfo) println(io) diff --git a/src/Registry/registry_instance.jl b/src/Registry/registry_instance.jl index 44825f7014..238ccf77ef 100644 --- a/src/Registry/registry_instance.jl +++ b/src/Registry/registry_instance.jl @@ -56,10 +56,10 @@ struct PkgInfo deps::Dict{VersionRange, Dict{String, UUID}} # WeakCompat.toml - glue_compat::Dict{VersionRange, Dict{String, VersionSpec}} + weak_compat::Dict{VersionRange, Dict{String, VersionSpec}} - # GlueDeps.toml - glue_deps::Dict{VersionRange, Dict{String, UUID}} + # WeakDeps.toml + weak_deps::Dict{VersionRange, Dict{String, UUID}} end isyanked(pkg::PkgInfo, v::VersionNumber) = pkg.version_info[v].yanked @@ -133,19 +133,19 @@ function initialize_glue_uncompressed!(pkg::PkgInfo, versions = keys(pkg.version sort!(versions) - glue_uncompressed_compat = uncompress(pkg.glue_compat, versions) - glue_uncompressed_deps = uncompress(pkg.glue_deps, versions) + glue_uncompressed_compat = uncompress(pkg.weak_compat, versions) + glue_uncompressed_deps = uncompress(pkg.weak_deps, versions) for v in versions vinfo = pkg.version_info[v] - glue_compat = Dict{UUID, VersionSpec}() + weak_compat = Dict{UUID, VersionSpec}() glue_uncompressed_deps_v = glue_uncompressed_deps[v] glue_uncompressed_compat_v = glue_uncompressed_compat[v] for (pkg, uuid) in glue_uncompressed_deps_v vspec = get(glue_uncompressed_compat_v, pkg, nothing) - glue_compat[uuid] = vspec === nothing ? VersionSpec() : vspec + weak_compat[uuid] = vspec === nothing ? VersionSpec() : vspec end - @init! vinfo.glue_uncompressed_compat = glue_compat + @init! vinfo.glue_uncompressed_compat = weak_compat end return pkg end @@ -155,8 +155,8 @@ function compat_info(pkg::PkgInfo) return Dict(v => info.uncompressed_compat for (v, info) in pkg.version_info) end -function glue_compat_info(pkg::PkgInfo) - if isempty(pkg.glue_deps) +function weak_compat_info(pkg::PkgInfo) + if isempty(pkg.weak_deps) return nothing end initialize_glue_uncompressed!(pkg) @@ -221,28 +221,28 @@ function init_package_info!(pkg::PkgEntry) deps[VersionRange()] = Dict("julia" => JULIA_UUID) # WeakCompat.toml - glue_compat_data_toml = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "WeakCompat.toml")) ? - parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "WeakCompat.toml")) : Dict{String, Any}() - glue_compat_data_toml = convert(Dict{String, Dict{String, Union{String, Vector{String}}}}, glue_compat_data_toml) - glue_compat = Dict{VersionRange, Dict{String, VersionSpec}}() - for (v, data) in glue_compat_data_toml + weak_compat_data_toml = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "WeakCompat.toml")) ? + parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "WeakCompat.toml")) : Dict{String, Any}() + weak_compat_data_toml = convert(Dict{String, Dict{String, Union{String, Vector{String}}}}, weak_compat_data_toml) + weak_compat = Dict{VersionRange, Dict{String, VersionSpec}}() + for (v, data) in weak_compat_data_toml vr = VersionRange(v) d = Dict{String, VersionSpec}(dep => VersionSpec(vr_dep) for (dep, vr_dep) in data) - glue_compat[vr] = d + weak_compat[vr] = d end - # GlueDeps.toml - glue_deps_data_toml = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "GlueDeps.toml")) ? - parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "GlueDeps.toml")) : Dict{String, Any}() - glue_deps_data_toml = convert(Dict{String, Dict{String, String}}, glue_deps_data_toml) - glue_deps = Dict{VersionRange, Dict{String, UUID}}() - for (v, data) in glue_deps_data_toml + # WeakDeps.toml + weak_deps_data_toml = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "WeakDeps.toml")) ? + parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "WeakDeps.toml")) : Dict{String, Any}() + weak_deps_data_toml = convert(Dict{String, Dict{String, String}}, weak_deps_data_toml) + weak_deps = Dict{VersionRange, Dict{String, UUID}}() + for (v, data) in weak_deps_data_toml vr = VersionRange(v) d = Dict{String, UUID}(dep => UUID(uuid) for (dep, uuid) in data) - glue_deps[vr] = d + weak_deps[vr] = d end - @init! pkg.info = PkgInfo(repo, subdir, version_info, compat, deps, glue_compat, glue_deps) + @init! pkg.info = PkgInfo(repo, subdir, version_info, compat, deps, weak_compat, weak_deps) return pkg.info end diff --git a/src/Types.jl b/src/Types.jl index 2e7ff22ff0..5233cf7dbc 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -239,7 +239,7 @@ Base.@kwdef mutable struct Project manifest::Union{String, Nothing} = nothing # Sections deps::Dict{String,UUID} = Dict{String,UUID}() - gluedeps::Dict{String,UUID} = Dict{String,UUID}() + weakdeps::Dict{String,UUID} = Dict{String,UUID}() gluepkgs::Dict{String,Union{Vector{String}, String}} = Dict{String,String}() extras::Dict{String,UUID} = Dict{String,UUID}() targets::Dict{String,Vector{String}} = Dict{String,Vector{String}}() @@ -264,7 +264,7 @@ Base.@kwdef mutable struct PackageEntry repo::GitRepo = GitRepo() tree_hash::Union{Nothing,SHA1} = nothing deps::Dict{String,UUID} = Dict{String,UUID}() - gluedeps::Dict{String,UUID} = Dict{String,UUID}() + weakdeps::Dict{String,UUID} = Dict{String,UUID}() gluepkgs::Dict{String,Union{Vector{String}, String}} = Dict{String,String}() uuid::Union{Nothing, UUID} = nothing other::Union{Dict,Nothing} = nothing @@ -276,11 +276,11 @@ Base.:(==)(t1::PackageEntry, t2::PackageEntry) = t1.name == t2.name && t1.repo == t2.repo && t1.tree_hash == t2.tree_hash && t1.deps == t2.deps && - t1.gluedeps == t2.gluedeps && + t1.weakdeps == t2.weakdeps && t1.gluepkgs == t2.gluepkgs && t1.uuid == t2.uuid # omits `other` -Base.hash(x::PackageEntry, h::UInt) = foldr(hash, [x.name, x.version, x.path, x.pinned, x.repo, x.tree_hash, x.deps, x.gluedeps,x.gluepkgs, x.uuid], init=h) # omits `other` +Base.hash(x::PackageEntry, h::UInt) = foldr(hash, [x.name, x.version, x.path, x.pinned, x.repo, x.tree_hash, x.deps, x.weakdeps,x.gluepkgs, x.uuid], init=h) # omits `other` Base.@kwdef mutable struct Manifest julia_version::Union{Nothing,VersionNumber} = nothing # only set to VERSION when resolving diff --git a/src/manifest.jl b/src/manifest.jl index 79521957a9..d7715917be 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -83,7 +83,7 @@ struct Stage1 uuid::UUID entry::PackageEntry deps::Union{Vector{String}, Dict{String,UUID}} - gluedeps::Union{Vector{String}, Dict{String,UUID}} + weakdeps::Union{Vector{String}, Dict{String,UUID}} end normalize_deps(name, uuid, deps, manifest; isglue=false) = deps @@ -114,7 +114,7 @@ function validate_manifest(julia_version::Union{Nothing,VersionNumber}, manifest info.entry.deps = normalize_deps(name, info.uuid, info.deps, stage1) end for (name, infos) in stage1, info in infos - info.entry.gluedeps = normalize_deps(name, info.uuid, info.gluedeps, stage1; isglue=true) + info.entry.weakdeps = normalize_deps(name, info.uuid, info.weakdeps, stage1; isglue=true) end # invariant: all dependencies are now normalized to Dict{String,UUID} deps = Dict{UUID, PackageEntry}() @@ -123,7 +123,7 @@ function validate_manifest(julia_version::Union{Nothing,VersionNumber}, manifest end # now just verify the graph structure for (entry_uuid, entry) in deps - for (deptype, isglue) in [(entry.deps, false), (entry.gluedeps, true)] + for (deptype, isglue) in [(entry.deps, false), (entry.weakdeps, true)] for (name, uuid) in deptype dep_entry = get(deps, uuid, nothing) if !isglue @@ -159,7 +159,7 @@ function Manifest(raw::Dict, f_or_io::Union{String, IO})::Manifest entry.name = name uuid = nothing deps = nothing - gluedeps = nothing + weakdeps = nothing try entry.pinned = read_pinned(get(info, "pinned", nothing)) uuid = read_field("uuid", nothing, info, safe_uuid)::UUID @@ -171,7 +171,7 @@ function Manifest(raw::Dict, f_or_io::Union{String, IO})::Manifest entry.tree_hash = read_field("git-tree-sha1", nothing, info, safe_SHA1) entry.uuid = uuid deps = read_deps(get(info::Dict, "deps", nothing)) - gluedeps = read_deps(get(info::Dict, "gluedeps", nothing)) + weakdeps = read_deps(get(info::Dict, "weakdeps", nothing)) entry.gluepkgs = get(Dict{String, String}, info::Dict, "gluepkgs") catch # TODO: Should probably not unconditionally log something @@ -179,7 +179,7 @@ function Manifest(raw::Dict, f_or_io::Union{String, IO})::Manifest rethrow() end entry.other = info::Union{Dict,Nothing} - stage1[name] = push!(get(stage1, name, Stage1[]), Stage1(uuid, entry, deps, gluedeps)) + stage1[name] = push!(get(stage1, name, Stage1[]), Stage1(uuid, entry, deps, weakdeps)) end # by this point, all the fields of the `PackageEntry`s have been type casted # but we have *not* verified the _graph_ structure of the manifest @@ -272,7 +272,7 @@ function destructure(manifest::Manifest)::Dict entry!(new_entry, "repo-url", repo_source) entry!(new_entry, "repo-rev", entry.repo.rev) entry!(new_entry, "repo-subdir", entry.repo.subdir) - for (deptype, depname) in [(entry.deps, "deps"), (entry.gluedeps, "gluedeps")] + for (deptype, depname) in [(entry.deps, "deps"), (entry.weakdeps, "weakdeps")] if isempty(deptype) delete!(new_entry, depname) else diff --git a/src/project.jl b/src/project.jl index 12c0f76c0a..93289eec60 100644 --- a/src/project.jl +++ b/src/project.jl @@ -2,7 +2,7 @@ # UTILS # ######### listed_deps(project::Project) = - append!(collect(keys(project.deps)), collect(keys(project.extras)), collect(keys(project.gluedeps))) + append!(collect(keys(project.deps)), collect(keys(project.extras)), collect(keys(project.weakdeps))) ########### # READING # @@ -80,7 +80,7 @@ function validate(project::Project; file=nothing) if length(dep_uuids) != length(unique(dep_uuids)) pkgerror("Two different dependencies can not have the same uuid" * location_string) end - glue_dep_uuids = collect(values(project.gluedeps)) + glue_dep_uuids = collect(values(project.weakdeps)) if length(glue_dep_uuids) != length(unique(glue_dep_uuids)) pkgerror("Two different glue dependencies can not have the same uuid" * location_string) end @@ -105,14 +105,14 @@ function validate(project::Project; file=nothing) pkgerror("A dependency was named twice in target `$target`") end dep in listed || pkgerror(""" - Dependency `$dep` in target `$target` not listed in `deps`, `gluedeps` or `extras` section + Dependency `$dep` in target `$target` not listed in `deps`, `weakdeps` or `extras` section """ * location_string) end # compat for (name, version) in project.compat name == "julia" && continue name in listed || - pkgerror("Compat `$name` not listed in `deps`, `gluedeps` or `extras` section" * location_string) + pkgerror("Compat `$name` not listed in `deps`, `weakdeps` or `extras` section" * location_string) end end @@ -124,7 +124,7 @@ function Project(raw::Dict; file=nothing) project.uuid = read_project_uuid(get(raw, "uuid", nothing)) project.version = read_project_version(get(raw, "version", nothing)) project.deps = read_project_deps(get(raw, "deps", nothing), "deps") - project.gluedeps = read_project_deps(get(raw, "gluedeps", nothing), "gluedeps") + project.weakdeps = read_project_deps(get(raw, "weakdeps", nothing), "weakdeps") project.gluepkgs = get(Dict{String, String}, raw, "gluepkgs") project.extras = read_project_deps(get(raw, "extras", nothing), "extras") project.compat = read_project_compat(get(raw, "compat", nothing), project) @@ -175,14 +175,14 @@ function destructure(project::Project)::Dict entry!("version", project.version) entry!("manifest", project.manifest) entry!("deps", project.deps) - entry!("gluedeps", project.gluedeps) + entry!("weakdeps", project.weakdeps) entry!("extras", project.extras) entry!("compat", Dict(name => x.str for (name, x) in project.compat)) entry!("targets", project.targets) return raw end -_project_key_order = ["name", "uuid", "keywords", "license", "desc", "deps", "gluedeps", "gluepkgs", "compat"] +_project_key_order = ["name", "uuid", "keywords", "license", "desc", "deps", "weakdeps", "gluepkgs", "compat"] project_key_order(key::String) = something(findfirst(x -> x == key, _project_key_order), length(_project_key_order) + 1) diff --git a/test/gluepkgs.jl b/test/gluepkgs.jl new file mode 100644 index 0000000000..cd75132106 --- /dev/null +++ b/test/gluepkgs.jl @@ -0,0 +1,38 @@ +using .Utils +using Test + +@testset "weak deps" begin + isolate(loaded_depot=true) do + Pkg.activate(; temp=true) + Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasGluePkgs.jl")) + Pkg.test("HasGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn + end + isolate(loaded_depot=true) do + Pkg.activate(; temp=true) + Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasDepWithGluePkgs.jl")) + Pkg.test("HasDepWithGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn + io = IOBuffer() + Pkg.status(; glue=true, mode=Pkg.PKGMODE_MANIFEST, io) + # TODO: Test output when glue deps are loaded etc. + str = String(take!(io)) + @test contains(str, "└─ GlueOffsetArrays [OffsetArrays]" ) + end + + isolate(loaded_depot=true) do + Pkg.activate(; temp=true) + Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasGluePkgs.jl")) + @test_throws Pkg.Resolve.ResolverError Pkg.add(; name = "OffsetArrays", version = "0.9.0") + end + + isolate(loaded_depot=false) do + depot = mktempdir(); empty!(DEPOT_PATH); push!(DEPOT_PATH, depot) + Pkg.activate(; temp=true) + Pkg.Registry.add(Pkg.RegistrySpec(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "GlueRegistry"))) + Pkg.Registry.add("General") + Pkg.add("HasGluePkgs") + Pkg.test("HasGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn + Pkg.add("HasDepWithGluePkgs") + Pkg.test("HasDepWithGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn + @test_throws Pkg.Resolve.ResolverError Pkg.add(; name = "OffsetArrays", version = "0.9.0") + end +end diff --git a/test/runtests.jl b/test/runtests.jl index a75bced3a6..51aaf0598c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -66,7 +66,7 @@ Logging.with_logger(hide_logs ? Logging.NullLogger() : Logging.current_logger()) "api.jl", "registry.jl", "subdir.jl", - "gluedeps.jl", + "gluepkgs.jl", "artifacts.jl", "binaryplatforms.jl", "platformengines.jl", diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Compat.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Compat.toml index bebba50133..2fa6d914ac 100644 --- a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Compat.toml +++ b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Compat.toml @@ -1,3 +1,5 @@ [0] -HasGluePkgs = "0.1" OffsetArrays = "1.12.0-1" + +["0.2-0"] +HasGluePkgs = "0.2" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Versions.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Versions.toml index e3c08bf782..d500a7c476 100644 --- a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Versions.toml +++ b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Versions.toml @@ -1,2 +1,2 @@ -["0.1.0"] -git-tree-sha1 = "c5337abfa2dc37e5b9dc967a93ffa05cfd642840" +["0.2.0"] +git-tree-sha1 = "5fbd46cdb98eb4c8873a042142e69fc21e77c061" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Versions.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Versions.toml index 41f9f13b02..b0ab2eecbc 100644 --- a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Versions.toml +++ b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Versions.toml @@ -1,2 +1,2 @@ -["0.1.0"] -git-tree-sha1 = "a854ae1bd07517256b369164c77d5f3d88451b3d" +["0.2.0"] +git-tree-sha1 = "3edae1d3d2002fa0d0187bcd2010fd7939bed591" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/GlueCompat.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/WeakCompat.toml similarity index 100% rename from test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/GlueCompat.toml rename to test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/WeakCompat.toml diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/GlueDeps.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/WeakDeps.toml similarity index 100% rename from test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/GlueDeps.toml rename to test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/WeakDeps.toml diff --git a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml index 4627434d62..ea93f6fb5b 100644 --- a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml +++ b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml @@ -28,7 +28,7 @@ deps = ["Example"] path = "../HasGluePkgs.jl" uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" version = "0.1.0" -gluedeps = ["OffsetArrays"] +weakdeps = ["OffsetArrays"] [deps.HasGluePkgs.gluepkgs] GlueOffsetArrays = "OffsetArrays" diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml index d3b8260a17..b6c8de9092 100644 --- a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml +++ b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml @@ -5,7 +5,7 @@ version = "0.1.0" [deps] Example = "7876af07-990d-54b4-ab0e-23690620f79a" -[gluedeps] +[weakdeps] OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" [gluepkgs] From fa6dbfe40516e14fb4d682861706ce9d8808e3af Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Wed, 30 Nov 2022 20:41:19 +0100 Subject: [PATCH 19/26] ignore packages both in weakdeps and deps for backwards compat --- docs/src/creating-packages.md | 60 ++++++++++++++++++++++++++--------- src/Operations.jl | 4 ++- src/Types.jl | 4 +++ src/project.jl | 5 +++ test/gluedeps.jl | 38 ---------------------- 5 files changed, 57 insertions(+), 54 deletions(-) delete mode 100644 test/gluedeps.jl diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index cc9af0dc57..b2741e49b8 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -263,6 +263,7 @@ SomePackage = "1.2" ``` The current usage of this is almost solely limited to "glue packages" which is described in the next section. + ## Conditional loading of code in packages (Glue packages) !!! note @@ -297,7 +298,7 @@ Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" # name of glue package to the left # glue dependencies required to load the glue pkg to the right # use a list for multiple glue dependencies -GlueContour = "Contour" +ContourGlue = "Contour" [compat] Contour = "0.6.2" @@ -314,9 +315,9 @@ end end # module ``` -`glue/GlueContour.jl` (can also be in `glue/GlueContour/GlueContour.jl`): +`glue/ContourGlue.jl` (can also be in `glue/ContourGlue/ContourGlue.jl`): ```julia -module GlueContour # Should be same name as the file (just like a normal package) +module ContourGlue # Should be same name as the file (just like a normal package) using Plotting, Contour @@ -327,12 +328,12 @@ end end # module ``` -A user that depends on `Plotting` will not pay the cost of the "glue code" inside the `GlueContour` module. +A user that depends on `Plotting` will not pay the cost of the "glue code" inside the `ContourGlue` module. It is only when the `Contour` package actually gets loaded that the `GlueCountour` glue package is loaded and provides the new functionality. -If one considers `GlueContour` as a completely separate package, it could be argued that defining `Plotting.plot(c::Contour.ContourCollection)` is -type piracy since `GlueContour` does not "own" neigher the method `Plotting.plot` nor the type `Contour.ContourCollection`. +If one considers `ContourGlue` as a completely separate package, it could be argued that defining `Plotting.plot(c::Contour.ContourCollection)` is +type piracy since `ContourGlue` does not "own" neigher the method `Plotting.plot` nor the type `Contour.ContourCollection`. However, for glue packages, it is ok to assume that the glue package owns the methdos in its parent package. In fact, this type of "type piracies" is one of the most standard use cases for glue packages. @@ -343,16 +344,13 @@ A glue package will only be loaded if the glue dependencies are loaded from the know about them, the glue packages will not load. This is because the manifest lacks some information that tells Julia when it should load these packages. So make sure you use a manifest generated at least the Julia version you are using. -!!! compat - In order to be compatible with earlier versions of Julia, some extra steps have to be taken. - These are: Duplicate the packages under `[weakdeps]` into `[extras]`. This is an unfortunate - duplication but without doing this the project verifier under older Julia versions will complain (error). +### Backwards compatibility -### Backwards compatibility with Requires.jl +This section discusses various methods for using glue packages on Julia versions that support them, +while simultaneously providing similar functionality on older Julia versions. +#### Requires.jl -Since glue packages and Requires.jl are solving the same problem, -it is useful to know how to use Requires.jl on older Julia versions while seamlessly -transitioning into using glue packages on newer Julia versions that support it. +This section is relevant if you are currently using Requries.jl but want to transition to using glue packages (while still having Requires be used on Julia versions that do not support glue packages). This is done by making the following changes (using the example above): - Add the following to the package file. This makes it so that Requires.jl loads and inserts the @@ -362,11 +360,24 @@ This is done by making the following changes (using the example above): if !isdefined(Base, :get_gluepkg) using Requires function __init__() - @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../glue/GlueContour.jl) + @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../glue/ContourGlue.jl) end end ``` + or if you have other things in your `__init__()` function: + ```julia + if !isdefined(Base, :get_gluepkg) + using Requires + end + function __init__() + # Other init functionality here + + @static if !isdefined(Base, :get_gluepkg) + @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../glue/ContourGlue.jl) + end + end + ``` - Do the following change in the glue packages for loading the glue dependency: ```julia isdefined(Base, :get_gluepkg) ? (using Contour) : (using ..Contour) @@ -375,6 +386,25 @@ This is done by making the following changes (using the example above): The package should now work with Requires.jl on Julia versions before glue packages were introduced and with glue packages afterward. +#### Transition from normal dependency to glue dependency + +This section is relevant if you have a normal dependency that you want to transition be a glue packages (while still having the dependency be a normal dependency on Julia versions that do not support glue packages). +This is done by making the following changes (using the example above): + +- Make sure that the package is **both** in the `[deps]` and `[weakdeps]` section. Newer Julia versions will ignore dependencis in `[deps]` that are also in `[weakdeps]`. +- Add the following to your main package file (typically at the bottom): + ```julia + if !isdefined(Base, :get_gluepkg) + include("../glue/ContourGlue.jl") + end + ``` + +#### Using a glue dependency while supporting older Julia version + +If you want to use use a glue dependency with compatibility constraints while supporting earlier Julia +versions you have to duplicate the packages under `[weakdeps]` into `[extras]`. This is an unfortunate +duplication but without doing this the project verifier under older Julia versions will complain (error). + ## Package naming guidelines Package names should be sensible to most Julia users, *even to those who are not domain experts*. diff --git a/src/Operations.jl b/src/Operations.jl index 74b28e0d1a..e0ea422a29 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -161,7 +161,9 @@ function fixup_glue!(env, pkgs) entry.weakdeps = p.weakdeps entry.gluepkgs = p.gluepkgs for (name, _) in p.weakdeps - delete!(entry.deps, name) + if !haskey(p.deps, name) + delete!(entry.deps, name) + end end end end diff --git a/src/Types.jl b/src/Types.jl index 5233cf7dbc..ebaa7866a2 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -239,6 +239,10 @@ Base.@kwdef mutable struct Project manifest::Union{String, Nothing} = nothing # Sections deps::Dict{String,UUID} = Dict{String,UUID}() + # deps that are also in weakdeps for backwards compat + # we do not store them in deps because we want to ignore them + # but for writing out the project file we need to remember them: + _deps_weak::Dict{String,UUID} = Dict{String,UUID}() weakdeps::Dict{String,UUID} = Dict{String,UUID}() gluepkgs::Dict{String,Union{Vector{String}, String}} = Dict{String,String}() extras::Dict{String,UUID} = Dict{String,UUID}() diff --git a/src/project.jl b/src/project.jl index 93289eec60..bfb94cc505 100644 --- a/src/project.jl +++ b/src/project.jl @@ -129,6 +129,10 @@ function Project(raw::Dict; file=nothing) project.extras = read_project_deps(get(raw, "extras", nothing), "extras") project.compat = read_project_compat(get(raw, "compat", nothing), project) project.targets = read_project_targets(get(raw, "targets", nothing), project) + + # Handle deps in both [deps] and [weakdeps] + project._deps_weak = Dict(intersect(project.deps, project.weakdeps)) + filter!(p->!haskey(project._deps_weak, p.first), project.deps) validate(project; file) return project end @@ -174,6 +178,7 @@ function destructure(project::Project)::Dict entry!("uuid", project.uuid) entry!("version", project.version) entry!("manifest", project.manifest) + merge!(project.deps, project._deps_weak) entry!("deps", project.deps) entry!("weakdeps", project.weakdeps) entry!("extras", project.extras) diff --git a/test/gluedeps.jl b/test/gluedeps.jl deleted file mode 100644 index cd75132106..0000000000 --- a/test/gluedeps.jl +++ /dev/null @@ -1,38 +0,0 @@ -using .Utils -using Test - -@testset "weak deps" begin - isolate(loaded_depot=true) do - Pkg.activate(; temp=true) - Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasGluePkgs.jl")) - Pkg.test("HasGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn - end - isolate(loaded_depot=true) do - Pkg.activate(; temp=true) - Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasDepWithGluePkgs.jl")) - Pkg.test("HasDepWithGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn - io = IOBuffer() - Pkg.status(; glue=true, mode=Pkg.PKGMODE_MANIFEST, io) - # TODO: Test output when glue deps are loaded etc. - str = String(take!(io)) - @test contains(str, "└─ GlueOffsetArrays [OffsetArrays]" ) - end - - isolate(loaded_depot=true) do - Pkg.activate(; temp=true) - Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasGluePkgs.jl")) - @test_throws Pkg.Resolve.ResolverError Pkg.add(; name = "OffsetArrays", version = "0.9.0") - end - - isolate(loaded_depot=false) do - depot = mktempdir(); empty!(DEPOT_PATH); push!(DEPOT_PATH, depot) - Pkg.activate(; temp=true) - Pkg.Registry.add(Pkg.RegistrySpec(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "GlueRegistry"))) - Pkg.Registry.add("General") - Pkg.add("HasGluePkgs") - Pkg.test("HasGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn - Pkg.add("HasDepWithGluePkgs") - Pkg.test("HasDepWithGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn - @test_throws Pkg.Resolve.ResolverError Pkg.add(; name = "OffsetArrays", version = "0.9.0") - end -end From abe7000540f370d1b0ba1d4ce85d78f44dbac48e Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Sat, 3 Dec 2022 22:29:03 +0200 Subject: [PATCH 20/26] glue -> extension --- docs/src/creating-packages.md | 87 ++++++++------- src/API.jl | 46 ++++---- src/Operations.jl | 102 +++++++++--------- src/Pkg.jl | 4 +- src/REPLMode/command_declarations.jl | 8 +- src/Registry/registry_instance.jl | 24 ++--- src/Resolve/graphtype.jl | 6 +- src/Types.jl | 8 +- src/manifest.jl | 18 ++-- src/project.jl | 10 +- test/extensions.jl | 38 +++++++ test/gluepkgs.jl | 38 ------- test/runtests.jl | 2 +- .../H/HasDepWithExtensions/Compat.toml | 3 + .../H/HasDepWithExtensions/Deps.toml | 3 + .../H/HasDepWithExtensions}/Package.toml | 4 +- .../H/HasDepWithExtensions/Versions.toml | 2 + .../H/HasExtensions/Compat.toml | 3 + .../H/HasExtensions}/Deps.toml | 2 +- .../H/HasExtensions}/Package.toml | 4 +- .../H/HasExtensions/Versions.toml | 2 + .../H/HasExtensions}/WeakCompat.toml | 1 - .../H/HasExtensions}/WeakDeps.toml | 1 - .../ExtensionRegistry/Registry.toml | 7 ++ .../HasDepWithExtensions.jl}/Manifest.toml | 10 +- .../HasDepWithExtensions.jl}/Project.toml | 6 +- .../src/HasDepWithExtensions.jl} | 10 +- .../HasDepWithExtensions.jl/test/runtests.jl | 5 + .../HasExtensions.jl}/Manifest.toml | 0 .../HasExtensions.jl}/Project.toml | 6 +- .../HasExtensions.jl/ext/OffsetArraysExt.jl | 13 +++ .../HasExtensions.jl/src/HasExtensions.jl} | 2 +- .../HasExtensions.jl/test/runtests.jl | 8 ++ .../H/HasDepWithGluePkgs/Compat.toml | 5 - .../H/HasDepWithGluePkgs/Versions.toml | 2 - .../GlueRegistry/H/HasGluePkgs/Compat.toml | 2 - .../GlueRegistry/H/HasGluePkgs/Deps.toml | 2 - .../GlueRegistry/H/HasGluePkgs/Versions.toml | 2 - .../GlueRegistry/Registry.toml | 6 -- .../HasDepWithGluePkgs.jl/test/runtests.jl | 5 - .../HasGluePkgs.jl/glue/GlueOffsetArrays.jl | 13 --- .../HasGluePkgs.jl/test/runtests.jl | 8 -- 42 files changed, 266 insertions(+), 262 deletions(-) create mode 100644 test/extensions.jl delete mode 100644 test/gluepkgs.jl create mode 100644 test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Compat.toml create mode 100644 test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Deps.toml rename test/test_packages/{GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs => ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions}/Package.toml (62%) create mode 100644 test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Versions.toml create mode 100644 test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/Compat.toml rename test/test_packages/{GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs => ExtensionExamples/ExtensionRegistry/H/HasExtensions}/Deps.toml (52%) rename test/test_packages/{GluePkgExamples/GlueRegistry/H/HasGluePkgs => ExtensionExamples/ExtensionRegistry/H/HasExtensions}/Package.toml (68%) create mode 100644 test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/Versions.toml rename test/test_packages/{GluePkgExamples/GlueRegistry/H/HasGluePkgs => ExtensionExamples/ExtensionRegistry/H/HasExtensions}/WeakCompat.toml (69%) rename test/test_packages/{GluePkgExamples/GlueRegistry/H/HasGluePkgs => ExtensionExamples/ExtensionRegistry/H/HasExtensions}/WeakDeps.toml (56%) create mode 100644 test/test_packages/ExtensionExamples/ExtensionRegistry/Registry.toml rename test/test_packages/{GluePkgExamples/HasDepWithGluePkgs.jl => ExtensionExamples/HasDepWithExtensions.jl}/Manifest.toml (88%) rename test/test_packages/{GluePkgExamples/HasDepWithGluePkgs.jl => ExtensionExamples/HasDepWithExtensions.jl}/Project.toml (69%) rename test/test_packages/{GluePkgExamples/HasDepWithGluePkgs.jl/src/HasDepWithGluePkgs.jl => ExtensionExamples/HasDepWithExtensions.jl/src/HasDepWithExtensions.jl} (50%) create mode 100644 test/test_packages/ExtensionExamples/HasDepWithExtensions.jl/test/runtests.jl rename test/test_packages/{GluePkgExamples/HasGluePkgs.jl => ExtensionExamples/HasExtensions.jl}/Manifest.toml (100%) rename test/test_packages/{GluePkgExamples/HasGluePkgs.jl => ExtensionExamples/HasExtensions.jl}/Project.toml (82%) create mode 100644 test/test_packages/ExtensionExamples/HasExtensions.jl/ext/OffsetArraysExt.jl rename test/test_packages/{GluePkgExamples/HasGluePkgs.jl/src/HasGluePkgs.jl => ExtensionExamples/HasExtensions.jl/src/HasExtensions.jl} (83%) create mode 100644 test/test_packages/ExtensionExamples/HasExtensions.jl/test/runtests.jl delete mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Compat.toml delete mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Versions.toml delete mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Compat.toml delete mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Deps.toml delete mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Versions.toml delete mode 100644 test/test_packages/GluePkgExamples/GlueRegistry/Registry.toml delete mode 100644 test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/test/runtests.jl delete mode 100644 test/test_packages/GluePkgExamples/HasGluePkgs.jl/glue/GlueOffsetArrays.jl delete mode 100644 test/test_packages/GluePkgExamples/HasGluePkgs.jl/test/runtests.jl diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index b2741e49b8..caf6a5c7db 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -262,25 +262,25 @@ SomePackage = "b3785f31-9d33-4cdf-bc73-f646780f1739" SomePackage = "1.2" ``` -The current usage of this is almost solely limited to "glue packages" which is described in the next section. +The current usage of this is almost solely limited to "extensions" which is described in the next section. -## Conditional loading of code in packages (Glue packages) +## Conditional loading of code in packages (Extensions) !!! note This is a somewhat advanced usage of Pkg which can be skipped for people new to Julia and Julia packages. It is sometimes desirable to be able to extend some functionality of a package without having to unconditionally take on the cost (in terms of e.g. load time) of adding a full dependency on that package. -A *glue package* is something resembling a package that gets automatically loaded when *some other set of packages* are +A package *extension* is a module in a file (similar to a package) that is automatically loaded when *some other set of packages* are loaded into the Julia session. This is very similar to functionality that the external package -Requires.jl used to provide, but which is now available directly through Julia. +Requires.jl provides, but which is now available directly through Julia. -A useful application of glue packages could be for a plotting package that should be able to plot +A useful application of extensions could be for a plotting package that should be able to plot objects from a wide variety of different Julia packages. Adding all those different Julia packages as dependencies could be expensive since they would end up getting loaded even if they were never used. Instead, the code required to plot objects for specific packages can be put into separate files -(glue packages) and these are loaded only when the packages that defines the type we want to plot +(extensions) and these are loaded only when the packages that defines the type we want to plot are loaded. Below is an example of how the code can be structured for a use case outlined above: @@ -294,11 +294,11 @@ uuid = "..." [weakdeps] Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" -[gluepkgs] -# name of glue package to the left -# glue dependencies required to load the glue pkg to the right -# use a list for multiple glue dependencies -ContourGlue = "Contour" +[extensions] +# name of extension to the left +# extension dependencies required to load the extension to the right +# use a list for multiple extension dependencies +ContourExt = "Contour" [compat] Contour = "0.6.2" @@ -315,9 +315,9 @@ end end # module ``` -`glue/ContourGlue.jl` (can also be in `glue/ContourGlue/ContourGlue.jl`): +`ext/ContourExt.jl` (can also be in `ext/ContourExt/ContourExt.jl`): ```julia -module ContourGlue # Should be same name as the file (just like a normal package) +module ContourExt # Should be same name as the file (just like a normal package) using Plotting, Contour @@ -328,80 +328,85 @@ end end # module ``` -A user that depends on `Plotting` will not pay the cost of the "glue code" inside the `ContourGlue` module. -It is only when the `Contour` package actually gets loaded that the `GlueCountour` glue package is loaded +A user that depends on `Plotting` will not pay the cost of the "extension" inside the `ContourExt` module. +It is only when the `Contour` package actually gets loaded that the `ContourExt` extension is loaded and provides the new functionality. -If one considers `ContourGlue` as a completely separate package, it could be argued that defining `Plotting.plot(c::Contour.ContourCollection)` is -type piracy since `ContourGlue` does not "own" neigher the method `Plotting.plot` nor the type `Contour.ContourCollection`. -However, for glue packages, it is ok to assume that the glue package owns the methdos in its parent package. -In fact, this type of "type piracies" is one of the most standard use cases for glue packages. +If one considers `ContourExt` as a completely separate package, it could be argued that defining `Plotting.plot(c::Contour.ContourCollection)` is +type piracy since `ContourExt` does not "own" neigher the method `Plotting.plot` nor the type `Contour.ContourCollection`. +However, for extensions, it is ok to assume that the extension owns the methods in its parent package. +In fact, this type of "type piracies" is one of the most standard use cases for extensions. -A glue package will only be loaded if the glue dependencies are loaded from the same environment or environments higher in the environment stack than the package itself. +An extension will only be loaded if the extension dependencies are loaded from the same environment or environments higher in the environment stack than the package itself. + +!!! compat + Often you will put the extension dependencies into the `test` target so they are loaded when running e.g. `Pkg.test()`. On earlier Julia versions + this requires you to also put the package in the `[extras]` section. This is unfortunate but the project verifier on older Julia versions will + complain if this is not done. !!! note - If you use a manifest generated by a Julia version that does not know about glue packages with a Julia version that does - know about them, the glue packages will not load. This is because the manifest lacks some information that tells Julia + If you use a manifest generated by a Julia version that does not know about extensions with a Julia version that does + know about them, the extensions will not load. This is because the manifest lacks some information that tells Julia when it should load these packages. So make sure you use a manifest generated at least the Julia version you are using. ### Backwards compatibility -This section discusses various methods for using glue packages on Julia versions that support them, +This section discusses various methods for using extensions on Julia versions that support them, while simultaneously providing similar functionality on older Julia versions. #### Requires.jl -This section is relevant if you are currently using Requries.jl but want to transition to using glue packages (while still having Requires be used on Julia versions that do not support glue packages). +This section is relevant if you are currently using Requries.jl but want to transition to using extensions (while still having Requires be used on Julia versions that do not support extensions). This is done by making the following changes (using the example above): - Add the following to the package file. This makes it so that Requires.jl loads and inserts the - callback only when glue packages are not supported + callback only when extensions are not supported ```julia - # This symbol is only defined on Julia versions that support glue packages - if !isdefined(Base, :get_gluepkg) + # This symbol is only defined on Julia versions that support extensions + if !isdefined(Base, :get_extension) using Requires function __init__() - @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../glue/ContourGlue.jl) + @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../ext/ContourExt.jl) end end ``` or if you have other things in your `__init__()` function: ```julia - if !isdefined(Base, :get_gluepkg) + if !isdefined(Base, :get_extension) using Requires end function __init__() # Other init functionality here - @static if !isdefined(Base, :get_gluepkg) - @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../glue/ContourGlue.jl) + @static if !isdefined(Base, :get_extension) + @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../ext/ContourExt.jl) end end ``` -- Do the following change in the glue packages for loading the glue dependency: +- Do the following change in the extensions for loading the extension dependency: ```julia - isdefined(Base, :get_gluepkg) ? (using Contour) : (using ..Contour) + isdefined(Base, :get_extension) ? (using Contour) : (using ..Contour) ``` -The package should now work with Requires.jl on Julia versions before glue packages were introduced -and with glue packages afterward. +The package should now work with Requires.jl on Julia versions before extensions were introduced +and with extensions afterward. -#### Transition from normal dependency to glue dependency +#### Transition from normal dependency to extension -This section is relevant if you have a normal dependency that you want to transition be a glue packages (while still having the dependency be a normal dependency on Julia versions that do not support glue packages). +This section is relevant if you have a normal dependency that you want to transition be an extension (while still having the dependency be a normal dependency on Julia versions that do not support extensions). This is done by making the following changes (using the example above): - Make sure that the package is **both** in the `[deps]` and `[weakdeps]` section. Newer Julia versions will ignore dependencis in `[deps]` that are also in `[weakdeps]`. - Add the following to your main package file (typically at the bottom): ```julia - if !isdefined(Base, :get_gluepkg) - include("../glue/ContourGlue.jl") + if !isdefined(Base, :get_extension) + include("../ext/ContourExt.jl") end ``` -#### Using a glue dependency while supporting older Julia version +#### Using an extension while supporting older Julia version -If you want to use use a glue dependency with compatibility constraints while supporting earlier Julia +If you want to use use an extension with compatibility constraints while supporting earlier Julia versions you have to duplicate the packages under `[weakdeps]` into `[extras]`. This is an unfortunate duplication but without doing this the project verifier under older Julia versions will complain (error). diff --git a/src/API.jl b/src/API.jl index 3d303023d7..a8816e531a 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1096,7 +1096,7 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: for (name, uuid) in ctx.env.project.deps if !Base.in_sysimage(Base.PkgId(uuid, name)) ] - gluepkgs = Dict{Base.PkgId, String}() # gluepkg -> parent + exts = Dict{Base.PkgId, String}() # ext -> parent # make a flat map of each dep and its deps depsmap = Dict{Base.PkgId, Vector{Base.PkgId}}() pkg_specs = PackageSpec[] @@ -1105,28 +1105,28 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: Base.in_sysimage(pkg) && continue deps = [Base.PkgId(last(x), first(x)) for x in last(dep).deps] depsmap[pkg] = filter!(!Base.in_sysimage, deps) - # add any glue packages + # add any extensions weakdeps = last(dep).weakdeps - for (gluepkg_name, gluedep_names) in last(dep).gluepkgs - gluepkg_deps = Base.PkgId[] - push!(gluepkg_deps, pkg) # depends on parent package - all_gluedeps_available = true - gluedep_names = gluedep_names isa String ? String[gluedep_names] : gluedep_names - for gluedep_name in gluedep_names - gluedep_uuid = weakdeps[gluedep_name] - if gluedep_uuid in keys(ctx.env.manifest.deps) - push!(gluepkg_deps, Base.PkgId(gluedep_uuid, gluedep_name)) + for (ext_name, extdep_names) in last(dep).exts + ext_deps = Base.PkgId[] + push!(ext_deps, pkg) # depends on parent package + all_extdeps_available = true + extdep_names = extdep_names isa String ? String[extdep_names] : extdep_names + for extdep_name in extdep_names + extdep_uuid = weakdeps[extdep_name] + if extdep_uuid in keys(ctx.env.manifest.deps) + push!(ext_deps, Base.PkgId(extdep_uuid, extdep_name)) else - all_gluedeps_available = false + all_extdeps_available = false break end end - all_gluedeps_available || continue - gluepkg_uuid = Base.uuid5(pkg.uuid, gluepkg_name) - gluepkg = Base.PkgId(gluepkg_uuid, gluepkg_name) - push!(pkg_specs, PackageSpec(uuid = gluepkg_uuid, name = gluepkg_name)) # create this here as the name cannot be looked up easily later via the uuid - depsmap[gluepkg] = filter!(!Base.in_sysimage, gluepkg_deps) - gluepkgs[gluepkg] = pkg.name + all_extdeps_available || continue + ext_uuid = Base.uuid5(pkg.uuid, ext_name) + ext = Base.PkgId(ext_uuid, ext_name) + push!(pkg_specs, PackageSpec(uuid = ext_uuid, name = ext_name)) # create this here as the name cannot be looked up easily later via the uuid + depsmap[ext] = filter!(!Base.in_sysimage, ext_deps) + exts[ext] = pkg.name end end @@ -1268,7 +1268,7 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: final_loop || print(iostr, sprint(io -> show_progress(io, bar; termwidth = displaysize(ctx.io)[2]); context=io), "\n") for dep in pkg_queue_show loaded = warn_loaded && haskey(Base.loaded_modules, dep) - _name = haskey(gluepkgs, dep) ? string(gluepkgs[dep], " → ", dep.name) : dep.name + _name = haskey(exts, dep) ? string(exts[dep], " → ", dep.name) : dep.name name = dep in direct_deps ? _name : string(color_string(_name, :light_black)) if dep in precomperr_deps print(iostr, color_string(" ? ", Base.warn_color()), name, "\n") @@ -1343,7 +1343,7 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call:: Base.acquire(parallel_limiter) is_direct_dep = pkg in direct_deps iob = IOBuffer() - _name = haskey(gluepkgs, pkg) ? string(gluepkgs[pkg], " → ", pkg.name) : pkg.name + _name = haskey(exts, pkg) ? string(exts[pkg], " → ", pkg.name) : pkg.name name = is_direct_dep ? _name : string(color_string(_name, :light_black)) !fancyprint && lock(print_lock) do isempty(pkg_queue) && printpkgstyle(io, :Precompiling, target) @@ -1646,14 +1646,14 @@ end @deprecate status(mode::PackageMode) status(mode=mode) -function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=PKGMODE_PROJECT, outdated::Bool=false, compat::Bool=false, glue::Bool=false, io::IO=stdout_f(), kwargs...) +function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=PKGMODE_PROJECT, outdated::Bool=false, compat::Bool=false, ext::Bool=false, io::IO=stdout_f(), kwargs...) if compat diff && pkgerror("Compat status has no `diff` mode") outdated && pkgerror("Compat status has no `outdated` mode") - glue && pkgerror("Compat status has no `glue` mode") + ext && pkgerror("Compat status has no `ext` mode") Operations.print_compat(ctx, pkgs; io) else - Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated, glue) + Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated, ext) end return nothing end diff --git a/src/Operations.jl b/src/Operations.jl index e0ea422a29..56c01363b9 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -150,8 +150,8 @@ end # This has to be done after the packages have been downloaded # since we need access to the Projet file to read the information -# about glue packages -function fixup_glue!(env, pkgs) +# about extensions +function fixup_ext!(env, pkgs) for pkg in pkgs v = joinpath(source_path(env.project_file, pkg), "Project.toml") if haskey(env.manifest, pkg.uuid) @@ -159,7 +159,7 @@ function fixup_glue!(env, pkgs) if isfile(v) p = Types.read_project(v) entry.weakdeps = p.weakdeps - entry.gluepkgs = p.gluepkgs + entry.exts = p.exts for (name, _) in p.weakdeps if !haskey(p.deps, name) delete!(entry.deps, name) @@ -221,7 +221,7 @@ end function collect_project(pkg::PackageSpec, path::String) deps = PackageSpec[] - glues = Set{UUID}() + weakdeps = Set{UUID}() project_file = projectfile_path(path; strict=true) if project_file === nothing pkgerror("could not find project file for package $(err_rep(pkg)) at `$path`") @@ -241,7 +241,7 @@ function collect_project(pkg::PackageSpec, path::String) for (name, uuid) in project.weakdeps vspec = get_compat(project, name) push!(deps, PackageSpec(name, uuid, vspec)) - push!(glues, uuid) + push!(weakdeps, uuid) end if project.version !== nothing pkg.version = project.version @@ -249,7 +249,7 @@ function collect_project(pkg::PackageSpec, path::String) # @warn("project file for $(pkg.name) is missing a `version` entry") pkg.version = VersionNumber(0) end - return deps, glues + return deps, weakdeps end is_tracking_path(pkg) = pkg.path !== nothing @@ -284,12 +284,12 @@ end function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UUID, String}) deps_map = Dict{UUID,Vector{PackageSpec}}() - glue_map = Dict{UUID,Set{UUID}}() + weak_map = Dict{UUID,Set{UUID}}() if env.pkg !== nothing pkg = env.pkg - deps, glues = collect_project(pkg, dirname(env.project_file)) + deps, weakdeps = collect_project(pkg, dirname(env.project_file)) deps_map[pkg.uuid] = deps - glue_map[pkg.uuid] = glues + weak_map[pkg.uuid] = weakdeps names[pkg.uuid] = pkg.name end for pkg in pkgs @@ -297,9 +297,9 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU if !isdir(path) pkgerror("expected package $(err_rep(pkg)) to exist at path `$path`") end - deps, glues = collect_project(pkg, path) + deps, weakdeps = collect_project(pkg, path) deps_map[pkg.uuid] = deps - glue_map[pkg.uuid] = glues + weak_map[pkg.uuid] = weakdeps end fixed = Dict{UUID,Resolve.Fixed}() @@ -316,7 +316,7 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU idx = findfirst(pkg -> pkg.uuid == uuid, pkgs) fix_pkg = pkgs[idx] end - fixed[uuid] = Resolve.Fixed(fix_pkg.version, q, glue_map[uuid]) + fixed[uuid] = Resolve.Fixed(fix_pkg.version, q, weak_map[uuid]) end return fixed end @@ -481,11 +481,11 @@ function deps_graph(env::EnvCache, registries::Vector{Registry.RegistryInstance} end if !isempty(proj.weakdeps) - glue_all_compat_u_vr = get_or_make!(weak_compat_u, v) + weak_all_compat_u_vr = get_or_make!(weak_compat_u, v) for (_, other_uuid) in proj.weakdeps push!(uuids, other_uuid) all_compat_u_vr[other_uuid] = VersionSpec() - push!(glue_all_compat_u_vr, other_uuid) + push!(weak_all_compat_u_vr, other_uuid) end end else @@ -1326,7 +1326,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}(); pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version) update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new_apply = download_source(ctx) - fixup_glue!(ctx.env, pkgs) + fixup_ext!(ctx.env, pkgs) # After downloading resolutionary packages, search for (Julia)Artifacts.toml files # and ensure they are all downloaded and unpacked as well: @@ -1349,7 +1349,7 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}, new_git::Set{UUID}; pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version) update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new_apply = download_source(ctx) - fixup_glue!(ctx.env, pkgs) + fixup_ext!(ctx.env, pkgs) download_artifacts(ctx.env; platform=platform, julia_version=ctx.julia_version, io=ctx.io) write_env(ctx.env) # write env before building show_update(ctx.env, ctx.registries; io=ctx.io) @@ -1477,7 +1477,7 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}, level::UpgradeLevel; end update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new_apply = download_source(ctx) - fixup_glue!(ctx.env, pkgs) + fixup_ext!(ctx.env, pkgs) download_artifacts(ctx.env, julia_version=ctx.julia_version, io=ctx.io) write_env(ctx.env; skip_writing_project) # write env before building show_update(ctx.env, ctx.registries; io=ctx.io, hidden_upgrades_info = true) @@ -1519,7 +1519,7 @@ function pin(ctx::Context, pkgs::Vector{PackageSpec}) pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, PRESERVE_TIERED, ctx.julia_version) update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new = download_source(ctx) - fixup_glue!(ctx.env, pkgs) + fixup_ext!(ctx.env, pkgs) download_artifacts(ctx.env; julia_version=ctx.julia_version, io=ctx.io) write_env(ctx.env) # write env before building show_update(ctx.env, ctx.registries; io=ctx.io) @@ -1561,7 +1561,7 @@ function free(ctx::Context, pkgs::Vector{PackageSpec}; err_if_free=true) update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new = download_source(ctx) - fixup_glue!(ctx.env, pkgs) + fixup_ext!(ctx.env, pkgs) download_artifacts(ctx.env, io=ctx.io) write_env(ctx.env) # write env before building show_update(ctx.env, ctx.registries; io=ctx.io) @@ -2096,33 +2096,33 @@ function is_package_downloaded(project_file::String, pkg::PackageSpec; platform= return true end -function status_glue_info(pkg::PackageSpec, env::EnvCache) +function status_ext_info(pkg::PackageSpec, env::EnvCache) manifest = env.manifest manifest_info = get(manifest, pkg.uuid, nothing) manifest_info === nothing && return nothing weakdepses = manifest_info.weakdeps - gluepkgs = manifest_info.gluepkgs - if !isempty(weakdepses) && !isempty(gluepkgs) - v = GlueInfo[] - for (gluepkg, gluedeps) in gluepkgs - gluedeps isa String && (gluedeps = String[gluedeps]) - gluepkg_loaded = (Base.get_gluepkg(Base.PkgId(pkg.uuid, pkg.name), Symbol(gluepkg)) !== nothing) + exts = manifest_info.exts + if !isempty(weakdepses) && !isempty(exts) + v = ExtInfo[] + for (ext, extdeps) in exts + extdeps isa String && (extdeps = String[extdeps]) + ext_loaded = (Base.get_extension(Base.PkgId(pkg.uuid, pkg.name), Symbol(ext)) !== nothing) # Check if deps are loaded - gluedeps_info= Tuple{String, Bool}[] - for gluedep in gluedeps - uuid = weakdepses[gluedep] - loaded = haskey(Base.loaded_modules, Base.PkgId(uuid, gluedep)) - push!(gluedeps_info, (gluedep, loaded)) + extdeps_info= Tuple{String, Bool}[] + for extdep in extdeps + uuid = weakdepses[extdep] + loaded = haskey(Base.loaded_modules, Base.PkgId(uuid, extdep)) + push!(extdeps_info, (extdep, loaded)) end - push!(v, GlueInfo((gluepkg, gluepkg_loaded), gluedeps_info)) + push!(v, ExtInfo((ext, ext_loaded), extdeps_info)) end return v end return nothing end -struct GlueInfo - gluepkg::Tuple{String, Bool} # name, loaded +struct ExtInfo + ext::Tuple{String, Bool} # name, loaded weakdeps::Vector{Tuple{String, Bool}} # name, loaded end struct PackageStatusData @@ -2134,11 +2134,11 @@ struct PackageStatusData heldback::Bool compat_data::Union{Nothing, Tuple{Vector{String}, VersionNumber, VersionNumber}} changed::Bool - glueinfo::Union{Nothing, Vector{GlueInfo}} + extinfo::Union{Nothing, Vector{ExtInfo}} end function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registries::Vector{Registry.RegistryInstance}, header::Symbol, - uuids::Vector, names::Vector; manifest=true, diff=false, ignore_indent::Bool, outdated::Bool, glue::Bool, io::IO, + uuids::Vector, names::Vector; manifest=true, diff=false, ignore_indent::Bool, outdated::Bool, ext::Bool, io::IO, mode::PackageMode, hidden_upgrades_info::Bool, show_usagetips::Bool=true) not_installed_indicator = sprint((io, args) -> printstyled(io, args...; color=Base.error_color()), "→", context=io) upgradable_indicator = sprint((io, args) -> printstyled(io, args...; color=:green), "⌃", context=io) @@ -2182,7 +2182,7 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie latest_version = true # Outdated info cinfo = nothing - glue_info = nothing + ext_info = nothing if !isnothing(new) && !is_stdlib(new.uuid) cinfo = status_compat_info(new, env, registries) if cinfo !== nothing @@ -2195,15 +2195,15 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie end if !isnothing(new) && !is_stdlib(new.uuid) - glue_info = status_glue_info(new, env) + ext_info = status_ext_info(new, env) end - if glue && glue_info == nothing + if ext && ext_info == nothing continue end - # TODO: Show glue deps for project as well + # TODO: Show extension deps for project as well? pkg_downloaded = !is_instantiated(new) || is_package_downloaded(env.project_file, new) @@ -2221,7 +2221,7 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie no_visible_packages_heldback &= (!changed || !pkg_heldback) no_packages_heldback &= !pkg_heldback - push!(package_statuses, PackageStatusData(uuid, old, new, pkg_downloaded, pkg_upgradable, pkg_heldback, cinfo, changed, glue_info)) + push!(package_statuses, PackageStatusData(uuid, old, new, pkg_downloaded, pkg_upgradable, pkg_heldback, cinfo, changed, ext_info)) end for pkg in package_statuses @@ -2267,21 +2267,21 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie end end - if glue && !diff && pkg.glueinfo !== nothing + if ext && !diff && pkg.extinfo !== nothing println(io) - for (i, glue) in enumerate(pkg.glueinfo) - sym = i == length(pkg.glueinfo) ? '└' : '├' - function print_glue_entry(io, (name, installed)) + for (i, ext) in enumerate(pkg.extinfo) + sym = i == length(pkg.extinfo) ? '└' : '├' + function print_ext_entry(io, (name, installed)) color = installed ? :light_green : :light_black printstyled(io, name, ;color) end print(io, " ", sym, "─ ") - print_glue_entry(io, glue.gluepkg) + print_ext_entry(io, ext.ext) print(io, " [") - join(io,sprint.(print_glue_entry, glue.weakdeps; context=io), ", ") + join(io,sprint.(print_ext_entry, ext.weakdeps; context=io), ", ") print(io, "]") - if i != length(pkg.glueinfo) + if i != length(pkg.extinfo) println(io) end end @@ -2341,7 +2341,7 @@ end function status(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}=PackageSpec[]; header=nothing, mode::PackageMode=PKGMODE_PROJECT, git_diff::Bool=false, env_diff=nothing, ignore_indent=true, - io::IO, outdated::Bool=false, glue::Bool=false, hidden_upgrades_info::Bool=false, show_usagetips::Bool=true) + io::IO, outdated::Bool=false, ext::Bool=false, hidden_upgrades_info::Bool=false, show_usagetips::Bool=true) io == Base.devnull && return # if a package, print header if header === nothing && env.pkg !== nothing @@ -2368,10 +2368,10 @@ function status(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pk diff = old_env !== nothing header = something(header, diff ? :Diff : :Status) if mode == PKGMODE_PROJECT || mode == PKGMODE_COMBINED - print_status(env, old_env, registries, header, filter_uuids, filter_names; manifest=false, diff, ignore_indent, io, outdated, glue, mode, hidden_upgrades_info, show_usagetips) + print_status(env, old_env, registries, header, filter_uuids, filter_names; manifest=false, diff, ignore_indent, io, outdated, ext, mode, hidden_upgrades_info, show_usagetips) end if mode == PKGMODE_MANIFEST || mode == PKGMODE_COMBINED - print_status(env, old_env, registries, header, filter_uuids, filter_names; diff, ignore_indent, io, outdated, glue, mode, hidden_upgrades_info, show_usagetips) + print_status(env, old_env, registries, header, filter_uuids, filter_names; diff, ignore_indent, io, outdated, ext, mode, hidden_upgrades_info, show_usagetips) end if is_manifest_current(env) === false tip = show_usagetips ? " It is recommended to `Pkg.resolve()` or consider `Pkg.update()` if necessary." : "" diff --git a/src/Pkg.jl b/src/Pkg.jl index 76eebe7732..dc72a1e668 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -419,7 +419,7 @@ from packages that are tracking a path. const resolve = API.resolve """ - Pkg.status([pkgs...]; outdated::Bool=false, mode::PackageMode=PKGMODE_PROJECT, diff::Bool=false, compat::Bool=false, glue::Bool=false io::IO=stdout) + Pkg.status([pkgs...]; outdated::Bool=false, mode::PackageMode=PKGMODE_PROJECT, diff::Bool=false, compat::Bool=false, ext::Bool=false io::IO=stdout) Print out the status of the project/manifest. @@ -450,7 +450,7 @@ that are in the project (explicitly added). If `mode` is `PKGMODE_MANIFEST`, print status also about those in the manifest (recursive dependencies). If there are any packages listed as arguments, the output will be limited to those packages. -Setting `glue=true` will show dependencies with glue packages and what glue dependencies +Setting `ext=true` will show dependencies with extensions and what extension dependencies of those that are currently loaded. Setting `diff=true` will, if the environment is in a git repository, limit diff --git a/src/REPLMode/command_declarations.jl b/src/REPLMode/command_declarations.jl index e3209b068e..5f17fc509e 100644 --- a/src/REPLMode/command_declarations.jl +++ b/src/REPLMode/command_declarations.jl @@ -376,7 +376,7 @@ PSA[:name => "status", PSA[:name => "diff", :short_name => "d", :api => :diff => true], PSA[:name => "outdated", :short_name => "o", :api => :outdated => true], PSA[:name => "compat", :short_name => "c", :api => :compat => true], - PSA[:name => "glue", :short_name => "g", :api => :glue => true], + PSA[:name => "extensions", :short_name => "e", :api => :Ext => true], ], :completions => complete_installed_packages, :description => "summarize contents of and changes to environment", @@ -384,8 +384,8 @@ PSA[:name => "status", [st|status] [-d|--diff] [-o|--outdated] [pkgs...] [st|status] [-d|--diff] [-o|--outdated] [-p|--project] [pkgs...] [st|status] [-d|--diff] [-o|--outdated] [-m|--manifest] [pkgs...] - [st|status] [-d|--diff] [-g|--glue] [-p|--project] [pkgs...] - [st|status] [-d|--diff] [-g|--glue] [-m|--manifest] [pkgs...] + [st|status] [-d|--diff] [-g|--extensions] [-p|--project] [pkgs...] + [st|status] [-d|--diff] [-g|--extensions] [-m|--manifest] [pkgs...] [st|status] [-c|--compat] [pkgs...] Show the status of the current environment. Packages marked with `⌃` have new @@ -394,7 +394,7 @@ new versions available, but cannot be installed due to compatibility constraints. To see why use `pkg> status --outdated` which shows any packages that are not at their latest version and if any packages are holding them back. -Use `pkg> status --glue` to show dependencies with glue packages and what glue dependencies +Use `pkg> status --extensions` to show dependencies with extensions and what extension dependencies of those that are currently loaded. In `--project` mode (default), the status of the project file is summarized. In `--manifest` diff --git a/src/Registry/registry_instance.jl b/src/Registry/registry_instance.jl index 238ccf77ef..bf4d9fba1e 100644 --- a/src/Registry/registry_instance.jl +++ b/src/Registry/registry_instance.jl @@ -36,7 +36,7 @@ custom_isfile(in_memory_registry::Union{Dict, Nothing}, folder::AbstractString, git_tree_sha1::Base.SHA1 yanked::Bool @lazy uncompressed_compat::Union{Dict{UUID, VersionSpec}} - @lazy glue_uncompressed_compat::Union{Dict{UUID, VersionSpec}} + @lazy weak_uncompressed_compat::Union{Dict{UUID, VersionSpec}} end VersionInfo(git_tree_sha1::Base.SHA1, yanked::Bool) = VersionInfo(git_tree_sha1, yanked, uninit, uninit) @@ -126,26 +126,26 @@ function initialize_uncompressed!(pkg::PkgInfo, versions = keys(pkg.version_info return pkg end -function initialize_glue_uncompressed!(pkg::PkgInfo, versions = keys(pkg.version_info)) +function initialize_weak_uncompressed!(pkg::PkgInfo, versions = keys(pkg.version_info)) # Only valid to call this with existing versions of the package # Remove all versions we have already uncompressed - versions = filter!(v -> !isinit(pkg.version_info[v], :glue_uncompressed_compat), collect(versions)) + versions = filter!(v -> !isinit(pkg.version_info[v], :weak_uncompressed_compat), collect(versions)) sort!(versions) - glue_uncompressed_compat = uncompress(pkg.weak_compat, versions) - glue_uncompressed_deps = uncompress(pkg.weak_deps, versions) + weak_uncompressed_compat = uncompress(pkg.weak_compat, versions) + weak_uncompressed_deps = uncompress(pkg.weak_deps, versions) for v in versions vinfo = pkg.version_info[v] weak_compat = Dict{UUID, VersionSpec}() - glue_uncompressed_deps_v = glue_uncompressed_deps[v] - glue_uncompressed_compat_v = glue_uncompressed_compat[v] - for (pkg, uuid) in glue_uncompressed_deps_v - vspec = get(glue_uncompressed_compat_v, pkg, nothing) + weak_uncompressed_deps_v = weak_uncompressed_deps[v] + weak_uncompressed_compat_v = weak_uncompressed_compat[v] + for (pkg, uuid) in weak_uncompressed_deps_v + vspec = get(weak_uncompressed_compat_v, pkg, nothing) weak_compat[uuid] = vspec === nothing ? VersionSpec() : vspec end - @init! vinfo.glue_uncompressed_compat = weak_compat + @init! vinfo.weak_uncompressed_compat = weak_compat end return pkg end @@ -159,8 +159,8 @@ function weak_compat_info(pkg::PkgInfo) if isempty(pkg.weak_deps) return nothing end - initialize_glue_uncompressed!(pkg) - return Dict(v => info.glue_uncompressed_compat for (v, info) in pkg.version_info) + initialize_weak_uncompressed!(pkg) + return Dict(v => info.weak_uncompressed_compat for (v, info) in pkg.version_info) end @lazy struct PkgEntry diff --git a/src/Resolve/graphtype.jl b/src/Resolve/graphtype.jl index 94d2d1882b..efe5593048 100644 --- a/src/Resolve/graphtype.jl +++ b/src/Resolve/graphtype.jl @@ -236,7 +236,7 @@ mutable struct Graph function Graph( compat::Dict{UUID,Dict{VersionNumber,Dict{UUID,VersionSpec}}}, - compat_glue::Dict{UUID,Dict{VersionNumber,Set{UUID}}}, + compat_weak::Dict{UUID,Dict{VersionNumber,Set{UUID}}}, uuid_to_name::Dict{UUID,String}, reqs::Requires, fixed::Dict{UUID,Fixed}, @@ -274,14 +274,14 @@ mutable struct Graph # Translate the requirements into bit masks # Hot code, measure performance before changing req_msk = Dict{Int,BitVector}() - maybe_weak = haskey(compat_glue, uuid0) && haskey(compat_glue[uuid0], vn) + maybe_weak = haskey(compat_weak, uuid0) && haskey(compat_weak[uuid0], vn) for (p1, vs) in req pv = pvers[p1] req_msk_p1 = BitVector(undef, spp[p1]) @inbounds for i in 1:spp[p1] - 1 req_msk_p1[i] = pv[i] ∈ vs end - weak = maybe_weak && (pkgs[p1] ∈ compat_glue[uuid0][vn]) + weak = maybe_weak && (pkgs[p1] ∈ compat_weak[uuid0][vn]) req_msk_p1[end] = weak req_msk[p1] = req_msk_p1 end diff --git a/src/Types.jl b/src/Types.jl index ebaa7866a2..5d1d6987f8 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -244,7 +244,7 @@ Base.@kwdef mutable struct Project # but for writing out the project file we need to remember them: _deps_weak::Dict{String,UUID} = Dict{String,UUID}() weakdeps::Dict{String,UUID} = Dict{String,UUID}() - gluepkgs::Dict{String,Union{Vector{String}, String}} = Dict{String,String}() + exts::Dict{String,Union{Vector{String}, String}} = Dict{String,String}() extras::Dict{String,UUID} = Dict{String,UUID}() targets::Dict{String,Vector{String}} = Dict{String,Vector{String}}() compat::Dict{String,Compat} = Dict{String,Compat}() @@ -269,7 +269,7 @@ Base.@kwdef mutable struct PackageEntry tree_hash::Union{Nothing,SHA1} = nothing deps::Dict{String,UUID} = Dict{String,UUID}() weakdeps::Dict{String,UUID} = Dict{String,UUID}() - gluepkgs::Dict{String,Union{Vector{String}, String}} = Dict{String,String}() + exts::Dict{String,Union{Vector{String}, String}} = Dict{String,String}() uuid::Union{Nothing, UUID} = nothing other::Union{Dict,Nothing} = nothing end @@ -281,10 +281,10 @@ Base.:(==)(t1::PackageEntry, t2::PackageEntry) = t1.name == t2.name && t1.tree_hash == t2.tree_hash && t1.deps == t2.deps && t1.weakdeps == t2.weakdeps && - t1.gluepkgs == t2.gluepkgs && + t1.exts == t2.exts && t1.uuid == t2.uuid # omits `other` -Base.hash(x::PackageEntry, h::UInt) = foldr(hash, [x.name, x.version, x.path, x.pinned, x.repo, x.tree_hash, x.deps, x.weakdeps,x.gluepkgs, x.uuid], init=h) # omits `other` +Base.hash(x::PackageEntry, h::UInt) = foldr(hash, [x.name, x.version, x.path, x.pinned, x.repo, x.tree_hash, x.deps, x.weakdeps,x.exts, x.uuid], init=h) # omits `other` Base.@kwdef mutable struct Manifest julia_version::Union{Nothing,VersionNumber} = nothing # only set to VERSION when resolving diff --git a/src/manifest.jl b/src/manifest.jl index d7715917be..ff25782126 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -86,15 +86,15 @@ struct Stage1 weakdeps::Union{Vector{String}, Dict{String,UUID}} end -normalize_deps(name, uuid, deps, manifest; isglue=false) = deps -function normalize_deps(name, uuid, deps::Vector{String}, manifest::Dict{String,Vector{Stage1}}; isglue=false) +normalize_deps(name, uuid, deps, manifest; isext=false) = deps +function normalize_deps(name, uuid, deps::Vector{String}, manifest::Dict{String,Vector{Stage1}}; isext=false) if length(deps) != length(unique(deps)) pkgerror("Duplicate entry in `$name=$uuid`'s `deps` field.") end final = Dict{String,UUID}() for dep in deps infos = get(manifest, dep, nothing) - if !isglue + if !isext if infos === nothing pkgerror("`$name=$uuid` depends on `$dep`, ", "but no such entry exists in the manifest.") @@ -114,7 +114,7 @@ function validate_manifest(julia_version::Union{Nothing,VersionNumber}, manifest info.entry.deps = normalize_deps(name, info.uuid, info.deps, stage1) end for (name, infos) in stage1, info in infos - info.entry.weakdeps = normalize_deps(name, info.uuid, info.weakdeps, stage1; isglue=true) + info.entry.weakdeps = normalize_deps(name, info.uuid, info.weakdeps, stage1; isext=true) end # invariant: all dependencies are now normalized to Dict{String,UUID} deps = Dict{UUID, PackageEntry}() @@ -123,10 +123,10 @@ function validate_manifest(julia_version::Union{Nothing,VersionNumber}, manifest end # now just verify the graph structure for (entry_uuid, entry) in deps - for (deptype, isglue) in [(entry.deps, false), (entry.weakdeps, true)] + for (deptype, isext) in [(entry.deps, false), (entry.weakdeps, true)] for (name, uuid) in deptype dep_entry = get(deps, uuid, nothing) - if !isglue + if !isext if dep_entry === nothing pkgerror("`$(entry.name)=$(entry_uuid)` depends on `$name=$uuid`, ", "but no such entry exists in the manifest.") @@ -172,7 +172,7 @@ function Manifest(raw::Dict, f_or_io::Union{String, IO})::Manifest entry.uuid = uuid deps = read_deps(get(info::Dict, "deps", nothing)) weakdeps = read_deps(get(info::Dict, "weakdeps", nothing)) - entry.gluepkgs = get(Dict{String, String}, info::Dict, "gluepkgs") + entry.exts = get(Dict{String, String}, info::Dict, "extensions") catch # TODO: Should probably not unconditionally log something @debug "Could not parse manifest entry for `$name`" f_or_io @@ -288,8 +288,8 @@ function destructure(manifest::Manifest)::Dict end # TODO: Write this inline - if !isempty(entry.gluepkgs) - entry!(new_entry, "gluepkgs", entry.gluepkgs) + if !isempty(entry.exts) + entry!(new_entry, "extensions", entry.exts) end if manifest.manifest_format.major == 1 push!(get!(raw, entry.name, Dict{String,Any}[]), new_entry) diff --git a/src/project.jl b/src/project.jl index bfb94cc505..4febc6417f 100644 --- a/src/project.jl +++ b/src/project.jl @@ -80,9 +80,9 @@ function validate(project::Project; file=nothing) if length(dep_uuids) != length(unique(dep_uuids)) pkgerror("Two different dependencies can not have the same uuid" * location_string) end - glue_dep_uuids = collect(values(project.weakdeps)) - if length(glue_dep_uuids) != length(unique(glue_dep_uuids)) - pkgerror("Two different glue dependencies can not have the same uuid" * location_string) + weak_dep_uuids = collect(values(project.weakdeps)) + if length(weak_dep_uuids) != length(unique(weak_dep_uuids)) + pkgerror("Two different weak dependencies can not have the same uuid" * location_string) end # extras extra_uuids = collect(values(project.extras)) @@ -125,7 +125,7 @@ function Project(raw::Dict; file=nothing) project.version = read_project_version(get(raw, "version", nothing)) project.deps = read_project_deps(get(raw, "deps", nothing), "deps") project.weakdeps = read_project_deps(get(raw, "weakdeps", nothing), "weakdeps") - project.gluepkgs = get(Dict{String, String}, raw, "gluepkgs") + project.exts = get(Dict{String, String}, raw, "extensions") project.extras = read_project_deps(get(raw, "extras", nothing), "extras") project.compat = read_project_compat(get(raw, "compat", nothing), project) project.targets = read_project_targets(get(raw, "targets", nothing), project) @@ -187,7 +187,7 @@ function destructure(project::Project)::Dict return raw end -_project_key_order = ["name", "uuid", "keywords", "license", "desc", "deps", "weakdeps", "gluepkgs", "compat"] +_project_key_order = ["name", "uuid", "keywords", "license", "desc", "deps", "weakdeps", "extensions", "compat"] project_key_order(key::String) = something(findfirst(x -> x == key, _project_key_order), length(_project_key_order) + 1) diff --git a/test/extensions.jl b/test/extensions.jl new file mode 100644 index 0000000000..cf4cd3956c --- /dev/null +++ b/test/extensions.jl @@ -0,0 +1,38 @@ +using .Utils +using Test + +@testset "weak deps" begin + isolate(loaded_depot=true) do + Pkg.activate(; temp=true) + Pkg.develop(path=joinpath(@__DIR__, "test_packages", "ExtensionExamples", "HasExtensions.jl")) + Pkg.test("HasExtensions", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn + end + isolate(loaded_depot=true) do + Pkg.activate(; temp=true) + Pkg.develop(path=joinpath(@__DIR__, "test_packages", "ExtensionExamples", "HasDepWithExtensions.jl")) + Pkg.test("HasDepWithExtensions", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn + io = IOBuffer() + Pkg.status(; ext=true, mode=Pkg.PKGMODE_MANIFEST, io) + # TODO: Test output when ext deps are loaded etc. + str = String(take!(io)) + @test contains(str, "└─ OffsetArraysExt [OffsetArrays]" ) + end + + isolate(loaded_depot=true) do + Pkg.activate(; temp=true) + Pkg.develop(path=joinpath(@__DIR__, "test_packages", "ExtensionExamples", "HasExtensions.jl")) + @test_throws Pkg.Resolve.ResolverError Pkg.add(; name = "OffsetArrays", version = "0.9.0") + end + + isolate(loaded_depot=false) do + depot = mktempdir(); empty!(DEPOT_PATH); push!(DEPOT_PATH, depot) + Pkg.activate(; temp=true) + Pkg.Registry.add(Pkg.RegistrySpec(path=joinpath(@__DIR__, "test_packages", "ExtensionExamples", "ExtensionRegistry"))) + Pkg.Registry.add("General") + Pkg.add("HasExtensions") + Pkg.test("HasExtensions", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn + Pkg.add("HasDepWithExtensions") + Pkg.test("HasDepWithExtensions", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn + @test_throws Pkg.Resolve.ResolverError Pkg.add(; name = "OffsetArrays", version = "0.9.0") + end +end diff --git a/test/gluepkgs.jl b/test/gluepkgs.jl deleted file mode 100644 index cd75132106..0000000000 --- a/test/gluepkgs.jl +++ /dev/null @@ -1,38 +0,0 @@ -using .Utils -using Test - -@testset "weak deps" begin - isolate(loaded_depot=true) do - Pkg.activate(; temp=true) - Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasGluePkgs.jl")) - Pkg.test("HasGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn - end - isolate(loaded_depot=true) do - Pkg.activate(; temp=true) - Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasDepWithGluePkgs.jl")) - Pkg.test("HasDepWithGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn - io = IOBuffer() - Pkg.status(; glue=true, mode=Pkg.PKGMODE_MANIFEST, io) - # TODO: Test output when glue deps are loaded etc. - str = String(take!(io)) - @test contains(str, "└─ GlueOffsetArrays [OffsetArrays]" ) - end - - isolate(loaded_depot=true) do - Pkg.activate(; temp=true) - Pkg.develop(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "HasGluePkgs.jl")) - @test_throws Pkg.Resolve.ResolverError Pkg.add(; name = "OffsetArrays", version = "0.9.0") - end - - isolate(loaded_depot=false) do - depot = mktempdir(); empty!(DEPOT_PATH); push!(DEPOT_PATH, depot) - Pkg.activate(; temp=true) - Pkg.Registry.add(Pkg.RegistrySpec(path=joinpath(@__DIR__, "test_packages", "GluePkgExamples", "GlueRegistry"))) - Pkg.Registry.add("General") - Pkg.add("HasGluePkgs") - Pkg.test("HasGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn - Pkg.add("HasDepWithGluePkgs") - Pkg.test("HasDepWithGluePkgs", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn - @test_throws Pkg.Resolve.ResolverError Pkg.add(; name = "OffsetArrays", version = "0.9.0") - end -end diff --git a/test/runtests.jl b/test/runtests.jl index 51aaf0598c..97eb47e58e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -66,7 +66,7 @@ Logging.with_logger(hide_logs ? Logging.NullLogger() : Logging.current_logger()) "api.jl", "registry.jl", "subdir.jl", - "gluepkgs.jl", + "extensions.jl", "artifacts.jl", "binaryplatforms.jl", "platformengines.jl", diff --git a/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Compat.toml b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Compat.toml new file mode 100644 index 0000000000..f66053e93f --- /dev/null +++ b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Compat.toml @@ -0,0 +1,3 @@ +[0] +HasExtensions = "0.2" +OffsetArrays = "1.12.0-1" diff --git a/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Deps.toml b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Deps.toml new file mode 100644 index 0000000000..7df787b6bf --- /dev/null +++ b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Deps.toml @@ -0,0 +1,3 @@ +[0] +HasExtensions = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" +OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Package.toml b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Package.toml similarity index 62% rename from test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Package.toml rename to test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Package.toml index cfc83472eb..d9c99cf3f4 100644 --- a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Package.toml +++ b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Package.toml @@ -1,4 +1,4 @@ -name = "HasDepWithGluePkgs" +name = "HasDepWithExtensions" uuid = "d4ef3d4a-8e22-4710-85d8-c6cf2eb9efca" repo = "https://github.com/KristofferC/GluePkgExamples.jl.git" -subdir = "HasDepWithGluePkgs.jl" +subdir = "HasDepWithExtensions.jl" diff --git a/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Versions.toml b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Versions.toml new file mode 100644 index 0000000000..c5e7b2e77b --- /dev/null +++ b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasDepWithExtensions/Versions.toml @@ -0,0 +1,2 @@ +["0.2.0"] +git-tree-sha1 = "cafe9d753be789ca618c7951a653f2927f0f7bcf" diff --git a/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/Compat.toml b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/Compat.toml new file mode 100644 index 0000000000..dc5e9a5b8c --- /dev/null +++ b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/Compat.toml @@ -0,0 +1,3 @@ +[0] +Example = "0.5" +OffsetArrays = "1.12.0-1" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Deps.toml b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/Deps.toml similarity index 52% rename from test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Deps.toml rename to test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/Deps.toml index 6949deaaa8..c6be33e2fc 100644 --- a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Deps.toml +++ b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/Deps.toml @@ -1,3 +1,3 @@ [0] -HasGluePkgs = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" +Example = "7876af07-990d-54b4-ab0e-23690620f79a" OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Package.toml b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/Package.toml similarity index 68% rename from test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Package.toml rename to test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/Package.toml index 8c4873fb49..a19681dcee 100644 --- a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Package.toml +++ b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/Package.toml @@ -1,4 +1,4 @@ -name = "HasGluePkgs" +name = "HasExtensions" uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" repo = "https://github.com/KristofferC/GluePkgExamples.jl.git" -subdir = "HasGluePkgs.jl" +subdir = "HasExtensions.jl" diff --git a/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/Versions.toml b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/Versions.toml new file mode 100644 index 0000000000..da794c5f1f --- /dev/null +++ b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/Versions.toml @@ -0,0 +1,2 @@ +["0.2.0"] +git-tree-sha1 = "f72a3690d75877ad5b6b45048dd5d1aad9264f52" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/WeakCompat.toml b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/WeakCompat.toml similarity index 69% rename from test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/WeakCompat.toml rename to test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/WeakCompat.toml index 7b6d607bfd..af8c7eba01 100644 --- a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/WeakCompat.toml +++ b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/WeakCompat.toml @@ -1,3 +1,2 @@ [0] -CSV = "0.10" OffsetArrays = "1.12.0-1" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/WeakDeps.toml b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/WeakDeps.toml similarity index 56% rename from test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/WeakDeps.toml rename to test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/WeakDeps.toml index f6d210c744..59ec79f6c8 100644 --- a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/WeakDeps.toml +++ b/test/test_packages/ExtensionExamples/ExtensionRegistry/H/HasExtensions/WeakDeps.toml @@ -1,3 +1,2 @@ [0] -CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" diff --git a/test/test_packages/ExtensionExamples/ExtensionRegistry/Registry.toml b/test/test_packages/ExtensionExamples/ExtensionRegistry/Registry.toml new file mode 100644 index 0000000000..1a4492c2a8 --- /dev/null +++ b/test/test_packages/ExtensionExamples/ExtensionRegistry/Registry.toml @@ -0,0 +1,7 @@ +name = "ExtensionRegistry" +uuid = "e84ee987-529c-45f3-abc6-f0c1e5b43514" +repo = "" + +[packages] +4d3288b3-3afc-4bb6-85f3-489fffe514c8 = { name = "HasExtensions", path = "H/HasExtensions" } +d4ef3d4a-8e22-4710-85d8-c6cf2eb9efca = { name = "HasDepWithExtensions", path = "H/HasDepWithExtensions" } diff --git a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml b/test/test_packages/ExtensionExamples/HasDepWithExtensions.jl/Manifest.toml similarity index 88% rename from test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml rename to test/test_packages/ExtensionExamples/HasDepWithExtensions.jl/Manifest.toml index ea93f6fb5b..e7908cbab3 100644 --- a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Manifest.toml +++ b/test/test_packages/ExtensionExamples/HasDepWithExtensions.jl/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.10.0-DEV" manifest_format = "2.0" -project_hash = "1af04d62b804224fdf802daf4a4b96e213299d30" +project_hash = "51e63007fdb8e0cae2b36f0e7b2acebf7b7432a5" [[deps.Adapt]] deps = ["LinearAlgebra"] @@ -23,15 +23,15 @@ git-tree-sha1 = "46e44e869b4d90b96bd8ed1fdcf32244fddfb6cc" uuid = "7876af07-990d-54b4-ab0e-23690620f79a" version = "0.5.3" -[[deps.HasGluePkgs]] +[[deps.HasExtensions]] deps = ["Example"] -path = "../HasGluePkgs.jl" +path = "../HasExtensions.jl" uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" version = "0.1.0" weakdeps = ["OffsetArrays"] - [deps.HasGluePkgs.gluepkgs] - GlueOffsetArrays = "OffsetArrays" + [deps.HasExtensions.extensions] + OffsetArraysExt = "OffsetArrays" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Project.toml b/test/test_packages/ExtensionExamples/HasDepWithExtensions.jl/Project.toml similarity index 69% rename from test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Project.toml rename to test/test_packages/ExtensionExamples/HasDepWithExtensions.jl/Project.toml index 2036b85044..0e84d99671 100644 --- a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/Project.toml +++ b/test/test_packages/ExtensionExamples/HasDepWithExtensions.jl/Project.toml @@ -1,13 +1,13 @@ -name = "HasDepWithGluePkgs" +name = "HasDepWithExtensions" uuid = "d4ef3d4a-8e22-4710-85d8-c6cf2eb9efca" version = "0.1.0" [deps] -HasGluePkgs = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" +HasExtensions = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" [compat] -HasGluePkgs = "0.1" +HasExtensions = "0.1" OffsetArrays = "1.12" [extras] diff --git a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/src/HasDepWithGluePkgs.jl b/test/test_packages/ExtensionExamples/HasDepWithExtensions.jl/src/HasDepWithExtensions.jl similarity index 50% rename from test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/src/HasDepWithGluePkgs.jl rename to test/test_packages/ExtensionExamples/HasDepWithExtensions.jl/src/HasDepWithExtensions.jl index 66a109b9a8..ecf1a92cbc 100644 --- a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/src/HasDepWithGluePkgs.jl +++ b/test/test_packages/ExtensionExamples/HasDepWithExtensions.jl/src/HasDepWithExtensions.jl @@ -1,15 +1,15 @@ -module HasDepWithGluePkgs +module HasDepWithExtensions -using HasGluePkgs +using HasExtensions using OffsetArrays: OffsetArray -# Loading OffsetArrays makes the glue module "GlueOffsetArrays" to load +# Loading OffsetArrays makes the extesion "OffsetArraysExt" to load function do_something() # @info "First do something with the basic array support in B" - HasGluePkgs.foo(rand(Float64, 2)) + HasExtensions.foo(rand(Float64, 2)) # @info "Now do something with extended OffsetArray support in B" - HasGluePkgs.foo(OffsetArray(rand(Float64, 2), 0:1)) + HasExtensions.foo(OffsetArray(rand(Float64, 2), 0:1)) end end # module diff --git a/test/test_packages/ExtensionExamples/HasDepWithExtensions.jl/test/runtests.jl b/test/test_packages/ExtensionExamples/HasDepWithExtensions.jl/test/runtests.jl new file mode 100644 index 0000000000..3830320b01 --- /dev/null +++ b/test/test_packages/ExtensionExamples/HasDepWithExtensions.jl/test/runtests.jl @@ -0,0 +1,5 @@ +using HasDepWithExtensions +using Test + +@test HasDepWithExtensions.HasExtensions.offsetarrays_loaded +HasDepWithExtensions.do_something() diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Manifest.toml b/test/test_packages/ExtensionExamples/HasExtensions.jl/Manifest.toml similarity index 100% rename from test/test_packages/GluePkgExamples/HasGluePkgs.jl/Manifest.toml rename to test/test_packages/ExtensionExamples/HasExtensions.jl/Manifest.toml diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml b/test/test_packages/ExtensionExamples/HasExtensions.jl/Project.toml similarity index 82% rename from test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml rename to test/test_packages/ExtensionExamples/HasExtensions.jl/Project.toml index b6c8de9092..0687e1fed8 100644 --- a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/Project.toml +++ b/test/test_packages/ExtensionExamples/HasExtensions.jl/Project.toml @@ -1,4 +1,4 @@ -name = "HasGluePkgs" +name = "HasExtensions" uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" version = "0.1.0" @@ -8,8 +8,8 @@ Example = "7876af07-990d-54b4-ab0e-23690620f79a" [weakdeps] OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" -[gluepkgs] -GlueOffsetArrays = "OffsetArrays" +[extensions] +OffsetArraysExt = "OffsetArrays" [compat] Example = "0.5" diff --git a/test/test_packages/ExtensionExamples/HasExtensions.jl/ext/OffsetArraysExt.jl b/test/test_packages/ExtensionExamples/HasExtensions.jl/ext/OffsetArraysExt.jl new file mode 100644 index 0000000000..12e130a4f7 --- /dev/null +++ b/test/test_packages/ExtensionExamples/HasExtensions.jl/ext/OffsetArraysExt.jl @@ -0,0 +1,13 @@ +module OffsetArraysExt + +using HasExtensions, OffsetArrays + +function foo(::OffsetArray) + return 2 +end + +function __init__() + HasExtensions.offsetarrays_loaded = true +end + +end diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/src/HasGluePkgs.jl b/test/test_packages/ExtensionExamples/HasExtensions.jl/src/HasExtensions.jl similarity index 83% rename from test/test_packages/GluePkgExamples/HasGluePkgs.jl/src/HasGluePkgs.jl rename to test/test_packages/ExtensionExamples/HasExtensions.jl/src/HasExtensions.jl index 212512cd73..4ab13b1327 100644 --- a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/src/HasGluePkgs.jl +++ b/test/test_packages/ExtensionExamples/HasExtensions.jl/src/HasExtensions.jl @@ -1,4 +1,4 @@ -module HasGluePkgs +module HasExtensions using Example diff --git a/test/test_packages/ExtensionExamples/HasExtensions.jl/test/runtests.jl b/test/test_packages/ExtensionExamples/HasExtensions.jl/test/runtests.jl new file mode 100644 index 0000000000..f0d4793edb --- /dev/null +++ b/test/test_packages/ExtensionExamples/HasExtensions.jl/test/runtests.jl @@ -0,0 +1,8 @@ +using HasExtensions +using Test + +@test !HasExtensions.offsetarrays_loaded + +using OffsetArrays + +@test HasExtensions.offsetarrays_loaded diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Compat.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Compat.toml deleted file mode 100644 index 2fa6d914ac..0000000000 --- a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Compat.toml +++ /dev/null @@ -1,5 +0,0 @@ -[0] -OffsetArrays = "1.12.0-1" - -["0.2-0"] -HasGluePkgs = "0.2" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Versions.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Versions.toml deleted file mode 100644 index d500a7c476..0000000000 --- a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasDepWithGluePkgs/Versions.toml +++ /dev/null @@ -1,2 +0,0 @@ -["0.2.0"] -git-tree-sha1 = "5fbd46cdb98eb4c8873a042142e69fc21e77c061" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Compat.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Compat.toml deleted file mode 100644 index 6acf004c98..0000000000 --- a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Compat.toml +++ /dev/null @@ -1,2 +0,0 @@ -[0] -Example = "0.5" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Deps.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Deps.toml deleted file mode 100644 index 4a437a62fc..0000000000 --- a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Deps.toml +++ /dev/null @@ -1,2 +0,0 @@ -[0] -Example = "7876af07-990d-54b4-ab0e-23690620f79a" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Versions.toml b/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Versions.toml deleted file mode 100644 index b0ab2eecbc..0000000000 --- a/test/test_packages/GluePkgExamples/GlueRegistry/H/HasGluePkgs/Versions.toml +++ /dev/null @@ -1,2 +0,0 @@ -["0.2.0"] -git-tree-sha1 = "3edae1d3d2002fa0d0187bcd2010fd7939bed591" diff --git a/test/test_packages/GluePkgExamples/GlueRegistry/Registry.toml b/test/test_packages/GluePkgExamples/GlueRegistry/Registry.toml deleted file mode 100644 index 32f7335209..0000000000 --- a/test/test_packages/GluePkgExamples/GlueRegistry/Registry.toml +++ /dev/null @@ -1,6 +0,0 @@ -name = "GlueRegistry" -uuid = "c22930c5-f744-4ce3-9117-e610c805c832" - -[packages] -4d3288b3-3afc-4bb6-85f3-489fffe514c8 = { name = "HasGluePkgs", path = "H/HasGluePkgs" } -d4ef3d4a-8e22-4710-85d8-c6cf2eb9efca = { name = "HasDepWithGluePkgs", path = "H/HasDepWithGluePkgs" } diff --git a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/test/runtests.jl b/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/test/runtests.jl deleted file mode 100644 index 318c70c3f5..0000000000 --- a/test/test_packages/GluePkgExamples/HasDepWithGluePkgs.jl/test/runtests.jl +++ /dev/null @@ -1,5 +0,0 @@ -using HasDepWithGluePkgs -using Test - -@test HasDepWithGluePkgs.HasGluePkgs.offsetarrays_loaded -HasDepWithGluePkgs.do_something() diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/glue/GlueOffsetArrays.jl b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/glue/GlueOffsetArrays.jl deleted file mode 100644 index 897a7b8188..0000000000 --- a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/glue/GlueOffsetArrays.jl +++ /dev/null @@ -1,13 +0,0 @@ -module GlueOffsetArrays - -using HasGluePkgs, OffsetArrays - -function foo(::OffsetArray) - return 2 -end - -function __init__() - HasGluePkgs.offsetarrays_loaded = true -end - -end diff --git a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/test/runtests.jl b/test/test_packages/GluePkgExamples/HasGluePkgs.jl/test/runtests.jl deleted file mode 100644 index 8042719f35..0000000000 --- a/test/test_packages/GluePkgExamples/HasGluePkgs.jl/test/runtests.jl +++ /dev/null @@ -1,8 +0,0 @@ -using HasGluePkgs -using Test - -@test !HasGluePkgs.offsetarrays_loaded - -using OffsetArrays - -@test HasGluePkgs.offsetarrays_loaded From 5085eb7ccfcf59f4bb3538e57c7827e4dd234f5e Mon Sep 17 00:00:00 2001 From: KristofferC Date: Tue, 6 Dec 2022 15:43:42 +0100 Subject: [PATCH 21/26] fix REPL for extension option --- src/API.jl | 6 +++--- src/Operations.jl | 14 +++++++------- src/Pkg.jl | 2 +- src/REPLMode/command_declarations.jl | 2 +- test/extensions.jl | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/API.jl b/src/API.jl index a8816e531a..0e1d0fd561 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1646,14 +1646,14 @@ end @deprecate status(mode::PackageMode) status(mode=mode) -function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=PKGMODE_PROJECT, outdated::Bool=false, compat::Bool=false, ext::Bool=false, io::IO=stdout_f(), kwargs...) +function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=PKGMODE_PROJECT, outdated::Bool=false, compat::Bool=false, extensions::Bool=false, io::IO=stdout_f()) if compat diff && pkgerror("Compat status has no `diff` mode") outdated && pkgerror("Compat status has no `outdated` mode") - ext && pkgerror("Compat status has no `ext` mode") + extensions && pkgerror("Compat status has no `extensions` mode") Operations.print_compat(ctx, pkgs; io) else - Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated, ext) + Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated, extensions) end return nothing end diff --git a/src/Operations.jl b/src/Operations.jl index 56c01363b9..89e2642095 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1327,7 +1327,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}(); update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version) new_apply = download_source(ctx) fixup_ext!(ctx.env, pkgs) - + # After downloading resolutionary packages, search for (Julia)Artifacts.toml files # and ensure they are all downloaded and unpacked as well: download_artifacts(ctx.env, platform=platform, julia_version=ctx.julia_version, io=ctx.io) @@ -2138,7 +2138,7 @@ struct PackageStatusData end function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registries::Vector{Registry.RegistryInstance}, header::Symbol, - uuids::Vector, names::Vector; manifest=true, diff=false, ignore_indent::Bool, outdated::Bool, ext::Bool, io::IO, + uuids::Vector, names::Vector; manifest=true, diff=false, ignore_indent::Bool, outdated::Bool, extensions::Bool, io::IO, mode::PackageMode, hidden_upgrades_info::Bool, show_usagetips::Bool=true) not_installed_indicator = sprint((io, args) -> printstyled(io, args...; color=Base.error_color()), "→", context=io) upgradable_indicator = sprint((io, args) -> printstyled(io, args...; color=:green), "⌃", context=io) @@ -2198,7 +2198,7 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie ext_info = status_ext_info(new, env) end - if ext && ext_info == nothing + if extensions && ext_info === nothing continue end @@ -2267,7 +2267,7 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registrie end end - if ext && !diff && pkg.extinfo !== nothing + if extensions && !diff && pkg.extinfo !== nothing println(io) for (i, ext) in enumerate(pkg.extinfo) sym = i == length(pkg.extinfo) ? '└' : '├' @@ -2341,7 +2341,7 @@ end function status(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}=PackageSpec[]; header=nothing, mode::PackageMode=PKGMODE_PROJECT, git_diff::Bool=false, env_diff=nothing, ignore_indent=true, - io::IO, outdated::Bool=false, ext::Bool=false, hidden_upgrades_info::Bool=false, show_usagetips::Bool=true) + io::IO, outdated::Bool=false, extensions::Bool=false, hidden_upgrades_info::Bool=false, show_usagetips::Bool=true) io == Base.devnull && return # if a package, print header if header === nothing && env.pkg !== nothing @@ -2368,10 +2368,10 @@ function status(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pk diff = old_env !== nothing header = something(header, diff ? :Diff : :Status) if mode == PKGMODE_PROJECT || mode == PKGMODE_COMBINED - print_status(env, old_env, registries, header, filter_uuids, filter_names; manifest=false, diff, ignore_indent, io, outdated, ext, mode, hidden_upgrades_info, show_usagetips) + print_status(env, old_env, registries, header, filter_uuids, filter_names; manifest=false, diff, ignore_indent, io, outdated, extensions, mode, hidden_upgrades_info, show_usagetips) end if mode == PKGMODE_MANIFEST || mode == PKGMODE_COMBINED - print_status(env, old_env, registries, header, filter_uuids, filter_names; diff, ignore_indent, io, outdated, ext, mode, hidden_upgrades_info, show_usagetips) + print_status(env, old_env, registries, header, filter_uuids, filter_names; diff, ignore_indent, io, outdated, extensions, mode, hidden_upgrades_info, show_usagetips) end if is_manifest_current(env) === false tip = show_usagetips ? " It is recommended to `Pkg.resolve()` or consider `Pkg.update()` if necessary." : "" diff --git a/src/Pkg.jl b/src/Pkg.jl index dc72a1e668..bde2b50066 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -419,7 +419,7 @@ from packages that are tracking a path. const resolve = API.resolve """ - Pkg.status([pkgs...]; outdated::Bool=false, mode::PackageMode=PKGMODE_PROJECT, diff::Bool=false, compat::Bool=false, ext::Bool=false io::IO=stdout) + Pkg.status([pkgs...]; outdated::Bool=false, mode::PackageMode=PKGMODE_PROJECT, diff::Bool=false, compat::Bool=false, extensions::Bool=false, io::IO=stdout) Print out the status of the project/manifest. diff --git a/src/REPLMode/command_declarations.jl b/src/REPLMode/command_declarations.jl index 5f17fc509e..06faf59055 100644 --- a/src/REPLMode/command_declarations.jl +++ b/src/REPLMode/command_declarations.jl @@ -376,7 +376,7 @@ PSA[:name => "status", PSA[:name => "diff", :short_name => "d", :api => :diff => true], PSA[:name => "outdated", :short_name => "o", :api => :outdated => true], PSA[:name => "compat", :short_name => "c", :api => :compat => true], - PSA[:name => "extensions", :short_name => "e", :api => :Ext => true], + PSA[:name => "extensions", :short_name => "e", :api => :extensions => true], ], :completions => complete_installed_packages, :description => "summarize contents of and changes to environment", diff --git a/test/extensions.jl b/test/extensions.jl index cf4cd3956c..a6bb7c2e38 100644 --- a/test/extensions.jl +++ b/test/extensions.jl @@ -12,7 +12,7 @@ using Test Pkg.develop(path=joinpath(@__DIR__, "test_packages", "ExtensionExamples", "HasDepWithExtensions.jl")) Pkg.test("HasDepWithExtensions", julia_args=`--depwarn=no`) # OffsetArrays errors from depwarn io = IOBuffer() - Pkg.status(; ext=true, mode=Pkg.PKGMODE_MANIFEST, io) + Pkg.status(; extensions=true, mode=Pkg.PKGMODE_MANIFEST, io) # TODO: Test output when ext deps are loaded etc. str = String(take!(io)) @test contains(str, "└─ OffsetArraysExt [OffsetArrays]" ) From 644a17640fe917c914d29f5dbe3da61588e0e6c9 Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Fri, 25 Nov 2022 16:46:46 +0600 Subject: [PATCH 22/26] Clarify RegistrySpec docstring (#3155) It is possible to think that `Registry` may be a standalone Pkg command with the previous docstring (e.g. a command that constructs or displays registry info). --- src/Pkg.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Pkg.jl b/src/Pkg.jl index bde2b50066..b18982f145 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -630,12 +630,12 @@ on all the registries in the vector. Below is a comparison between the REPL mode and the functional API:: -| `REPL` | `API` | -|:---------------------|:------------------------------------------------| -| `Registry` | `RegistrySpec("Registry")` | -| `Registry=a67d...` | `RegistrySpec(name="Registry", uuid="a67d...")` | -| `local/path` | `RegistrySpec(path="local/path")` | -| `www.myregistry.com` | `RegistrySpec(url="www.myregistry.com")` | +| `REPL` | `API` | +|:---------------------|:--------------------------------------------------| +| `MyRegistry` | `RegistrySpec("MyRegistry")` | +| `MyRegistry=a67d...` | `RegistrySpec(name="MyRegistry", uuid="a67d...")` | +| `local/path` | `RegistrySpec(path="local/path")` | +| `www.myregistry.com` | `RegistrySpec(url="www.myregistry.com")` | """ const RegistrySpec = Registry.RegistrySpec From 50a093850d3b24272f1fb73eb77bfb82cdeef83c Mon Sep 17 00:00:00 2001 From: Lilith Orion Hafner Date: Fri, 25 Nov 2022 22:18:11 +0600 Subject: [PATCH 23/26] slightly simplify API internals (#3156) Co-authored-by: Lilith Hafner --- src/API.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/API.jl b/src/API.jl index 0e1d0fd561..6abfbb64e0 100644 --- a/src/API.jl +++ b/src/API.jl @@ -162,12 +162,12 @@ for f in (:develop, :add, :rm, :up, :pin, :free, :test, :build, :status, :why) function $f(; name::Union{Nothing,AbstractString}=nothing, uuid::Union{Nothing,String,UUID}=nothing, version::Union{VersionNumber, String, VersionSpec, Nothing}=nothing, url=nothing, rev=nothing, path=nothing, mode=PKGMODE_PROJECT, subdir=nothing, kwargs...) - pkg = PackageSpec(; name=name, uuid=uuid, version=version, url=url, rev=rev, path=path, subdir=subdir) + pkg = PackageSpec(; name, uuid, version, url, rev, path, subdir) if $f === status || $f === rm || $f === up kwargs = merge((;kwargs...), (:mode => mode,)) end # Handle $f() case - if unique([name,uuid,version,url,rev,path,subdir]) == [nothing] + if all(isnothing, [name,uuid,version,url,rev,path,subdir]) $f(PackageSpec[]; kwargs...) else $f(pkg; kwargs...) From 0c2bafdd29343c750f296441d245ac564d731a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Denis=20Baru=C4=8Di=C4=87?= Date: Wed, 16 Nov 2022 15:39:05 +0100 Subject: [PATCH 24/26] REPLMode: Remove constructor for `APIOptions` `APIOptions` is defined as an alias for `Dict{Symbol, Any}`. Consequently, the definition of a constructor for `APIOptions` introduced a new constructor for Dict. This commit removes that constructor. --- src/REPLMode/REPLMode.jl | 19 +++++++++++-------- src/REPLMode/completions.jl | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/REPLMode/REPLMode.jl b/src/REPLMode/REPLMode.jl index 60a4941fa9..d8556bb3bb 100644 --- a/src/REPLMode/REPLMode.jl +++ b/src/REPLMode/REPLMode.jl @@ -32,7 +32,7 @@ end # TODO assert names matching lex regex # assert now so that you don't fail at user time -# see function `REPLMode.APIOptions` +# see function `REPLMode.api_options` function OptionSpec(;name::String, short_name::Union{Nothing,String}=nothing, takes_arg::Bool=false, @@ -293,20 +293,23 @@ parse(input::String) = #------------# # APIOptions # #------------# + +# Do NOT introduce a constructor for APIOptions +# as long as it's an alias for Dict const APIOptions = Dict{Symbol, Any} -function APIOptions(options::Vector{Option}, - specs::Dict{String, OptionSpec}, - )::APIOptions - api_options = Dict{Symbol, Any}() +function api_options(options::Vector{Option}, + specs::Dict{String, OptionSpec}) + api_opts = APIOptions() enforce_option(options, specs) for option in options spec = specs[option.val] - api_options[spec.api.first] = spec.takes_arg ? + api_opts[spec.api.first] = spec.takes_arg ? spec.api.second(option.argument) : spec.api.second end - return api_options + return api_opts end + Context!(ctx::APIOptions)::Context = Types.Context!(collect(ctx)) #---------# @@ -361,7 +364,7 @@ This step is distinct from `parse` in that it relies on the command specificatio """ function Command(statement::Statement)::Command # options - options = APIOptions(statement.options, statement.spec.option_specs) + options = api_options(statement.options, statement.spec.option_specs) # arguments arg_spec = statement.spec.argument_spec arguments = arg_spec.parser(statement.arguments, options) diff --git a/src/REPLMode/completions.jl b/src/REPLMode/completions.jl index 57369fc9fc..ea6949741c 100644 --- a/src/REPLMode/completions.jl +++ b/src/REPLMode/completions.jl @@ -173,7 +173,7 @@ function complete_argument(spec::CommandSpec, options::Vector{String}, # finish parsing opts local opts try - opts = APIOptions(map(parse_option, options), spec.option_specs) + opts = api_options(map(parse_option, options), spec.option_specs) catch e e isa PkgError && return String[] rethrow() From acefa5f6f00746b52420704d0e4d9998698b38fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Denis=20Baru=C4=8Di=C4=87?= Date: Wed, 16 Nov 2022 15:44:07 +0100 Subject: [PATCH 25/26] REPLMode: Remove constructor for `PackageToken` `PackageToken` is a Union of multiple struct types. This commit removes a constructor for `PackageToken` and replaces it with a function `packagetoken`. Furthermore, this commit removes `PackageIdentifier` as an alias for String and replaces that with a struct type, which wraps a String containing a package identifier. --- src/REPLMode/argument_parsers.jl | 53 ++++++++++++++++++-------------- test/repl.jl | 5 +-- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/src/REPLMode/argument_parsers.jl b/src/REPLMode/argument_parsers.jl index f3914e33a5..880726a043 100644 --- a/src/REPLMode/argument_parsers.jl +++ b/src/REPLMode/argument_parsers.jl @@ -1,5 +1,32 @@ import ..isdir_nothrow, ..Registry.RegistrySpec, ..isurl +struct PackageIdentifier + val::String +end + +struct VersionToken + version::String +end + +struct Rev + rev::String +end + +struct Subdir + dir::String +end + +const PackageToken = Union{PackageIdentifier, + VersionToken, + Rev, + Subdir} + +packagetoken(word::String)::PackageToken = + first(word) == '@' ? VersionToken(word[2:end]) : + first(word) == '#' ? Rev(word[2:end]) : + first(word) == ':' ? Subdir(word[2:end]) : + PackageIdentifier(word) + ############### # PackageSpec # ############### @@ -17,26 +44,11 @@ function parse_package(args::Vector{QString}, options; add_or_dev=false)::Vector push!(words, word) end end - args = PackageToken[PackageToken(pkgword) for pkgword in words] + args = PackageToken[packagetoken(pkgword) for pkgword in words] return parse_package_args(args; add_or_dev=add_or_dev) end -struct VersionToken - version::String -end - -struct Rev - rev::String -end - -struct Subdir - dir::String -end - -const PackageIdentifier = String -const PackageToken = Union{PackageIdentifier, VersionToken, Rev, Subdir} - # Match a git repository URL. This includes uses of `@` and `:` but # requires that it has `.git` at the end. let url = raw"((git|ssh|http(s)?)|(git@[\w\-\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git$)(/)?", @@ -78,12 +90,6 @@ function package_lex(qwords::Vector{QString})::Vector{String} return words end -PackageToken(word::String)::PackageToken = - first(word) == '@' ? VersionToken(word[2:end]) : - first(word) == '#' ? Rev(word[2:end]) : - first(word) == ':' ? Subdir(word[2:end]) : - String(word) - function parse_package_args(args::Vector{PackageToken}; add_or_dev=false)::Vector{PackageSpec} # check for and apply PackageSpec modifier (e.g. `#foo` or `@v1.0.2`) function apply_modifier!(pkg::PackageSpec, args::Vector{PackageToken}) @@ -131,7 +137,8 @@ let uuid = raw"(?i)[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}( end # packages can be identified through: uuid, name, or name+uuid # additionally valid for add/develop are: local path, url -function parse_package_identifier(word::AbstractString; add_or_develop=false)::PackageSpec +function parse_package_identifier(pkg_id::PackageIdentifier; add_or_develop=false)::PackageSpec + word = pkg_id.val if add_or_develop if isurl(word) return PackageSpec(; url=word) diff --git a/test/repl.jl b/test/repl.jl index 9678e9a365..78d1c15bfb 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -571,8 +571,9 @@ temp_pkg_dir() do project_path end @testset "parse package url win" begin - @test typeof(Pkg.REPLMode.parse_package_identifier("https://github.com/abc/ABC.jl"; - add_or_develop=true)) == Pkg.Types.PackageSpec + pkg_id = Pkg.REPLMode.PackageIdentifier("https://github.com/abc/ABC.jl") + pkg_spec = Pkg.REPLMode.parse_package_identifier(pkg_id; add_or_develop=true) + @test typeof(pkg_spec) == Pkg.Types.PackageSpec end @testset "parse git url (issue #1935) " begin From a2aef897a2a737f84b48885e2f4a5a6f1b230cfe Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Wed, 7 Dec 2022 18:31:35 +0100 Subject: [PATCH 26/26] review fixes --- docs/src/creating-packages.md | 6 +++--- src/Operations.jl | 2 +- src/Pkg.jl | 1 + src/Types.jl | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index caf6a5c7db..f6149a380d 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -251,7 +251,7 @@ This is an important topic so there is a separate chapter about it: [Compatibili This is a somewhat advanced usage of Pkg which can be skipped for people new to Julia and Julia packages. A weak dependency is a dependency that will not automatically install when the package is installed but -you can still control what versions of that package is allowed to install by setting compatibility on it. +you can still control what versions of that package are allowed to be installed by setting compatibility on it. These are listed in the project file under the `[weakdeps]` section: ```toml @@ -355,7 +355,7 @@ This section discusses various methods for using extensions on Julia versions th while simultaneously providing similar functionality on older Julia versions. #### Requires.jl -This section is relevant if you are currently using Requries.jl but want to transition to using extensions (while still having Requires be used on Julia versions that do not support extensions). +This section is relevant if you are currently using Requires.jl but want to transition to using extensions (while still having Requires be used on Julia versions that do not support extensions). This is done by making the following changes (using the example above): - Add the following to the package file. This makes it so that Requires.jl loads and inserts the @@ -365,7 +365,7 @@ This is done by making the following changes (using the example above): if !isdefined(Base, :get_extension) using Requires function __init__() - @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../ext/ContourExt.jl) + @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../ext/ContourExt.jl") end end ``` diff --git a/src/Operations.jl b/src/Operations.jl index 89e2642095..8bf0ba8211 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -149,7 +149,7 @@ function update_manifest!(env::EnvCache, pkgs::Vector{PackageSpec}, deps_map, ju end # This has to be done after the packages have been downloaded -# since we need access to the Projet file to read the information +# since we need access to the Project file to read the information # about extensions function fixup_ext!(env, pkgs) for pkg in pkgs diff --git a/src/Pkg.jl b/src/Pkg.jl index b18982f145..451154db06 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -421,6 +421,7 @@ const resolve = API.resolve """ Pkg.status([pkgs...]; outdated::Bool=false, mode::PackageMode=PKGMODE_PROJECT, diff::Bool=false, compat::Bool=false, extensions::Bool=false, io::IO=stdout) + Print out the status of the project/manifest. Packages marked with `⌃` have new versions that can be installed, e.g. via [`Pkg.up`](@ref). diff --git a/src/Types.jl b/src/Types.jl index 5d1d6987f8..e65dda0beb 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -284,7 +284,7 @@ Base.:(==)(t1::PackageEntry, t2::PackageEntry) = t1.name == t2.name && t1.exts == t2.exts && t1.uuid == t2.uuid # omits `other` -Base.hash(x::PackageEntry, h::UInt) = foldr(hash, [x.name, x.version, x.path, x.pinned, x.repo, x.tree_hash, x.deps, x.weakdeps,x.exts, x.uuid], init=h) # omits `other` +Base.hash(x::PackageEntry, h::UInt) = foldr(hash, [x.name, x.version, x.path, x.pinned, x.repo, x.tree_hash, x.deps, x.weakdeps, x.exts, x.uuid], init=h) # omits `other` Base.@kwdef mutable struct Manifest julia_version::Union{Nothing,VersionNumber} = nothing # only set to VERSION when resolving