add support for package extensions

docs/make.jl

@@ -35,6 +35,7 @@ makedocs(
         "managing-packages.md",
         "environments.md",
         "creating-packages.md",
+        "gluedeps.md",
         "compatibility.md",
         "registries.md",
         "artifacts.md",

docs/src/creating-packages.md

@@ -245,6 +245,111 @@ using Test
 Every dependency should in general have a compatibility constraint on it.
 This is an important topic so there is a separate chapter about it: [Compatibility](@ref Compatibility).
 
+## Conditional loading of code in packages (Glue packages)
+
+!!! note
+    This is a somewhat advanced section which can be skipped for people new to Julia and Julia packages.
+
+It is sometimes desirable to be able to extend some functionality of a package without having to
+unconditionally take on the cost (in terms of e.g. load time) of adding an extra dependency.
+A *glue package* is a file that gets automatically loaded when some other set of packages are
+loaded into the Julia session. This is very similar to functionality that the external package
+Requires.jl used to provide, but which is now available directly through Julia.
+
+A useful application of glue packages could be for a plotting package that should be able to plot
+objects from a wide variety of different Julia packages.
+Adding all those different Julia packages as dependencies
+could be expensive since they would end up getting loaded even if they were never used.
+Instead, these code required to plot objects for specific packages can be put into separate files
+(glue packages) which is only loaded when 
+
+Below is an example of how the code can be structured for a use case as outlined above.
+
+ `Project.toml`:
+ ```toml
+name = "Plotting"
+version = "0.1.0"
+uuid = "..."
+
+[gluedeps]
+Contour = "d38c429a-6771-53c6-b99e-75d170b6e991"
+
+[gluepkgs] 
+# name of glue package to the left
+# glue dependencies required to load the glue pkg to the right
+# use a list for multiple glue dependencies
+GlueContour = "Contour" 
+
+[compat] # compat can also be given on glue dependencies
+Contour = "0.6.2"
+```
+
+`src/Plotting.jl`:
+```julia
+module Plotting
+
+function plot(x::Vector)
+    # Some functionality for plotting a vector here
+end
+
+end # module
+```
+
+`glue/GlueContour.jl` (can also be in `glue/GlueContour/GlueContour.jl`):
+```julia
+module GlueContour # Should be same name as the file (just like a normal package)
+
+using Plotting, Contour
+
+function Plotting.plot(c::Contour.ContourCollection)
+    # Some functionality for plotting a contour here
+end
+
+end # module
+```
+
+A user that depends on `Plotting` will not pay the cost of the "glue code" inside the `GlueContour` module.
+It is only when the `Contour` package actually gets loaded that the `GlueCountour` glue package is loaded
+and provide the new functionality.
+
+Compatibility can be set on glue dependencies just like normal dependencies and they can be used in the `test`
+target to make them available when testing.
+
+A glue package will only be loaded if the glue dependencies are loaded from the same environment or environments higher in the environment stack than the package itself.
+
+
+!!! compat
+    In order to be compatible with earlier versions of Julia, some extra steps have to be taken.
+    These are: Duplicate the packages under `[gluedeps]` into `[extras]`. This is an unfortunate
+    duplication but without doing this the project verifier under older Julia versions will complain (error).
+
+### Backwards compatibility with Requires.jl
+
+Since glue packages and Requires.jl are solving the same problem,
+it is useful to know how to use Requires.jl on older Julia versions while seamlessly
+transitioning into using glue packages on newer Julia versions that support it.
+This is done by making the following changes (using the example above):
+
+- Add the following to the package file. This makes it so that Requires.jl loads and inserts the
+  callback only when glue packages are not supported
+  ```julia
+  # This symbol is only defined on Julia versions that support glue packages
+  if isdefined(Base, :get_gluepkg)
+  using Requires
+  function __init__()
+      @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../glue/GlueContour.jl)
+  end
+  end
+  ```
+
+- Do the following change in the glue packages for loading the glue dependency:
+ ```julia
+ isdefined(Base, :get_gluepkg) ? (using Contour) : (using ..Contour)
+ ```
+
+The package should now work with Requires.jl on Julia versions before glue packages were introduced
+and with glue packages afterward.
+
 ## Package naming guidelines
 
 Package names should be sensible to most Julia users, *even to those who are not domain experts*.

src/API.jl

@@ -301,9 +301,11 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode=PKGMODE_PROJECT, all_p
     ensure_resolved(ctx, ctx.env.manifest, pkgs)
 
     Operations.rm(ctx, pkgs; mode)
+
     return
 end
 
+
 function append_all_pkgs!(pkgs, ctx, mode)
     if mode == PKGMODE_PROJECT || mode == PKGMODE_COMBINED
         for (name::String, uuid::UUID) in ctx.env.project.deps
@@ -1094,15 +1096,41 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call::
         for (name, uuid) in ctx.env.project.deps if !Base.in_sysimage(Base.PkgId(uuid, name))
     ]
 
-    man = ctx.env.manifest
-    deps_pair_or_nothing = Iterators.map(man) do dep
+    gluepkgs = Dict{Base.PkgId, String}() # gluepkg -> parent
+    # make a flat map of each dep and its deps
+    depsmap = Dict{Base.PkgId, Vector{Base.PkgId}}()
+    pkg_specs = PackageSpec[]
+    for dep in ctx.env.manifest
         pkg = Base.PkgId(first(dep), last(dep).name)
-        Base.in_sysimage(pkg) && return nothing
+        Base.in_sysimage(pkg) && continue
         deps = [Base.PkgId(last(x), first(x)) for x in last(dep).deps]
-        return pkg => filter!(!Base.in_sysimage, deps)
+        depsmap[pkg] = filter!(!Base.in_sysimage, deps)
+        # add any glue packages
+        gluedeps = last(dep).gluedeps
+        for (gluepkg_name, gluedep_names) in last(dep).gluepkgs
+            gluepkg_deps = Base.PkgId[]
+            push!(gluepkg_deps, pkg) # depends on parent package
+            all_gluedeps_available = true
+            gluedep_names = gluedep_names isa String ? String[gluedep_names] : gluedep_names
+            for gluedep_name in gluedep_names
+                gluedep_uuid = gluedeps[gluedep_name]
+                if gluedep_uuid in keys(ctx.env.manifest.deps)
+                    push!(gluepkg_deps, Base.PkgId(gluedep_uuid, gluedep_name))
+                else
+                    all_gluedeps_available = false
+                    break
+                end
+            end
+            all_gluedeps_available || continue
+            gluepkg_uuid = Base.uuid5(pkg.uuid, gluepkg_name)
+            gluepkg = Base.PkgId(gluepkg_uuid, gluepkg_name)
+            push!(pkg_specs, PackageSpec(uuid = gluepkg_uuid, name = gluepkg_name)) # create this here as the name cannot be looked up easily later via the uuid
+            depsmap[gluepkg] = filter!(!Base.in_sysimage, gluepkg_deps)
+            gluepkgs[gluepkg] = pkg.name
+        end
     end
-    depsmap = Dict{Base.PkgId, Vector{Base.PkgId}}(Iterators.filter(!isnothing, deps_pair_or_nothing)) #flat map of each dep and its deps
 
+    # if the active environment is a package, add that
     ctx_env_pkg = ctx.env.pkg
     if ctx_env_pkg !== nothing && isfile( joinpath( dirname(ctx.env.project_file), "src", "$(ctx_env_pkg.name).jl") )
         depsmap[Base.PkgId(ctx_env_pkg.uuid, ctx_env_pkg.name)] = [
@@ -1112,21 +1140,25 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call::
         push!(direct_deps, Base.PkgId(ctx_env_pkg.uuid, ctx_env_pkg.name))
     end
 
+    # return early if no deps
     isempty(depsmap) && return
 
+    # initialize signalling
     started = Dict{Base.PkgId,Bool}()
     was_processed = Dict{Base.PkgId,Base.Event}()
     was_recompiled = Dict{Base.PkgId,Bool}()
-    pkg_specs = PackageSpec[]
     for pkgid in keys(depsmap)
         started[pkgid] = false
         was_processed[pkgid] = Base.Event()
         was_recompiled[pkgid] = false
         push!(pkg_specs, get_or_make_pkgspec(pkg_specs, ctx, pkgid.uuid))
     end
+
+    # remove packages that are suspended because they errored before
+    # note that when `Pkg.precompile` is manually called, all suspended packages are unsuspended
     precomp_prune_suspended!(pkg_specs)
 
-    # guarding against circular deps
+    # find and guard against circular deps
     circular_deps = Base.PkgId[]
     function in_deps(_pkgs, deps, dmap)
         isempty(deps) && return false
@@ -1143,8 +1175,8 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call::
         @warn """Circular dependency detected. Precompilation will be skipped for:\n  $(join(string.(circular_deps), "\n  "))"""
     end
 
+    # if a list of packages is given, restrict to dependencies of given packages
     if !isempty(pkgs)
-        # if a list of packages is given, restrict to dependencies of given packages
         function collect_all_deps(depsmap, dep, alldeps=Base.PkgId[])
             append!(alldeps, depsmap[dep])
             for _dep in depsmap[dep]
@@ -1162,6 +1194,7 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call::
         filter!(d->in(first(d), keep), depsmap)
         isempty(depsmap) && pkgerror("No direct dependencies found matching $(repr(pkgs))")
     end
+    target = string(isempty(pkgs) ? "project" : join(pkgs, ", "), "...")
 
     pkg_queue = Base.PkgId[]
     failed_deps = Dict{Base.PkgId, String}()
@@ -1199,7 +1232,8 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call::
         end
     end
 
-    t_print = @async begin # fancy print loop
+    ## fancy print loop
+    t_print = @async begin
         try
             wait(first_started)
             (isempty(pkg_queue) || interrupted_or_done.set) && return
@@ -1234,7 +1268,8 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call::
                         final_loop || print(iostr, sprint(io -> show_progress(io, bar; termwidth = displaysize(ctx.io)[2]); context=io), "\n")
                         for dep in pkg_queue_show
                             loaded = warn_loaded && haskey(Base.loaded_modules, dep)
-                            name = dep in direct_deps ? dep.name : string(color_string(dep.name, :light_black))
+                            _name = haskey(gluepkgs, dep) ? string(gluepkgs[dep], " → ", dep.name) : dep.name
+                            name = dep in direct_deps ? _name : string(color_string(_name, :light_black))
                             if dep in precomperr_deps
                                 print(iostr, color_string("  ? ", Base.warn_color()), name, "\n")
                             elseif haskey(failed_deps, dep)
@@ -1274,7 +1309,8 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call::
     end
     tasks = Task[]
     Base.LOADING_CACHE[] = Base.LoadingCache()
-    for (pkg, deps) in depsmap # precompilation loop
+    ## precompilation loop
+    for (pkg, deps) in depsmap
         paths = Base.find_all_in_cache_path(pkg)
         sourcepath = Base.locate_package(pkg)
         if sourcepath === nothing
@@ -1307,9 +1343,10 @@ function precompile(ctx::Context, pkgs::Vector{String}=String[]; internal_call::
                     Base.acquire(parallel_limiter)
                     is_direct_dep = pkg in direct_deps
                     iob = IOBuffer()
-                    name = is_direct_dep ? pkg.name : string(color_string(pkg.name, :light_black))
+                    _name = haskey(gluepkgs, pkg) ? string(gluepkgs[pkg], " → ", pkg.name) : pkg.name
+                    name = is_direct_dep ? _name : string(color_string(_name, :light_black))
                     !fancyprint && lock(print_lock) do
-                        isempty(pkg_queue) && printpkgstyle(io, :Precompiling, "environment...")
+                        isempty(pkg_queue) && printpkgstyle(io, :Precompiling, target)
                     end
                     push!(pkg_queue, pkg)
                     started[pkg] = true
@@ -1609,13 +1646,14 @@ end
 
 @deprecate status(mode::PackageMode) status(mode=mode)
 
-function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=PKGMODE_PROJECT, outdated::Bool=false, compat::Bool=false, io::IO=stdout_f(), kwargs...)
+function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=PKGMODE_PROJECT, outdated::Bool=false, compat::Bool=false, glue::Bool=false, io::IO=stdout_f(), kwargs...)
     if compat
         diff && pkgerror("Compat status has no `diff` mode")
         outdated && pkgerror("Compat status has no `outdated` mode")
+        glue && pkgerror("Compat status has no `glue` mode")
         Operations.print_compat(ctx, pkgs; io)
     else
-        Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated)
+        Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated, glue)
     end
     return nothing
 end