Add support for "package extensions" to code loading (#47695)

* Add support for "glue packages" to code loading

This allows packages to define "glue packages" which
are modules that are automatically loaded when
a set of other packages are loaded into the Julia
session.

(cherry picked from commit 495a004)

NEWS.md

@@ -98,6 +98,10 @@ Standard library changes
 
 #### Package Manager
 
+- "Package Extensions": support for loading a piece of code based on other
+  packages being loaded in the Julia session.
+  This has similar applications as the Requires.jl package but also
+  supports precompilation and setting compatibility.
 #### LinearAlgebra
 
 * The methods `a / b` and `b \ a` with `a` a scalar and `b` a vector, which were equivalent to `a * pinv(b)`,

doc/src/manual/code-loading.md

@@ -348,7 +348,46 @@ The subscripted `rootsᵢ`, `graphᵢ` and `pathsᵢ` variables correspond to th
 2. Packages in non-primary environments can end up using incompatible versions of their dependencies even if their own environments are entirely compatible. This can happen when one of their dependencies is shadowed by a version in an earlier environment in the stack (either by graph or path, or both).
 
 Since the primary environment is typically the environment of a project you're working on, while environments later in the stack contain additional tools, this is the right trade-off: it's better to break your development tools but keep the project working. When such incompatibilities occur, you'll typically want to upgrade your dev tools to versions that are compatible with the main project.
+### "Extension"s
 
+An "extension" is a module that is automatically loaded when a specified set of other packages (its "extension dependencies") are loaded in the current Julia session. The extension dependencies of an extension are a subset of those packages listed under the `[weakdeps]` section of a Project file. Extensions are defined under the `[extensions]` section in the project file:
+
+```toml
+name = "MyPackage"
+
+[weakdeps]
+ExtDep = "c9a23..." # uuid
+OtherExtDep = "862e..." # uuid
+
+[extensions]
+BarExt = ["ExtDep", "OtherExtDep"]
+FooExt = "ExtDep"
+...
+```
+
+The keys under `extensions` are the name of the extensions.
+They are loaded when all the packages on the right hand side (the extension dependencies) of that extension are loaded.
+If an extension only has one extension dependency the list of extension dependencies can be written as just a string for brevity.
+The location for the entry point of the extension is either in `ext/FooExt.jl` or `ext/FooExt/FooExt.jl` for
+extension `FooExt`.
+The content of an extension is often structured as:
+
+```
+module FooExt
+
+# Load main package and extension dependencies
+using MyPackage, ExtDep
+
+# Extend functionality in main package with types from the extension dependencies
+MyPackage.func(x::ExtDep.SomeStruct) = ...
+
+end
+```
+
+When a package with extensions is added to an environment, the `weakdeps` and `extensions` sections
+are stored in the manifest file in the section for that package. The dependency lookup rules for
+a package are the same as for its "parent" except that the listed extension dependencies are also considered as
+dependencies.
 ### Package/Environment Preferences
 
 Preferences are dictionaries of metadata that influence package behavior within an environment.

test/loading.jl

@@ -991,5 +991,36 @@ end
     end
 end
 
+@testset "Extensions" begin
+    old_depot_path = copy(DEPOT_PATH)
+    try
+        tmp = mktempdir()
+        push!(empty!(DEPOT_PATH), joinpath(tmp, "depot"))
+
+        proj = joinpath(@__DIR__, "project", "Extensions", "HasDepWithExtensions.jl")
+        for i in 1:2 # Once when requiring precomilation, once where it is already precompiled
+            cmd = `$(Base.julia_cmd()) --project=$proj --startup-file=no -e '
+                begin
+                using HasExtensions
+                # Base.get_extension(HasExtensions, :Extension) === nothing || error("unexpectedly got an extension")
+                HasExtensions.ext_loaded && error("ext_loaded set")
+                using HasDepWithExtensions
+                # Base.get_extension(HasExtensions, :Extension).extvar == 1 || error("extvar in Extension not set")
+                HasExtensions.ext_loaded || error("ext_loaded not set")
+                HasExtensions.ext_folder_loaded && error("ext_folder_loaded set")
+                HasDepWithExtensions.do_something() || error("do_something errored")
+                using ExtDep2
+                HasExtensions.ext_folder_loaded || error("ext_folder_loaded not set")
+
+                end
+            '`
+            @test success(cmd)
+        end
+    finally
+        copy!(DEPOT_PATH, old_depot_path)
+    end
+end
+
+
 empty!(Base.DEPOT_PATH)
 append!(Base.DEPOT_PATH, original_depot_path)

test/project/Extensions/ExtDep.jl/Project.toml

@@ -0,0 +1,3 @@
+name = "ExtDep"
+uuid = "fa069be4-f60b-4d4c-8b95-f8008775090c"
+version = "0.1.0"

test/project/Extensions/ExtDep.jl/src/ExtDep.jl

@@ -0,0 +1,5 @@
+module ExtDep
+
+struct ExtDepStruct end
+
+end # module ExtDep

test/project/Extensions/ExtDep2/Project.toml

@@ -0,0 +1,3 @@
+name = "ExtDep2"
+uuid = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
+version = "0.1.0"

test/project/Extensions/ExtDep2/src/ExtDep2.jl

@@ -0,0 +1,5 @@
+module ExtDep2
+
+greet() = print("Hello World!")
+
+end # module ExtDep2

test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml

@@ -0,0 +1,25 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.10.0-DEV"
+manifest_format = "2.0"
+project_hash = "7cbe1857ecc6692a8cc8be428a5ad5073531ff98"
+
+[[deps.ExtDep]]
+path = "../ExtDep.jl"
+uuid = "fa069be4-f60b-4d4c-8b95-f8008775090c"
+version = "0.1.0"
+
+[[deps.ExtDep2]]
+path = "../ExtDep2"
+uuid = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
+version = "0.1.0"
+
+[[deps.HasExtensions]]
+weakdeps = ["ExtDep", "ExtDep2"]
+path = "../HasExtensions.jl"
+uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8"
+version = "0.1.0"
+
+    [deps.HasExtensions.extensions]
+    Extension = "ExtDep"
+    ExtensionFolder = ["ExtDep", "ExtDep2"]

test/project/Extensions/HasDepWithExtensions.jl/Project.toml

@@ -0,0 +1,8 @@
+name = "HasDepWithExtensions"
+uuid = "d4ef3d4a-8e22-4710-85d8-c6cf2eb9efca"
+version = "0.1.0"
+
+[deps]
+ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c"
+ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
+HasExtensions = "4d3288b3-3afc-4bb6-85f3-489fffe514c8"

test/project/Extensions/HasDepWithExtensions.jl/src/HasDepWithExtensions.jl

@@ -0,0 +1,13 @@
+module HasDepWithExtensions
+
+using HasExtensions: HasExtensions, HasExtensionsStruct
+using ExtDep: ExtDepStruct
+# Loading ExtDep makes the extension "Extension" load
+
+function do_something()
+    HasExtensions.foo(HasExtensionsStruct()) == 1 || error()
+    HasExtensions.foo(ExtDepStruct()) == 2 || error()
+    return true
+end
+
+end # module

test/project/Extensions/HasExtensions.jl/Manifest.toml

@@ -0,0 +1,7 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.10.0-DEV"
+manifest_format = "2.0"
+project_hash = "c87947f1f1f070eea848950c304d668a112dec3d"
+
+[deps]

test/project/Extensions/HasExtensions.jl/Project.toml

@@ -0,0 +1,11 @@
+name = "HasExtensions"
+uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8"
+version = "0.1.0"
+
+[weakdeps]
+ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c"
+ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
+
+[extensions]
+Extension = "ExtDep"
+ExtensionFolder = ["ExtDep", "ExtDep2"]

test/project/Extensions/HasExtensions.jl/ext/Extension.jl

@@ -0,0 +1,13 @@
+module Extension
+
+using HasExtensions, ExtDep
+
+HasExtensions.foo(::ExtDep.ExtDepStruct) = 2
+
+function __init__()
+    HasExtensions.ext_loaded = true
+end
+
+const extvar = 1
+
+end

test/project/Extensions/HasExtensions.jl/ext/ExtensionFolder/ExtensionFolder.jl

@@ -0,0 +1,9 @@
+module ExtensionFolder
+
+using ExtDep, ExtDep2, HasExtensions
+
+function __init__()
+    HasExtensions.ext_folder_loaded = true
+end
+
+end

test/project/Extensions/HasExtensions.jl/src/HasExtensions.jl

@@ -0,0 +1,10 @@
+module HasExtensions
+
+struct HasExtensionsStruct end
+
+foo(::HasExtensionsStruct) = 1
+
+ext_loaded = false
+ext_folder_loaded = false
+
+end # module