Merge pull request #26854 from JuliaLang/kc/bump_pkg3_6

Bump Pkg3

stdlib/Pkg3/.travis.yml

@@ -19,8 +19,9 @@ branches:
 
 script:
   - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
-  - julia --check-bounds=yes -e 'Pkg.clone(pwd()); Pkg.build("Pkg3"); Pkg.test("Pkg3"; coverage=true)'
+  - julia --check-bounds=yes -e 'using UUIDs; write("Project.toml", replace(read("Project.toml", String), r"uuid = .*?\n" =>"uuid = \"$(uuid4())\"")); import Pkg3; Pkg3.build(); Pkg3.test(; coverage=true)'
 
 after_success:
-  - julia -e 'cd(Pkg.dir("Pkg3")); Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())'
+  - julia -e 'import Pkg3; Pkg3.add("Documenter"); include("docs/make.jl")'
+  - julia -e 'import Pkg3; Pkg3.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())'
 

stdlib/Pkg3/appveyor.yml

@@ -32,8 +32,8 @@ build_script:
 # Need to convert from shallow to complete for Pkg.clone to work
   - IF EXIST .git\shallow (git fetch --unshallow)
   - C:\projects\julia\bin\julia -e "versioninfo();
-      Pkg.clone(pwd(), \"Pkg3\"); Pkg.build(\"Pkg3\")"
+      using UUIDs; write(\"Project.toml\", replace(read(\"Project.toml\", String), r\"uuid = .*?\\n\" => \"uuid = \\\"\$(uuid4())\\\"\")); import Pkg3; Pkg3.build()"
 
 test_script:
-  - C:\projects\julia\bin\julia -e "Pkg.test(\"Pkg3\")"
+  - C:\projects\julia\bin\julia -e "import Pkg3; Pkg3.test(; coverage=true)"
 

stdlib/Pkg3/docs/src/index.md

@@ -322,6 +322,7 @@ Downloaded MacroTools ─ v0.4.0
 The dependencies of the unregistered package (here `MacroTools`) got installed.
 For unregistered packages we could have given a branch (or commit SHA) to track using `#`, just like for registered packages.
 
+
 ## Developing packages
 
 Let’s say we found a bug in one of our dependencies, e.g. `JSON` that we want to fix. We can get the full git-repo using the `develop` command
@@ -352,7 +353,7 @@ It is also possible to give a local path as the argument to `develop` which will
 
 Overriding the dependency path for a non registered package is done by giving the git-repo url as an argument to `develop`.
 
-### Updating dependencies
+## Updating dependencies
 
 When new versions of packages the project is using  are released, it is a good idea to update. Simply calling `up` will try to update *all* the dependencies of the project. Sometimes this is not what you want. You can specify a subset of the dependencies to upgrade by giving them as arguments to `up`, e.g:
 
@@ -375,7 +376,17 @@ If you just want install the packages that are given by the current `Manifest.to
 (HelloWorld) pkg> up --manifest --fixed
 ```
 
-### Preview mode
+## Precompiling the project
+
+The REPL command `precompile` can be used to precompile all the dependencies in the project. You can for example do
+
+```
+(HelloWorld) pkg> update; precompile
+```
+
+do update the dependencies and then precompile them.
+
+## Preview mode
 
 If you just want to see the effects of running a command, but not change your state you can `preview` a command.
 For example:
@@ -393,7 +404,7 @@ or
 will show you the effects adding `Plots`, or doing a full upgrade, respectively, would have on your project.
 However, nothing would be installed and your `Project.toml` and `Manfiest.toml` are untouched.
 
-### Using someone elses project
+## Using someone elses project
 
 Simple clone their project using e.g. `git clone`, `cd` to the project directory and call
 
@@ -402,3 +413,5 @@ Simple clone their project using e.g. `git clone`, `cd` to the project directory
 ```
 
 This will install the packages at the same state that the project you cloned was using.
+
+

stdlib/Pkg3/src/API.jl

@@ -7,16 +7,18 @@ import Dates
 import LibGit2
 
 import ..depots, ..logdir, ..devdir, ..print_first_command_header
-import ..Operations, ..Display, ..GitTools
+import ..Operations, ..Display, ..GitTools, ..Pkg3
 using ..Types, ..TOML
 
 
 preview_info() = printstyled("───── Preview mode ─────\n"; color=Base.info_color(), bold=true)
 
 include("generate.jl")
 
+parse_package(pkg) = Pkg3.REPLMode.parse_package(pkg; context=Pkg3.REPLMode.CMD_ADD)
+
 add_or_develop(pkg::Union{String, PackageSpec}; kwargs...) = add_or_develop([pkg]; kwargs...)
-add_or_develop(pkgs::Vector{String}; kwargs...)            = add_or_develop([PackageSpec(pkg) for pkg in pkgs]; kwargs...)
+add_or_develop(pkgs::Vector{String}; kwargs...)            = add_or_develop([parse_package(pkg) for pkg in pkgs]; kwargs...)
 add_or_develop(pkgs::Vector{PackageSpec}; kwargs...)       = add_or_develop(Context(), pkgs; kwargs...)
 
 function add_or_develop(ctx::Context, pkgs::Vector{PackageSpec}; mode::Symbol, kwargs...)
@@ -27,13 +29,15 @@ function add_or_develop(ctx::Context, pkgs::Vector{PackageSpec}; mode::Symbol, k
         new_git = handle_repos_develop!(ctx, pkgs)
     else
         new_git = handle_repos_add!(ctx, pkgs; upgrade_or_add=true)
+        update_registry(ctx)
     end
     project_deps_resolve!(ctx.env, pkgs)
     registry_resolve!(ctx.env, pkgs)
     stdlib_resolve!(ctx, pkgs)
     ensure_resolved(ctx.env, pkgs, registry=true)
     Operations.add_or_develop(ctx, pkgs; new_git=new_git)
     ctx.preview && preview_info()
+    return
 end
 
 add(args...; kwargs...) = add_or_develop(args...; mode = :add, kwargs...)
@@ -53,24 +57,15 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; kwargs...)
     manifest_resolve!(ctx.env, pkgs)
     Operations.rm(ctx, pkgs)
     ctx.preview && preview_info()
+    return
 end
 
 
-up(;kwargs...)                           = up(PackageSpec[]; kwargs...)
-up(pkg::Union{String, PackageSpec}; kwargs...)               = up([pkg]; kwargs...)
-up(pkgs::Vector{String}; kwargs...)      = up([PackageSpec(pkg) for pkg in pkgs]; kwargs...)
-up(pkgs::Vector{PackageSpec}; kwargs...) = up(Context(), pkgs; kwargs...)
-
-function up(ctx::Context, pkgs::Vector{PackageSpec};
-            level::UpgradeLevel=UPLEVEL_MAJOR, mode::PackageMode=PKGMODE_PROJECT, kwargs...)
-    print_first_command_header()
-    Context!(ctx; kwargs...)
-    ctx.preview && preview_info()
-
+function update_registry(ctx)
     # Update the registry
     errors = Tuple{String, String}[]
     if ctx.preview
-        info("Skipping updating registry in preview mode")
+        @info("Skipping updating registry in preview mode")
     else
         for reg in registries()
             if isdir(joinpath(reg, ".git"))
@@ -86,7 +81,13 @@ function up(ctx::Context, pkgs::Vector{PackageSpec};
                         return
                     end
                     branch = LibGit2.headname(repo)
-                    GitTools.fetch(repo)
+                    try
+                        GitTools.fetch(repo)
+                    catch e
+                        e isa LibGit2.GitError || rethrow(e)
+                        push!(errors, (reg, "failed to fetch from repo"))
+                        return
+                    end
                     ff_succeeded = try
                         LibGit2.merge!(repo; branch="refs/remotes/origin/$branch", fastforward=true)
                     catch e
@@ -107,15 +108,27 @@ function up(ctx::Context, pkgs::Vector{PackageSpec};
             end
         end
     end
-
     if !isempty(errors)
         warn_str = "Some registries failed to update:"
         for (reg, err) in errors
             warn_str *= "\n    — $reg — $err"
         end
         @warn warn_str
     end
+    return
+end
 
+up(;kwargs...)                           = up(PackageSpec[]; kwargs...)
+up(pkg::Union{String, PackageSpec}; kwargs...)               = up([pkg]; kwargs...)
+up(pkgs::Vector{String}; kwargs...)      = up([PackageSpec(pkg) for pkg in pkgs]; kwargs...)
+up(pkgs::Vector{PackageSpec}; kwargs...) = up(Context(), pkgs; kwargs...)
+
+function up(ctx::Context, pkgs::Vector{PackageSpec};
+            level::UpgradeLevel=UPLEVEL_MAJOR, mode::PackageMode=PKGMODE_PROJECT, kwargs...)
+    print_first_command_header()
+    Context!(ctx; kwargs...)
+    ctx.preview && preview_info()
+    update_registry(ctx)
     if isempty(pkgs)
         if mode == PKGMODE_PROJECT
             for (name::String, uuidstr::String) in ctx.env.project["deps"]
@@ -135,6 +148,7 @@ function up(ctx::Context, pkgs::Vector{PackageSpec};
     end
     Operations.up(ctx, pkgs)
     ctx.preview && preview_info()
+    return
 end
 
 
@@ -149,6 +163,7 @@ function pin(ctx::Context, pkgs::Vector{PackageSpec}; kwargs...)
     project_deps_resolve!(ctx.env, pkgs)
     ensure_resolved(ctx.env, pkgs)
     Operations.pin(ctx, pkgs)
+    return
 end
 
 
@@ -163,11 +178,12 @@ function free(ctx::Context, pkgs::Vector{PackageSpec}; kwargs...)
     registry_resolve!(ctx.env, pkgs)
     ensure_resolved(ctx.env, pkgs; registry=true)
     Operations.free(ctx, pkgs)
+    return
 end
 
 
 
-test(;kwargs...)                                  = test(PackageSpec[], kwargs...)
+test(;kwargs...)                                  = test(PackageSpec[]; kwargs...)
 test(pkg::Union{String, PackageSpec}; kwargs...)  = test([pkg]; kwargs...)
 test(pkgs::Vector{String}; kwargs...)             = test([PackageSpec(pkg) for pkg in pkgs]; kwargs...)
 test(pkgs::Vector{PackageSpec}; kwargs...)        = test(Context(), pkgs; kwargs...)
@@ -186,6 +202,7 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; coverage=false, kwargs...
     manifest_resolve!(ctx.env, pkgs)
     ensure_resolved(ctx.env, pkgs)
     Operations.test(ctx, pkgs; coverage=coverage)
+    return
 end
 
 
@@ -299,6 +316,7 @@ function gc(ctx::Context=Context(); period = Dates.Week(6), kwargs...)
     byte_save_str = length(paths_to_delete) == 0 ? "" : ("saving " * @sprintf("%.3f %s", bytes, Base._mem_units[mb]))
     @info("Deleted $(length(paths_to_delete)) package installations $byte_save_str")
     ctx.preview && preview_info()
+    return
 end
 
 
@@ -317,6 +335,7 @@ function _get_deps!(ctx::Context, pkgs::Vector{PackageSpec}, uuids::Vector{UUID}
         end
         _get_deps!(ctx, pkgs, uuids)
     end
+    return
 end
 
 build(pkgs...) = build([PackageSpec(pkg) for pkg in pkgs])
@@ -349,6 +368,7 @@ function build(ctx::Context, pkgs::Vector{PackageSpec}; kwargs...)
     length(uuids) == 0 && (@info("no packages to build"); return)
     Operations.build_versions(ctx, uuids; might_need_to_resolve=true)
     ctx.preview && preview_info()
+    return
 end
 
 init() = init(Context())
@@ -357,6 +377,82 @@ function init(ctx::Context, path::String=pwd())
     print_first_command_header()
     Context!(ctx; env = EnvCache(joinpath(path, "Project.toml")))
     Operations.init(ctx)
+    return
+end
+
+#####################################
+# Backwards compatibility with Pkg2 #
+#####################################
+
+function clone(pkg::String...)
+    @warn "Pkg.clone is only kept for legacy CI script reasons, please use `add`" maxlog=1
+    add(joinpath(pkg...))
+end
+
+function dir(pkg::String, paths::String...)
+    @warn "Pkg.dir is only kept for legacy CI script reasons" maxlog=1
+    pkgid = Base.identify_package(pkg)
+    pkgid == nothing && return nothing
+    path = Base.locate_package(pkgid)
+    pkgid == nothing && return nothing
+    return joinpath(abspath(path, "..", "..", paths...))
+end
+
+
+function precompile(ctx::Context)
+    printpkgstyle(ctx, :Precompiling, "project...")
+
+    pkgids = [Base.PkgId(UUID(uuid), name) for (name, uuid) in ctx.env.project["deps"] if !(UUID(uuid) in  keys(ctx.stdlibs))]
+    if ctx.env.pkg !== nothing && isfile( joinpath( dirname(ctx.env.project_file), "src", ctx.env.pkg.name * ".jl"))
+        push!(pkgids, Base.PkgId(ctx.env.pkg.uuid, ctx.env.pkg.name))
+    end
+
+    needs_to_be_precompiled = String[]
+    for pkg in pkgids
+        paths = Base.find_all_in_cache_path(pkg)
+        sourcepath = Base.locate_package(pkg)
+        if sourcepath == nothing
+            cmderror("couldn't find path to $(pkg.name) when trying to precompilie project")
+        end
+        found_matching_precompile = false
+        for path_to_try in paths::Vector{String}
+            staledeps = Base.stale_cachefile(sourcepath, path_to_try)
+            if !(staledeps isa Bool)
+                found_matching_precompile = true
+            end
+        end
+        if !found_matching_precompile
+            # Only precompile packages that has contains `__precompile__` or `__precompile__(true)`
+            source = read(sourcepath, String)
+            if occursin(r"__precompile__\(\)|__precompile__\(true\)", source)
+                push!(needs_to_be_precompiled, pkg.name)
+            end
+        end
+    end
+
+    # Perhaps running all the imports in the same process would avoid some overheda.
+    # Julia starts pretty fast though (0.3 seconds)
+    code = join(["import " * pkg for pkg in needs_to_be_precompiled], '\n') * "\nexit(0)"
+    for (i, pkg) in enumerate(needs_to_be_precompiled)
+        code = """
+            empty!(Base.DEPOT_PATH)
+            append!(Base.DEPOT_PATH, $(repr(map(abspath, DEPOT_PATH))))
+            empty!(Base.DL_LOAD_PATH)
+            append!(Base.DL_LOAD_PATH, $(repr(map(abspath, Base.DL_LOAD_PATH))))
+            empty!(Base.LOAD_PATH)
+            append!(Base.LOAD_PATH, $(repr(Base.LOAD_PATH)))
+            import $pkg
+        """
+        printpkgstyle(ctx, :Precompiling, pkg, " [$i of $(length(needs_to_be_precompiled))]")
+        run(pipeline(ignorestatus(```
+        $(Base.julia_cmd()) -O$(Base.JLOptions().opt_level) --color=no --history-file=no
+        --startup-file=$(Base.JLOptions().startupfile != 2 ? "yes" : "no")
+        --compiled-modules="yes"
+        --depwarn=no
+        --eval $code
+        ```)))
+    end
+    return nothing
 end
 
 end # module

stdlib/Pkg3/src/Operations.jl

@@ -106,7 +106,7 @@ function collect_fixed!(ctx::Context, pkgs::Vector{PackageSpec}, uuid_to_name::D
 
         uuid_to_pkg[pkg.uuid] = pkg
         uuid_to_name[pkg.uuid] = pkg.name
-        found_project = collect_project!(pkg, path, fix_deps_map)
+        found_project = collect_project!(ctx, pkg, path, fix_deps_map)
         if !found_project
             collect_require!(ctx, pkg, path, fix_deps_map)
         end
@@ -127,7 +127,7 @@ function collect_fixed!(ctx::Context, pkgs::Vector{PackageSpec}, uuid_to_name::D
     return fixed
 end
 
-function collect_project!(pkg::PackageSpec, path::String, fix_deps_map::Dict{UUID,Vector{PackageSpec}})
+function collect_project!(ctx::Context, pkg::PackageSpec, path::String, fix_deps_map::Dict{UUID,Vector{PackageSpec}})
     project_file = joinpath(path, "Project.toml")
     fix_deps_map[pkg.uuid] = valtype(fix_deps_map)()
     !isfile(project_file) && return false
@@ -137,7 +137,12 @@ function collect_project!(pkg::PackageSpec, path::String, fix_deps_map::Dict{UUI
         deppkg = PackageSpec(deppkg_name, UUID(uuid), vspec)
         push!(fix_deps_map[pkg.uuid], deppkg)
     end
-    pkg.version = VersionNumber(get(project, "version", "0.0"))
+    if haskey(project, "version")
+        pkg.version = VersionNumber(project["version"])
+    else
+        @warn "project file for $(pkg.name) is missing a `version` entry"
+        set_maximum_version_registry!(ctx.env, pkg)
+    end
     return true
 end
 

stdlib/Pkg3/src/Pkg3.jl

@@ -40,6 +40,10 @@ include("REPLMode.jl")
 
 import .API: add, rm, up, test, gc, init, build, installed, pin, free, checkout, develop, generate
 const update = up
+# legacy CI script support
+import .API: clone, dir
+
+
 import .REPLMode: @pkg_str
 export @pkg_str