From 92e2f86654252db5aa4a0aa74ffe8671a34dd818 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Tue, 10 Aug 2021 10:02:10 +0200 Subject: [PATCH] only attempt to use Pkg server for package downloads if the server tracks a registry containing that package --- src/Operations.jl | 19 +++++++++++++--- src/Registry/Registry.jl | 47 ++++++++++++++++++---------------------- test/utils.jl | 2 +- 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index 3d8fd13e7d..9492080ff2 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -669,6 +669,9 @@ function download_source(ctx::Context; readonly=true) widths = [textwidth(pkg.name) for (pkg, _) in pkgs_to_install] max_name = maximum(widths; init=0) + # Check what registries the current pkg server tracks + server_registry_info = Registry.pkg_server_registry_info() + @sync begin jobs = Channel{eltype(pkgs_to_install)}(ctx.num_concurrent_downloads) results = Channel(ctx.num_concurrent_downloads) @@ -688,9 +691,19 @@ function download_source(ctx::Context; readonly=true) end try archive_urls = Pair{String,Bool}[] - if (server = pkg_server()) !== nothing - url = "$server/package/$(pkg.uuid)/$(pkg.tree_hash)" - push!(archive_urls, url => true) + # Check if the current package is available in one of the registries being tracked by the pkg server + # In that case, download from the package server + if server_registry_info !== nothing + server, registry_info = server_registry_info + for reg in ctx.registries + if reg.uuid in keys(registry_info) + if haskey(reg, pkg.uuid) + url = "$server/package/$(pkg.uuid)/$(pkg.tree_hash)" + push!(archive_urls, url => true) + break + end + end + end end for repo_url in urls url = get_archive_url_for_version(repo_url, pkg.tree_hash) diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index 7aa1eb8edb..92018795e0 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -53,26 +53,10 @@ const DEFAULT_REGISTRIES = uuid = UUID("23338594-aafe-5451-b93e-139f81909106"), url = "https://github.com/JuliaRegistries/General.git")] -# Use the pattern -# -# registry_urls = nothing -# for ... -# url, registry_urls = pkg_server_registry_url(uuid, registry_urls) -# end -# -# to query the pkg server at most once for registries. -pkg_server_registry_url(uuid::UUID, ::Nothing) = - pkg_server_registry_url(uuid, pkg_server_registry_urls()) - -pkg_server_registry_url(uuid::UUID, registry_urls::Dict{UUID, String}) = - get(registry_urls, uuid, nothing), registry_urls - -pkg_server_registry_url(::Nothing, registry_urls) = nothing, registry_urls - -function pkg_server_registry_urls() - registry_urls = Dict{UUID, String}() +function pkg_server_registry_info() + registry_info = Dict{UUID, Base.SHA1}() server = pkg_server() - server === nothing && return registry_urls + server === nothing && return registry_info tmp_path = tempname() download_ok = false try @@ -85,13 +69,24 @@ function pkg_server_registry_urls() open(tmp_path) do io for line in eachline(io) if (m = match(r"^/registry/([^/]+)/([^/]+)$", line)) !== nothing - uuid = UUID(m.captures[1]) - hash = String(m.captures[2]) - registry_urls[uuid] = "$server/registry/$uuid/$hash" + uuid = UUID(m.captures[1]::SubString{String}) + hash = Base.SHA1(m.captures[2]::SubString{String}) + registry_info[uuid] = hash end end end Base.rm(tmp_path, force=true) + return server, registry_info +end + +function pkg_server_registry_urls() + server_registry_info = pkg_server_registry_info() + server_registry_info === nothing && return nothing + registry_urls = Dict{UUID, String}() + server, registry_info = server_registry_info + for (uuid, hash) in registry_info + registry_urls[uuid] = "$server/registry/$uuid/$hash" + end return registry_urls end @@ -166,12 +161,12 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depot::String=d populate_known_registries_with_urls!(regs) regdir = joinpath(depot, "registries") isdir(regdir) || mkpath(regdir) - registry_urls = nothing + registry_urls = pkg_server_registry_urls() for reg in regs if reg.path !== nothing && reg.url !== nothing Pkg.Types.pkgerror("ambiguous registry specification; both url and path is set.") end - url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls) + url = get(registry_urls, reg.uuid, nothing) if url !== nothing && registry_read_from_tarball() tmp = tempname() try @@ -347,14 +342,14 @@ update(regs::Vector{String}; kwargs...) = update([RegistrySpec(name = name) for function update(regs::Vector{RegistrySpec} = RegistrySpec[]; io::IO=stderr_f(), force::Bool=true) isempty(regs) && (regs = reachable_registries(; depots=depots1())) errors = Tuple{String, String}[] - registry_urls = nothing + registry_urls = pkg_server_registry_urls() for reg in unique(r -> r.uuid, find_installed_registries(io, regs); seen=Set{UUID}()) let reg=reg regpath = pathrepr(reg.path) if reg.tree_info !== nothing printpkgstyle(io, :Updating, "registry at " * regpath) old_hash = reg.tree_info - url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls) + url = get(registry_urls, reg.uuid, nothing) if url !== nothing check_registry_state(reg) end diff --git a/test/utils.jl b/test/utils.jl index a64e9a2307..b7b0e1c755 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -20,7 +20,7 @@ const REGISTRY_DIR = joinpath(REGISTRY_DEPOT, "registries", "General") const GENERAL_UUID = UUID("23338594-aafe-5451-b93e-139f81909106") function init_reg() - url, _ = Pkg.Registry.pkg_server_registry_url(GENERAL_UUID, nothing) + url = Pkg.Registry.pkg_server_registry_urls[GENERAL_UUID] mkpath(REGISTRY_DIR) if Pkg.Registry.registry_use_pkg_server() @info "Downloading General registry from $url"