Parallelize artifact downloads. Update progress bar styling. (#3952)

* parallelize artifact downloads

* show largest first

* non fancyprint mode &. formatting

* update progress bar styling

* fix alignment

* fix color

* fix non-fancy print formatting

* fix scope

* rpad artifact names on CI

* improve error handling

* error handling improvements

* comment

* fix test

* comments & tidies

---------

Co-authored-by: KristofferC <kristoffer.carlsson@juliacomputing.com>

src/API.jl

@@ -1268,7 +1268,7 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing,
     # Install all packages
     new_apply = Operations.download_source(ctx)
     # Install all artifacts
-    Operations.download_artifacts(ctx.env; platform, verbose, io=ctx.io)
+    Operations.download_artifacts(ctx; platform, verbose)
     # Run build scripts
     allow_build && Operations.build_versions(ctx, union(new_apply, new_git); verbose=verbose)
 

src/Artifacts.jl

@@ -301,6 +301,7 @@ function download_artifact(
     verbose::Bool = false,
     quiet_download::Bool = false,
     io::IO=stderr_f(),
+    progress::Union{Function, Nothing} = nothing,
 )
     if artifact_exists(tree_hash)
         return true
@@ -323,8 +324,8 @@ function download_artifact(
     temp_dir = mktempdir(artifacts_dir)
 
     try
-        download_verify_unpack(tarball_url, tarball_hash, temp_dir, ignore_existence=true, verbose=verbose,
-                quiet_download=quiet_download, io=io)
+        download_verify_unpack(tarball_url, tarball_hash, temp_dir;
+                                ignore_existence=true, verbose, quiet_download, io, progress)
         calc_hash = SHA1(GitTools.tree_hash(temp_dir))
 
         # Did we get what we expected?  If not, freak out.
@@ -394,82 +395,101 @@ function ensure_artifact_installed(name::String, artifacts_toml::String;
                                    pkg_uuid::Union{Base.UUID,Nothing}=nothing,
                                    verbose::Bool = false,
                                    quiet_download::Bool = false,
+                                   progress::Union{Function,Nothing} = nothing,
                                    io::IO=stderr_f())
     meta = artifact_meta(name, artifacts_toml; pkg_uuid=pkg_uuid, platform=platform)
     if meta === nothing
         error("Cannot locate artifact '$(name)' in '$(artifacts_toml)'")
     end
 
-    return ensure_artifact_installed(name, meta, artifacts_toml; platform=platform,
-                                     verbose=verbose, quiet_download=quiet_download, io=io)
+    return ensure_artifact_installed(name, meta, artifacts_toml;
+                                    platform, verbose, quiet_download, progress, io)
 end
 
 function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::String;
                                    platform::AbstractPlatform = HostPlatform(),
                                    verbose::Bool = false,
                                    quiet_download::Bool = false,
+                                   progress::Union{Function,Nothing} = nothing,
                                    io::IO=stderr_f())
-    hash = SHA1(meta["git-tree-sha1"])
 
+    hash = SHA1(meta["git-tree-sha1"])
     if !artifact_exists(hash)
-        errors = Any[]
-        # first try downloading from Pkg server
-        # TODO: only do this if Pkg server knows about this package
-        if (server = pkg_server()) !== nothing
-            url = "$server/artifact/$hash"
-            download_success = let url=url
-                @debug "Downloading artifact from Pkg server" name artifacts_toml platform url
-                with_show_download_info(io, name, quiet_download) do
-                    download_artifact(hash, url; verbose=verbose, quiet_download=quiet_download, io=io)
-                end
-            end
-            # download_success is either `true` or an error object
-            if download_success === true
-                return artifact_path(hash)
-            else
-                @debug "Failed to download artifact from Pkg server" download_success
-                push!(errors, (url, download_success))
-            end
+        if isnothing(progress) || verbose == true
+            return try_artifact_download_sources(name, hash, meta, artifacts_toml; platform, verbose, quiet_download, io)
+        else
+            # if a custom progress handler is given it is taken to mean the caller wants to handle the download scheduling
+            return () -> try_artifact_download_sources(name, hash, meta, artifacts_toml; platform, quiet_download=true, io, progress)
         end
+    else
+        return artifact_path(hash)
+    end
+end
 
-        # If this artifact does not exist on-disk already, ensure it has download
-        # information, then download it!
-        if !haskey(meta, "download")
-            error("Cannot automatically install '$(name)'; no download section in '$(artifacts_toml)'")
+function try_artifact_download_sources(
+            name::String, hash::SHA1, meta::Dict, artifacts_toml::String;
+            platform::AbstractPlatform=HostPlatform(),
+            verbose::Bool=false,
+            quiet_download::Bool=false,
+            io::IO=stderr_f(),
+            progress::Union{Function,Nothing}=nothing)
+
+    errors = Any[]
+    # first try downloading from Pkg server
+    # TODO: only do this if Pkg server knows about this package
+    if (server = pkg_server()) !== nothing
+        url = "$server/artifact/$hash"
+        download_success = let url = url
+            @debug "Downloading artifact from Pkg server" name artifacts_toml platform url
+            with_show_download_info(io, name, quiet_download) do
+                download_artifact(hash, url; verbose, quiet_download, io, progress)
+            end
         end
+        # download_success is either `true` or an error object
+        if download_success === true
+            return artifact_path(hash)
+        else
+            @debug "Failed to download artifact from Pkg server" download_success
+            push!(errors, (url, download_success))
+        end
+    end
 
-        # Attempt to download from all sources
-        for entry in meta["download"]
-            url = entry["url"]
-            tarball_hash = entry["sha256"]
-            download_success = let url=url
-                @debug "Downloading artifact" name artifacts_toml platform url
-                with_show_download_info(io, name, quiet_download) do
-                    download_artifact(hash, url, tarball_hash; verbose=verbose, quiet_download=quiet_download, io=io)
-                end
-            end
-            # download_success is either `true` or an error object
-            if download_success === true
-                return artifact_path(hash)
-            else
-                @debug "Failed to download artifact" download_success
-                push!(errors, (url, download_success))
+    # If this artifact does not exist on-disk already, ensure it has download
+    # information, then download it!
+    if !haskey(meta, "download")
+        error("Cannot automatically install '$(name)'; no download section in '$(artifacts_toml)'")
+    end
+
+    # Attempt to download from all sources
+    for entry in meta["download"]
+        url = entry["url"]
+        tarball_hash = entry["sha256"]
+        download_success = let url = url
+            @debug "Downloading artifact" name artifacts_toml platform url
+            with_show_download_info(io, name, quiet_download) do
+                download_artifact(hash, url, tarball_hash; verbose, quiet_download, io, progress)
             end
         end
-        errmsg = """
-        Unable to automatically download/install artifact '$(name)' from sources listed in '$(artifacts_toml)'.
-        Sources attempted:
-        """
-        for (url, err) in errors
-            errmsg *= "- $(url)\n"
-            errmsg *= "    Error: $(sprint(showerror, err))\n"
+        # download_success is either `true` or an error object
+        if download_success === true
+            return artifact_path(hash)
+        else
+            @debug "Failed to download artifact" download_success
+            push!(errors, (url, download_success))
         end
-        error(errmsg)
-    else
-        return artifact_path(hash)
     end
+    errmsg = """
+    Unable to automatically download/install artifact '$(name)' from sources listed in '$(artifacts_toml)'.
+    Sources attempted:
+    """
+    for (url, err) in errors
+        errmsg *= "- $(url)\n"
+        errmsg *= "    Error: $(sprint(showerror, err))\n"
+    end
+    error(errmsg)
 end
 
+
 function with_show_download_info(f, io, name, quiet_download)
     fancyprint = can_fancyprint(io)
     if !quiet_download
@@ -485,7 +505,7 @@ function with_show_download_info(f, io, name, quiet_download)
         if !quiet_download
             fancyprint && print(io, "\033[1A") # move cursor up one line
             fancyprint && print(io, "\033[2K") # clear line
-            if success 
+            if success
                 fancyprint && printpkgstyle(io, :Downloaded, "artifact: $name")
             else
                 printpkgstyle(io, :Failure, "artifact: $name", color = :red)

src/MiniProgressBars.jl

@@ -4,18 +4,31 @@ export MiniProgressBar, start_progress, end_progress, show_progress, print_progr
 
 using Printf
 
+# Until Base.format_bytes supports sigdigits
+function pkg_format_bytes(bytes; binary=true, sigdigits::Integer=3)
+    units = binary ? Base._mem_units : Base._cnt_units
+    factor = binary ? 1024 : 1000
+    bytes, mb = Base.prettyprint_getunits(bytes, length(units), Int64(factor))
+    if mb == 1
+        return string(Int(bytes), " ", Base._mem_units[mb], bytes==1 ? "" : "s")
+    else
+        return string(Base.Ryu.writefixed(Float64(bytes), sigdigits), binary ? " $(units[mb])" : "$(units[mb])B")
+    end
+end
+
 Base.@kwdef mutable struct MiniProgressBar
     max::Int = 1.0
     header::String = ""
     color::Symbol = :nothing
     width::Int = 40
-    current::Int = 0.0
-    prev::Int = 0.0
+    current::Int = 0
+    prev::Int = 0
     has_shown::Bool = false
     time_shown::Float64 = 0.0
-    percentage::Bool = true
+    mode::Symbol = :percentage # :percentage :int :data
     always_reprint::Bool = false
     indent::Int = 4
+    main::Bool = true
 end
 
 const PROGRESS_BAR_TIME_GRANULARITY = Ref(1 / 30.0) # 30 fps
@@ -47,21 +60,32 @@ function show_progress(io::IO, p::MiniProgressBar; termwidth=nothing, carriagere
     p.prev = p.current
     p.has_shown = true
 
-    progress_text = if p.percentage
+    progress_text = if p.mode == :percentage
         @sprintf "%2.1f %%" perc
-    else
+    elseif p.mode == :int
         string(p.current, "/",  p.max)
+    elseif p.mode == :data
+        lpad(string(pkg_format_bytes(p.current; sigdigits=1), "/", pkg_format_bytes(p.max; sigdigits=1)), 20)
+    else
+        error("Unknown mode $(p.mode)")
     end
     termwidth = @something termwidth displaysize(io)[2]
     max_progress_width = max(0, min(termwidth - textwidth(p.header) - textwidth(progress_text) - 10 , p.width))
     n_filled = ceil(Int, max_progress_width * perc / 100)
     n_left = max_progress_width - n_filled
+    headers = split(p.header)
     to_print = sprint(; context=io) do io
         print(io, " "^p.indent)
-        printstyled(io, p.header, color=p.color, bold=true)
-        print(io, " [")
-        print(io, "="^n_filled, ">")
-        print(io, " "^n_left, "]  ", )
+        if p.main
+            printstyled(io, headers[1], " "; color=:green, bold=true)
+            printstyled(io, join(headers[2:end], ' '))
+        else
+            print(io, p.header)
+        end
+        print(io, " ")
+        printstyled(io, "━"^n_filled; color=p.color)
+        printstyled(io, perc >= 95 ? "━" : "╸"; color=p.color)
+        printstyled(io, "━"^n_left, " "; color=:light_black)
         print(io, progress_text)
         carriagereturn && print(io, "\r")
     end
@@ -80,10 +104,10 @@ end
 # prog = MiniProgressBar(...)
 # prog.end = n
 # for progress in 1:n
-#     print_progree_bottom(io)
+#     print_progress_bottom(io)
 #     println("stuff")
 #     prog.current = progress
-#     showproress(io, prog)
+#     showprogress(io, prog)
 #  end
 #
 function print_progress_bottom(io::IO)