Setup buildkite as an alternative to AZP

Co-authored-by: Mosè Giordano <giordano@users.noreply.github.com>

.buildkite/0_webui.yml

@@ -0,0 +1,23 @@
+# This file represents what is put into the webUI.
+# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect.
+steps:
+  - label: ":buildkite: Pipeline upload"
+    command: buildkite-agent pipeline upload
+    branches: "!gh-pages"
+    agents:
+      queue: "yggdrasil"
+    plugins:
+      - JuliaCI/merge-commit: ~
+      - staticfloat/cryptic#v2:
+          signed_pipelines:
+            - pipeline: .buildkite/pipeline.yml
+              signature_file: .buildkite/pipeline.yml.signature
+              inputs:
+                - .buildkite/generator.jl
+                - .buildkite/utils.jl
+                - .buildkite/Project.toml
+                - .buildkite/Manifest.toml
+                - .buildkite/path_processors/per-project
+                - .buildkite/init.sh
+                - .buildkite/register.sh
+    command: "true"

.buildkite/Manifest.toml

@@ -0,0 +1,162 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.8.0-beta3"
+manifest_format = "2.0"
+project_hash = "85881da52ac27331aa61f08c30c93ab8b41b5a1c"
+
+[[deps.ArgTools]]
+uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
+version = "1.1.1"
+
+[[deps.Artifacts]]
+uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+
+[[deps.Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+
+[[deps.Dates]]
+deps = ["Printf"]
+uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
+
+[[deps.Downloads]]
+deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
+uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+version = "1.6.0"
+
+[[deps.FileWatching]]
+uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
+
+[[deps.InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+
+[[deps.JLLWrappers]]
+deps = ["Preferences"]
+git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1"
+uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
+version = "1.4.1"
+
+[[deps.LibCURL]]
+deps = ["LibCURL_jll", "MozillaCACerts_jll"]
+uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
+version = "0.6.3"
+
+[[deps.LibCURL_jll]]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
+uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
+version = "7.81.0+0"
+
+[[deps.LibGit2]]
+deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
+uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
+
+[[deps.LibSSH2_jll]]
+deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
+uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
+version = "1.10.2+0"
+
+[[deps.Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+
+[[deps.Libiconv_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
+git-tree-sha1 = "42b62845d70a619f063a7da093d995ec8e15e778"
+uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531"
+version = "1.16.1+1"
+
+[[deps.Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+
+[[deps.Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+
+[[deps.MbedTLS_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
+version = "2.28.0+0"
+
+[[deps.MozillaCACerts_jll]]
+uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
+version = "2022.2.1"
+
+[[deps.NetworkOptions]]
+uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
+version = "1.2.0"
+
+[[deps.Pkg]]
+deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
+uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+version = "1.8.0"
+
+[[deps.Preferences]]
+deps = ["TOML"]
+git-tree-sha1 = "d3538e7f8a790dc8903519090857ef8e1283eecd"
+uuid = "21216c6a-2e73-6563-6e65-726566657250"
+version = "1.2.5"
+
+[[deps.Printf]]
+deps = ["Unicode"]
+uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+
+[[deps.REPL]]
+deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
+uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+
+[[deps.Random]]
+deps = ["SHA", "Serialization"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+
+[[deps.SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+version = "0.7.0"
+
+[[deps.Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+
+[[deps.Sockets]]
+uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
+
+[[deps.StringEncodings]]
+deps = ["Libiconv_jll"]
+git-tree-sha1 = "50ccd5ddb00d19392577902f0079267a72c5ab04"
+uuid = "69024149-9ee7-55f6-a4c4-859efe599b68"
+version = "0.3.5"
+
+[[deps.TOML]]
+deps = ["Dates"]
+uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+version = "1.0.0"
+
+[[deps.Tar]]
+deps = ["ArgTools", "SHA"]
+uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+version = "1.10.0"
+
+[[deps.UUIDs]]
+deps = ["Random", "SHA"]
+uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[[deps.Unicode]]
+uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
+
+[[deps.YAML]]
+deps = ["Base64", "Dates", "Printf", "StringEncodings"]
+git-tree-sha1 = "3c6e8b9f5cdaaa21340f841653942e1a6b6561e5"
+uuid = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
+version = "0.4.7"
+
+[[deps.Zlib_jll]]
+deps = ["Libdl"]
+uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
+version = "1.2.12+1"
+
+[[deps.nghttp2_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
+version = "1.41.0+1"
+
+[[deps.p7zip_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
+version = "16.2.1+1"

.buildkite/Project.toml

@@ -0,0 +1,6 @@
+[deps]
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"

.buildkite/build.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Fail on error
+set -e
+
+export JULIA_PROJECT="${BUILDKITE_BUILD_CHECKOUT_PATH}/.ci"
+
+echo "--- Setup Julia packages"
+julia --color=yes -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()'
+
+# Cleanup temporary things that might have been left-over
+echo "--- Cleanup"
+./clean_builds.sh
+./clean_products.sh
+
+echo "+++ Build"
+cd "${PROJECT}"
+julia ./build_tarballs.jl --verbose "${PLATFORM}"

.buildkite/cryptic_repo_keys/.gitignore

@@ -0,0 +1,7 @@
+
+# Ignore the unencrypted repo_key
+repo_key
+
+# Ignore any agent keys (public or private) we have stored
+agent_key*
+

.buildkite/generator.jl

@@ -0,0 +1,115 @@
+#!/bin/env julia
+if VERSION < v"1.8.0"
+    Base.ACTIVE_PROJECT[] = @__DIR__
+else
+    Base.set_active_project(@__DIR__)
+end
+import Pkg
+Pkg.instantiate()
+
+using Downloads, Dates, SHA
+
+const PROJECT = only(ARGS)
+const DEBUG = !haskey(ENV, "BUILDKITE")
+
+include("utils.jl")
+
+if !isnothing(Pkg.pkg_server())
+    resp = try
+        headers = Pkg.PlatformEngines.get_metadata_headers(Pkg.pkg_server())
+        Downloads.request("$(Pkg.pkg_server())/registries"; headers)
+    catch e
+        # Let us know the download of the registry went wrong, but do not hard fail
+        @error "Could not download the registry" exception=(e, catch_backtrace())
+    end
+    last_mod_idx = findfirst(h -> first(h) == "last-modified", resp.headers)
+    msg = "PkgServer: " * resp.url
+    delay = if !isnothing(last_mod_idx)
+        last_mod = last(resp.headers[last_mod_idx])
+        msg *= " -- last updated: " * last_mod
+        # Manually strip the "GMT" timezone and hope it never changes.
+        # Do not error out if parsing fails.
+        dt = tryparse(DateTime, replace(last_mod, " GMT"=>""), dateformat"e, d u y H:M:S")
+        # If parsing did fail, set the delay to 0.
+        isnothing(dt) ? Second(0) : now(UTC) - dt
+    else
+        Second(0)
+    end
+    delay > Second(0) && (msg *= " (" * string(Dates.canonicalize(round(delay, Second))) * " ago)")
+    @info(msg)
+    annotate(msg, style="info", context="pkg")
+    tolerance = Hour(1)
+    if delay > tolerance
+        @warn "The PkgServer registry is older than $(tolerance)"
+        annotate("The PkgServer registry is older than $(tolerance)", style = "warning", context="pkg")
+    end
+end
+
+# If there are scary projects we need to exclude, we list them here. (Used to contain `LLVM`)
+EXCLUDED_NAMES = Set{String}([])
+
+if PROJECT ∈ EXCLUDED_NAMES
+    @info "Skipping project since it is excluded." PROJECT
+    exit()
+end
+
+# Remove secret from environment
+sanitize(cmd) = addenv(cmd, Dict("BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET" => nothing))
+exec(cmd) = @assert success(pipeline(sanitize(cmd), stderr=stderr, stdout=stdout))
+
+YGGDRASIL_BASE = dirname(@__DIR__)
+julia(args) = `$(Base.julia_cmd()) --project=$(YGGDRASIL_BASE)/.ci $args`
+
+# Next, we're going to ensure that our BB is up to date and precompiled
+julia(`-e "import Pkg; Pkg.instantiate(); Pkg.precompile()"`) |> exec
+
+# Next, for each project, download its sources. We do this by generating meta.json
+# files, then parsing them with `download_sources.jl`
+
+TEMP = mktempdir()
+
+const NAME = basename(PROJECT)
+
+# We always invoke a `build_tarballs.jl` file from its own directory
+cd(PROJECT) do
+    println("Generating meta.json...")
+    JSON_PATH = "$(TEMP)/$(NAME).meta.json"
+    julia(`--compile=min ./build_tarballs.jl --meta-json="$(JSON_PATH)"`) |> exec
+
+    println("Downloading sources...")
+    julia(`$(YGGDRASIL_BASE)/.ci/download_sources.jl "$(JSON_PATH)" $(TEMP)/$(NAME).platforms.list`) |> exec
+end
+
+println("Determining builds to queue...")
+
+# Load in the platforms
+PLATFORMS = split(readchomp(joinpath(TEMP, "$(NAME).platforms.list")))
+if isempty(PLATFORMS)
+    @error "Unable to determine the proper platforms" NAME
+end
+
+# Create the BUILD_STEPS
+BUILD_STEPS = Any[]
+for PLATFORM in PLATFORMS
+    println("    $(PLATFORM): building")
+
+    push!(BUILD_STEPS, build_step(NAME, PLATFORM, PROJECT))
+end
+
+const IS_PR = get(ENV, "BUILDKITE_PULL_REQUEST", "false") != "false"
+
+STEPS = Any[]
+if !IS_PR
+    push!(STEPS, jll_init_step(NAME, PROJECT))
+    push!(STEPS, wait_step())
+end
+push!(STEPS, group_step(NAME, BUILD_STEPS))
+if !IS_PR
+    push!(STEPS, wait_step())
+    push!(STEPS, register_step(NAME, PROJECT))
+end
+
+definition = Dict(
+    :steps => STEPS
+)
+upload_pipeline(definition)

.buildkite/init.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Fail on error
+set -e
+
+export JULIA_PROJECT="${BUILDKITE_BUILD_CHECKOUT_PATH}/.ci"
+
+echo "--- Setup Julia packages"
+GITHUB_TOKEN="" julia --color=yes -e 'import Pkg; Pkg.instantiate(); Pkg.precompile()'
+
+cd ${PROJECT}
+echo "--- Generating meta.json..."
+GITHUB_TOKEN="" julia --compile=min ./build_tarballs.jl --meta-json=${NAME}.meta.json
+echo "--- Initializing JLL package..."
+julia ${BUILDKITE_BUILD_CHECKOUT_PATH}/.ci/jll_init.jl ${NAME}.meta.json

.buildkite/path_processors/per-project

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# De-duplicate directories
+declare -A DIRS
+for f in "$@"; do
+    DIR="$(dirname ${f})"
+    DIRS["${DIR}"]=1
+done
+echo "${!DIRS[@]}"

.buildkite/pipeline.yml

@@ -0,0 +1,28 @@
+steps:
+  - label: ":runner: Dynamically launch Pipelines"
+    plugins:
+      - JuliaCI/julia#v1:
+          persist_depot_dirs: packages,artifacts,compiled
+          version: "1.7"
+          depot_hard_size_limit: "68719476736" # 64GB
+      - JuliaCI/merge-commit: ~
+      - staticfloat/forerunner#v1:
+          watch:
+            - "**/*/build_tarballs.jl"
+          path_processor: .buildkite/path_processors/per-project
+          target: .buildkite/generator.jl
+          target_type: command
+    agents:
+      queue: "yggdrasil"
+      arch:  "x86_64"
+      os:    "linux"
+      # Only run on "sandbox.jl" machines
+      sandbox_capable: "true"
+    if: build.message !~ /\[skip tests\]/
+    timeout_in_minutes: 10
+    env:
+      JULIA_PKG_SERVER: us-east.pkg.julialang.org
+      # Use eager registry to not have to wait for updates of the conservative registry
+      JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
+      # Forward JOB_ID_SECRET
+      BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?}

.buildkite/pipeline.yml.signature

@@ -0,0 +1 @@
+Salted__Ί2��rY�k%��x���'Bz�D@�`p���/e$�Vb�%Q9�g2����B
K��iEC�77�Hk�8Mo�у.����|
/p���i)��