From e6e79f7ab4549ab4d6ffbad22038ebfce1368f93 Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sun, 25 Jul 2021 05:16:24 -0400 Subject: [PATCH] CI (Buildkite): Update all rootfs images to version v3.1, add some documentation, and reorganize/rename/move a bunch of configuration files (#41698) --- .buildkite/README.md | 5 ++ .buildkite/coverage-linux64/0_webui.yml | 16 ------ .buildkite/coverage-linux64/README.md | 6 --- .buildkite/coverage-linux64/pipeline.yml | 53 ------------------- .../coverage-linux64/run_tests_parallel.jl | 36 ------------- .../coverage-linux64/run_tests_serial.jl | 16 ------ .buildkite/experimental/pipeline.yml | 19 ------- .buildkite/pipeline.yml | 28 ---------- .../{ => pipelines}/experimental/0_webui.yml | 8 ++- .buildkite/pipelines/experimental/README.md | 7 +++ .../experimental/launch_unsigned_builders.yml | 6 +++ .../experimental/misc}/sanitizers.yml | 13 ++--- .buildkite/{ => pipelines/main}/0_webui.yml | 4 +- .buildkite/pipelines/main/README.md | 15 ++++++ .../main/launch_unsigned_builders.yml | 29 ++++++++++ .../{ => pipelines/main}/misc/doctest.yml | 9 ++-- .../{ => pipelines/main}/misc/embedding.yml | 7 +-- .../{ => pipelines/main}/misc/llvmpasses.yml | 7 +-- .../main/misc}/signed_pipeline_test.yml | 0 .../{ => pipelines/main}/misc/whitespace.yml | 7 +-- .../main}/platforms/linux64.yml | 14 +++-- .buildkite/pipelines/scheduled/0_webui.yml | 24 +++++++++ .buildkite/pipelines/scheduled/README.md | 5 ++ .../scheduled/coverage/coverage_linux64.yml | 40 ++++++++++++++ .../scheduled/coverage/run_tests_parallel.jl | 25 +++++++++ .../scheduled/coverage}/upload_coverage.jl | 0 .../scheduled/launch_unsigned_builders.yml | 0 .buildkite/{ => utilities/rr}/rr_capture.jl | 0 28 files changed, 185 insertions(+), 214 deletions(-) create mode 100644 .buildkite/README.md delete mode 100644 .buildkite/coverage-linux64/0_webui.yml delete mode 100644 .buildkite/coverage-linux64/README.md delete mode 100644 .buildkite/coverage-linux64/pipeline.yml delete mode 100644 .buildkite/coverage-linux64/run_tests_parallel.jl delete mode 100644 .buildkite/coverage-linux64/run_tests_serial.jl delete mode 100644 .buildkite/experimental/pipeline.yml delete mode 100644 .buildkite/pipeline.yml rename .buildkite/{ => pipelines}/experimental/0_webui.yml (67%) create mode 100644 .buildkite/pipelines/experimental/README.md create mode 100644 .buildkite/pipelines/experimental/launch_unsigned_builders.yml rename .buildkite/{experimental => pipelines/experimental/misc}/sanitizers.yml (67%) rename .buildkite/{ => pipelines/main}/0_webui.yml (89%) create mode 100644 .buildkite/pipelines/main/README.md create mode 100644 .buildkite/pipelines/main/launch_unsigned_builders.yml rename .buildkite/{ => pipelines/main}/misc/doctest.yml (71%) rename .buildkite/{ => pipelines/main}/misc/embedding.yml (76%) rename .buildkite/{ => pipelines/main}/misc/llvmpasses.yml (83%) rename .buildkite/{ => pipelines/main/misc}/signed_pipeline_test.yml (100%) rename .buildkite/{ => pipelines/main}/misc/whitespace.yml (63%) rename .buildkite/{ => pipelines/main}/platforms/linux64.yml (79%) create mode 100644 .buildkite/pipelines/scheduled/0_webui.yml create mode 100644 .buildkite/pipelines/scheduled/README.md create mode 100644 .buildkite/pipelines/scheduled/coverage/coverage_linux64.yml create mode 100644 .buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl rename .buildkite/{coverage-linux64 => pipelines/scheduled/coverage}/upload_coverage.jl (100%) create mode 100644 .buildkite/pipelines/scheduled/launch_unsigned_builders.yml rename .buildkite/{ => utilities/rr}/rr_capture.jl (100%) diff --git a/.buildkite/README.md b/.buildkite/README.md new file mode 100644 index 0000000000000..12887536b90c1 --- /dev/null +++ b/.buildkite/README.md @@ -0,0 +1,5 @@ +# Buildkite + +This directory contains the Buildkite configuration files for Base Julia CI. + +The rootfs image definitions are located in the [rootfs-images](https://github.com/JuliaCI/rootfs-images) repository. diff --git a/.buildkite/coverage-linux64/0_webui.yml b/.buildkite/coverage-linux64/0_webui.yml deleted file mode 100644 index b0df8ec41de68..0000000000000 --- a/.buildkite/coverage-linux64/0_webui.yml +++ /dev/null @@ -1,16 +0,0 @@ -# This file represents what is put into the webUI. -# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. -# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. -agents: - queue: "julia" - sandbox.jl: "true" - -steps: - - label: ":unlock: Unlock secrets, launch pipelines" - plugins: - - staticfloat/cryptic: - # Our signed pipelines must have a `signature` or `signature_file` parameter that - # verifies the treehash of the pipeline itself and the inputs listed in `inputs` - signed_pipelines: - - pipeline: .buildkite/coverage-linux64/pipeline.yml - signature: U2FsdGVkX190BiaBGCTT6GNBDe5cHX3ZAP0IXbBfOWo7ys+1IuD5ymf4ImJbRIeE8NQac+iqud+bqCPHjii6DKmqzX+Jz6vax1NY3AxROhYlO5spUClvKr1wdngUCQON diff --git a/.buildkite/coverage-linux64/README.md b/.buildkite/coverage-linux64/README.md deleted file mode 100644 index 8cfb31d698225..0000000000000 --- a/.buildkite/coverage-linux64/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Coverage pipeline - -We run coverage on a separate pipeline, that uses a scheduled build rather than webhooks. -The pipeline is here: https://buildkite.com/julialang/julia-coverage-linux64 - -It contains [its own webui steps](0_webuiy.ml) (listed here in this repository for clarity) and its own [pipeline.yml](pipeline.yml). diff --git a/.buildkite/coverage-linux64/pipeline.yml b/.buildkite/coverage-linux64/pipeline.yml deleted file mode 100644 index 218212d24ac2a..0000000000000 --- a/.buildkite/coverage-linux64/pipeline.yml +++ /dev/null @@ -1,53 +0,0 @@ -# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones -# since we need nestable sandboxing. The rootfs images being used here are built from -# the `.buildkite/rootfs_images/llvm-passes.jl` file. -agents: - queue: "julia" - # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing - sandbox.jl: "true" - os: "linux" - -steps: - - label: ":unlock: :coverage: Run coverage test" - plugins: - - staticfloat/cryptic: - variables: - - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA==" - - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw==" - - JuliaCI/julia#v1: - version: 1.6 - - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz - rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8" - uid: 1000 - gid: 1000 - commands: | - echo "--- Build Julia from source" - make -j 6 - - echo "--- Print Julia version info" - ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' - ./julia -e '@info "" Sys.CPU_THREADS' - # this is necessary to make sure that the LibGit2 tests passes - git config --global init.defaultBranch master - - echo "--- Run some Julia tests in serial" - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/codegen.jl - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/contextual.jl - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/inference.jl - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/inline.jl - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/interpreter_exec.jl - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/irpasses.jl - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/ssair.jl - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_serial.jl test/compiler/validation.jl - - echo "--- Run Julia tests in parallel with code coverage enabled" - ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_parallel.jl - - echo "--- Process and upload coverage information" - ./julia .buildkite/coverage-linux64/upload_coverage.jl - timeout_in_minutes: 600 # 600 minutes = 10 hours - -# We must accept the signed job id secret in order to propagate secrets -env: - BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/.buildkite/coverage-linux64/run_tests_parallel.jl b/.buildkite/coverage-linux64/run_tests_parallel.jl deleted file mode 100644 index 5386828aa4e14..0000000000000 --- a/.buildkite/coverage-linux64/run_tests_parallel.jl +++ /dev/null @@ -1,36 +0,0 @@ -# When running this file, make sure to set the `--code-coverage=all` command-line flag. - -# Important note: even if one or more tests fail, we will still exit with status code 0. - -# The reason for this is that we always want to upload code coverage, even if some of the -# tests fail. Therefore, even if the `coverage-linux64` pipeline passes, you should not -# assume that all of the tests passed. If you want to know if all of the tests are passing, -# please look at the status of the `tester_linux64` pipeline. - -const include_tests = String[] - -const exclude_tests = String[] - -empty!(Base.DEPOT_PATH) -push!(Base.DEPOT_PATH, mktempdir(; cleanup = true)) - -module ChooseTests - include(joinpath(dirname(dirname(@__DIR__)), "test", "choosetests.jl")) -end - -const tests = ChooseTests.choosetests() |> - first |> - x -> setdiff(x, exclude_tests) |> - x -> vcat(x, include_tests) |> - unique |> - sort - -const ncores = min(Sys.CPU_THREADS, Threads.nthreads()) - -@info "" ncores Sys.CPU_THREADS Threads.nthreads() - -try - Base.runtests(tests; ncores) -catch ex - @error "" exception=(ex, catch_backtrace()) -end diff --git a/.buildkite/coverage-linux64/run_tests_serial.jl b/.buildkite/coverage-linux64/run_tests_serial.jl deleted file mode 100644 index 6d7380a55e402..0000000000000 --- a/.buildkite/coverage-linux64/run_tests_serial.jl +++ /dev/null @@ -1,16 +0,0 @@ -using Test - -const repository_root = dirname(dirname(@__DIR__)) - -for filename in ARGS - path = joinpath(repository_root, filename) - @info "Starting $(filename)" - try - @testset "$(filename)" begin - include(path) - end - catch ex - @error "" exception=(ex, catch_backtrace()) - end - @info "Finished $(filename)" -end diff --git a/.buildkite/experimental/pipeline.yml b/.buildkite/experimental/pipeline.yml deleted file mode 100644 index 6f86f7ef243e5..0000000000000 --- a/.buildkite/experimental/pipeline.yml +++ /dev/null @@ -1,19 +0,0 @@ -# This file launches all the build jobs that _don't_ require secrets access. -# These jobs can pass their output off to jobs that do require secrets access, -# but those privileged steps require signing before they can be run. -# -# Yes, this is creating another layer of indirection; the flow now looks like: -# -# [webui] -> pipeline.yml -> llvm_passes.yml -# -# when we could theoretically just have the `webui` launch `llvm_passes.yml`, -# however this raises the bar for contributors to add new (unsigned) steps to -# our CI configuration, so I'd rather live with an extra layer of indirection -# and only need to touch the webui configuration when we need to alter -# something about the privileged steps. -steps: - - label: ":buildkite: Launch unsigned pipelines" - commands: | - buildkite-agent pipeline upload .buildkite/experimental/sanitizers.yml - agents: - queue: julia diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 3a3ebc718cbaa..0000000000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,28 +0,0 @@ -# This file launches all the build jobs that _don't_ require secrets access. -# These jobs can pass their output off to jobs that do require secrets access, -# but those privileged steps require signing before they can be run. -# -# Yes, this is creating another layer of indirection; the flow now looks like: -# -# [webui] -> pipeline.yml -> misc/whitespace.yml -# -# when we could theoretically just have the `webui` launch `misc/whitespace.yml`, -# however this raises the bar for contributors to add new (unsigned) steps to -# our CI configuration, so I'd rather live with an extra layer of indirection -# and only need to touch the webui configuration when we need to alter -# something about the privileged steps. -steps: - - label: ":buildkite: Launch unsigned pipelines" - commands: | - # First, we launch whitespace, because we want that pipeline to finish as quickly as possible. - buildkite-agent pipeline upload .buildkite/misc/whitespace.yml - - # Next, we launch the miscellaneous pipelines in alphabetical order. - buildkite-agent pipeline upload .buildkite/misc/doctest.yml - buildkite-agent pipeline upload .buildkite/misc/embedding.yml - buildkite-agent pipeline upload .buildkite/misc/llvmpasses.yml - - # Finally, we launch the platform pipelines in alphabetical order. - buildkite-agent pipeline upload .buildkite/platforms/linux64.yml - agents: - queue: julia diff --git a/.buildkite/experimental/0_webui.yml b/.buildkite/pipelines/experimental/0_webui.yml similarity index 67% rename from .buildkite/experimental/0_webui.yml rename to .buildkite/pipelines/experimental/0_webui.yml index ab3f68db4c393..54dbbc59d4256 100644 --- a/.buildkite/experimental/0_webui.yml +++ b/.buildkite/pipelines/experimental/0_webui.yml @@ -15,4 +15,10 @@ steps: # but some of our signed pipelines can wait upon the completion of these unsigned # pipelines. unsigned_pipelines: - - .buildkite/experimental/pipeline.yml + - .buildkite/pipelines/experimental/launch_unsigned_builders.yml + + # Our signed pipelines must have a `signature` or `signature_file` parameter that + # verifies the treehash of the pipeline itself and the inputs listed in `inputs` + # signed_pipelines: + # - pipeline: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml + # signature: "my_signature" diff --git a/.buildkite/pipelines/experimental/README.md b/.buildkite/pipelines/experimental/README.md new file mode 100644 index 0000000000000..f92aac7a1af02 --- /dev/null +++ b/.buildkite/pipelines/experimental/README.md @@ -0,0 +1,7 @@ +## Experimental pipeline (`master` branch only) + +This is the [`julia-master->experimental`](https://buildkite.com/julialang/julia-master-experimental) pipeline. + +We use this pipeline for builders that are not yet stable enough to go into the main pipeline. + +These builders are triggered by GitHub webhook events, such as pushes and pull requests. diff --git a/.buildkite/pipelines/experimental/launch_unsigned_builders.yml b/.buildkite/pipelines/experimental/launch_unsigned_builders.yml new file mode 100644 index 0000000000000..f023e19a5c940 --- /dev/null +++ b/.buildkite/pipelines/experimental/launch_unsigned_builders.yml @@ -0,0 +1,6 @@ +steps: + - label: ":buildkite: Launch unsigned pipelines" + commands: | + buildkite-agent pipeline upload .buildkite/pipelines/experimental/misc/sanitizers.yml + agents: + queue: julia diff --git a/.buildkite/experimental/sanitizers.yml b/.buildkite/pipelines/experimental/misc/sanitizers.yml similarity index 67% rename from .buildkite/experimental/sanitizers.yml rename to .buildkite/pipelines/experimental/misc/sanitizers.yml index 534dac81e53f3..67c0b547d4b20 100644 --- a/.buildkite/experimental/sanitizers.yml +++ b/.buildkite/pipelines/experimental/misc/sanitizers.yml @@ -1,6 +1,3 @@ -# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones -# since we need nestable sandboxing. The rootfs images being used here are built from -# the `.buildkite/rootfs_images/llvm-passes.jl` file. agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -14,8 +11,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v2.0/llvm_passes.tar.gz - rootfs_treehash: "0b5a80c1d0ab110a57fbdb7f4bc042a5ede310f3" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/llvm_passes.x86_64.tar.gz + rootfs_treehash: "9dd715500b117a16fcfa419ea0bca0c0ca902cee" uid: 1000 gid: 1000 workspaces: @@ -29,6 +26,6 @@ steps: echo "--- Test that ASAN is enabled" contrib/asan/check.jl ./tmp/test-asan/asan/usr/bin/julia-debug timeout_in_minutes: 120 - # notify: - # - github_commit_status: - # context: "asan" + # notify: # TODO: uncomment this line + # - github_commit_status: # TODO: uncomment this line + # context: "asan" # TODO: uncomment this line diff --git a/.buildkite/0_webui.yml b/.buildkite/pipelines/main/0_webui.yml similarity index 89% rename from .buildkite/0_webui.yml rename to .buildkite/pipelines/main/0_webui.yml index 440d2d443ce7e..96dc81cef36fe 100644 --- a/.buildkite/0_webui.yml +++ b/.buildkite/pipelines/main/0_webui.yml @@ -15,10 +15,10 @@ steps: # but some of our signed pipelines can wait upon the completion of these unsigned # pipelines. unsigned_pipelines: - - .buildkite/pipeline.yml + - .buildkite/pipelines/main/launch_unsigned_builders.yml # Our signed pipelines must have a `signature` or `signature_file` parameter that # verifies the treehash of the pipeline itself and the inputs listed in `inputs` signed_pipelines: - - pipeline: .buildkite/signed_pipeline_test.yml + - pipeline: .buildkite/pipelines/main/misc/signed_pipeline_test.yml signature: "U2FsdGVkX18ZrMlORSIi0TvW99eZ3JyUEhntMyXjrOSJ9jAtUohgHc8+dMnWUc6qWjYt8k63wfPrth33SGWDiF814Bm1W5Zk3+R6HgVu6UCrQEI5sLm+SJPbrvET+Jkn" diff --git a/.buildkite/pipelines/main/README.md b/.buildkite/pipelines/main/README.md new file mode 100644 index 0000000000000..6b9d67bd7cc3a --- /dev/null +++ b/.buildkite/pipelines/main/README.md @@ -0,0 +1,15 @@ +## Main pipeline + +This is the main pipeline. It contains most of the builders. These builders are triggered by GitHub webhook events, such as pushes and pull requests. + +We have a different main pipeline for each permanent branch. + +For example: + +| Permanent Branch | Pipeline | +| ---------------- | -------------------------------------------------------------------------------- | +| `master` | [`julia-master`](https://buildkite.com/julialang/julia-master) | +| `release-1.6` | [`julia-release-1.6`](https://buildkite.com/julialang/julia-release-1-dot-6) | +| `release-1.7` | [`julia-release-1.7`](https://buildkite.com/julialang/julia-release-1-dot-7) | + +(This is not a complete list.) diff --git a/.buildkite/pipelines/main/launch_unsigned_builders.yml b/.buildkite/pipelines/main/launch_unsigned_builders.yml new file mode 100644 index 0000000000000..6e9f0f0d8fa23 --- /dev/null +++ b/.buildkite/pipelines/main/launch_unsigned_builders.yml @@ -0,0 +1,29 @@ +# This file launches all the build jobs that _don't_ require secrets access. +# These jobs can pass their output off to jobs that do require secrets access, +# but those privileged steps require signing before they can be run. +# +# Yes, this is creating another layer of indirection; the flow now looks like: +# +# [webui] -> launch_unsigned_builders.yml -> misc/whitespace.yml +# +# when we could theoretically just have the `webui` launch `misc/whitespace.yml`, +# however this raises the bar for contributors to add new (unsigned) steps to +# our CI configuration, so I'd rather live with an extra layer of indirection +# and only need to touch the webui configuration when we need to alter +# something about the privileged steps. + +steps: + - label: ":buildkite: Launch unsigned builders" + commands: | + # First, we launch the `whitespace` builder, because we want that builder to finish as quickly as possible. + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml + + # Next, we launch the miscellaneous builders in alphabetical order. + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml + + # Finally, we launch the platform builders (`package_*`) and (`tester_*`) in alphabetical order. + buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux64.yml + agents: + queue: julia diff --git a/.buildkite/misc/doctest.yml b/.buildkite/pipelines/main/misc/doctest.yml similarity index 71% rename from .buildkite/misc/doctest.yml rename to .buildkite/pipelines/main/misc/doctest.yml index 234e49f5dcf53..0a5dc29bcb1c7 100644 --- a/.buildkite/misc/doctest.yml +++ b/.buildkite/pipelines/main/misc/doctest.yml @@ -1,6 +1,3 @@ -# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones -# since we need nestable sandboxing. The rootfs images being used here are built from -# the `.buildkite/rootfs_images/llvm-passes.jl` file. agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -14,8 +11,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz - rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz + rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" uid: 1000 gid: 1000 workspaces: @@ -33,7 +30,7 @@ steps: echo "--- Run Julia doctests" JULIA_NUM_THREADS=1 make -C doc doctest=true - timeout_in_minutes: 120 + timeout_in_minutes: 45 notify: - github_commit_status: context: "doctest" diff --git a/.buildkite/misc/embedding.yml b/.buildkite/pipelines/main/misc/embedding.yml similarity index 76% rename from .buildkite/misc/embedding.yml rename to .buildkite/pipelines/main/misc/embedding.yml index 31f85633479a4..087ca0f68eb3d 100644 --- a/.buildkite/misc/embedding.yml +++ b/.buildkite/pipelines/main/misc/embedding.yml @@ -1,6 +1,3 @@ -# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones -# since we need nestable sandboxing. The rootfs images being used here are built from -# the `.buildkite/rootfs_images/llvm-passes.jl` file. agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -14,8 +11,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz - rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz + rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" uid: 1000 gid: 1000 workspaces: diff --git a/.buildkite/misc/llvmpasses.yml b/.buildkite/pipelines/main/misc/llvmpasses.yml similarity index 83% rename from .buildkite/misc/llvmpasses.yml rename to .buildkite/pipelines/main/misc/llvmpasses.yml index 7660b3978e967..eed2f957855bb 100644 --- a/.buildkite/misc/llvmpasses.yml +++ b/.buildkite/pipelines/main/misc/llvmpasses.yml @@ -1,6 +1,3 @@ -# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones -# since we need nestable sandboxing. The rootfs images being used here are built from -# the `.buildkite/rootfs_images/llvm-passes.jl` file. agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -36,8 +33,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz - rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz + rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" uid: 1000 gid: 1000 workspaces: diff --git a/.buildkite/signed_pipeline_test.yml b/.buildkite/pipelines/main/misc/signed_pipeline_test.yml similarity index 100% rename from .buildkite/signed_pipeline_test.yml rename to .buildkite/pipelines/main/misc/signed_pipeline_test.yml diff --git a/.buildkite/misc/whitespace.yml b/.buildkite/pipelines/main/misc/whitespace.yml similarity index 63% rename from .buildkite/misc/whitespace.yml rename to .buildkite/pipelines/main/misc/whitespace.yml index 01b4c79e7822b..3f9bf13421d8e 100644 --- a/.buildkite/misc/whitespace.yml +++ b/.buildkite/pipelines/main/misc/whitespace.yml @@ -1,6 +1,3 @@ -# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones -# since we need nestable sandboxing. The rootfs images being used here are built from -# the `.buildkite/rootfs_images/llvm-passes.jl` file. agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -14,8 +11,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz - rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz + rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" workspaces: - "/cache/repos:/cache/repos" commands: | diff --git a/.buildkite/platforms/linux64.yml b/.buildkite/pipelines/main/platforms/linux64.yml similarity index 79% rename from .buildkite/platforms/linux64.yml rename to .buildkite/pipelines/main/platforms/linux64.yml index 69befe1064df7..7780c574951e8 100644 --- a/.buildkite/platforms/linux64.yml +++ b/.buildkite/pipelines/main/platforms/linux64.yml @@ -1,6 +1,3 @@ -# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones -# since we need nestable sandboxing. The rootfs images being used here are built from -# the `.buildkite/rootfs_images/llvm-passes.jl` file. agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing @@ -14,8 +11,8 @@ steps: - JuliaCI/julia#v1: version: 1.6 - staticfloat/sandbox#v1: - rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz - rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz + rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" uid: 1000 gid: 1000 workspaces: @@ -42,6 +39,7 @@ steps: - github_commit_status: context: "package_linux64" + # TODO: uncomment the following lines in order to enable the `tester_linux64` builder # - label: "tester_linux64" # key: tester_linux64 # depends_on: package_linux64 @@ -50,8 +48,8 @@ steps: # version: 1.6 # - staticfloat/sandbox#v1: # # TODO: use a separate `tester_linux` image, instead of using the `package_linux` image. - # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.0/package_linux.x86_64.tar.gz - # rootfs_treehash: "d5722d586b93eb307bb6340d275afdbf7578a756" + # rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz + # rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" # uid: 1000 # gid: 1000 # workspaces: @@ -73,7 +71,7 @@ steps: # # echo "--- Run the Julia test suite" # unset JULIA_DEPOT_PATH - # julia-artifact/bin/julia .buildkite/rr_capture.jl julia-artifact/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)' + # julia-artifact/bin/julia .buildkite/utilities/rr/rr_capture.jl julia-artifact/bin/julia -e 'Base.runtests(["all"]; ncores = Sys.CPU_THREADS)' # timeout_in_minutes: 120 # notify: # - github_commit_status: diff --git a/.buildkite/pipelines/scheduled/0_webui.yml b/.buildkite/pipelines/scheduled/0_webui.yml new file mode 100644 index 0000000000000..8aaf812376b5c --- /dev/null +++ b/.buildkite/pipelines/scheduled/0_webui.yml @@ -0,0 +1,24 @@ +# This file represents what is put into the webUI. +# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect. +# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps. +agents: + queue: "julia" + sandbox.jl: "true" + +steps: + - label: ":unlock: Unlock secrets, launch pipelines" + plugins: + - staticfloat/cryptic: + # Our list of pipelines that should be launched (but don't require a signature) + # These pipelines can be modified by any contributor and CI will still run. + # Build secrets will not be available in these pipelines (or their children) + # but some of our signed pipelines can wait upon the completion of these unsigned + # pipelines. + # unsigned_pipelines: + # - .buildkite/pipelines/scheduled/launch_unsigned_builders.yml + + # Our signed pipelines must have a `signature` or `signature_file` parameter that + # verifies the treehash of the pipeline itself and the inputs listed in `inputs` + signed_pipelines: + - pipeline: .buildkite/pipelines/scheduled/coverage/coverage_linux64.yml + signature: "U2FsdGVkX1+lpFo/nKzx3c6xCZPKYTAuunXpOsZG4+s4+iU5LfEpMvtNvpKQjDugRoxQxCItMqB6vr4KZN3KtKhjkLbr8ExAyaPil/N/uFhrLlpwNem9dxHbPrU2l7qo" diff --git a/.buildkite/pipelines/scheduled/README.md b/.buildkite/pipelines/scheduled/README.md new file mode 100644 index 0000000000000..ca071dceb2a44 --- /dev/null +++ b/.buildkite/pipelines/scheduled/README.md @@ -0,0 +1,5 @@ +## Scheduled pipeline (`master` branch only) + +This is the [`julia-master->scheduled`](https://buildkite.com/julialang/julia-master-scheduled) pipeline. + +We use this pipeline for scheduled builds. The builders in this pipeline run on a schedule once per day. They are not triggered by GitHub webhooks. diff --git a/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml b/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml new file mode 100644 index 0000000000000..ce7a3aca4227d --- /dev/null +++ b/.buildkite/pipelines/scheduled/coverage/coverage_linux64.yml @@ -0,0 +1,40 @@ +agents: + queue: "julia" + # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing + sandbox.jl: "true" + os: "linux" + +steps: + - label: ":unlock: :coverage: Run coverage test" + plugins: + - staticfloat/cryptic: + variables: + - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA==" + - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw==" + - JuliaCI/julia#v1: + version: 1.6 + - staticfloat/sandbox#v1: + rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz + rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e" + uid: 1000 + gid: 1000 + commands: | + echo "--- Build Julia from source" + make -j 6 + + echo "--- Print Julia version info" + ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()' + ./julia -e '@info "" Sys.CPU_THREADS' + # this is necessary to make sure that the LibGit2 tests passes + git config --global init.defaultBranch master + + echo "--- Run Julia tests in parallel with code coverage enabled" + ./julia --code-coverage=all --sysimage-native-code=no .buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl + + echo "--- Process and upload coverage information" + ./julia .buildkite/pipelines/scheduled/coverage/upload_coverage.jl + timeout_in_minutes: 240 # 240 minutes = 4 hours + +# We must accept the signed job id secret in order to propagate secrets +env: + BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?} diff --git a/.buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl b/.buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl new file mode 100644 index 0000000000000..6da608b5e8be9 --- /dev/null +++ b/.buildkite/pipelines/scheduled/coverage/run_tests_parallel.jl @@ -0,0 +1,25 @@ +# Important note: even if one or more tests fail, we will still exit with status code 0. + +# The reason for this is that we always want to upload code coverage, even if some of the +# tests fail. Therefore, even if the `coverage_linux64` builder passes, you should not +# assume that all of the tests passed. If you want to know if all of the tests are passing, +# please look at the status of the `tester_*` builders (e.g. `tester_linux64`). + +# When running this file, make sure to set all of the following command-line flags: +# 1. `--code-coverage=all` +# 2. `--sysimage-native-code=no` + +empty!(Base.DEPOT_PATH) +push!(Base.DEPOT_PATH, mktempdir(; cleanup = true)) + +const tests = "all" +const ncores = Sys.CPU_THREADS + +@info "" Sys.CPU_THREADS +@info "" tests ncores + +try + Base.runtests(tests; ncores) +catch ex + @error "" exception=(ex, catch_backtrace()) +end diff --git a/.buildkite/coverage-linux64/upload_coverage.jl b/.buildkite/pipelines/scheduled/coverage/upload_coverage.jl similarity index 100% rename from .buildkite/coverage-linux64/upload_coverage.jl rename to .buildkite/pipelines/scheduled/coverage/upload_coverage.jl diff --git a/.buildkite/pipelines/scheduled/launch_unsigned_builders.yml b/.buildkite/pipelines/scheduled/launch_unsigned_builders.yml new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/.buildkite/rr_capture.jl b/.buildkite/utilities/rr/rr_capture.jl similarity index 100% rename from .buildkite/rr_capture.jl rename to .buildkite/utilities/rr/rr_capture.jl