From 482b882502f6c756ecaa7a35f4507ef9938b745e Mon Sep 17 00:00:00 2001 From: Dilum Aluthge Date: Sat, 30 Oct 2021 03:17:36 -0400 Subject: [PATCH] CI (Buildkite): add a separate Buildkite job that runs the tests of the network-related stdlibs (e.g. Downloads.jl), and automatically retries that job up to a maximum number of tries --- .../main/platforms/package_linux.arches | 14 ++--- .../main/platforms/package_linux.yml | 2 +- .../main/platforms/tester_linux.arches | 23 +++++--- .../pipelines/main/platforms/tester_linux.yml | 59 ++++++++++--------- .../no_bb/no_bb_package_linux.arches | 4 +- .../scheduled/no_bb/no_bb_tester_linux.arches | 5 +- .buildkite/utilities/platforms/platforms.sh | 59 +++++++++++++++---- .buildkite/utilities/rr/rr_capture.jl | 15 ++--- 8 files changed, 112 insertions(+), 69 deletions(-) diff --git a/.buildkite/pipelines/main/platforms/package_linux.arches b/.buildkite/pipelines/main/platforms/package_linux.arches index 966d5f6f09c3e..ebf57471d59fd 100644 --- a/.buildkite/pipelines/main/platforms/package_linux.arches +++ b/.buildkite/pipelines/main/platforms/package_linux.arches @@ -1,7 +1,7 @@ -# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -# linux _aarch64 false _aarch64 aarch64 none 60 no no no v0.0 0000000000000000000000000000000000000000 -# linux _armv7l false _armv7l armv7l none 60 no no no v0.0 0000000000000000000000000000000000000000 -linux 32 false 32 i686 none 60 no no no v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 -# linux _ppc64le false _ppc64le powerpc64le none 60 no no no v0.0 0000000000000000000000000000000000000000 -linux 64 false 64 x86_64 none 60 no no no v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -musl 64 false 64 x86_64 none 60 no no no v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 +# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS GROUP MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH +linux 32 . 32 i686 . . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 +linux 64 . 64 x86_64 . . . . . . . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +# linux _aarch64 . _aarch64 aarch64 . . . . . . . . .... ........................................ +# linux _armv7l . _armv7l armv7l . . . . . . . . .... ........................................ +# linux _ppc64le . _ppc64le powerpc64le . . . . . . . . .... ........................................ +musl 64 . 64 x86_64 . . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 diff --git a/.buildkite/pipelines/main/platforms/package_linux.yml b/.buildkite/pipelines/main/platforms/package_linux.yml index 66c0f4784d942..ce778c393b064 100644 --- a/.buildkite/pipelines/main/platforms/package_linux.yml +++ b/.buildkite/pipelines/main/platforms/package_linux.yml @@ -19,7 +19,7 @@ steps: workspaces: # Include `/cache/repos` so that our `git` version introspection works. - "/cache/repos:/cache/repos" - timeout_in_minutes: ${TIMEOUT?} + timeout_in_minutes: ${TIMEOUT_BK?} commands: | echo "--- Print the full and short commit hashes" SHORT_COMMIT_LENGTH=10 diff --git a/.buildkite/pipelines/main/platforms/tester_linux.arches b/.buildkite/pipelines/main/platforms/tester_linux.arches index 1d79860a93dc3..9c55266e56307 100644 --- a/.buildkite/pipelines/main/platforms/tester_linux.arches +++ b/.buildkite/pipelines/main/platforms/tester_linux.arches @@ -1,9 +1,14 @@ -# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -# linux _aarch64 false _aarch64 aarch64 none 60 no no no v0.0 0000000000000000000000000000000000000000 -# linux _armv7l false _armv7l armv7l none 60 no no no v0.0 0000000000000000000000000000000000000000 -linux 32 false 32 i686 none 60 no no no v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 -# linux _ppc64le false _ppc64le powerpc64le none 60 no no no v0.0 0000000000000000000000000000000000000000 -linux 64_rr false 64 x86_64 none 180 yes no no v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_st false 64 x86_64 none 60 no yes no v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -linux 64_mt false 64 x86_64 none 60 no no yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a -musl 64 true 64 x86_64 none 60 no no no v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 +# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS GROUP MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH + +linux 32 . 32 i686 no-net . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 +linux 32_net . 32 i686 net . . . 3 . yes . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957 + +linux 64_rr . 64 x86_64 no-net . 180 120 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +linux 64_net . 64 x86_64 net . 120 60 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +linux 64_st . 64 x86_64 no-net . . . . . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +linux 64_mt . 64 x86_64 no-net . . . . . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a + +# linux _aarch64 . _aarch64 aarch64 . . . . . . . . .... ........................................ +# linux _armv7l . _armv7l armv7l . . . . . . . . .... ........................................ +# linux _ppc64le . _ppc64le powerpc64le . . . . . . . . .... ........................................ +musl 64 true 64 x86_64 . . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90 diff --git a/.buildkite/pipelines/main/platforms/tester_linux.yml b/.buildkite/pipelines/main/platforms/tester_linux.yml index 123462458cfa1..5b9b5ff475da1 100644 --- a/.buildkite/pipelines/main/platforms/tester_linux.yml +++ b/.buildkite/pipelines/main/platforms/tester_linux.yml @@ -23,7 +23,11 @@ steps: - "/cache/repos:/cache/repos" env: JULIA_SHELL: "/bin/bash" - timeout_in_minutes: ${TIMEOUT?} + timeout_in_minutes: ${TIMEOUT_BK?} + retry: + automatic: + - exit_status: "*" + limit: ${RETRIES?} soft_fail: ${ALLOW_FAIL?} commands: | echo "--- Print the full and short commit hashes" @@ -55,45 +59,42 @@ steps: echo "--- Set some environment variables in preparation for running the Julia test suite" unset JULIA_DEPOT_PATH export OPENBLAS_NUM_THREADS=8 - export TESTS="[\"all\"]" - if [[ "${IS_RR?}" == "yes" ]]; then - export JULIA_BINARY_UNDER_RR="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}" + if [[ "${GROUP?}" == "all" ]]; then + export TESTS="all" + elif [[ "${GROUP?}" == "net" ]]; then + export TESTS="Downloads LibGit2 LibGit2/online download" + elif [[ "${GROUP?}" == "no-net" ]]; then + export TESTS="all --skip Downloads LibGit2/online download" + else + echo "Invalid value for GROUP: ${GROUP?}" + exit 1 + fi - export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY_UNDER_RR:?}" - export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])" + export JULIA_TEST_RR_TIMEOUT="${TIMEOUT_RR?}" - # For the `rr` job, we disable multi-threading. - export JULIA_NUM_THREADS=1 + if [[ "${IS_RR?}" == "yes" ]]; then + export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}" + export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])" else - export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY:?}" + export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?}" export NCORES_FOR_TESTS="Sys.CPU_THREADS" + fi - if [[ "${IS_ST?}" == "yes" ]]; then - # "ST" = single-threaded - export JULIA_NUM_THREADS=1 - elif [[ "${IS_MT?}" == "yes" ]]; then - # "MT" = multi-threaded - export JULIA_NUM_THREADS=16 + if [[ "${IS_ST?}" == "yes" ]]; then + export JULIA_NUM_THREADS=1 + fi - if [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-release-1-dot-6" ]]; then - # On Julia 1.6, we skip the Distributed test suite if multithreading is enabled. - export TESTS="[\"all\", \"--skip\", \"Distributed\"]" - elif [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-release-1-dot-7" ]]; then - # On Julia 1.7, we skip the Distributed test suite if multithreading is enabled. - export TESTS="[\"all\", \"--skip\", \"Distributed\"]" - else - # On Julia 1.8 and later, we do not skip any test suites. - export TESTS="[\"all\"]" - fi - fi + if [[ "${IS_MT?}" == "yes" ]]; then + export JULIA_NUM_THREADS=16 fi - echo "JULIA_BINARY_FOR_TESTS is: $${JULIA_BINARY_FOR_TESTS:?}" - echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # this variable might not be set + echo "JULIA_CMD_FOR_TESTS is: $${JULIA_CMD_FOR_TESTS:?}" + echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set echo "NCORES_FOR_TESTS is: $${NCORES_FOR_TESTS:?}" echo "OPENBLAS_NUM_THREADS is: $${OPENBLAS_NUM_THREADS:?}" + echo "GROUP is: ${GROUP?}" echo "TESTS is: $${TESTS:?}" echo "--- Run the Julia test suite" - $${JULIA_BINARY_FOR_TESTS:?} -e "Base.runtests($${TESTS:?}; ncores = $${NCORES_FOR_TESTS:?})" + $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})" diff --git a/.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches b/.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches index 8375e7bf8da61..b2e6856c02af5 100644 --- a/.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches +++ b/.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches @@ -1,2 +1,2 @@ -# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 64_no_bb false 64_no_bb x86_64 USE_BINARYBUILDER=0 240 no no no v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS GROUP MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH +linux 64_no_bb false 64_no_bb x86_64 . USE_BINARYBUILDER=0 180 . . . . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a diff --git a/.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches b/.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches index 5e10a829913e0..4309db01e1050 100644 --- a/.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches +++ b/.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches @@ -1,2 +1,3 @@ -# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH -linux 64_st_no_bb false 64_no_bb x86_64 none 60 no yes no v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS GROUP MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH +linux 64_no_bb . 64_no_bb x86_64 no-net . 180 120 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a +linux 64_no_bb_net . 64_no_bb x86_64 net . 180 120 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a diff --git a/.buildkite/utilities/platforms/platforms.sh b/.buildkite/utilities/platforms/platforms.sh index 87daa8a2a9e28..75674d4428394 100755 --- a/.buildkite/utilities/platforms/platforms.sh +++ b/.buildkite/utilities/platforms/platforms.sh @@ -4,12 +4,12 @@ ARCHES="$1" YAML="$2" if [[ ! -f "${ARCHES:?}" ]] ; then - echo "File does not exist: ${ARCHES:?}" + echo "Arches file does not exist: ${ARCHES:?}" exit 1 fi if [[ ! -f "${YAML:?}" ]] ; then - echo "File does not exist: ${YAML:?}" + echo "YAML file does not exist: ${YAML:?}" exit 1 fi @@ -17,27 +17,62 @@ cat "${ARCHES:?}" | tr -s ' ' | while read _line; do # Remove whitespace from the beginning and end of each line line=`echo $_line | tr -s ' '` - # Skip all lines that begin with `#` + # Skip any line that begins with the `#` character if [[ $line == \#* ]]; then continue fi + # Skip any empty line + if [[ $line == "" ]]; then + continue + fi + export PLATFORM=`echo $line | cut -d ' ' -f 1 | tr -s ' '` export LABEL=`echo $line | cut -d ' ' -f 2 | tr -s ' '` export ALLOW_FAIL=`echo $line | cut -d ' ' -f 3 | tr -s ' '` export ARCH=`echo $line | cut -d ' ' -f 4 | tr -s ' '` export ARCH_ROOTFS=`echo $line | cut -d ' ' -f 5 | tr -s ' '` - export MAKE_FLAGS=`echo $line | cut -d ' ' -f 6 | tr -s ' '` - export TIMEOUT=`echo $line | cut -d ' ' -f 7 | tr -s ' '` - export IS_RR=`echo $line | cut -d ' ' -f 8 | tr -s ' '` - export IS_ST=`echo $line | cut -d ' ' -f 9 | tr -s ' '` - export IS_MT=`echo $line | cut -d ' ' -f 10 | tr -s ' '` - export ROOTFS_TAG=`echo $line | cut -d ' ' -f 11 | tr -s ' '` - export ROOTFS_HASH=`echo $line | cut -d ' ' -f 12 | tr -s ' '` - - if [[ "${MAKE_FLAGS:?}" == "none" ]]; then + export GROUP=`echo $line | cut -d ' ' -f 6 | tr -s ' '` + export MAKE_FLAGS=`echo $line | cut -d ' ' -f 7 | tr -s ' '` + export TIMEOUT_BK=`echo $line | cut -d ' ' -f 8 | tr -s ' '` + export TIMEOUT_RR=`echo $line | cut -d ' ' -f 9 | tr -s ' '` + export RETRIES=`echo $line | cut -d ' ' -f 10 | tr -s ' '` + export IS_RR=`echo $line | cut -d ' ' -f 11 | tr -s ' '` + export IS_ST=`echo $line | cut -d ' ' -f 12 | tr -s ' '` + export IS_MT=`echo $line | cut -d ' ' -f 13 | tr -s ' '` + export ROOTFS_TAG=`echo $line | cut -d ' ' -f 14 | tr -s ' '` + export ROOTFS_HASH=`echo $line | cut -d ' ' -f 15 | tr -s ' '` + + if [[ "${ALLOW_FAIL:?}" == "." ]]; then + export ALLOW_FAIL="false" + fi + + if [[ "${GROUP:?}" == "." ]]; then + export GROUP="all" + fi + + if [[ "${MAKE_FLAGS:?}" == "." ]]; then export MAKE_FLAGS="" fi + if [[ "${TIMEOUT_BK:?}" == "." ]]; then + export TIMEOUT_BK="60" + fi + + if [[ "${TIMEOUT_RR:?}" == "." ]]; then + export TIMEOUT_RR="30" + fi + + if [[ "${RETRIES:?}" == "." ]]; then + export RETRIES="0" + fi + + if [[ "${IS_ST:?}" == "yes" ]]; then + if [[ "${IS_MT:?}" == "yes" ]]; then + echo "You cannot set both IS_ST and IS_MT to yes" + exit 1 + fi + fi + buildkite-agent pipeline upload "${YAML:?}" done diff --git a/.buildkite/utilities/rr/rr_capture.jl b/.buildkite/utilities/rr/rr_capture.jl index 5d60377a72bb6..37bf3ca124271 100644 --- a/.buildkite/utilities/rr/rr_capture.jl +++ b/.buildkite/utilities/rr/rr_capture.jl @@ -55,13 +55,14 @@ end @info "We will run the command under rr" -const build_number = get_from_env("BUILDKITE_BUILD_NUMBER") -const job_name = get_from_env("BUILDKITE_STEP_KEY") -const commit_full = get_from_env("BUILDKITE_COMMIT") -const commit_short = first(commit_full, 10) -const timeout_minutes = 120 -const JULIA_TEST_NUM_CORES = get(ENV, "JULIA_TEST_NUM_CORES", "8") -const julia_test_num_cores_int = parse(Int, JULIA_TEST_NUM_CORES) +const build_number = get_from_env("BUILDKITE_BUILD_NUMBER") +const job_name = get_from_env("BUILDKITE_STEP_KEY") +const commit_full = get_from_env("BUILDKITE_COMMIT") +const commit_short = first(commit_full, 10) +const JULIA_TEST_RR_TIMEOUT = get(ENV, "JULIA_TEST_RR_TIMEOUT", "120") +const timeout_minutes = parse(Int, JULIA_TEST_RR_TIMEOUT) +const JULIA_TEST_NUM_CORES = get(ENV, "JULIA_TEST_NUM_CORES", "8") +const julia_test_num_cores_int = parse(Int, JULIA_TEST_NUM_CORES) const num_cores = min( 8, Sys.CPU_THREADS,