CI (Buildkite): add a separate Buildkite job that runs the tests of t…

…he network-related stdlibs (e.g. Downloads.jl), and automatically retries that job up to a maximum number of tries (#42861) (cherry picked from commit 3003742)
JuliaLang · Nov 12, 2021 · 655163f · 655163f
1 parent 09a49c6
commit 655163f
Show file tree

Hide file tree

Showing 8 changed files with 112 additions and 69 deletions.
diff --git a/.buildkite/pipelines/main/platforms/package_linux.arches b/.buildkite/pipelines/main/platforms/package_linux.arches
@@ -1,7 +1,7 @@
-# PLATFORM    LABEL       ALLOW_FAIL    ARCH        ARCH_ROOTFS    MAKE_FLAGS     TIMEOUT    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
-# linux       _aarch64    false         _aarch64    aarch64        none           60         no       no       no       v0.0          0000000000000000000000000000000000000000
-# linux       _armv7l     false         _armv7l     armv7l         none           60         no       no       no       v0.0          0000000000000000000000000000000000000000
-linux         32          false         32          i686           none           60         no       no       no       v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
-# linux       _ppc64le    false         _ppc64le    powerpc64le    none           60         no       no       no       v0.0          0000000000000000000000000000000000000000
-linux         64          false         64          x86_64         none           60         no       no       no       v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
-musl          64          false         64          x86_64         none           60         no       no       no       v4.8          d13a47c87c38005bd5d97132e51789cafd852f90
+# PLATFORM    LABEL       ALLOW_FAIL    ARCH        ARCH_ROOTFS    GROUP    MAKE_FLAGS     TIMEOUT_BK    TIMEOUT_RR     RETRIES    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
+linux         32          .             32          i686           .        .              .             .              .          .        .        .        v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
+linux         64          .             64          x86_64         .        .              .             .              .          .        .        .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+# linux       _aarch64    .             _aarch64    aarch64        .        .              .             .              .          .        .        .        ....          ........................................
+# linux       _armv7l     .             _armv7l     armv7l         .        .              .             .              .          .        .        .        ....          ........................................
+# linux       _ppc64le    .             _ppc64le    powerpc64le    .        .              .             .              .          .        .        .        ....          ........................................
+musl          64          .             64          x86_64         .        .              .             .              .          .        .        .        v4.8          d13a47c87c38005bd5d97132e51789cafd852f90
diff --git a/.buildkite/pipelines/main/platforms/package_linux.yml b/.buildkite/pipelines/main/platforms/package_linux.yml
@@ -19,7 +19,7 @@ steps:
           workspaces:
             # Include `/cache/repos` so that our `git` version introspection works.
             - "/cache/repos:/cache/repos"
-    timeout_in_minutes: ${TIMEOUT?}
+    timeout_in_minutes: ${TIMEOUT_BK?}
     commands: |
       echo "--- Print the full and short commit hashes"
       SHORT_COMMIT_LENGTH=10

diff --git a/.buildkite/pipelines/main/platforms/tester_linux.arches b/.buildkite/pipelines/main/platforms/tester_linux.arches
@@ -1,9 +1,14 @@
-# PLATFORM    LABEL       ALLOW_FAIL    ARCH        ARCH_ROOTFS    MAKE_FLAGS     TIMEOUT    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
-# linux       _aarch64    false         _aarch64    aarch64        none           60         no       no       no       v0.0          0000000000000000000000000000000000000000
-# linux       _armv7l     false         _armv7l     armv7l         none           60         no       no       no       v0.0          0000000000000000000000000000000000000000
-linux         32          false         32          i686           none           60         no       no       no       v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
-# linux       _ppc64le    false         _ppc64le    powerpc64le    none           60         no       no       no       v0.0          0000000000000000000000000000000000000000
-linux         64_rr       false         64          x86_64         none           180        yes      no       no       v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
-linux         64_st       false         64          x86_64         none           60         no       yes      no       v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
-linux         64_mt       false         64          x86_64         none           60         no       no       yes      v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
-musl          64          true          64          x86_64         none           60         no       no       no       v4.8          d13a47c87c38005bd5d97132e51789cafd852f90
+# PLATFORM    LABEL       ALLOW_FAIL    ARCH        ARCH_ROOTFS    GROUP    MAKE_FLAGS     TIMEOUT_BK    TIMEOUT_RR     RETRIES    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
+
+linux         32          .             32          i686           no-net   .              .             .              .          .        .        .        v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
+linux         32_net      .             32          i686           net      .              .             .              3          .        yes      .        v4.8          b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
+
+linux         64_rr       .             64          x86_64         no-net   .              180           120            .          yes      yes      .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+linux         64_net      .             64          x86_64         net      .              120           60             3          yes      yes      .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+linux         64_st       .             64          x86_64         no-net   .              .             .              .          .        yes      .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+linux         64_mt       .             64          x86_64         no-net   .              .             .              .          .        .        yes      v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+
+# linux       _aarch64    .             _aarch64    aarch64        .        .              .             .              .          .        .        .        ....          ........................................
+# linux       _armv7l     .             _armv7l     armv7l         .        .              .             .              .          .        .        .        ....          ........................................
+# linux       _ppc64le    .             _ppc64le    powerpc64le    .        .              .             .              .          .        .        .        ....          ........................................
+musl          64          true          64          x86_64         .        .              .             .              .          .        .        .        v4.8          d13a47c87c38005bd5d97132e51789cafd852f90
diff --git a/.buildkite/pipelines/main/platforms/tester_linux.yml b/.buildkite/pipelines/main/platforms/tester_linux.yml
@@ -23,7 +23,11 @@ steps:
             - "/cache/repos:/cache/repos"
     env:
       JULIA_SHELL: "/bin/bash"
-    timeout_in_minutes: ${TIMEOUT?}
+    timeout_in_minutes: ${TIMEOUT_BK?}
+    retry:
+      automatic:
+        - exit_status: "*"
+          limit: ${RETRIES?}
     soft_fail: ${ALLOW_FAIL?}
     commands: |
       echo "--- Print the full and short commit hashes"
@@ -55,45 +59,42 @@ steps:
       echo "--- Set some environment variables in preparation for running the Julia test suite"
       unset JULIA_DEPOT_PATH
       export OPENBLAS_NUM_THREADS=8
-      export TESTS="[\"all\"]"
 
-      if [[ "${IS_RR?}" == "yes" ]]; then
-        export JULIA_BINARY_UNDER_RR="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}"
+      if [[   "${GROUP?}" == "all" ]]; then
+        export TESTS="all"
+      elif [[ "${GROUP?}" == "net" ]]; then
+        export TESTS="Downloads LibGit2 LibGit2/online download"
+      elif [[ "${GROUP?}" == "no-net" ]]; then
+        export TESTS="all --skip Downloads LibGit2/online download"
+      else
+        echo "Invalid value for GROUP: ${GROUP?}"
+        exit 1
+      fi
 
-        export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY_UNDER_RR:?}"
-        export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])"
+      export JULIA_TEST_RR_TIMEOUT="${TIMEOUT_RR?}"
 
-        # For the `rr` job, we disable multi-threading.
-        export JULIA_NUM_THREADS=1
+      if [[ "${IS_RR?}" == "yes" ]]; then
+        export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}"
+        export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])"
       else
-        export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY:?}"
+        export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?}"
         export NCORES_FOR_TESTS="Sys.CPU_THREADS"
+      fi
 
-        if [[ "${IS_ST?}"   == "yes" ]]; then
-          # "ST" = single-threaded
-          export JULIA_NUM_THREADS=1
-        elif [[ "${IS_MT?}" == "yes" ]]; then
-          # "MT" = multi-threaded
-          export JULIA_NUM_THREADS=16
+      if [[ "${IS_ST?}"   == "yes" ]]; then
+        export JULIA_NUM_THREADS=1
+      fi
 
-          if [[ "$${BUILDKITE_PIPELINE_SLUG:?}"   == "julia-release-1-dot-6" ]]; then
-            # On Julia 1.6, we skip the Distributed test suite if multithreading is enabled.
-            export TESTS="[\"all\", \"--skip\", \"Distributed\"]"
-          elif [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-release-1-dot-7" ]]; then
-            # On Julia 1.7, we skip the Distributed test suite if multithreading is enabled.
-            export TESTS="[\"all\", \"--skip\", \"Distributed\"]"
-          else
-            # On Julia 1.8 and later, we do not skip any test suites.
-            export TESTS="[\"all\"]"
-          fi
-        fi
+      if [[ "${IS_MT?}" == "yes" ]]; then
+        export JULIA_NUM_THREADS=16
       fi
 
-      echo "JULIA_BINARY_FOR_TESTS is: $${JULIA_BINARY_FOR_TESTS:?}"
-      echo "JULIA_NUM_THREADS is:      $${JULIA_NUM_THREADS}" # this variable might not be set
+      echo "JULIA_CMD_FOR_TESTS is:    $${JULIA_CMD_FOR_TESTS:?}"
+      echo "JULIA_NUM_THREADS is:      $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set
       echo "NCORES_FOR_TESTS is:       $${NCORES_FOR_TESTS:?}"
       echo "OPENBLAS_NUM_THREADS is:   $${OPENBLAS_NUM_THREADS:?}"
+      echo "GROUP is:                  ${GROUP?}"
       echo "TESTS is:                  $${TESTS:?}"
 
       echo "--- Run the Julia test suite"
-      $${JULIA_BINARY_FOR_TESTS:?} -e "Base.runtests($${TESTS:?}; ncores = $${NCORES_FOR_TESTS:?})"
+      $${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})"
diff --git a/.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches b/.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches
@@ -1,2 +1,2 @@
-# PLATFORM    LABEL       ALLOW_FAIL    ARCH        ARCH_ROOTFS    MAKE_FLAGS             TIMEOUT    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
-linux         64_no_bb    false         64_no_bb    x86_64         USE_BINARYBUILDER=0    240        no       no       no       v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+# PLATFORM    LABEL       ALLOW_FAIL    ARCH        ARCH_ROOTFS    GROUP    MAKE_FLAGS             TIMEOUT_BK    TIMEOUT_RR     RETRIES    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
+linux         64_no_bb    false         64_no_bb    x86_64         .        USE_BINARYBUILDER=0    180           .              .          .        .        .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
diff --git a/.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches b/.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches
@@ -1,2 +1,3 @@
-# PLATFORM    LABEL          ALLOW_FAIL    ARCH        ARCH_ROOTFS    MAKE_FLAGS     TIMEOUT    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
-linux         64_st_no_bb    false         64_no_bb    x86_64         none           60         no       yes      no       v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+# PLATFORM    LABEL              ALLOW_FAIL    ARCH        ARCH_ROOTFS    GROUP     MAKE_FLAGS     TIMEOUT_BK    TIMEOUT_RR     RETRIES    IS_RR    IS_ST    IS_MT    ROOTFS_TAG    ROOTFS_HASH
+linux         64_no_bb           .             64_no_bb    x86_64         no-net    .              180           120            .          yes      yes      .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
+linux         64_no_bb_net       .             64_no_bb    x86_64         net       .              180           120            3          yes      yes      .        v4.8          2a058481b567f0e91b9aa3ce4ad4f09e6419355a
diff --git a/.buildkite/utilities/platforms/platforms.sh b/.buildkite/utilities/platforms/platforms.sh
@@ -4,40 +4,75 @@ ARCHES="$1"
 YAML="$2"
 
 if [[ ! -f "${ARCHES:?}" ]] ; then
-  echo "File does not exist: ${ARCHES:?}"
+  echo "Arches file does not exist: ${ARCHES:?}"
   exit 1
 fi
 
 if [[ ! -f "${YAML:?}" ]] ; then
-  echo "File does not exist: ${YAML:?}"
+  echo "YAML file does not exist: ${YAML:?}"
   exit 1
 fi
 
 cat "${ARCHES:?}" | tr -s ' ' | while read _line; do
   # Remove whitespace from the beginning and end of each line
   line=`echo $_line | tr -s ' '`
 
-  # Skip all lines that begin with `#`
+  # Skip any line that begins with the `#` character
   if [[ $line == \#* ]]; then
     continue
   fi
 
+  # Skip any empty line
+  if [[ $line == "" ]]; then
+    continue
+  fi
+
   export PLATFORM=`echo $line    | cut -d ' ' -f 1  | tr -s ' '`
   export LABEL=`echo $line       | cut -d ' ' -f 2  | tr -s ' '`
   export ALLOW_FAIL=`echo $line  | cut -d ' ' -f 3  | tr -s ' '`
   export ARCH=`echo $line        | cut -d ' ' -f 4  | tr -s ' '`
   export ARCH_ROOTFS=`echo $line | cut -d ' ' -f 5  | tr -s ' '`
-  export MAKE_FLAGS=`echo $line  | cut -d ' ' -f 6  | tr -s ' '`
-  export TIMEOUT=`echo $line     | cut -d ' ' -f 7  | tr -s ' '`
-  export IS_RR=`echo $line       | cut -d ' ' -f 8  | tr -s ' '`
-  export IS_ST=`echo $line       | cut -d ' ' -f 9  | tr -s ' '`
-  export IS_MT=`echo $line       | cut -d ' ' -f 10 | tr -s ' '`
-  export ROOTFS_TAG=`echo $line  | cut -d ' ' -f 11 | tr -s ' '`
-  export ROOTFS_HASH=`echo $line | cut -d ' ' -f 12 | tr -s ' '`
-
-  if [[ "${MAKE_FLAGS:?}" == "none" ]]; then
+  export GROUP=`echo $line       | cut -d ' ' -f 6  | tr -s ' '`
+  export MAKE_FLAGS=`echo $line  | cut -d ' ' -f 7  | tr -s ' '`
+  export TIMEOUT_BK=`echo $line  | cut -d ' ' -f 8  | tr -s ' '`
+  export TIMEOUT_RR=`echo $line  | cut -d ' ' -f 9  | tr -s ' '`
+  export RETRIES=`echo $line     | cut -d ' ' -f 10  | tr -s ' '`
+  export IS_RR=`echo $line       | cut -d ' ' -f 11 | tr -s ' '`
+  export IS_ST=`echo $line       | cut -d ' ' -f 12 | tr -s ' '`
+  export IS_MT=`echo $line       | cut -d ' ' -f 13 | tr -s ' '`
+  export ROOTFS_TAG=`echo $line  | cut -d ' ' -f 14 | tr -s ' '`
+  export ROOTFS_HASH=`echo $line | cut -d ' ' -f 15 | tr -s ' '`
+
+  if [[ "${ALLOW_FAIL:?}" == "." ]]; then
+    export ALLOW_FAIL="false"
+  fi
+
+  if [[ "${GROUP:?}" == "." ]]; then
+    export GROUP="all"
+  fi
+
+  if [[ "${MAKE_FLAGS:?}" == "." ]]; then
     export MAKE_FLAGS=""
   fi
 
+  if [[ "${TIMEOUT_BK:?}" == "." ]]; then
+    export TIMEOUT_BK="60"
+  fi
+
+  if [[ "${TIMEOUT_RR:?}" == "." ]]; then
+    export TIMEOUT_RR="30"
+  fi
+
+  if [[ "${RETRIES:?}" == "." ]]; then
+    export RETRIES="0"
+  fi
+
+  if [[   "${IS_ST:?}"   == "yes" ]]; then
+    if [[ "${IS_MT:?}"   == "yes" ]]; then
+      echo "You cannot set both IS_ST and IS_MT to yes"
+      exit 1
+    fi
+  fi
+
   buildkite-agent pipeline upload "${YAML:?}"
 done
diff --git a/.buildkite/utilities/rr/rr_capture.jl b/.buildkite/utilities/rr/rr_capture.jl
@@ -55,13 +55,14 @@ end
 
 @info "We will run the command under rr"
 
-const build_number             = get_from_env("BUILDKITE_BUILD_NUMBER")
-const job_name                 = get_from_env("BUILDKITE_STEP_KEY")
-const commit_full              = get_from_env("BUILDKITE_COMMIT")
-const commit_short             = first(commit_full, 10)
-const timeout_minutes          = 120
-const JULIA_TEST_NUM_CORES     = get(ENV, "JULIA_TEST_NUM_CORES", "8")
-const julia_test_num_cores_int = parse(Int, JULIA_TEST_NUM_CORES)
+const build_number                      = get_from_env("BUILDKITE_BUILD_NUMBER")
+const job_name                          = get_from_env("BUILDKITE_STEP_KEY")
+const commit_full                       = get_from_env("BUILDKITE_COMMIT")
+const commit_short                      = first(commit_full, 10)
+const JULIA_TEST_RR_TIMEOUT             = get(ENV,  "JULIA_TEST_RR_TIMEOUT", "120")
+const timeout_minutes                   = parse(Int, JULIA_TEST_RR_TIMEOUT)
+const JULIA_TEST_NUM_CORES              = get(ENV,  "JULIA_TEST_NUM_CORES", "8")
+const julia_test_num_cores_int          = parse(Int, JULIA_TEST_NUM_CORES)
 const num_cores = min(
     8,
     Sys.CPU_THREADS,