From 71e50180713eaaaa3bedc827e6890cede03baa0b Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Oct 2019 12:26:02 +0100 Subject: [PATCH 1/9] ci: replace MINGW_URL with CUSTOM_MINGW=1 This commit replaces the mirrors base URL contained in the MINGW_URL with a CUSTOM_MINGW=1 environment variable. The mirrors base URL will be fetched instead through the MIRRORS_BASE environment variable, defined in src/ci/shared.sh. --- src/ci/azure-pipelines/auto.yml | 12 ++++++------ src/ci/scripts/install-clang.sh | 2 +- src/ci/scripts/install-mingw.sh | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 836f81a781656..f69808c3702bc 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -284,7 +284,7 @@ jobs: MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw-subset-1 - MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc + CUSTOM_MINGW: 1 MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 # FIXME(#59637) @@ -294,14 +294,14 @@ jobs: MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw-subset-2 - MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc + CUSTOM_MINGW: 1 MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 x86_64-mingw-1: MSYS_BITS: 64 SCRIPT: make ci-mingw-subset-1 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc + CUSTOM_MINGW: 1 MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z MINGW_DIR: mingw64 # FIXME(#59637) @@ -311,7 +311,7 @@ jobs: MSYS_BITS: 64 SCRIPT: make ci-mingw-subset-2 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc + CUSTOM_MINGW: 1 MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z MINGW_DIR: mingw64 @@ -340,7 +340,7 @@ jobs: MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler SCRIPT: python x.py dist - MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc + CUSTOM_MINGW: 1 MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 DIST_REQUIRE_ALL_TOOLS: 1 @@ -349,7 +349,7 @@ jobs: MSYS_BITS: 64 SCRIPT: python x.py dist RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler - MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc + CUSTOM_MINGW: 1 MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z MINGW_DIR: mingw64 DIST_REQUIRE_ALL_TOOLS: 1 diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh index b1e9bf92ca5d2..f0179994e8e4c 100755 --- a/src/ci/scripts/install-clang.sh +++ b/src/ci/scripts/install-clang.sh @@ -17,7 +17,7 @@ if isMacOS; then # Configure `AR` specifically so rustbuild doesn't try to infer it as # `clang-ar` by accident. ciCommandSetEnv AR "ar" -elif isWindows && [[ -z ${MINGW_URL+x} ]]; then +elif isWindows && [[ ${CUSTOM_MINGW-0} -ne 1 ]]; then # If we're compiling for MSVC then we, like most other distribution builders, # switch to clang as the compiler. This'll allow us eventually to enable LTO # amongst LLVM and rustc. Note that we only do this on MSVC as I don't think diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh index b4e8b889f520a..8b11e59fb639d 100755 --- a/src/ci/scripts/install-mingw.sh +++ b/src/ci/scripts/install-mingw.sh @@ -28,7 +28,7 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if isWindows; then - if [[ -z "${MINGW_URL+x}" ]]; then + if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then arch=i686 if [ "$MSYS_BITS" = "64" ]; then arch=x86_64 @@ -37,9 +37,9 @@ if isWindows; then mingw-w64-$arch-gcc mingw-w64-$arch-python2 ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/mingw${MSYS_BITS}/bin" else - curl -o mingw.7z "${MINGW_URL}/${MINGW_ARCHIVE}" + curl -o mingw.7z "${MIRRORS_BASE}/${MINGW_ARCHIVE}" 7z x -y mingw.7z > /dev/null - curl -o "${MINGW_DIR}/bin/gdborig.exe" "${MINGW_URL}/2017-04-20-${MSYS_BITS}bit-gdborig.exe" + curl -o "${MINGW_DIR}/bin/gdborig.exe" "${MIRRORS_BASE}/2017-04-20-${MSYS_BITS}bit-gdborig.exe" ciCommandAddPath "$(pwd)/${MINGW_DIR}/bin" fi fi From 53c2c04d63b3c043ec5b1b4f401e04868a1139a4 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Oct 2019 12:35:55 +0100 Subject: [PATCH 2/9] ci: remove the MINGW_DIR and MINGW_ARCHIVE env vars --- src/ci/azure-pipelines/auto.yml | 12 ------------ src/ci/scripts/install-mingw.sh | 15 ++++++++++++--- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index f69808c3702bc..74fb237eebd1e 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -285,8 +285,6 @@ jobs: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw-subset-1 CUSTOM_MINGW: 1 - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 # FIXME(#59637) NO_DEBUG_ASSERTIONS: 1 NO_LLVM_ASSERTIONS: 1 @@ -295,15 +293,11 @@ jobs: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw-subset-2 CUSTOM_MINGW: 1 - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 x86_64-mingw-1: MSYS_BITS: 64 SCRIPT: make ci-mingw-subset-1 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu CUSTOM_MINGW: 1 - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 # FIXME(#59637) NO_DEBUG_ASSERTIONS: 1 NO_LLVM_ASSERTIONS: 1 @@ -312,8 +306,6 @@ jobs: SCRIPT: make ci-mingw-subset-2 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu CUSTOM_MINGW: 1 - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 # 32/64 bit MSVC and GNU deployment dist-x86_64-msvc: @@ -341,8 +333,6 @@ jobs: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler SCRIPT: python x.py dist CUSTOM_MINGW: 1 - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 dist-x86_64-mingw: @@ -350,8 +340,6 @@ jobs: SCRIPT: python x.py dist RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler CUSTOM_MINGW: 1 - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh index 8b11e59fb639d..631d5e575a5dc 100755 --- a/src/ci/scripts/install-mingw.sh +++ b/src/ci/scripts/install-mingw.sh @@ -27,6 +27,9 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" +MINGW_ARCHIVE_32="i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z" +MINGW_ARCHIVE_64="x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z" + if isWindows; then if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then arch=i686 @@ -37,9 +40,15 @@ if isWindows; then mingw-w64-$arch-gcc mingw-w64-$arch-python2 ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/mingw${MSYS_BITS}/bin" else - curl -o mingw.7z "${MIRRORS_BASE}/${MINGW_ARCHIVE}" + mingw_archive="${MINGW_ARCHIVE_32}" + if [[ "${MSYS_BITS}" = "64" ]]; then + mingw_archive="${MINGW_ARCHIVE_64}" + fi + mingw_dir="mingw${MSYS_BITS}" + + curl -o mingw.7z "${MIRRORS_BASE}/${mingw_archive}" 7z x -y mingw.7z > /dev/null - curl -o "${MINGW_DIR}/bin/gdborig.exe" "${MIRRORS_BASE}/2017-04-20-${MSYS_BITS}bit-gdborig.exe" - ciCommandAddPath "$(pwd)/${MINGW_DIR}/bin" + curl -o "${mingw_dir}/bin/gdborig.exe" "${MIRRORS_BASE}/2017-04-20-${MSYS_BITS}bit-gdborig.exe" + ciCommandAddPath "$(pwd)/${mingw_dir}/bin" fi fi From af6b26646b41e1b9614be12ae126f7033a3fa908 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Oct 2019 12:45:03 +0100 Subject: [PATCH 3/9] ci: remove the MSYS_BITS env var --- src/ci/azure-pipelines/auto.yml | 17 --------------- src/ci/azure-pipelines/steps/run.yml | 14 ++++++------ src/ci/azure-pipelines/try.yml | 1 - src/ci/scripts/install-mingw.sh | 32 ++++++++++++++++++---------- 4 files changed, 28 insertions(+), 36 deletions(-) diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 74fb237eebd1e..946eb483c2946 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -223,25 +223,21 @@ jobs: matrix: # 32/64 bit MSVC tests x86_64-msvc-1: - MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: make ci-subset-1 # FIXME(#59637) NO_DEBUG_ASSERTIONS: 1 NO_LLVM_ASSERTIONS: 1 x86_64-msvc-2: - MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: make ci-subset-2 i686-msvc-1: - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc SCRIPT: make ci-subset-1 # FIXME(#59637) NO_DEBUG_ASSERTIONS: 1 NO_LLVM_ASSERTIONS: 1 i686-msvc-2: - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc SCRIPT: make ci-subset-2 # FIXME(#59637) @@ -249,11 +245,9 @@ jobs: NO_LLVM_ASSERTIONS: 1 # MSVC aux tests x86_64-msvc-aux: - MSYS_BITS: 64 RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc x86_64-msvc-cargo: - MSYS_BITS: 64 SCRIPT: python x.py test src/tools/cargotest src/tools/cargo RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc VCVARS_BAT: vcvars64.bat @@ -262,10 +256,8 @@ jobs: NO_LLVM_ASSERTIONS: 1 # MSVC tools tests x86_64-msvc-tools: - MSYS_BITS: 64 SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json - DEPLOY_TOOLSTATES_JSON: toolstates-windows.json # 32/64-bit MinGW builds. # @@ -281,7 +273,6 @@ jobs: # came from the mingw-w64 SourceForge download site. Unfortunately # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. i686-mingw-1: - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw-subset-1 CUSTOM_MINGW: 1 @@ -289,12 +280,10 @@ jobs: NO_DEBUG_ASSERTIONS: 1 NO_LLVM_ASSERTIONS: 1 i686-mingw-2: - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw-subset-2 CUSTOM_MINGW: 1 x86_64-mingw-1: - MSYS_BITS: 64 SCRIPT: make ci-mingw-subset-1 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu CUSTOM_MINGW: 1 @@ -302,14 +291,12 @@ jobs: NO_DEBUG_ASSERTIONS: 1 NO_LLVM_ASSERTIONS: 1 x86_64-mingw-2: - MSYS_BITS: 64 SCRIPT: make ci-mingw-subset-2 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu CUSTOM_MINGW: 1 # 32/64 bit MSVC and GNU deployment dist-x86_64-msvc: - MSYS_BITS: 64 RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc @@ -319,7 +306,6 @@ jobs: DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 dist-i686-msvc: - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: >- --build=i686-pc-windows-msvc --target=i586-pc-windows-msvc @@ -329,14 +315,12 @@ jobs: DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 dist-i686-mingw: - MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler SCRIPT: python x.py dist CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 dist-x86_64-mingw: - MSYS_BITS: 64 SCRIPT: python x.py dist RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler CUSTOM_MINGW: 1 @@ -345,7 +329,6 @@ jobs: # "alternate" deployment, see .travis.yml for more info dist-x86_64-msvc-alt: - MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler SCRIPT: python x.py dist DEPLOY_ALT: 1 diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index cef2d235602f1..4d7c68593223a 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -8,6 +8,13 @@ steps: +# Configure our CI_JOB_NAME variable which log analyzers can use for the main +# step to see what's going on. +- bash: | + builder=$(echo $AGENT_JOBNAME | cut -d ' ' -f 2) + echo "##vso[task.setvariable variable=CI_JOB_NAME]$builder" + displayName: Configure Job Name + # Disable automatic line ending conversion, which is enabled by default on # Azure's Windows image. Having the conversion enabled caused regressions both # in our test suite (it broke miri tests) and in the ecosystem, since we @@ -135,13 +142,6 @@ steps: condition: and(succeeded(), not(variables.SKIP_JOB)) displayName: Install awscli -# Configure our CI_JOB_NAME variable which log analyzers can use for the main -# step to see what's going on. -- bash: | - builder=$(echo $AGENT_JOBNAME | cut -d ' ' -f 2) - echo "##vso[task.setvariable variable=CI_JOB_NAME]$builder" - displayName: Configure Job Name - # As a quick smoke check on the otherwise very fast mingw-check linux builder # check our own internal scripts. - bash: | diff --git a/src/ci/azure-pipelines/try.yml b/src/ci/azure-pipelines/try.yml index c919b1023a0eb..fe39ce3e24116 100644 --- a/src/ci/azure-pipelines/try.yml +++ b/src/ci/azure-pipelines/try.yml @@ -72,7 +72,6 @@ jobs: # DEPLOY: 1 # # dist-x86_64-msvc-alt: -# MSYS_BITS: 64 # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler # SCRIPT: python x.py dist # DEPLOY_ALT: 1 diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh index 631d5e575a5dc..8b579587b9e1f 100755 --- a/src/ci/scripts/install-mingw.sh +++ b/src/ci/scripts/install-mingw.sh @@ -31,24 +31,34 @@ MINGW_ARCHIVE_32="i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z" MINGW_ARCHIVE_64="x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z" if isWindows; then + case "${CI_JOB_NAME}" in + *i686*) + bits=32 + arch=i686 + mingw_archive="${MINGW_ARCHIVE_32}" + ;; + *x86_64*) + bits=64 + arch=x86_64 + mingw_archive="${MINGW_ARCHIVE_64}" + ;; + *) + echo "src/ci/scripts/install-mingw.sh can't detect the builder's architecture" + echo "please tweak it to recognize the builder named '${CI_JOB_NAME}'" + exit 1 + ;; + esac + if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then - arch=i686 - if [ "$MSYS_BITS" = "64" ]; then - arch=x86_64 - fi pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake \ mingw-w64-$arch-gcc mingw-w64-$arch-python2 - ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/mingw${MSYS_BITS}/bin" + ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/mingw${bits}/bin" else - mingw_archive="${MINGW_ARCHIVE_32}" - if [[ "${MSYS_BITS}" = "64" ]]; then - mingw_archive="${MINGW_ARCHIVE_64}" - fi - mingw_dir="mingw${MSYS_BITS}" + mingw_dir="mingw${bits}" curl -o mingw.7z "${MIRRORS_BASE}/${mingw_archive}" 7z x -y mingw.7z > /dev/null - curl -o "${mingw_dir}/bin/gdborig.exe" "${MIRRORS_BASE}/2017-04-20-${MSYS_BITS}bit-gdborig.exe" + curl -o "${mingw_dir}/bin/gdborig.exe" "${MIRRORS_BASE}/2017-04-20-${bits}bit-gdborig.exe" ciCommandAddPath "$(pwd)/${mingw_dir}/bin" fi fi From 6104aa77eb97050c3b302a0d13934040de84cda9 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Oct 2019 18:13:18 +0100 Subject: [PATCH 4/9] ci: extract validate-toolstate into a script --- src/ci/azure-pipelines/steps/run.yml | 13 +------------ src/ci/scripts/validate-toolstate.sh | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 12 deletions(-) create mode 100755 src/ci/scripts/validate-toolstate.sh diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 4d7c68593223a..4df315d5d5cd3 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -144,18 +144,7 @@ steps: # As a quick smoke check on the otherwise very fast mingw-check linux builder # check our own internal scripts. -- bash: | - set -e - git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git - cd rust-toolstate - python2.7 "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "" "" - # Only check maintainers if this build is supposed to publish toolstate. - # Builds that are not supposed to publish don't have the access token. - if [ -n "${TOOLSTATE_PUBLISH+is_set}" ]; then - TOOLSTATE_VALIDATE_MAINTAINERS_REPO=rust-lang/rust python2.7 "${BUILD_SOURCESDIRECTORY}/src/tools/publish_toolstate.py" - fi - cd .. - rm -rf rust-toolstate +- bash: src/ci/scripts/validate-toolstate.sh env: TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN) condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['IMAGE'], 'mingw-check')) diff --git a/src/ci/scripts/validate-toolstate.sh b/src/ci/scripts/validate-toolstate.sh new file mode 100755 index 0000000000000..93e645b24a859 --- /dev/null +++ b/src/ci/scripts/validate-toolstate.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# A quick smoke test to make sure publish_tooolstate.py works. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git +cd rust-toolstate +python2.7 "${BUILD_SOURCESDIRECTORY}/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "" "" +# Only check maintainers if this build is supposed to publish toolstate. +# Builds that are not supposed to publish don't have the access token. +if [ -n "${TOOLSTATE_PUBLISH+is_set}" ]; then + TOOLSTATE_VALIDATE_MAINTAINERS_REPO=rust-lang/rust python2.7 "${BUILD_SOURCESDIRECTORY}/src/tools/publish_toolstate.py" +fi +cd .. +rm -rf rust-toolstate From d623c56f4c2eab00a9cb5ba12e9e708430945f1a Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Oct 2019 18:18:49 +0100 Subject: [PATCH 5/9] ci: extract running the build into a script --- src/ci/azure-pipelines/steps/run.yml | 15 +-------------- src/ci/scripts/run-build-from-ci.sh | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 14 deletions(-) create mode 100755 src/ci/scripts/run-build-from-ci.sh diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 4df315d5d5cd3..fe2bee507152d 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -150,22 +150,9 @@ steps: condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['IMAGE'], 'mingw-check')) displayName: Verify the publish_toolstate script works -- bash: | - set -e - # Remove any preexisting rustup installation since it can interfere - # with the cargotest step and its auto-detection of things like Clippy in - # the environment - rustup self uninstall -y || true - if [ "$IMAGE" = "" ]; then - src/ci/run.sh - else - src/ci/docker/run.sh $IMAGE - fi - #timeoutInMinutes: 180 +- bash: src/ci/scripts/run-build-from-ci.sh timeoutInMinutes: 600 env: - CI: true - SRC: . AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY_ID) AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY) TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN) diff --git a/src/ci/scripts/run-build-from-ci.sh b/src/ci/scripts/run-build-from-ci.sh new file mode 100755 index 0000000000000..c02117f459de0 --- /dev/null +++ b/src/ci/scripts/run-build-from-ci.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Start the CI build. You shouldn't run this locally: call either src/ci/run.sh +# or src/ci/docker/run.sh instead. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +export CI="true" +export SRC=. + +# Remove any preexisting rustup installation since it can interfere +# with the cargotest step and its auto-detection of things like Clippy in +# the environment +rustup self uninstall -y || true +if [ -z "${IMAGE+x}" ]; then + src/ci/run.sh +else + src/ci/docker/run.sh "${IMAGE}" +fi From e209ee42e99cd76d7fefd0c53fbd84f0d2123935 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 30 Oct 2019 18:56:40 +0100 Subject: [PATCH 6/9] ci: extract collecting cpu stats into a script --- src/ci/azure-pipelines/steps/run.yml | 7 ++----- src/ci/scripts/collect-cpu-stats.sh | 9 +++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) create mode 100755 src/ci/scripts/collect-cpu-stats.sh diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index fe2bee507152d..e856207b17d49 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -31,11 +31,8 @@ steps: - bash: src/ci/scripts/should-skip-this.sh displayName: Decide whether to run this job -# Spawn a background process to collect CPU usage statistics which we'll upload -# at the end of the build. See the comments in the script here for more -# information. -- bash: python src/ci/cpu-usage-over-time.py &> cpu-usage.csv & - displayName: "Collect CPU-usage statistics in the background" +- bash: src/ci/scripts/collect-cpu-stats.sh + displayName: Collect CPU-usage statistics in the background - bash: src/ci/scripts/dump-environment.sh displayName: Show the current environment diff --git a/src/ci/scripts/collect-cpu-stats.sh b/src/ci/scripts/collect-cpu-stats.sh new file mode 100755 index 0000000000000..08065431f9816 --- /dev/null +++ b/src/ci/scripts/collect-cpu-stats.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Spawn a background process to collect CPU usage statistics which we'll upload +# at the end of the build. See the comments in the script here for more +# information. + +set -euo pipefail +IFS=$'\n\t' + +python src/ci/cpu-usage-over-time.py &> cpu-usage.csv & From c90cc12b0721cc6e80ba62041bf0b82058428580 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Thu, 31 Oct 2019 16:56:11 +0100 Subject: [PATCH 7/9] ci: move validate-toolstate.sh in the mingw-check image The task was already run just there, so this cleans things up. --- src/ci/azure-pipelines/steps/run.yml | 8 -------- src/ci/docker/mingw-check/Dockerfile | 5 ++++- .../mingw-check}/validate-toolstate.sh | 9 +++++---- 3 files changed, 9 insertions(+), 13 deletions(-) rename src/ci/{scripts => docker/mingw-check}/validate-toolstate.sh (57%) diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index e856207b17d49..214c11fd69024 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -139,14 +139,6 @@ steps: condition: and(succeeded(), not(variables.SKIP_JOB)) displayName: Install awscli -# As a quick smoke check on the otherwise very fast mingw-check linux builder -# check our own internal scripts. -- bash: src/ci/scripts/validate-toolstate.sh - env: - TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN) - condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['IMAGE'], 'mingw-check')) - displayName: Verify the publish_toolstate script works - - bash: src/ci/scripts/run-build-from-ci.sh timeoutInMinutes: 600 env: diff --git a/src/ci/docker/mingw-check/Dockerfile b/src/ci/docker/mingw-check/Dockerfile index 24e2dea4ca773..b2d96aed2a9e8 100644 --- a/src/ci/docker/mingw-check/Dockerfile +++ b/src/ci/docker/mingw-check/Dockerfile @@ -19,7 +19,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +COPY mingw-check/validate-toolstate.sh /scripts/ + ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1 ENV SCRIPT python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \ python2.7 ../x.py build --stage 0 src/tools/build-manifest && \ - python2.7 ../x.py test --stage 0 src/tools/compiletest + python2.7 ../x.py test --stage 0 src/tools/compiletest && \ + /scripts/validate-toolstate.sh diff --git a/src/ci/scripts/validate-toolstate.sh b/src/ci/docker/mingw-check/validate-toolstate.sh similarity index 57% rename from src/ci/scripts/validate-toolstate.sh rename to src/ci/docker/mingw-check/validate-toolstate.sh index 93e645b24a859..2ebf1d6d5ae7f 100755 --- a/src/ci/scripts/validate-toolstate.sh +++ b/src/ci/docker/mingw-check/validate-toolstate.sh @@ -4,15 +4,16 @@ set -euo pipefail IFS=$'\n\t' -source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" - +rm -rf rust-toolstate git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git cd rust-toolstate -python2.7 "${BUILD_SOURCESDIRECTORY}/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "" "" +python2.7 "../../src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" \ + "$(git log --format=%s -n1 HEAD)" "" "" # Only check maintainers if this build is supposed to publish toolstate. # Builds that are not supposed to publish don't have the access token. if [ -n "${TOOLSTATE_PUBLISH+is_set}" ]; then - TOOLSTATE_VALIDATE_MAINTAINERS_REPO=rust-lang/rust python2.7 "${BUILD_SOURCESDIRECTORY}/src/tools/publish_toolstate.py" + TOOLSTATE_VALIDATE_MAINTAINERS_REPO=rust-lang/rust python2.7 \ + "../../src/tools/publish_toolstate.py" fi cd .. rm -rf rust-toolstate From 14da85c68e6b5cd8a64a1f2f5ba5cd50c725e38b Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 4 Nov 2019 17:40:18 +0100 Subject: [PATCH 8/9] ci: move mirrors to https://ci-mirrors.rust-lang.org --- src/ci/docker/armhf-gnu/Dockerfile | 2 +- src/ci/docker/dist-armv7-linux/crosstool-ng.sh | 2 +- src/ci/docker/dist-various-1/install-mips-musl.sh | 2 +- src/ci/docker/dist-various-1/install-mipsel-musl.sh | 2 +- src/ci/docker/dist-various-2/build-wasi-toolchain.sh | 2 +- src/ci/docker/dist-x86_64-linux/build-curl.sh | 2 +- src/ci/docker/dist-x86_64-linux/build-openssl.sh | 2 +- src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh | 2 +- src/ci/docker/scripts/freebsd-toolchain.sh | 2 +- src/ci/docker/scripts/sccache.sh | 2 +- src/ci/shared.sh | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/ci/docker/armhf-gnu/Dockerfile b/src/ci/docker/armhf-gnu/Dockerfile index 9493b33698708..5373612279bca 100644 --- a/src/ci/docker/armhf-gnu/Dockerfile +++ b/src/ci/docker/armhf-gnu/Dockerfile @@ -72,7 +72,7 @@ RUN arm-linux-gnueabihf-gcc addentropy.c -o rootfs/addentropy -static # TODO: What is this?! # Source of the file: https://github.com/vfdev-5/qemu-rpi2-vexpress/raw/master/vexpress-v2p-ca15-tc1.dtb -RUN curl -O https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/vexpress-v2p-ca15-tc1.dtb +RUN curl -O https://ci-mirrors.rust-lang.org/rustc/vexpress-v2p-ca15-tc1.dtb COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh diff --git a/src/ci/docker/dist-armv7-linux/crosstool-ng.sh b/src/ci/docker/dist-armv7-linux/crosstool-ng.sh index ae737d9677d87..fb067a79a5c85 100644 --- a/src/ci/docker/dist-armv7-linux/crosstool-ng.sh +++ b/src/ci/docker/dist-armv7-linux/crosstool-ng.sh @@ -1,7 +1,7 @@ set -ex # Mirrored from https://github.com/crosstool-ng/crosstool-ng/archive/crosstool-ng-1.24.0.tar.gz -url="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/crosstool-ng-1.24.0.tar.gz" +url="https://ci-mirrors.rust-lang.org/rustc/crosstool-ng-1.24.0.tar.gz" curl -Lf $url | tar xzf - cd crosstool-ng-crosstool-ng-1.24.0 ./bootstrap diff --git a/src/ci/docker/dist-various-1/install-mips-musl.sh b/src/ci/docker/dist-various-1/install-mips-musl.sh index 29cfb5d96083e..9584258d23403 100755 --- a/src/ci/docker/dist-various-1/install-mips-musl.sh +++ b/src/ci/docker/dist-various-1/install-mips-musl.sh @@ -5,7 +5,7 @@ mkdir /usr/local/mips-linux-musl # originally from # https://downloads.openwrt.org/snapshots/trunk/ar71xx/generic/ # OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2 -URL="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc" +URL="https://ci-mirrors.rust-lang.org/rustc" FILE="OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2" curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mips-linux-musl --strip-components=2 diff --git a/src/ci/docker/dist-various-1/install-mipsel-musl.sh b/src/ci/docker/dist-various-1/install-mipsel-musl.sh index de8c359d16757..50a8e554b1675 100755 --- a/src/ci/docker/dist-various-1/install-mipsel-musl.sh +++ b/src/ci/docker/dist-various-1/install-mipsel-musl.sh @@ -5,7 +5,7 @@ mkdir /usr/local/mipsel-linux-musl # Note that this originally came from: # https://downloads.openwrt.org/snapshots/trunk/malta/generic/ # OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2 -URL="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc" +URL="https://ci-mirrors.rust-lang.org/rustc" FILE="OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2" curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mipsel-linux-musl --strip-components=2 diff --git a/src/ci/docker/dist-various-2/build-wasi-toolchain.sh b/src/ci/docker/dist-various-2/build-wasi-toolchain.sh index c63ea6facca64..17aa78945cf21 100755 --- a/src/ci/docker/dist-various-2/build-wasi-toolchain.sh +++ b/src/ci/docker/dist-various-2/build-wasi-toolchain.sh @@ -5,7 +5,7 @@ set -ex # Originally from https://releases.llvm.org/9.0.0/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz -curl https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/clang%2Bllvm-9.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz | \ +curl https://ci-mirrors.rust-lang.org/rustc/clang%2Bllvm-9.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz | \ tar xJf - export PATH=`pwd`/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-14.04/bin:$PATH diff --git a/src/ci/docker/dist-x86_64-linux/build-curl.sh b/src/ci/docker/dist-x86_64-linux/build-curl.sh index 8200bbe2fdce5..a65e3de5cfc26 100755 --- a/src/ci/docker/dist-x86_64-linux/build-curl.sh +++ b/src/ci/docker/dist-x86_64-linux/build-curl.sh @@ -5,7 +5,7 @@ source shared.sh VERSION=7.66.0 -curl https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/curl-$VERSION.tar.xz \ +curl https://ci-mirrors.rust-lang.org/rustc/curl-$VERSION.tar.xz \ | xz --decompress \ | tar xf - diff --git a/src/ci/docker/dist-x86_64-linux/build-openssl.sh b/src/ci/docker/dist-x86_64-linux/build-openssl.sh index be8a6c93945e9..0d9bdf7a09929 100755 --- a/src/ci/docker/dist-x86_64-linux/build-openssl.sh +++ b/src/ci/docker/dist-x86_64-linux/build-openssl.sh @@ -4,7 +4,7 @@ set -ex source shared.sh VERSION=1.0.2k -URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/openssl-$VERSION.tar.gz +URL=https://ci-mirrors.rust-lang.org/rustc/openssl-$VERSION.tar.gz curl $URL | tar xzf - diff --git a/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh b/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh index 797f674b954f2..f8697c698b9fb 100755 --- a/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh +++ b/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh @@ -25,7 +25,7 @@ cd netbsd mkdir -p /x-tools/x86_64-unknown-netbsd/sysroot -URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc +URL=https://ci-mirrors.rust-lang.org/rustc # Originally from ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-$BSD/source/sets/*.tgz curl $URL/2018-03-01-netbsd-src.tgz | tar xzf - diff --git a/src/ci/docker/scripts/freebsd-toolchain.sh b/src/ci/docker/scripts/freebsd-toolchain.sh index 70155e770a960..5670e10be23cf 100755 --- a/src/ci/docker/scripts/freebsd-toolchain.sh +++ b/src/ci/docker/scripts/freebsd-toolchain.sh @@ -59,7 +59,7 @@ done # Originally downloaded from: # https://download.freebsd.org/ftp/releases/${freebsd_arch}/${freebsd_version}-RELEASE/base.txz -URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2019-04-04-freebsd-${freebsd_arch}-${freebsd_version}-RELEASE-base.txz +URL=https://ci-mirrors.rust-lang.org/rustc/2019-04-04-freebsd-${freebsd_arch}-${freebsd_version}-RELEASE-base.txz curl "$URL" | tar xJf - -C "$sysroot" --wildcards "${files_to_extract[@]}" # Fix up absolute symlinks from the system image. This can be removed diff --git a/src/ci/docker/scripts/sccache.sh b/src/ci/docker/scripts/sccache.sh index efeb0ed0d72d0..552afbfee7c17 100644 --- a/src/ci/docker/scripts/sccache.sh +++ b/src/ci/docker/scripts/sccache.sh @@ -1,6 +1,6 @@ set -ex curl -fo /usr/local/bin/sccache \ - https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-02-sccache-x86_64-unknown-linux-musl + https://ci-mirrors.rust-lang.org/rustc/2018-04-02-sccache-x86_64-unknown-linux-musl chmod +x /usr/local/bin/sccache diff --git a/src/ci/shared.sh b/src/ci/shared.sh index 718a5379ae558..862ded0d5dbf0 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -4,7 +4,7 @@ # `source shared.sh`, hence the invalid shebang and not being # marked as an executable file in git. -export MIRRORS_BASE="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc" +export MIRRORS_BASE="https://ci-mirrors.rust-lang.org/rustc" # See http://unix.stackexchange.com/questions/82598 # Duplicated in docker/dist-various-2/shared.sh From 85132b20c4b3f6858ea273ec276c4ad9162603f8 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 6 Nov 2019 18:58:57 +0100 Subject: [PATCH 9/9] ci: download curl and openssl from s3 for dist-x86_64-linux CentOS 5 only supports SSLv3 without SNI, and to get newer protocols working we need to download and compile OpenSSL and cURL from our mirror. Because of that, we can't use the CDN, as CloudFront requires TLSv1 with SNI. This commit changes the dist-x86_64-linux image to bypass the CDN for OpenSSL and cURL. --- src/ci/docker/dist-x86_64-linux/build-curl.sh | 5 ++++- src/ci/docker/dist-x86_64-linux/build-openssl.sh | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/ci/docker/dist-x86_64-linux/build-curl.sh b/src/ci/docker/dist-x86_64-linux/build-curl.sh index a65e3de5cfc26..0c7eb5fdac9be 100755 --- a/src/ci/docker/dist-x86_64-linux/build-curl.sh +++ b/src/ci/docker/dist-x86_64-linux/build-curl.sh @@ -5,7 +5,10 @@ source shared.sh VERSION=7.66.0 -curl https://ci-mirrors.rust-lang.org/rustc/curl-$VERSION.tar.xz \ +# This needs to be downloaded directly from S3, it can't go through the CDN. +# That's because the CDN is backed by CloudFront, which requires SNI and TLSv1 +# (without paying an absurd amount of money). +curl https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/curl-$VERSION.tar.xz \ | xz --decompress \ | tar xf - diff --git a/src/ci/docker/dist-x86_64-linux/build-openssl.sh b/src/ci/docker/dist-x86_64-linux/build-openssl.sh index 0d9bdf7a09929..d8f6bdb51b8d5 100755 --- a/src/ci/docker/dist-x86_64-linux/build-openssl.sh +++ b/src/ci/docker/dist-x86_64-linux/build-openssl.sh @@ -4,7 +4,11 @@ set -ex source shared.sh VERSION=1.0.2k -URL=https://ci-mirrors.rust-lang.org/rustc/openssl-$VERSION.tar.gz + +# This needs to be downloaded directly from S3, it can't go through the CDN. +# That's because the CDN is backed by CloudFront, which requires SNI and TLSv1 +# (without paying an absurd amount of money). +URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/openssl-$VERSION.tar.gz curl $URL | tar xzf -