From f21a9042a0075e46f6fee03ed7784048bba5ec7b Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 25 Mar 2019 11:39:45 -0700 Subject: [PATCH] ci: Conditionally build parallel compiler on `try` This commit configures Travis/AppVeyor to conditionally compile parallel compilers on `@bors: try`. This is an experiment currently to see how this plays out, but the intention is that if the commit message contains the term "parallel-compiler" then when `@bors: try` is issued it will perform differently than the try branch does today, building three compilers: Linux, macOS, and Windows. We currently have no `try` builders for macOS or Windows due to typical capacity issues, so it's intended that this is only very sparingly used from time to time when necessary. [parallel-compiler] --- .travis.yml | 23 +++- appveyor.yml | 250 ++++++++++++++++++++++++------------------- src/ci/docker/run.sh | 1 + src/ci/run.sh | 4 + 4 files changed, 165 insertions(+), 113 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7a8772d7abd63..1c7c7d10530ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,14 +26,14 @@ matrix: - env: IMAGE=dist-x86_64-linux DEPLOY=1 name: dist-x86_64-linux - if: branch = try OR branch = auto + if: (branch = try AND NOT commit_message ~= parallel-compiler) OR branch = auto # "alternate" deployments, these are "nightlies" but have LLVM assertions # turned on, they're deployed to a different location primarily for # additional testing. - env: IMAGE=dist-x86_64-linux DEPLOY_ALT=1 name: dist-x86_64-linux-alt - if: branch = try OR branch = auto + if: (branch = try AND NOT commit_message ~= parallel-compiler) OR branch = auto - env: > RUST_CHECK_TARGET=dist @@ -239,6 +239,25 @@ matrix: . src/ci/docker/x86_64-gnu-tools/repo.sh; commit_toolstate_change "$MESSAGE_FILE" "$TRAVIS_BUILD_DIR/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE" "$TOOLSTATE_REPO_ACCESS_TOKEN"; + - env: IMAGE=dist-x86_64-linux DEPLOY=1 DIST_PARALLEL_COMPILER=1 + name: dist-x86_64-linux-parallel-compiler + if: branch = try AND commit_message ~= parallel-compiler + - env: > + RUST_CHECK_TARGET=dist + RUST_CONFIGURE_ARGS="--enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb --set rust.jemalloc" + SRC=. + DEPLOY=1 + RUSTC_RETRY_LINKER_ON_SEGFAULT=1 + MACOSX_DEPLOYMENT_TARGET=10.7 + NO_LLVM_ASSERTIONS=1 + NO_DEBUG_ASSERTIONS=1 + DIST_REQUIRE_ALL_TOOLS=1 + DIST_PARALLEL_COMPILER=1 + os: osx + osx_image: xcode9.3-moar + name: dist-x86_64-apple-parallel-compiler + if: branch = try AND commit_message ~= parallel-compiler + before_install: # We'll use the AWS cli to download/upload cached docker layers as well as # push our deployments, so download that here. diff --git a/appveyor.yml b/appveyor.yml index ab0a29d2752b5..a754074c3da69 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,116 +8,150 @@ environment: # server goes down presumably. See #43333 for more info CARGO_HTTP_CHECK_REVOKE: false - matrix: - # 32/64 bit MSVC tests - - CI_JOB_NAME: x86_64-msvc - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - SCRIPT: python x.py test - - CI_JOB_NAME: i686-msvc-1 - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make appveyor-subset-1 - - CI_JOB_NAME: i686-msvc-2 - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make appveyor-subset-2 - # MSVC aux tests - - CI_JOB_NAME: x86_64-msvc-aux - MSYS_BITS: 64 - RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - - CI_JOB_NAME: x86_64-msvc-cargo - MSYS_BITS: 64 - SCRIPT: python x.py test src/tools/cargotest src/tools/cargo - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - # MSVC tools tests - - CI_JOB_NAME: x86_64-msvc-tools - MSYS_BITS: 64 - SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri +for: +- + branches: + only: + - auto + environment: + matrix: + # 32/64 bit MSVC tests + - CI_JOB_NAME: x86_64-msvc + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: python x.py test + - CI_JOB_NAME: i686-msvc-1 + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make appveyor-subset-1 + - CI_JOB_NAME: i686-msvc-2 + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make appveyor-subset-2 - # 32/64-bit MinGW builds. - # - # We are using MinGW with posix threads since LLVM does not compile with - # the win32 threads version due to missing support for C++'s std::thread. - # - # Instead of relying on the MinGW version installed on appveryor we download - # and install one ourselves so we won't be surprised by changes to appveyor's - # build image. - # - # Finally, note that the downloads below are all in the `rust-lang-ci` S3 - # bucket, but they cleraly didn't originate there! The downloads originally - # came from the mingw-w64 SourceForge download site. Unfortunately - # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. - - CI_JOB_NAME: i686-mingw-1 - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make appveyor-subset-1 - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - - CI_JOB_NAME: i686-mingw-2 - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make appveyor-subset-2 - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - - CI_JOB_NAME: x86_64-mingw - MSYS_BITS: 64 - SCRIPT: python x.py test - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 + # MSVC aux tests + - CI_JOB_NAME: x86_64-msvc-aux + MSYS_BITS: 64 + RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + - CI_JOB_NAME: x86_64-msvc-cargo + MSYS_BITS: 64 + SCRIPT: python x.py test src/tools/cargotest src/tools/cargo + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + + # MSVC tools tests + - CI_JOB_NAME: x86_64-msvc-tools + MSYS_BITS: 64 + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri + + # 32/64-bit MinGW builds. + # + # We are using MinGW with posix threads since LLVM does not compile with + # the win32 threads version due to missing support for C++'s std::thread. + # + # Instead of relying on the MinGW version installed on appveryor we download + # and install one ourselves so we won't be surprised by changes to appveyor's + # build image. + # + # Finally, note that the downloads below are all in the `rust-lang-ci` S3 + # bucket, but they cleraly didn't originate there! The downloads originally + # came from the mingw-w64 SourceForge download site. Unfortunately + # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. + - CI_JOB_NAME: i686-mingw-1 + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make appveyor-subset-1 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + - CI_JOB_NAME: i686-mingw-2 + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make appveyor-subset-2 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + - CI_JOB_NAME: x86_64-mingw + MSYS_BITS: 64 + SCRIPT: python x.py test + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + + # 32/64 bit MSVC and GNU deployment + - CI_JOB_NAME: dist-x86_64-msvc + RUST_CONFIGURE_ARGS: > + --build=x86_64-pc-windows-msvc + --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + - CI_JOB_NAME: dist-i686-msvc + RUST_CONFIGURE_ARGS: > + --build=i686-pc-windows-msvc + --target=i586-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + - CI_JOB_NAME: dist-i686-mingw + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools + SCRIPT: python x.py dist + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + - CI_JOB_NAME: dist-x86_64-mingw + MSYS_BITS: 64 + SCRIPT: python x.py dist + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 - # 32/64 bit MSVC and GNU deployment - - CI_JOB_NAME: dist-x86_64-msvc - RUST_CONFIGURE_ARGS: > - --build=x86_64-pc-windows-msvc - --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc - --enable-full-tools - --enable-profiler - SCRIPT: python x.py dist - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - - CI_JOB_NAME: dist-i686-msvc - RUST_CONFIGURE_ARGS: > - --build=i686-pc-windows-msvc - --target=i586-pc-windows-msvc - --enable-full-tools - --enable-profiler - SCRIPT: python x.py dist - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - - CI_JOB_NAME: dist-i686-mingw - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools - SCRIPT: python x.py dist - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - - CI_JOB_NAME: dist-x86_64-mingw - MSYS_BITS: 64 - SCRIPT: python x.py dist - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 + # "alternate" deployment, see .travis.yml for more info + - CI_JOB_NAME: dist-x86_64-msvc-alt + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler + SCRIPT: python x.py dist + DEPLOY_ALT: 1 - # "alternate" deployment, see .travis.yml for more info - - CI_JOB_NAME: dist-x86_64-msvc-alt - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler - SCRIPT: python x.py dist - DEPLOY_ALT: 1 +# Try builder +- + branches: + only: + - try + environment: + matrix: + - CI_JOB_NAME: dist-x86_64-msvc-parallel + RUST_CONFIGURE_ARGS: > + --build=x86_64-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + DIST_PARALLEL_COMPILER: 1 + DEPLOY: 1 + +# Skip builds on all other branches +- + branches: + except: + - try + - auto + only_commits: + message: /nonexistent-message-to-fool-appveyor-to-not-run-a-build/ matrix: fast_finish: true @@ -216,10 +250,6 @@ on_failure: - set PATH=%PATH%;"C:\Program Files (x86)\Windows Kits\10\Debuggers\X64" - if exist %LOCALAPPDATA%\CrashDumps for %%f in (%LOCALAPPDATA%\CrashDumps\*) do cdb -c "k;q" -G -z "%%f" -branches: - only: - - auto - before_deploy: - ps: | New-Item -Path deploy -ItemType directory @@ -239,7 +269,6 @@ deploy: artifact: /.*/ folder: rustc-builds on: - branch: auto DEPLOY: 1 max_error_retry: 5 @@ -254,7 +283,6 @@ deploy: artifact: /.*/ folder: rustc-builds-alt on: - branch: auto DEPLOY_ALT: 1 max_error_retry: 5 diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index ef151750af9bf..a05c0fce77649 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -133,6 +133,7 @@ exec docker \ --env CARGO_HOME=/cargo \ --env DEPLOY \ --env DEPLOY_ALT \ + --env DIST_PARALLEL_COMPILER \ --env LOCAL_USER_ID=`id -u` \ --env TRAVIS \ --env TRAVIS_BRANCH \ diff --git a/src/ci/run.sh b/src/ci/run.sh index 42d0d7db5964c..327bcd48adf29 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -37,6 +37,10 @@ if [ "$DIST_SRC" = "" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-dist-src" fi +if [ "$DIST_PARALLEL_COMPILER" = "" ]; then + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.parallel-compiler" +fi + # If we're deploying artifacts then we set the release channel, otherwise if # we're not deploying then we want to be sure to enable all assertions because # we'll be running tests