From 14d50bfecb569aee7ee42430f50d46252b303d24 Mon Sep 17 00:00:00 2001 From: kennytm Date: Sat, 9 Jun 2018 15:48:44 +0800 Subject: [PATCH 1/5] Allow some tools to be run without first building LLVM. Conservatively only disable LLVM for rust-installer. This should shave 5 minutes from the x86_64-gnu-distcheck job by not building LLVM twice. --- src/bootstrap/tool.rs | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index d4a2e96cc0d3f..595fe0ab9ad7a 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -254,7 +254,7 @@ pub fn prepare_tool_cargo( } macro_rules! tool { - ($($name:ident, $path:expr, $tool_name:expr, $mode:expr;)+) => { + ($($name:ident, $path:expr, $tool_name:expr, $mode:expr $(,llvm_tools = $llvm:expr)*;)+) => { #[derive(Copy, Clone)] pub enum Tool { $( @@ -269,6 +269,13 @@ macro_rules! tool { }; mode } + + /// Whether this tool requires LLVM to run + pub fn uses_llvm_tools(&self) -> bool { + match self { + $(Tool::$name => true $(&& $llvm)*,)+ + } + } } impl<'a> Builder<'a> { @@ -343,7 +350,7 @@ tool!( Compiletest, "src/tools/compiletest", "compiletest", Mode::ToolTest; BuildManifest, "src/tools/build-manifest", "build-manifest", Mode::ToolStd; RemoteTestClient, "src/tools/remote-test-client", "remote-test-client", Mode::ToolStd; - RustInstaller, "src/tools/rust-installer", "fabricate", Mode::ToolStd; + RustInstaller, "src/tools/rust-installer", "fabricate", Mode::ToolStd, llvm_tools = false; RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes", Mode::ToolStd; ); @@ -586,7 +593,7 @@ impl<'a> Builder<'a> { pub fn tool_cmd(&self, tool: Tool) -> Command { let mut cmd = Command::new(self.tool_exe(tool)); let compiler = self.compiler(self.tool_default_stage(tool), self.config.build); - self.prepare_tool_cmd(compiler, tool.get_mode(), &mut cmd); + self.prepare_tool_cmd(compiler, tool, &mut cmd); cmd } @@ -594,11 +601,11 @@ impl<'a> Builder<'a> { /// /// Notably this munges the dynamic library lookup path to point to the /// right location to run `compiler`. - fn prepare_tool_cmd(&self, compiler: Compiler, mode: Mode, cmd: &mut Command) { + fn prepare_tool_cmd(&self, compiler: Compiler, tool: Tool, cmd: &mut Command) { let host = &compiler.host; let mut lib_paths: Vec = vec![ PathBuf::from(&self.sysroot_libdir(compiler, compiler.host)), - self.cargo_out(compiler, mode, *host).join("deps"), + self.cargo_out(compiler, tool.get_mode(), *host).join("deps"), ]; // On MSVC a tool may invoke a C compiler (e.g. compiletest in run-make @@ -621,17 +628,19 @@ impl<'a> Builder<'a> { // Add the llvm/bin directory to PATH since it contains lots of // useful, platform-independent tools - if let Some(llvm_bin_path) = self.llvm_bin_path() { - if host.contains("windows") { - // On Windows, PATH and the dynamic library path are the same, - // so we just add the LLVM bin path to lib_path - lib_paths.push(llvm_bin_path); - } else { - let old_path = env::var_os("PATH").unwrap_or_default(); - let new_path = env::join_paths(iter::once(llvm_bin_path) - .chain(env::split_paths(&old_path))) - .expect("Could not add LLVM bin path to PATH"); - cmd.env("PATH", new_path); + if tool.uses_llvm_tools() { + if let Some(llvm_bin_path) = self.llvm_bin_path() { + if host.contains("windows") { + // On Windows, PATH and the dynamic library path are the same, + // so we just add the LLVM bin path to lib_path + lib_paths.push(llvm_bin_path); + } else { + let old_path = env::var_os("PATH").unwrap_or_default(); + let new_path = env::join_paths(iter::once(llvm_bin_path) + .chain(env::split_paths(&old_path))) + .expect("Could not add LLVM bin path to PATH"); + cmd.env("PATH", new_path); + } } } From d2b5b7603b6b7ecb4ff93981c785aef640015e68 Mon Sep 17 00:00:00 2001 From: kennytm Date: Sat, 9 Jun 2018 22:49:02 +0800 Subject: [PATCH 2/5] Do not push a commit if the toolstate is unchanged. This should greatly reduce the commits on the rust-toolstate repository. `publish_toolstate.py` defaults to keep the old status if a new one is not found, so nothing needs to be changed to that file. --- .../x86_64-gnu-tools/checkregression.py | 14 ++++++++--- src/ci/docker/x86_64-gnu-tools/checktools.sh | 25 ++++++++++++------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/ci/docker/x86_64-gnu-tools/checkregression.py b/src/ci/docker/x86_64-gnu-tools/checkregression.py index df791d12645fd..208aab434ce1f 100755 --- a/src/ci/docker/x86_64-gnu-tools/checkregression.py +++ b/src/ci/docker/x86_64-gnu-tools/checkregression.py @@ -18,6 +18,7 @@ os_name = sys.argv[1] toolstate_file = sys.argv[2] current_state = sys.argv[3] + verb = sys.argv[4] # 'regressed' or 'changed' with open(toolstate_file, 'r') as f: toolstate = json.load(f) @@ -29,10 +30,17 @@ tool = cur['tool'] state = cur[os_name] new_state = toolstate.get(tool, '') - if new_state < state: + if verb == 'regressed': + updated = new_state < state + elif verb == 'changed': + updated = new_state != state + else: + print('Unknown verb {}'.format(updated)) + sys.exit(2) + if updated: print( - 'Error: The state of "{}" has regressed from "{}" to "{}"' - .format(tool, state, new_state) + 'The state of "{}" has {} from "{}" to "{}"' + .format(tool, verb, state, new_state) ) regressed = True diff --git a/src/ci/docker/x86_64-gnu-tools/checktools.sh b/src/ci/docker/x86_64-gnu-tools/checktools.sh index d71d5daf8113b..d26be93861c0f 100755 --- a/src/ci/docker/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/x86_64-gnu-tools/checktools.sh @@ -91,19 +91,26 @@ status_check() { status_check "submodule_changed" -if [ "$RUST_RELEASE_CHANNEL" = nightly -a -n "${TOOLSTATE_REPO_ACCESS_TOKEN+is_set}" ]; then - . "$(dirname $0)/repo.sh" - MESSAGE_FILE=$(mktemp -t msg.XXXXXX) - echo "($OS CI update)" > "$MESSAGE_FILE" - commit_toolstate_change "$MESSAGE_FILE" \ +CHECK_NOT="$(dirname $0)/checkregression.py" +change_toolstate() { + # only update the history + if python2.7 "$CHECK_NOT" "$OS" "$TOOLSTATE_FILE" "_data/latest.json" changed; then + echo 'Toolstate is not changed. Not updating.' + else + if [ $SIX_WEEK_CYCLE -eq 5 ]; then + python2.7 "$CHECK_NOT" "$OS" "$TOOLSTATE_FILE" "_data/latest.json" regressed + fi sed -i "1 a\\ $COMMIT\t$(cat "$TOOLSTATE_FILE") " "history/$OS.tsv" - # if we are at the last week in the 6-week release cycle, reject any kind of regression. - if [ $SIX_WEEK_CYCLE -eq 5 ]; then - python2.7 "$(dirname $0)/checkregression.py" \ - "$OS" "$TOOLSTATE_FILE" "rust-toolstate/_data/latest.json" fi +} + +if [ "$RUST_RELEASE_CHANNEL" = nightly -a -n "${TOOLSTATE_REPO_ACCESS_TOKEN+is_set}" ]; then + . "$(dirname $0)/repo.sh" + MESSAGE_FILE=$(mktemp -t msg.XXXXXX) + echo "($OS CI update)" > "$MESSAGE_FILE" + commit_toolstate_change "$MESSAGE_FILE" change_toolstate rm -f "$MESSAGE_FILE" exit 0 fi From b00ea6c2da3cf723c36328fe91057c6568fab688 Mon Sep 17 00:00:00 2001 From: kennytm Date: Sun, 10 Jun 2018 01:27:24 +0800 Subject: [PATCH 3/5] Allowing building the codegen backend specifically. Use `./x.py build src/librustc_codegen_llvm` to build the codegen backend. --- src/bootstrap/builder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index be9c926bedf22..d482a0d565027 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -339,6 +339,7 @@ impl<'a> Builder<'a> { compile::Std, compile::Test, compile::Rustc, + compile::CodegenBackend, compile::StartupObjects, tool::BuildManifest, tool::Rustbook, From 4a0e92eee5efbb362096a39f99aaaa69c0169ae5 Mon Sep 17 00:00:00 2001 From: kennytm Date: Sun, 10 Jun 2018 01:31:55 +0800 Subject: [PATCH 4/5] Log the clang_rt.asan-dynamic-i386.vers on failure to track #50887 --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index c154f3d8e251f..2e6722cf85563 100644 --- a/.travis.yml +++ b/.travis.yml @@ -284,6 +284,9 @@ after_failure: -exec head -750 {} \; -exec echo travis_fold":"end:crashlog \; || true + # see #50887 + - head -30 ./obj/build/x86_64-unknown-linux-gnu/native/asan/build/lib/asan/clang_rt.asan-dynamic-i386.vers || true + # attempt to debug anything killed by the oom killer on linux, just to see if # it happened - dmesg | grep -i kill From ab5e3e66b57cded1127303d32adf35502d3e1fa0 Mon Sep 17 00:00:00 2001 From: kennytm Date: Sun, 10 Jun 2018 04:50:42 +0800 Subject: [PATCH 5/5] Added comment to explain why only RustIstaller has `llvm_tools = false`. --- src/bootstrap/tool.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 595fe0ab9ad7a..0c164d86332a8 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -340,6 +340,9 @@ macro_rules! tool { } } +// FIXME(#51459): We have only checked that RustInstaller does not require +// the LLVM binaries when running. We should go through all tools to determine +// if they really need LLVM binaries, and make `llvm_tools` a required argument. tool!( Rustbook, "src/tools/rustbook", "rustbook", Mode::ToolRustc; ErrorIndex, "src/tools/error_index_generator", "error_index_generator", Mode::ToolRustc;