From 0fcd3e7b07e0f50a75f50b2e7dc935ae06fc5abd Mon Sep 17 00:00:00 2001 From: Mark Simulacrum Date: Mon, 16 Oct 2017 11:40:47 -0600 Subject: [PATCH 1/2] Make sure to clear out the stageN-{rustc,std,tools} directories. We copy built tool binaries into a dedicated directory to avoid deleting them, stageN-tools-bin. These aren't ever cleared out by code, since there should be no reason to do so, and we'll simply overwrite them as necessary. When clearing out the stageN-{std,rustc,tools} directories, make sure to delete both Cargo directories -- per-target and build scripts. This ensures that changing libstd doesn't cause problems due to build scripts not being rebuilt, even though they should be. --- src/bootstrap/builder.rs | 2 +- src/bootstrap/compile.rs | 23 ++++++++------------- src/bootstrap/lib.rs | 13 ++++++++++-- src/bootstrap/tool.rs | 44 +++++++++++++++++++++++++++++----------- src/bootstrap/util.rs | 21 +++++++++++++++++-- 5 files changed, 72 insertions(+), 31 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 6480b5a619c03..dee4999879ca3 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -612,7 +612,7 @@ impl<'a> Builder<'a> { // Set this for all builds to make sure doc builds also get it. cargo.env("CFG_RELEASE_CHANNEL", &self.build.config.channel); - if self.is_verbose() { + if self.is_very_verbose() { cargo.arg("-v"); } // FIXME: cargo bench does not accept `--release` diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index b1c630a8de9e5..f837371bebecb 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -29,7 +29,7 @@ use build_helper::{output, mtime, up_to_date}; use filetime::FileTime; use serde_json; -use util::{exe, libdir, is_dylib, copy}; +use util::{exe, libdir, is_dylib, copy, read_stamp_file}; use {Build, Compiler, Mode}; use native; use tool; @@ -102,7 +102,7 @@ impl Step for Std { copy_musl_third_party_objects(build, target, &libdir); } - let out_dir = build.cargo_out(compiler, Mode::Libstd, target); + let out_dir = build.stage_out(compiler, Mode::Libstd); build.clear_if_dirty(&out_dir, &builder.rustc(compiler)); let mut cargo = builder.cargo(compiler, Mode::Libstd, target, "build"); std_cargo(build, &compiler, target, &mut cargo); @@ -354,7 +354,7 @@ impl Step for Test { let _folder = build.fold_output(|| format!("stage{}-test", compiler.stage)); println!("Building stage{} test artifacts ({} -> {})", compiler.stage, &compiler.host, target); - let out_dir = build.cargo_out(compiler, Mode::Libtest, target); + let out_dir = build.stage_out(compiler, Mode::Libtest); build.clear_if_dirty(&out_dir, &libstd_stamp(build, compiler, target)); let mut cargo = builder.cargo(compiler, Mode::Libtest, target, "build"); test_cargo(build, &compiler, target, &mut cargo); @@ -480,8 +480,9 @@ impl Step for Rustc { println!("Building stage{} compiler artifacts ({} -> {})", compiler.stage, &compiler.host, target); - let out_dir = build.cargo_out(compiler, Mode::Librustc, target); - build.clear_if_dirty(&out_dir, &libtest_stamp(build, compiler, target)); + let stage_out = builder.stage_out(compiler, Mode::Librustc); + build.clear_if_dirty(&stage_out, &libstd_stamp(build, compiler, target)); + build.clear_if_dirty(&stage_out, &libtest_stamp(build, compiler, target)); let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "build"); rustc_cargo(build, &compiler, target, &mut cargo); @@ -757,15 +758,7 @@ impl Step for Assemble { /// `sysroot_dst` provided. fn add_to_sysroot(sysroot_dst: &Path, stamp: &Path) { t!(fs::create_dir_all(&sysroot_dst)); - let mut contents = Vec::new(); - t!(t!(File::open(stamp)).read_to_end(&mut contents)); - // This is the method we use for extracting paths from the stamp file passed to us. See - // run_cargo for more information (in this file). - for part in contents.split(|b| *b == 0) { - if part.is_empty() { - continue - } - let path = Path::new(t!(str::from_utf8(part))); + for path in read_stamp_file(stamp) { copy(&path, &sysroot_dst.join(path.file_name().unwrap())); } } @@ -938,6 +931,8 @@ fn run_cargo(build: &Build, cargo: &mut Command, stamp: &Path) { let max = max.unwrap(); let max_path = max_path.unwrap(); if stamp_contents == new_contents && max <= stamp_mtime { + build.verbose(&format!("not updating {:?}; contents equal and {} <= {}", + stamp, max, stamp_mtime)); return } if max > stamp_mtime { diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 6ac919d3fbdda..479283b359554 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -385,16 +385,19 @@ impl Build { /// Clear out `dir` if `input` is newer. /// /// After this executes, it will also ensure that `dir` exists. - fn clear_if_dirty(&self, dir: &Path, input: &Path) { + fn clear_if_dirty(&self, dir: &Path, input: &Path) -> bool { let stamp = dir.join(".stamp"); + let mut cleared = false; if mtime(&stamp) < mtime(input) { self.verbose(&format!("Dirty - {}", dir.display())); let _ = fs::remove_dir_all(dir); + cleared = true; } else if stamp.exists() { - return + return cleared; } t!(fs::create_dir_all(dir)); t!(File::create(stamp)); + cleared } /// Get the space-separated set of activated features for the standard @@ -435,6 +438,12 @@ impl Build { if self.config.rust_optimize {"release"} else {"debug"} } + fn tools_dir(&self, compiler: Compiler) -> PathBuf { + let out = self.out.join(&*compiler.host).join(format!("stage{}-tools-bin", compiler.stage)); + t!(fs::create_dir_all(&out)); + out + } + /// Get the directory for incremental by-products when using the /// given compiler. fn incremental_dir(&self, compiler: Compiler) -> PathBuf { diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 662c56d728dfe..bdbef3a943b90 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -38,24 +38,40 @@ impl Step for CleanTools { run.never() } - /// Build a tool in `src/tools` - /// - /// This will build the specified tool with the specified `host` compiler in - /// `stage` into the normal cargo output directory. fn run(self, builder: &Builder) { let build = builder.build; let compiler = self.compiler; let target = self.target; let mode = self.mode; - let stamp = match mode { - Mode::Libstd => libstd_stamp(build, compiler, target), - Mode::Libtest => libtest_stamp(build, compiler, target), - Mode::Librustc => librustc_stamp(build, compiler, target), - _ => panic!(), + // This is for the original compiler, but if we're forced to use stage 1, then + // std/test/rustc stamps won't exist in stage 2, so we need to get those from stage 1, since + // we copy the libs forward. + let tools_dir = build.stage_out(compiler, Mode::Tool); + let compiler = if builder.force_use_stage1(compiler, target) { + builder.compiler(1, compiler.host) + } else { + compiler }; - let out_dir = build.cargo_out(compiler, Mode::Tool, target); - build.clear_if_dirty(&out_dir, &stamp); + + for &cur_mode in &[Mode::Libstd, Mode::Libtest, Mode::Librustc] { + let stamp = match cur_mode { + Mode::Libstd => libstd_stamp(build, compiler, target), + Mode::Libtest => libtest_stamp(build, compiler, target), + Mode::Librustc => librustc_stamp(build, compiler, target), + _ => panic!(), + }; + + if build.clear_if_dirty(&tools_dir, &stamp) { + break; + } + + // If we are a rustc tool, and std changed, we also need to clear ourselves out -- our + // dependencies depend on std. Therefore, we iterate up until our own mode. + if mode == cur_mode { + break; + } + } } } @@ -100,7 +116,11 @@ impl Step for ToolBuild { let mut cargo = prepare_tool_cargo(builder, compiler, target, "build", path); build.run_expecting(&mut cargo, expectation); - build.cargo_out(compiler, Mode::Tool, target).join(exe(tool, &compiler.host)) + let cargo_out = build.cargo_out(compiler, Mode::Tool, target) + .join(exe(tool, &compiler.host)); + let bin = build.tools_dir(compiler).join(exe(tool, &compiler.host)); + copy(&cargo_out, &bin); + bin } } diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index a521dd0945391..2506048858f2b 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -14,8 +14,9 @@ //! not a lot of interesting happenings here unfortunately. use std::env; -use std::fs; -use std::io::{self, Write}; +use std::str; +use std::fs::{self, File}; +use std::io::{self, Read, Write}; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::{SystemTime, Instant}; @@ -50,6 +51,22 @@ pub fn copy(src: &Path, dst: &Path) { t!(filetime::set_file_times(dst, atime, mtime)); } +pub fn read_stamp_file(stamp: &Path) -> Vec { + let mut paths = Vec::new(); + let mut contents = Vec::new(); + t!(t!(File::open(stamp)).read_to_end(&mut contents)); + // This is the method we use for extracting paths from the stamp file passed to us. See + // run_cargo for more information (in compile.rs). + for part in contents.split(|b| *b == 0) { + if part.is_empty() { + continue + } + let path = PathBuf::from(t!(str::from_utf8(part))); + paths.push(path); + } + paths +} + /// Copies the `src` directory recursively to `dst`. Both are assumed to exist /// when this function is called. pub fn cp_r(src: &Path, dst: &Path) { From 686c1015cfa341748b6de20114e8caba4626931b Mon Sep 17 00:00:00 2001 From: Mark Simulacrum Date: Thu, 19 Oct 2017 17:30:37 -0600 Subject: [PATCH 2/2] Make tools which may not build return Option. This makes it mandatory for other steps to have to handle the potential failure instead of failing in an odd way later down the road. --- src/bootstrap/check.rs | 74 +++++++++++++++++++++++------------------- src/bootstrap/dist.rs | 4 ++- src/bootstrap/tool.rs | 40 +++++++++++++---------- 3 files changed, 65 insertions(+), 53 deletions(-) diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index d9ee63eef8cdd..11bc9f7d21772 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -340,25 +340,28 @@ impl Step for Miri { let host = self.host; let compiler = builder.compiler(1, host); - let miri = builder.ensure(tool::Miri { compiler, target: self.host }); - let mut cargo = builder.cargo(compiler, Mode::Tool, host, "test"); - cargo.arg("--manifest-path").arg(build.src.join("src/tools/miri/Cargo.toml")); - - // Don't build tests dynamically, just a pain to work with - cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1"); - // miri tests need to know about the stage sysroot - cargo.env("MIRI_SYSROOT", builder.sysroot(compiler)); - cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler)); - cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler)); - cargo.env("MIRI_PATH", miri); - - builder.add_rustc_lib_path(compiler, &mut cargo); - - try_run_expecting( - build, - &mut cargo, - builder.build.config.toolstate.miri.passes(ToolState::Testing), - ); + if let Some(miri) = builder.ensure(tool::Miri { compiler, target: self.host }) { + let mut cargo = builder.cargo(compiler, Mode::Tool, host, "test"); + cargo.arg("--manifest-path").arg(build.src.join("src/tools/miri/Cargo.toml")); + + // Don't build tests dynamically, just a pain to work with + cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1"); + // miri tests need to know about the stage sysroot + cargo.env("MIRI_SYSROOT", builder.sysroot(compiler)); + cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler)); + cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler)); + cargo.env("MIRI_PATH", miri); + + builder.add_rustc_lib_path(compiler, &mut cargo); + + try_run_expecting( + build, + &mut cargo, + builder.build.config.toolstate.miri.passes(ToolState::Testing), + ); + } else { + eprintln!("failed to test miri: could not build"); + } } } @@ -391,24 +394,27 @@ impl Step for Clippy { let host = self.host; let compiler = builder.compiler(stage, host); - let clippy = builder.ensure(tool::Clippy { compiler, target: self.host }); - let mut cargo = builder.cargo(compiler, Mode::Tool, host, "test"); - cargo.arg("--manifest-path").arg(build.src.join("src/tools/clippy/Cargo.toml")); + if let Some(clippy) = builder.ensure(tool::Clippy { compiler, target: self.host }) { + let mut cargo = builder.cargo(compiler, Mode::Tool, host, "test"); + cargo.arg("--manifest-path").arg(build.src.join("src/tools/clippy/Cargo.toml")); - // Don't build tests dynamically, just a pain to work with - cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1"); - // clippy tests need to know about the stage sysroot - cargo.env("SYSROOT", builder.sysroot(compiler)); - // clippy tests need to find the driver - cargo.env("CLIPPY_DRIVER_PATH", clippy); + // Don't build tests dynamically, just a pain to work with + cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1"); + // clippy tests need to know about the stage sysroot + cargo.env("SYSROOT", builder.sysroot(compiler)); + // clippy tests need to find the driver + cargo.env("CLIPPY_DRIVER_PATH", clippy); - builder.add_rustc_lib_path(compiler, &mut cargo); + builder.add_rustc_lib_path(compiler, &mut cargo); - try_run_expecting( - build, - &mut cargo, - builder.build.config.toolstate.clippy.passes(ToolState::Testing), - ); + try_run_expecting( + build, + &mut cargo, + builder.build.config.toolstate.clippy.passes(ToolState::Testing), + ); + } else { + eprintln!("failed to test clippy: could not build"); + } } } diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 02dfa04d9203a..a6663a854840e 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1073,10 +1073,12 @@ impl Step for Rls { t!(fs::create_dir_all(&image)); // Prepare the image directory + // We expect RLS to build, because we've exited this step above if tool + // state for RLS isn't testing. let rls = builder.ensure(tool::Rls { compiler: builder.compiler(stage, build.build), target - }); + }).expect("Rls to build: toolstate is testing"); install(&rls, &image.join("bin"), 0o755); let doc = image.join("share/doc/rls"); install(&src.join("README.md"), &doc, 0o644); diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index bdbef3a943b90..912ffa519758c 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -86,7 +86,7 @@ struct ToolBuild { } impl Step for ToolBuild { - type Output = PathBuf; + type Output = Option; fn should_run(run: ShouldRun) -> ShouldRun { run.never() @@ -96,7 +96,7 @@ impl Step for ToolBuild { /// /// This will build the specified tool with the specified `host` compiler in /// `stage` into the normal cargo output directory. - fn run(self, builder: &Builder) -> PathBuf { + fn run(self, builder: &Builder) -> Option { let build = builder.build; let compiler = self.compiler; let target = self.target; @@ -116,11 +116,15 @@ impl Step for ToolBuild { let mut cargo = prepare_tool_cargo(builder, compiler, target, "build", path); build.run_expecting(&mut cargo, expectation); - let cargo_out = build.cargo_out(compiler, Mode::Tool, target) - .join(exe(tool, &compiler.host)); - let bin = build.tools_dir(compiler).join(exe(tool, &compiler.host)); - copy(&cargo_out, &bin); - bin + if expectation == BuildExpectation::Succeeding || expectation == BuildExpectation::None { + let cargo_out = build.cargo_out(compiler, Mode::Tool, target) + .join(exe(tool, &compiler.host)); + let bin = build.tools_dir(compiler).join(exe(tool, &compiler.host)); + copy(&cargo_out, &bin); + Some(bin) + } else { + None + } } } @@ -229,7 +233,7 @@ macro_rules! tool { mode: $mode, path: $path, expectation: BuildExpectation::None, - }) + }).expect("expected to build -- BuildExpectation::None") } } )+ @@ -277,7 +281,7 @@ impl Step for RemoteTestServer { mode: Mode::Libstd, path: "src/tools/remote-test-server", expectation: BuildExpectation::None, - }) + }).expect("expected to build -- BuildExpectation::None") } } @@ -395,7 +399,7 @@ impl Step for Cargo { mode: Mode::Librustc, path: "src/tools/cargo", expectation: BuildExpectation::None, - }) + }).expect("BuildExpectation::None - expected to build") } } @@ -406,7 +410,7 @@ pub struct Clippy { } impl Step for Clippy { - type Output = PathBuf; + type Output = Option; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -421,7 +425,7 @@ impl Step for Clippy { }); } - fn run(self, builder: &Builder) -> PathBuf { + fn run(self, builder: &Builder) -> Option { // Clippy depends on procedural macros (serde), which requires a full host // compiler to be available, so we need to depend on that. builder.ensure(compile::Rustc { @@ -446,7 +450,7 @@ pub struct Rls { } impl Step for Rls { - type Output = PathBuf; + type Output = Option; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -462,7 +466,7 @@ impl Step for Rls { }); } - fn run(self, builder: &Builder) -> PathBuf { + fn run(self, builder: &Builder) -> Option { builder.ensure(native::Openssl { target: self.target, }); @@ -490,7 +494,7 @@ pub struct Rustfmt { } impl Step for Rustfmt { - type Output = PathBuf; + type Output = Option; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -506,7 +510,7 @@ impl Step for Rustfmt { }); } - fn run(self, builder: &Builder) -> PathBuf { + fn run(self, builder: &Builder) -> Option { builder.ensure(ToolBuild { compiler: self.compiler, target: self.target, @@ -526,7 +530,7 @@ pub struct Miri { } impl Step for Miri { - type Output = PathBuf; + type Output = Option; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -542,7 +546,7 @@ impl Step for Miri { }); } - fn run(self, builder: &Builder) -> PathBuf { + fn run(self, builder: &Builder) -> Option { builder.ensure(ToolBuild { compiler: self.compiler, target: self.target,