diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80d150e1df..dfe6d5ff6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,15 +62,14 @@ jobs: # contains package information of crates installed via `cargo install`. ~/.cargo/.crates.toml ~/.cargo/.crates2.json - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', 'cargo-miri/src/version.rs') }} + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} restore-keys: ${{ runner.os }}-cargo - - name: Install rustup-toolchain-install-master and xargo + - name: Install rustup-toolchain-install-master if: ${{ steps.cache.outputs.cache-hit == 'false' }} shell: bash run: | cargo install rustup-toolchain-install-master - cargo install xargo - name: Install "master" toolchain shell: bash diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8d965ae8fc..a88e69115b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,7 +38,7 @@ for you. If you don't want all of these to happen, you can add individual `.auto ## Building and testing Miri Invoking Miri requires getting a bunch of flags right and setting up a custom -sysroot with xargo. The `miri` script takes care of that for you. With the +sysroot. The `miri` script takes care of that for you. With the build environment prepared, compiling Miri is just one command away: ``` diff --git a/README.md b/README.md index 2e27bb39af..a1fc5b8440 100644 --- a/README.md +++ b/README.md @@ -447,7 +447,7 @@ binaries, and as such worth documenting: some compiler flags to prepare the code for interpretation; with `host`, this is not done. This environment variable is useful to be sure that the compiled `rlib`s are compatible with Miri. -* `MIRI_CALLED_FROM_XARGO` is set during the Miri-induced `xargo` sysroot build, +* `MIRI_CALLED_FROM_SETUP` is set during the Miri sysroot build, which will re-invoke `cargo-miri` as the `rustc` to use for this build. * `MIRI_CALLED_FROM_RUSTDOC` when set to any value tells `cargo-miri` that it is running as a child process of `rustdoc`, which invokes it twice for each doc-test diff --git a/cargo-miri/Cargo.lock b/cargo-miri/Cargo.lock index 95c2bda505..e88f4683cf 100644 --- a/cargo-miri/Cargo.lock +++ b/cargo-miri/Cargo.lock @@ -35,6 +35,7 @@ version = "0.1.0" dependencies = [ "cargo_metadata", "directories", + "rustc-build-sysroot", "rustc-workspace-hack", "rustc_version", "serde", @@ -142,6 +143,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "getrandom" version = "0.2.3" @@ -322,6 +329,43 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.2.10" @@ -341,6 +385,26 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "rustc-build-sysroot" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af54db6a317a83d6ab05540865427de791673039ff329a546420550c30531ba" +dependencies = [ + "anyhow", + "rustc_version", + "tempdir", +] + [[package]] name = "rustc-workspace-hack" version = "1.0.0" @@ -419,6 +483,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand", + "remove_dir_all", +] + [[package]] name = "thiserror" version = "1.0.30" diff --git a/cargo-miri/Cargo.toml b/cargo-miri/Cargo.toml index 9ac170c5b5..3ab334d0b2 100644 --- a/cargo-miri/Cargo.toml +++ b/cargo-miri/Cargo.toml @@ -18,6 +18,7 @@ directories = "3" rustc_version = "0.4" serde_json = "1.0.40" cargo_metadata = "0.15.0" +rustc-build-sysroot = "0.1" # A noop dependency that changes in the Rust repository, it's a bit of a hack. # See the `src/tools/rustc-workspace-hack/README.md` file in `rust-lang/rust` diff --git a/cargo-miri/src/main.rs b/cargo-miri/src/main.rs index 331c4c9c2b..9b5fa7ae87 100644 --- a/cargo-miri/src/main.rs +++ b/cargo-miri/src/main.rs @@ -6,7 +6,6 @@ mod util; mod arg; mod phases; mod setup; -mod version; use std::{env, iter}; @@ -22,8 +21,8 @@ fn main() { // Dispatch to `cargo-miri` phase. Here is a rough idea of "who calls who". // // Initially, we are invoked as `cargo-miri miri run/test`. We first run the setup phase: - // - We call `xargo`, and set `RUSTC` back to us, together with `MIRI_CALLED_FROM_XARGO`, - // so that xargo's rustc invocations end up in `phase_rustc` with `RustcPhase::Setup`. + // - We use `rustc-build-sysroot`, and set `RUSTC` back to us, together with `MIRI_CALLED_FROM_SETUP`, + // so that the sysroot build rustc invocations end up in `phase_rustc` with `RustcPhase::Setup`. // There we then call the Miri driver with `MIRI_BE_RUSTC` to perform the actual build. // // Then we call `cargo run/test`, exactly forwarding all user flags, plus some configuration so @@ -52,7 +51,7 @@ fn main() { // the Miri driver for interpretation. // Dispatch running as part of sysroot compilation. - if env::var_os("MIRI_CALLED_FROM_XARGO").is_some() { + if env::var_os("MIRI_CALLED_FROM_SETUP").is_some() { phase_rustc(args, RustcPhase::Setup); return; } diff --git a/cargo-miri/src/phases.rs b/cargo-miri/src/phases.rs index 93eb3cb174..f771b2ae4a 100644 --- a/cargo-miri/src/phases.rs +++ b/cargo-miri/src/phases.rs @@ -7,6 +7,8 @@ use std::io::BufReader; use std::path::PathBuf; use std::process::Command; +use rustc_version::VersionMeta; + use crate::{setup::*, util::*}; const CARGO_MIRI_HELP: &str = r#"Runs binary crates and tests in Miri @@ -90,12 +92,14 @@ pub fn phase_cargo_miri(mut args: impl Iterator) { let verbose = num_arg_flag("-v"); // Determine the involved architectures. - let host = version_info().host; + let rustc_version = VersionMeta::for_command(miri_for_host()) + .expect("failed to determine underlying rustc version of Miri"); + let host = &rustc_version.host; let target = get_arg_flag_value("--target"); - let target = target.as_ref().unwrap_or(&host); + let target = target.as_ref().unwrap_or(host); // We always setup. - setup(&subcommand, &host, target); + setup(&subcommand, target, &rustc_version); // Invoke actual cargo for the job, but with different flags. // We re-use `cargo test` and `cargo run`, which makes target and binary handling very easy but @@ -204,7 +208,7 @@ pub fn phase_cargo_miri(mut args: impl Iterator) { #[derive(Debug, Copy, Clone, PartialEq)] pub enum RustcPhase { - /// `rustc` called via `xargo` for sysroot build. + /// `rustc` called during sysroot build. Setup, /// `rustc` called by `cargo` for regular build. Build, @@ -264,7 +268,7 @@ pub fn phase_rustc(mut args: impl Iterator, phase: RustcPhase) { let verbose = std::env::var("MIRI_VERBOSE") .map_or(0, |verbose| verbose.parse().expect("verbosity flag must be an integer")); let target_crate = is_target_crate(); - // Determine whether this is cargo/xargo invoking rustc to get some infos. + // Determine whether this is cargo invoking rustc to get some infos. let info_query = get_arg_flag_value("--print").is_some() || has_arg_flag("-vV"); let store_json = |info: CrateRunInfo| { diff --git a/cargo-miri/src/setup.rs b/cargo-miri/src/setup.rs index c27bb18631..60099ad019 100644 --- a/cargo-miri/src/setup.rs +++ b/cargo-miri/src/setup.rs @@ -1,64 +1,19 @@ -//! Implements `cargo miri setup` via xargo +//! Implements `cargo miri setup`. use std::env; use std::ffi::OsStr; -use std::fs::{self}; -use std::io::BufRead; -use std::ops::Not; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::process::{self, Command}; -use crate::{util::*, version::*}; +use rustc_build_sysroot::{BuildMode, Sysroot}; +use rustc_version::VersionMeta; -fn xargo_version() -> Option<(u32, u32, u32)> { - let out = xargo_check().arg("--version").output().ok()?; - if !out.status.success() { - return None; - } - // Parse output. The first line looks like "xargo 0.3.12 (b004f1c 2018-12-13)". - let line = out - .stderr - .lines() - .next() - .expect("malformed `xargo --version` output: not at least one line") - .expect("malformed `xargo --version` output: error reading first line"); - let (name, version) = { - let mut split = line.split(' '); - ( - split.next().expect("malformed `xargo --version` output: empty"), - split.next().expect("malformed `xargo --version` output: not at least two words"), - ) - }; - if name != "xargo" { - // This is some fork of xargo - return None; - } - let mut version_pieces = version.split('.'); - let major = version_pieces - .next() - .expect("malformed `xargo --version` output: not a major version piece") - .parse() - .expect("malformed `xargo --version` output: major version is not an integer"); - let minor = version_pieces - .next() - .expect("malformed `xargo --version` output: not a minor version piece") - .parse() - .expect("malformed `xargo --version` output: minor version is not an integer"); - let patch = version_pieces - .next() - .expect("malformed `xargo --version` output: not a patch version piece") - .parse() - .expect("malformed `xargo --version` output: patch version is not an integer"); - if version_pieces.next().is_some() { - panic!("malformed `xargo --version` output: more than three pieces in version"); - } - Some((major, minor, patch)) -} +use crate::util::*; /// Performs the setup required to make `cargo miri` work: Getting a custom-built libstd. Then sets /// `MIRI_SYSROOT`. Skipped if `MIRI_SYSROOT` is already set, in which case we expect the user has /// done all this already. -pub fn setup(subcommand: &MiriCommand, host: &str, target: &str) { +pub fn setup(subcommand: &MiriCommand, target: &str, rustc_version: &VersionMeta) { let only_setup = matches!(subcommand, MiriCommand::Setup); let ask_user = !only_setup; let print_sysroot = only_setup && has_arg_flag("--print-sysroot"); // whether we just print the sysroot path @@ -69,21 +24,8 @@ pub fn setup(subcommand: &MiriCommand, host: &str, target: &str) { return; } - // First, we need xargo. - if xargo_version().map_or(true, |v| v < XARGO_MIN_VERSION) { - if std::env::var_os("XARGO_CHECK").is_some() { - // The user manually gave us a xargo binary; don't do anything automatically. - show_error!("xargo is too old; please upgrade to the latest version") - } - let mut cmd = cargo(); - cmd.args(["install", "xargo"]); - ask_to_run(cmd, ask_user, "install a recent enough xargo"); - } - - // Determine where the rust sources are located. The env vars manually setting the source - // (`MIRI_LIB_SRC`, `XARGO_RUST_SRC`) trump auto-detection. - let rust_src_env_var = - std::env::var_os("MIRI_LIB_SRC").or_else(|| std::env::var_os("XARGO_RUST_SRC")); + // Determine where the rust sources are located. The env var trumps auto-detection. + let rust_src_env_var = std::env::var_os("MIRI_LIB_SRC"); let rust_src = match rust_src_env_var { Some(path) => { let path = PathBuf::from(path); @@ -92,22 +34,9 @@ pub fn setup(subcommand: &MiriCommand, host: &str, target: &str) { } None => { // Check for `rust-src` rustup component. - let output = miri_for_host() - .args(["--print", "sysroot"]) - .output() - .expect("failed to determine sysroot"); - if !output.status.success() { - show_error!( - "Failed to determine sysroot; Miri said:\n{}", - String::from_utf8_lossy(&output.stderr).trim_end() - ); - } - let sysroot = std::str::from_utf8(&output.stdout).unwrap(); - let sysroot = Path::new(sysroot.trim_end_matches('\n')); - // Check for `$SYSROOT/lib/rustlib/src/rust/library`; test if that contains `std/Cargo.toml`. - let rustup_src = - sysroot.join("lib").join("rustlib").join("src").join("rust").join("library"); - if !rustup_src.join("std").join("Cargo.toml").exists() { + let rustup_src = rustc_build_sysroot::rustc_sysroot_src(miri_for_host()) + .expect("could not determine sysroot source directory"); + if !rustup_src.exists() { // Ask the user to install the `rust-src` component, and use that. let mut cmd = Command::new("rustup"); cmd.args(["component", "add", "rust-src"]); @@ -131,115 +60,85 @@ pub fn setup(subcommand: &MiriCommand, host: &str, target: &str) { ); } - // Next, we need our own libstd. Prepare a xargo project for that purpose. - // We will do this work in whatever is a good cache dir for this platform. - let dirs = directories::ProjectDirs::from("org", "rust-lang", "miri").unwrap(); - let dir = dirs.cache_dir(); - if !dir.exists() { - fs::create_dir_all(dir).unwrap(); - } - // The interesting bit: Xargo.toml (only needs content if we actually need std) - let xargo_toml = if std::env::var_os("MIRI_NO_STD").is_some() { - "" - } else { - r#" -[dependencies.std] -default_features = false -# We support unwinding, so enable that panic runtime. -features = ["panic_unwind", "backtrace"] - -[dependencies.test] -"# - }; - write_to_file(&dir.join("Xargo.toml"), xargo_toml); - // The boring bits: a dummy project for xargo. - // FIXME: With xargo-check, can we avoid doing this? - write_to_file( - &dir.join("Cargo.toml"), - r#" -[package] -name = "miri-xargo" -description = "A dummy project for building libstd with xargo." -version = "0.0.0" - -[lib] -path = "lib.rs" -"#, - ); - write_to_file(&dir.join("lib.rs"), "#![no_std]"); - - // Figure out where xargo will build its stuff. - // Unfortunately, it puts things into a different directory when the - // architecture matches the host. - let sysroot = if target == host { dir.join("HOST") } else { PathBuf::from(dir) }; + // Determine where to put the sysroot. + let user_dirs = directories::ProjectDirs::from("org", "rust-lang", "miri").unwrap(); + let sysroot_dir = user_dirs.cache_dir(); // Make sure all target-level Miri invocations know their sysroot. - std::env::set_var("MIRI_SYSROOT", &sysroot); + std::env::set_var("MIRI_SYSROOT", &sysroot_dir); - // Now invoke xargo. - let mut command = xargo_check(); - command.arg("check").arg("-q"); - command.current_dir(dir); - command.env("XARGO_HOME", dir); - command.env("XARGO_RUST_SRC", &rust_src); - // We always need to set a target so rustc bootstrap can tell apart host from target crates. - command.arg("--target").arg(target); - // Use Miri as rustc to build a libstd compatible with us (and use the right flags). - // However, when we are running in bootstrap, we cannot just overwrite `RUSTC`, - // because we still need bootstrap to distinguish between host and target crates. - // In that case we overwrite `RUSTC_REAL` instead which determines the rustc used - // for target crates. - // We set ourselves (`cargo-miri`) instead of Miri directly to be able to patch the flags - // for `libpanic_abort` (usually this is done by bootstrap but we have to do it ourselves). - // The `MIRI_CALLED_FROM_XARGO` will mean we dispatch to `phase_setup_rustc`. - let cargo_miri_path = std::env::current_exe().expect("current executable path invalid"); - if env::var_os("RUSTC_STAGE").is_some() { - assert!(env::var_os("RUSTC").is_some()); - command.env("RUSTC_REAL", &cargo_miri_path); - } else { - command.env("RUSTC", &cargo_miri_path); - } - command.env("MIRI_CALLED_FROM_XARGO", "1"); - // Make sure there are no other wrappers getting in our way - // (Cc https://github.com/rust-lang/miri/issues/1421, https://github.com/rust-lang/miri/issues/2429). - // Looks like setting `RUSTC_WRAPPER` to the empty string overwrites `build.rustc-wrapper` set via `config.toml`. - command.env("RUSTC_WRAPPER", ""); - // Disable debug assertions in the standard library -- Miri is already slow enough. But keep the - // overflow checks, they are cheap. This completely overwrites flags the user might have set, - // which is consistent with normal `cargo build` that does not apply `RUSTFLAGS` to the sysroot - // either. - command.env("RUSTFLAGS", "-Cdebug-assertions=off -Coverflow-checks=on"); - // Manage the output the user sees. + // Do the build. if only_setup { // We want to be explicit. eprintln!("Preparing a sysroot for Miri (target: {target})..."); - if print_sysroot { - // Be extra sure there is no noise on stdout. - command.stdout(process::Stdio::null()); - } } else { // We want to be quiet, but still let the user know that something is happening. eprint!("Preparing a sysroot for Miri (target: {target})... "); - command.stdout(process::Stdio::null()); - command.stderr(process::Stdio::null()); } + Sysroot::new(sysroot_dir, target) + .build_from_source( + &rust_src, + BuildMode::Check, + &["panic_unwind", "backtrace"], + rustc_version, + || { + let mut command = cargo(); + // Use Miri as rustc to build a libstd compatible with us (and use the right flags). + // However, when we are running in bootstrap, we cannot just overwrite `RUSTC`, + // because we still need bootstrap to distinguish between host and target crates. + // In that case we overwrite `RUSTC_REAL` instead which determines the rustc used + // for target crates. + // We set ourselves (`cargo-miri`) instead of Miri directly to be able to patch the flags + // for `libpanic_abort` (usually this is done by bootstrap but we have to do it ourselves). + // The `MIRI_CALLED_FROM_SETUP` will mean we dispatch to `phase_setup_rustc`. + let cargo_miri_path = + std::env::current_exe().expect("current executable path invalid"); + if env::var_os("RUSTC_STAGE").is_some() { + assert!(env::var_os("RUSTC").is_some()); + command.env("RUSTC_REAL", &cargo_miri_path); + } else { + command.env("RUSTC", &cargo_miri_path); + } + command.env("MIRI_CALLED_FROM_SETUP", "1"); + // Make sure there are no other wrappers getting in our way (Cc + // https://github.com/rust-lang/miri/issues/1421, + // https://github.com/rust-lang/miri/issues/2429). Looks like setting + // `RUSTC_WRAPPER` to the empty string overwrites `build.rustc-wrapper` set via + // `config.toml`. + command.env("RUSTC_WRAPPER", ""); + // Disable debug assertions in the standard library -- Miri is already slow enough. + // But keep the overflow checks, they are cheap. This completely overwrites flags + // the user might have set, which is consistent with normal `cargo build` that does + // not apply `RUSTFLAGS` to the sysroot either. + command.env("RUSTFLAGS", "-Cdebug-assertions=off -Coverflow-checks=on"); - // Finally run it! - if command.status().expect("failed to run xargo").success().not() { - if only_setup { - show_error!("failed to run xargo, see error details above") - } else { - show_error!("failed to run xargo; run `cargo miri setup` to see the error details") - } - } - - // Figure out what to print. + if only_setup { + if print_sysroot { + // Be extra sure there is no noise on stdout. + command.stdout(process::Stdio::null()); + } + } else { + command.stdout(process::Stdio::null()); + command.stderr(process::Stdio::null()); + } + command + }, + ) + .unwrap_or_else(|_| { + if only_setup { + show_error!("failed to build sysroot, see error details above") + } else { + show_error!( + "failed to build sysroot; run `cargo miri setup` to see the error details" + ) + } + }); if only_setup { - eprintln!("A sysroot for Miri is now available in `{}`.", sysroot.display()); + eprintln!("A sysroot for Miri is now available in `{}`.", sysroot_dir.display()); } else { eprintln!("done"); } if print_sysroot { // Print just the sysroot and nothing else to stdout; this way we do not need any escaping. - println!("{}", sysroot.display()); + println!("{}", sysroot_dir.display()); } } diff --git a/cargo-miri/src/util.rs b/cargo-miri/src/util.rs index 8f29eebaac..aabe5547e5 100644 --- a/cargo-miri/src/util.rs +++ b/cargo-miri/src/util.rs @@ -2,14 +2,13 @@ use std::collections::HashMap; use std::env; use std::ffi::OsString; use std::fmt::Write as _; -use std::fs::{self, File}; +use std::fs::File; use std::io::{self, BufWriter, Read, Write}; use std::ops::Not; use std::path::{Path, PathBuf}; use std::process::Command; use cargo_metadata::{Metadata, MetadataCommand}; -use rustc_version::VersionMeta; use serde::{Deserialize, Serialize}; pub use crate::arg::*; @@ -111,19 +110,10 @@ pub fn miri_for_host() -> Command { cmd } -pub fn version_info() -> VersionMeta { - VersionMeta::for_command(miri_for_host()) - .expect("failed to determine underlying rustc version of Miri") -} - pub fn cargo() -> Command { Command::new(env::var_os("CARGO").unwrap_or_else(|| OsString::from("cargo"))) } -pub fn xargo_check() -> Command { - Command::new(env::var_os("XARGO_CHECK").unwrap_or_else(|| OsString::from("xargo-check"))) -} - /// Execute the `Command`, where possible by replacing the current process with a new process /// described by the `Command`. Then exit this process with the exit code of the new process. pub fn exec(mut cmd: Command) -> ! { @@ -203,23 +193,6 @@ pub fn ask_to_run(mut cmd: Command, ask: bool, text: &str) { } } -/// Writes the given content to the given file *cross-process atomically*, in the sense that another -/// process concurrently reading that file will see either the old content or the new content, but -/// not some intermediate (e.g., empty) state. -/// -/// We assume no other parts of this same process are trying to read or write that file. -pub fn write_to_file(filename: &Path, content: &str) { - // Create a temporary file with the desired contents. - let mut temp_filename = filename.as_os_str().to_os_string(); - temp_filename.push(&format!(".{}", std::process::id())); - let mut temp_file = File::create(&temp_filename).unwrap(); - temp_file.write_all(content.as_bytes()).unwrap(); - drop(temp_file); - - // Move file to the desired location. - fs::rename(temp_filename, filename).unwrap(); -} - // Computes the extra flags that need to be passed to cargo to make it behave like the current // cargo invocation. fn cargo_extra_flags() -> Vec { diff --git a/cargo-miri/src/version.rs b/cargo-miri/src/version.rs deleted file mode 100644 index 366e90df17..0000000000 --- a/cargo-miri/src/version.rs +++ /dev/null @@ -1,2 +0,0 @@ -// We put this in a separate file so that it can be hashed for GHA caching. -pub const XARGO_MIN_VERSION: (u32, u32, u32) = (0, 3, 26);