Auto merge of #73504 - RalfJung:rollup-iy8hsvl, r=RalfJung

Rollup of 10 pull requests

Successful merges:

 - #72280 (Fix up autoderef when reborrowing)
 - #72785 (linker: MSVC supports linking static libraries as a whole archive)
 - #73011 (first stage of implementing LLVM code coverage)
 - #73044 (compiletest: Add directives to detect sanitizer support)
 - #73054 (memory access sanity checks: abort instead of panic)
 - #73136 (Change how compiler-builtins gets many CGUs)
 - #73280 (Add E0763)
 - #73317 (bootstrap: read config from $RUST_BOOTSTRAP_CONFIG)
 - #73350 (bootstrap/install.rs: support a nonexistent `prefix` in `x.py install`)
 - #73352 (Speed up bootstrap a little.)

Failed merges:

r? @ghost

Cargo.toml

@@ -42,6 +42,19 @@ debug-assertions = false
 debug = false
 debug-assertions = false
 
+[profile.release.package.compiler_builtins]
+# For compiler-builtins we always use a high number of codegen units.
+# The goal here is to place every single intrinsic into its own object
+# file to avoid symbol clashes with the system libgcc if possible. Note
+# that this number doesn't actually produce this many object files, we
+# just don't create more than this number of object files.
+#
+# It's a bit of a bummer that we have to pass this here, unfortunately.
+# Ideally this would be specified through an env var to Cargo so Cargo
+# knows how many CGUs are for this specific crate, but for now
+# per-crate configuration isn't specifiable in the environment.
+codegen-units = 10000
+
 # We want the RLS to use the version of Cargo that we've got vendored in this
 # repository to ensure that the same exact version of Cargo is used by both the
 # RLS and the Cargo binary itself. The RLS depends on Cargo as a git repository

config.toml.example

@@ -209,7 +209,8 @@
 # Build the sanitizer runtimes
 #sanitizers = false
 
-# Build the profiler runtime
+# Build the profiler runtime (required when compiling with options that depend
+# on this runtime, such as `-C profile-generate` or `-Z instrument-coverage`).
 #profiler = false
 
 # Indicates whether the native libraries linked into Cargo will be statically

src/bootstrap/bootstrap.py

@@ -894,7 +894,7 @@ def bootstrap(help_triggered):
     build.clean = args.clean
 
     try:
-        toml_path = args.config or 'config.toml'
+        toml_path = os.getenv('RUST_BOOTSTRAP_CONFIG') or args.config or 'config.toml'
         if not os.path.exists(toml_path):
             toml_path = os.path.join(build.rust_root, toml_path)
 
@@ -947,6 +947,7 @@ def bootstrap(help_triggered):
     env["SRC"] = build.rust_root
     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
     env["BOOTSTRAP_PYTHON"] = sys.executable
+    env["BOOTSTRAP_CONFIG"] = toml_path
     env["BUILD_DIR"] = build.build_dir
     env["RUSTC_BOOTSTRAP"] = '1'
     env["CARGO"] = build.cargo()

src/bootstrap/builder.rs

@@ -99,9 +99,21 @@ struct StepDescription {
     name: &'static str,
 }
 
+/// Collection of paths used to match a task rule.
 #[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)]
 pub enum PathSet {
+    /// A collection of individual paths.
+    ///
+    /// These are generally matched as a path suffix. For example, a
+    /// command-line value of `libstd` will match if `src/libstd` is in the
+    /// set.
     Set(BTreeSet<PathBuf>),
+    /// A "suite" of paths.
+    ///
+    /// These can match as a path suffix (like `Set`), or as a prefix. For
+    /// example, a command-line value of `src/test/ui/abi/variadic-ffi.rs`
+    /// will match `src/test/ui`. A command-line value of `ui` would also
+    /// match `src/test/ui`.
     Suite(PathBuf),
 }
 
@@ -251,21 +263,33 @@ impl<'a> ShouldRun<'a> {
         self
     }
 
-    // Unlike `krate` this will create just one pathset. As such, it probably shouldn't actually
-    // ever be used, but as we transition to having all rules properly handle passing krate(...) by
-    // actually doing something different for every crate passed.
+    /// Indicates it should run if the command-line selects the given crate or
+    /// any of its (local) dependencies.
+    ///
+    /// Compared to `krate`, this treats the dependencies as aliases for the
+    /// same job. Generally it is preferred to use `krate`, and treat each
+    /// individual path separately. For example `./x.py test src/liballoc`
+    /// (which uses `krate`) will test just `liballoc`. However, `./x.py check
+    /// src/liballoc` (which uses `all_krates`) will check all of `libtest`.
+    /// `all_krates` should probably be removed at some point.
     pub fn all_krates(mut self, name: &str) -> Self {
         let mut set = BTreeSet::new();
         for krate in self.builder.in_tree_crates(name) {
-            set.insert(PathBuf::from(&krate.path));
+            let path = krate.local_path(self.builder);
+            set.insert(path);
         }
         self.paths.insert(PathSet::Set(set));
         self
     }
 
+    /// Indicates it should run if the command-line selects the given crate or
+    /// any of its (local) dependencies.
+    ///
+    /// `make_run` will be called separately for each matching command-line path.
     pub fn krate(mut self, name: &str) -> Self {
         for krate in self.builder.in_tree_crates(name) {
-            self.paths.insert(PathSet::one(&krate.path));
+            let path = krate.local_path(self.builder);
+            self.paths.insert(PathSet::one(path));
         }
         self
     }
@@ -488,13 +512,19 @@ impl<'a> Builder<'a> {
             should_run = (desc.should_run)(should_run);
         }
         let mut help = String::from("Available paths:\n");
+        let mut add_path = |path: &Path| {
+            help.push_str(&format!("    ./x.py {} {}\n", subcommand, path.display()));
+        };
         for pathset in should_run.paths {
-            if let PathSet::Set(set) = pathset {
-                set.iter().for_each(|path| {
-                    help.push_str(
-                        format!("    ./x.py {} {}\n", subcommand, path.display()).as_str(),
-                    )
-                })
+            match pathset {
+                PathSet::Set(set) => {
+                    for path in set {
+                        add_path(&path);
+                    }
+                }
+                PathSet::Suite(path) => {
+                    add_path(&path.join("..."));
+                }
             }
         }
         Some(help)

src/bootstrap/doc.rs

@@ -548,8 +548,8 @@ impl Step for Rustc {
         // Find dependencies for top level crates.
         let mut compiler_crates = HashSet::new();
         for root_crate in &["rustc_driver", "rustc_codegen_llvm", "rustc_codegen_ssa"] {
-            let interned_root_crate = INTERNER.intern_str(root_crate);
-            find_compiler_crates(builder, &interned_root_crate, &mut compiler_crates);
+            compiler_crates
+                .extend(builder.in_tree_crates(root_crate).into_iter().map(|krate| krate.name));
         }
 
         for krate in &compiler_crates {
@@ -564,22 +564,6 @@ impl Step for Rustc {
     }
 }
 
-fn find_compiler_crates(
-    builder: &Builder<'_>,
-    name: &Interned<String>,
-    crates: &mut HashSet<Interned<String>>,
-) {
-    // Add current crate.
-    crates.insert(*name);
-
-    // Look for dependencies.
-    for dep in builder.crates.get(name).unwrap().deps.iter() {
-        if builder.crates.get(dep).unwrap().is_local(builder) {
-            find_compiler_crates(builder, dep, crates);
-        }
-    }
-}
-
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Rustdoc {
     stage: u32,

src/bootstrap/flags.rs

@@ -3,19 +3,17 @@
 //! This module implements the command-line parsing of the build system which
 //! has various flags to configure how it's run.
 
-use std::fs;
+use std::env;
 use std::path::PathBuf;
 use std::process;
 
 use getopts::Options;
 
 use crate::builder::Builder;
+use crate::cache::{Interned, INTERNER};
 use crate::config::Config;
-use crate::metadata;
 use crate::{Build, DocTests};
 
-use crate::cache::{Interned, INTERNER};
-
 /// Deserialized version of all flags for this compile.
 pub struct Flags {
     pub verbose: usize, // number of -v args; each extra -v after the first is passed to Cargo
@@ -438,19 +436,12 @@ Arguments:
         // Get any optional paths which occur after the subcommand
         let paths = matches.free[1..].iter().map(|p| p.into()).collect::<Vec<PathBuf>>();
 
-        let cfg_file = matches.opt_str("config").map(PathBuf::from).or_else(|| {
-            if fs::metadata("config.toml").is_ok() {
-                Some(PathBuf::from("config.toml"))
-            } else {
-                None
-            }
-        });
+        let cfg_file = env::var_os("BOOTSTRAP_CONFIG").map(PathBuf::from);
 
         // All subcommands except `clean` can have an optional "Available paths" section
         if matches.opt_present("verbose") {
             let config = Config::parse(&["build".to_string()]);
-            let mut build = Build::new(config);
-            metadata::build(&mut build);
+            let build = Build::new(config);
 
             let maybe_rules_help = Builder::get_help(&build, subcommand.as_str());
             extra_help.push_str(maybe_rules_help.unwrap_or_default().as_str());

src/bootstrap/install.rs

@@ -70,7 +70,10 @@ fn install_sh(
     let libdir_default = PathBuf::from("lib");
     let mandir_default = datadir_default.join("man");
     let prefix = builder.config.prefix.as_ref().map_or(prefix_default, |p| {
-        fs::canonicalize(p).unwrap_or_else(|_| panic!("could not canonicalize {}", p.display()))
+        fs::create_dir_all(p)
+            .unwrap_or_else(|err| panic!("could not create {}: {}", p.display(), err));
+        fs::canonicalize(p)
+            .unwrap_or_else(|err| panic!("could not canonicalize {}: {}", p.display(), err))
     });
     let sysconfdir = builder.config.sysconfdir.as_ref().unwrap_or(&sysconfdir_default);
     let datadir = builder.config.datadir.as_ref().unwrap_or(&datadir_default);

src/bootstrap/lib.rs

@@ -270,12 +270,7 @@ struct Crate {
 }
 
 impl Crate {
-    fn is_local(&self, build: &Build) -> bool {
-        self.path.starts_with(&build.config.src) && !self.path.to_string_lossy().ends_with("_shim")
-    }
-
     fn local_path(&self, build: &Build) -> PathBuf {
-        assert!(self.is_local(build));
         self.path.strip_prefix(&build.config.src).unwrap().into()
     }
 }
@@ -1090,17 +1085,29 @@ impl Build {
         }
     }
 
+    /// Returns a Vec of all the dependencies of the given root crate,
+    /// including transitive dependencies and the root itself. Only includes
+    /// "local" crates (those in the local source tree, not from a registry).
     fn in_tree_crates(&self, root: &str) -> Vec<&Crate> {
         let mut ret = Vec::new();
         let mut list = vec![INTERNER.intern_str(root)];
         let mut visited = HashSet::new();
         while let Some(krate) = list.pop() {
             let krate = &self.crates[&krate];
-            if krate.is_local(self) {
-                ret.push(krate);
-            }
+            ret.push(krate);
             for dep in &krate.deps {
-                if visited.insert(dep) && dep != "build_helper" {
+                // Don't include optional deps if their features are not
+                // enabled. Ideally this would be computed from `cargo
+                // metadata --features …`, but that is somewhat slow. Just
+                // skip `build_helper` since there aren't any operations we
+                // want to perform on it. In the future, we may want to
+                // consider just filtering all build and dev dependencies in
+                // metadata::build.
+                if visited.insert(dep)
+                    && dep != "build_helper"
+                    && (dep != "profiler_builtins" || self.config.profiler)
+                    && (dep != "rustc_codegen_llvm" || self.config.llvm_enabled())
+                {
                     list.push(*dep);
                 }
             }

src/bootstrap/metadata.rs

@@ -1,5 +1,3 @@
-use std::collections::HashMap;
-use std::collections::HashSet;
 use std::path::PathBuf;
 use std::process::Command;
 
@@ -12,7 +10,6 @@ use crate::{Build, Crate};
 #[derive(Deserialize)]
 struct Output {
     packages: Vec<Package>,
-    resolve: Resolve,
 }
 
 #[derive(Deserialize)]
@@ -21,72 +18,39 @@ struct Package {
     name: String,
     source: Option<String>,
     manifest_path: String,
+    dependencies: Vec<Dependency>,
 }
 
 #[derive(Deserialize)]
-struct Resolve {
-    nodes: Vec<ResolveNode>,
-}
-
-#[derive(Deserialize)]
-struct ResolveNode {
-    id: String,
-    dependencies: Vec<String>,
+struct Dependency {
+    name: String,
+    source: Option<String>,
 }
 
 pub fn build(build: &mut Build) {
-    let mut resolves = Vec::new();
-    build_krate(&build.std_features(), build, &mut resolves, "src/libstd");
-    build_krate("", build, &mut resolves, "src/libtest");
-    build_krate(&build.rustc_features(), build, &mut resolves, "src/rustc");
-
-    let mut id2name = HashMap::with_capacity(build.crates.len());
-    for (name, krate) in build.crates.iter() {
-        id2name.insert(krate.id.clone(), name.clone());
-    }
-
-    for node in resolves {
-        let name = match id2name.get(&node.id) {
-            Some(name) => name,
-            None => continue,
-        };
-
-        let krate = build.crates.get_mut(name).unwrap();
-        for dep in node.dependencies.iter() {
-            let dep = match id2name.get(dep) {
-                Some(dep) => dep,
-                None => continue,
-            };
-            krate.deps.insert(*dep);
-        }
-    }
-}
-
-fn build_krate(features: &str, build: &mut Build, resolves: &mut Vec<ResolveNode>, krate: &str) {
     // Run `cargo metadata` to figure out what crates we're testing.
-    //
-    // Down below we're going to call `cargo test`, but to test the right set
-    // of packages we're going to have to know what `-p` arguments to pass it
-    // to know what crates to test. Here we run `cargo metadata` to learn about
-    // the dependency graph and what `-p` arguments there are.
     let mut cargo = Command::new(&build.initial_cargo);
     cargo
         .arg("metadata")
         .arg("--format-version")
         .arg("1")
-        .arg("--features")
-        .arg(features)
+        .arg("--no-deps")
         .arg("--manifest-path")
-        .arg(build.src.join(krate).join("Cargo.toml"));
+        .arg(build.src.join("Cargo.toml"));
     let output = output(&mut cargo);
     let output: Output = serde_json::from_str(&output).unwrap();
     for package in output.packages {
         if package.source.is_none() {
             let name = INTERNER.intern_string(package.name);
             let mut path = PathBuf::from(package.manifest_path);
             path.pop();
-            build.crates.insert(name, Crate { name, id: package.id, deps: HashSet::new(), path });
+            let deps = package
+                .dependencies
+                .into_iter()
+                .filter(|dep| dep.source.is_none())
+                .map(|dep| INTERNER.intern_string(dep.name))
+                .collect();
+            build.crates.insert(name, Crate { name, id: package.id, deps, path });
         }
     }
-    resolves.extend(output.resolve.nodes);
 }

src/bootstrap/test.rs

@@ -1648,14 +1648,8 @@ impl Step for Crate {
     type Output = ();
     const DEFAULT: bool = true;
 
-    fn should_run(mut run: ShouldRun<'_>) -> ShouldRun<'_> {
-        let builder = run.builder;
-        for krate in run.builder.in_tree_crates("test") {
-            if !(krate.name.starts_with("rustc_") && krate.name.ends_with("san")) {
-                run = run.path(krate.local_path(&builder).to_str().unwrap());
-            }
-        }
-        run
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.krate("test")
     }
 
     fn make_run(run: RunConfig<'_>) {