Auto merge of rust-lang#106168 - jyn514:clean-crates, r=Mark-Simulacrum

Allow cleaning individual crates

As a bonus, this stops special casing `clean` in `Builder`.

## Motivation

Cleaning artifacts isn't strictly necessary to get cargo to rebuild; `touch compiler/rustc_driver/src/lib.rs` (for example) will also work. There's two reasons I thought making this part of bootstrap proper was a better approach:
1. `touch` does not *remove* artifacts, it just causes a rebuild. This is unhelpful for when you want to measure how long the compiler itself takes to build (e.g. for rust-lang#65031).
2. It seems a little more discoverable; and I want to extend it in the future to things like `x clean --stage 1 rustc`, which makes it easier to work around rust-lang#76720 without having to completely wipe all the stage 0 artifacts, or having to be intimately familiar with which directories to remove.

src/bootstrap/builder.rs

@@ -13,7 +13,6 @@ use std::time::{Duration, Instant};
 
 use crate::cache::{Cache, Interned, INTERNER};
 use crate::config::{SplitDebuginfo, TargetSelection};
-use crate::dist;
 use crate::doc;
 use crate::flags::{Color, Subcommand};
 use crate::install;
@@ -25,6 +24,7 @@ use crate::tool::{self, SourceType};
 use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t};
 use crate::EXTRA_CHECK_CFGS;
 use crate::{check, compile, Crate};
+use crate::{clean, dist};
 use crate::{Build, CLang, DocTests, GitRepo, Mode};
 
 pub use crate::Compiler;
@@ -96,6 +96,17 @@ impl RunConfig<'_> {
     pub fn build_triple(&self) -> TargetSelection {
         self.builder.build.build
     }
+
+    /// Return a `-p=x -p=y` string suitable for passing to a cargo invocation.
+    pub fn cargo_crates_in_set(&self) -> Interned<Vec<String>> {
+        let mut crates = Vec::new();
+        for krate in &self.paths {
+            let path = krate.assert_single_path();
+            let crate_name = self.builder.crate_paths[&path.path];
+            crates.push(format!("-p={crate_name}"));
+        }
+        INTERNER.intern_list(crates)
+    }
 }
 
 struct StepDescription {
@@ -764,8 +775,9 @@ impl<'a> Builder<'a> {
                 run::GenerateCopyright,
             ),
             Kind::Setup => describe!(setup::Profile),
-            // These commands either don't use paths, or they're special-cased in Build::build()
-            Kind::Clean | Kind::Format => vec![],
+            Kind::Clean => describe!(clean::CleanAll, clean::Rustc, clean::Std),
+            // special-cased in Build::build()
+            Kind::Format => vec![],
         }
     }
 
@@ -827,14 +839,12 @@ impl<'a> Builder<'a> {
             Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]),
             Subcommand::Install { ref paths } => (Kind::Install, &paths[..]),
             Subcommand::Run { ref paths, .. } => (Kind::Run, &paths[..]),
+            Subcommand::Clean { ref paths, .. } => (Kind::Clean, &paths[..]),
             Subcommand::Format { .. } => (Kind::Format, &[][..]),
             Subcommand::Setup { profile: ref path } => (
                 Kind::Setup,
                 path.as_ref().map_or([].as_slice(), |path| std::slice::from_ref(path)),
             ),
-            Subcommand::Clean { .. } => {
-                panic!()
-            }
         };
 
         Self::new_internal(build, kind, paths.to_owned())
@@ -1077,6 +1087,62 @@ impl<'a> Builder<'a> {
         None
     }
 
+    /// Like `cargo`, but only passes flags that are valid for all commands.
+    pub fn bare_cargo(
+        &self,
+        compiler: Compiler,
+        mode: Mode,
+        target: TargetSelection,
+        cmd: &str,
+    ) -> Command {
+        let mut cargo = Command::new(&self.initial_cargo);
+        // Run cargo from the source root so it can find .cargo/config.
+        // This matters when using vendoring and the working directory is outside the repository.
+        cargo.current_dir(&self.src);
+
+        let out_dir = self.stage_out(compiler, mode);
+        cargo.env("CARGO_TARGET_DIR", &out_dir).arg(cmd);
+
+        // Found with `rg "init_env_logger\("`. If anyone uses `init_env_logger`
+        // from out of tree it shouldn't matter, since x.py is only used for
+        // building in-tree.
+        let color_logs = ["RUSTDOC_LOG_COLOR", "RUSTC_LOG_COLOR", "RUST_LOG_COLOR"];
+        match self.build.config.color {
+            Color::Always => {
+                cargo.arg("--color=always");
+                for log in &color_logs {
+                    cargo.env(log, "always");
+                }
+            }
+            Color::Never => {
+                cargo.arg("--color=never");
+                for log in &color_logs {
+                    cargo.env(log, "never");
+                }
+            }
+            Color::Auto => {} // nothing to do
+        }
+
+        if cmd != "install" {
+            cargo.arg("--target").arg(target.rustc_target_arg());
+        } else {
+            assert_eq!(target, compiler.host);
+        }
+
+        if self.config.rust_optimize {
+            // FIXME: cargo bench/install do not accept `--release`
+            if cmd != "bench" && cmd != "install" {
+                cargo.arg("--release");
+            }
+        }
+
+        // Remove make-related flags to ensure Cargo can correctly set things up
+        cargo.env_remove("MAKEFLAGS");
+        cargo.env_remove("MFLAGS");
+
+        cargo
+    }
+
     /// Prepares an invocation of `cargo` to be run.
     ///
     /// This will create a `Command` that represents a pending execution of
@@ -1092,11 +1158,8 @@ impl<'a> Builder<'a> {
         target: TargetSelection,
         cmd: &str,
     ) -> Cargo {
-        let mut cargo = Command::new(&self.initial_cargo);
+        let mut cargo = self.bare_cargo(compiler, mode, target, cmd);
         let out_dir = self.stage_out(compiler, mode);
-        // Run cargo from the source root so it can find .cargo/config.
-        // This matters when using vendoring and the working directory is outside the repository.
-        cargo.current_dir(&self.src);
 
         // Codegen backends are not yet tracked by -Zbinary-dep-depinfo,
         // so we need to explicitly clear out if they've been updated.
@@ -1121,8 +1184,6 @@ impl<'a> Builder<'a> {
             self.clear_if_dirty(&my_out, &rustdoc);
         }
 
-        cargo.env("CARGO_TARGET_DIR", &out_dir).arg(cmd);
-
         let profile_var = |name: &str| {
             let profile = if self.config.rust_optimize { "RELEASE" } else { "DEV" };
             format!("CARGO_PROFILE_{}_{}", profile, name)
@@ -1135,32 +1196,6 @@ impl<'a> Builder<'a> {
             cargo.env("REAL_LIBRARY_PATH", e);
         }
 
-        // Found with `rg "init_env_logger\("`. If anyone uses `init_env_logger`
-        // from out of tree it shouldn't matter, since x.py is only used for
-        // building in-tree.
-        let color_logs = ["RUSTDOC_LOG_COLOR", "RUSTC_LOG_COLOR", "RUST_LOG_COLOR"];
-        match self.build.config.color {
-            Color::Always => {
-                cargo.arg("--color=always");
-                for log in &color_logs {
-                    cargo.env(log, "always");
-                }
-            }
-            Color::Never => {
-                cargo.arg("--color=never");
-                for log in &color_logs {
-                    cargo.env(log, "never");
-                }
-            }
-            Color::Auto => {} // nothing to do
-        }
-
-        if cmd != "install" {
-            cargo.arg("--target").arg(target.rustc_target_arg());
-        } else {
-            assert_eq!(target, compiler.host);
-        }
-
         // Set a flag for `check`/`clippy`/`fix`, so that certain build
         // scripts can do less work (i.e. not building/requiring LLVM).
         if cmd == "check" || cmd == "clippy" || cmd == "fix" {
@@ -1341,9 +1376,6 @@ impl<'a> Builder<'a> {
         }
 
         cargo.arg("-j").arg(self.jobs().to_string());
-        // Remove make-related flags to ensure Cargo can correctly set things up
-        cargo.env_remove("MAKEFLAGS");
-        cargo.env_remove("MFLAGS");
 
         // FIXME: Temporary fix for https://github.com/rust-lang/cargo/issues/3005
         // Force cargo to output binaries with disambiguating hashes in the name
@@ -1827,13 +1859,6 @@ impl<'a> Builder<'a> {
             }
         }
 
-        if self.config.rust_optimize {
-            // FIXME: cargo bench/install do not accept `--release`
-            if cmd != "bench" && cmd != "install" {
-                cargo.arg("--release");
-            }
-        }
-
         if self.config.locked_deps {
             cargo.arg("--locked");
         }

src/bootstrap/clean.rs

@@ -9,10 +9,83 @@ use std::fs;
 use std::io::{self, ErrorKind};
 use std::path::Path;
 
+use crate::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::cache::Interned;
+use crate::config::TargetSelection;
 use crate::util::t;
-use crate::Build;
+use crate::{Build, Mode, Subcommand};
 
-pub fn clean(build: &Build, all: bool) {
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct CleanAll {}
+
+impl Step for CleanAll {
+    const DEFAULT: bool = true;
+    type Output = ();
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(CleanAll {})
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        let Subcommand::Clean { all, .. } = builder.config.cmd else { unreachable!("wrong subcommand?") };
+        clean_default(builder.build, all)
+    }
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.never() // handled by DEFAULT
+    }
+}
+
+macro_rules! clean_crate_tree {
+    ( $( $name:ident, $mode:path, $root_crate:literal);+ $(;)? ) => { $(
+        #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+        pub struct $name {
+            target: TargetSelection,
+            crates: Interned<Vec<String>>,
+        }
+
+        impl Step for $name {
+            type Output = ();
+
+            fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+                let crates = run.builder.in_tree_crates($root_crate, None);
+                run.crates(crates)
+            }
+
+            fn make_run(run: RunConfig<'_>) {
+                let builder = run.builder;
+                if builder.top_stage != 0 {
+                    panic!("non-stage-0 clean not supported for individual crates");
+                }
+                builder.ensure(Self { crates: run.cargo_crates_in_set(), target: run.target });
+            }
+
+            fn run(self, builder: &Builder<'_>) -> Self::Output {
+                let compiler = builder.compiler(0, self.target);
+                let mut cargo = builder.bare_cargo(compiler, $mode, self.target, "clean");
+                for krate in &*self.crates {
+                    cargo.arg(krate);
+                }
+
+                builder.info(&format!(
+                    "Cleaning stage{} {} artifacts ({} -> {})",
+                    compiler.stage, stringify!($name).to_lowercase(), &compiler.host, self.target
+                ));
+
+                // NOTE: doesn't use `run_cargo` because we don't want to save a stamp file,
+                // and doesn't use `stream_cargo` to avoid passing `--message-format` which `clean` doesn't accept.
+                builder.run(&mut cargo);
+            }
+        }
+    )+ }
+}
+
+clean_crate_tree! {
+    Rustc, Mode::Rustc, "rustc-main";
+    Std, Mode::Std, "test";
+}
+
+fn clean_default(build: &Build, all: bool) {
     rm_rf("tmp".as_ref());
 
     if all {

src/bootstrap/compile.rs

@@ -46,17 +46,6 @@ impl Std {
     }
 }
 
-/// Return a `-p=x -p=y` string suitable for passing to a cargo invocation.
-fn build_crates_in_set(run: &RunConfig<'_>) -> Interned<Vec<String>> {
-    let mut crates = Vec::new();
-    for krate in &run.paths {
-        let path = krate.assert_single_path();
-        let crate_name = run.builder.crate_paths[&path.path];
-        crates.push(format!("-p={crate_name}"));
-    }
-    INTERNER.intern_list(crates)
-}
-
 impl Step for Std {
     type Output = ();
     const DEFAULT: bool = true;
@@ -76,7 +65,7 @@ impl Step for Std {
         // Build all crates anyway, as if they hadn't passed the other args.
         let has_library =
             run.paths.iter().any(|set| set.assert_single_path().path.ends_with("library"));
-        let crates = if has_library { Default::default() } else { build_crates_in_set(&run) };
+        let crates = if has_library { Default::default() } else { run.cargo_crates_in_set() };
         run.builder.ensure(Std {
             compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),
             target: run.target,
@@ -603,7 +592,7 @@ impl Step for Rustc {
     }
 
     fn make_run(run: RunConfig<'_>) {
-        let crates = build_crates_in_set(&run);
+        let crates = run.cargo_crates_in_set();
         run.builder.ensure(Rustc {
             compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()),
             target: run.target,

src/bootstrap/flags.rs

@@ -130,6 +130,7 @@ pub enum Subcommand {
         test_args: Vec<String>,
     },
     Clean {
+        paths: Vec<PathBuf>,
         all: bool,
     },
     Dist {
@@ -611,14 +612,7 @@ Arguments:
                 open: matches.opt_present("open"),
                 json: matches.opt_present("json"),
             },
-            Kind::Clean => {
-                if !paths.is_empty() {
-                    println!("\nclean does not take a path argument\n");
-                    usage(1, &opts, verbose, &subcommand_help);
-                }
-
-                Subcommand::Clean { all: matches.opt_present("all") }
-            }
+            Kind::Clean => Subcommand::Clean { all: matches.opt_present("all"), paths },
             Kind::Format => Subcommand::Format { check: matches.opt_present("check"), paths },
             Kind::Dist => Subcommand::Dist { paths },
             Kind::Install => Subcommand::Install { paths },

src/bootstrap/lib.rs

@@ -727,10 +727,6 @@ impl Build {
             return format::format(&builder::Builder::new(&self), *check, &paths);
         }
 
-        if let Subcommand::Clean { all } = self.config.cmd {
-            return clean::clean(self, all);
-        }
-
         // Download rustfmt early so that it can be used in rust-analyzer configs.
         let _ = &builder::Builder::new(&self).initial_rustfmt();