Implement the Cargo half of pipelined compilation

src/cargo/core/compiler/build_config.rs

@@ -127,6 +127,11 @@ pub enum CompileMode {
     Test,
     /// Building a target with `rustc` (lib or bin).
     Build,
+    /// Building just the metadata of an rlib with `rustc`
+    ///
+    /// This is somewhat of a special mode, and for more information see the
+    /// documentation in `job_queue.rs`
+    BuildRmeta,
     /// Building a target with `rustc` to emit `rmeta` metadata only. If
     /// `test` is true, then it is also compiled with `--test` to check it like
     /// a test.
@@ -154,6 +159,7 @@ impl ser::Serialize for CompileMode {
         match *self {
             Test => "test".serialize(s),
             Build => "build".serialize(s),
+            BuildRmeta => "build-rmeta".serialize(s),
             Check { .. } => "check".serialize(s),
             Bench => "bench".serialize(s),
             Doc { .. } => "doc".serialize(s),
@@ -202,9 +208,10 @@ impl CompileMode {
     /// List of all modes (currently used by `cargo clean -p` for computing
     /// all possible outputs).
     pub fn all_modes() -> &'static [CompileMode] {
-        static ALL: [CompileMode; 9] = [
+        static ALL: &[CompileMode] = &[
             CompileMode::Test,
             CompileMode::Build,
+            CompileMode::BuildRmeta,
             CompileMode::Check { test: true },
             CompileMode::Check { test: false },
             CompileMode::Bench,
@@ -213,6 +220,6 @@ impl CompileMode {
             CompileMode::Doctest,
             CompileMode::RunCustomBuild,
         ];
-        &ALL
+        ALL
     }
 }

src/cargo/core/compiler/build_context/mod.rs

@@ -8,9 +8,9 @@ use crate::core::profiles::Profiles;
 use crate::core::{Dependency, Workspace};
 use crate::core::{PackageId, PackageSet, Resolve};
 use crate::util::errors::CargoResult;
-use crate::util::{profile, Cfg, Config, Rustc};
-use crate::core::compiler::{Unit, Kind, BuildConfig, BuildOutput};
+use crate::core::compiler::{Unit, Kind, BuildConfig, BuildOutput, CompileMode};
 use crate::core::compiler::unit::UnitInterner;
+use crate::util::{profile, Cfg, Config, Rustc};
 
 mod target_info;
 pub use self::target_info::{FileFlavor, TargetInfo};
@@ -179,7 +179,18 @@ impl<'a, 'cfg> BuildContext<'a, 'cfg> {
     }
 
     pub fn extra_args_for(&self, unit: &Unit<'a>) -> Option<&Vec<String>> {
-        self.extra_compiler_args.get(unit)
+        // Extra arguments are currently only registered for top-level units and
+        // this is how `cargo rustc` and `cargo rustdoc` are implemented. We may
+        // split a top level unit though for pipelining, and the actual work
+        // happens in the `BuildRmeta` stage and not the `Build` stage. To
+        // handle that difference and ensure arguments get to the right place be
+        // sure to switch `BuildRmeta` modes to `Build` for lookup.
+        if unit.mode == CompileMode::BuildRmeta {
+            let build = unit.with_mode(CompileMode::Build, self.units);
+            self.extra_compiler_args.get(&build)
+        } else {
+            self.extra_compiler_args.get(unit)
+        }
     }
 }
 

src/cargo/core/compiler/context/compilation_files.rs

@@ -8,7 +8,7 @@ use std::sync::Arc;
 use lazycell::LazyCell;
 use log::info;
 
-use super::{BuildContext, Context, FileFlavor, Kind, Layout};
+use super::{BuildContext, Context, FileFlavor, Kind, Layout, CompileMode};
 use crate::core::compiler::Unit;
 use crate::core::{TargetKind, Workspace};
 use crate::util::{self, CargoResult};
@@ -299,18 +299,19 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
 
         let mut ret = Vec::new();
         let mut unsupported = Vec::new();
-        {
-            if unit.mode.is_check() {
+        match unit.mode {
+            CompileMode::BuildRmeta | CompileMode::Check { .. } => {
                 // This may be confusing. rustc outputs a file named `lib*.rmeta`
                 // for both libraries and binaries.
                 let path = out_dir.join(format!("lib{}.rmeta", file_stem));
                 ret.push(OutputFile {
-                    path,
+                    path: path.clone(),
                     hardlink: None,
                     export_path: None,
                     flavor: FileFlavor::Linkable,
                 });
-            } else {
+            }
+            CompileMode::Test | CompileMode::Build | CompileMode::Bench => {
                 let mut add = |crate_type: &str, flavor: FileFlavor| -> CargoResult<()> {
                     let crate_type = if crate_type == "lib" {
                         "rlib"
@@ -324,34 +325,35 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
                         bcx.target_triple(),
                     )?;
 
-                    match file_types {
-                        Some(types) => {
-                            for file_type in types {
-                                let path = out_dir.join(file_type.filename(&file_stem));
-                                let hardlink = link_stem
-                                    .as_ref()
-                                    .map(|&(ref ld, ref ls)| ld.join(file_type.filename(ls)));
-                                let export_path = if unit.target.is_custom_build() {
-                                    None
-                                } else {
-                                    self.export_dir.as_ref().and_then(|export_dir| {
-                                        hardlink.as_ref().and_then(|hardlink| {
-                                            Some(export_dir.join(hardlink.file_name().unwrap()))
-                                        })
-                                    })
-                                };
-                                ret.push(OutputFile {
-                                    path,
-                                    hardlink,
-                                    export_path,
-                                    flavor: file_type.flavor,
-                                });
-                            }
-                        }
+                    let types = match file_types {
+                        Some(types) => types,
                         // Not supported; don't worry about it.
                         None => {
                             unsupported.push(crate_type.to_string());
+                            return Ok(());
                         }
+                    };
+
+                    for file_type in types {
+                        let path = out_dir.join(file_type.filename(&file_stem));
+                        let hardlink = link_stem
+                            .as_ref()
+                            .map(|&(ref ld, ref ls)| ld.join(file_type.filename(ls)));
+                        let export_path = if unit.target.is_custom_build() {
+                            None
+                        } else {
+                            self.export_dir.as_ref().and_then(|export_dir| {
+                                hardlink.as_ref().and_then(|hardlink| {
+                                    Some(export_dir.join(hardlink.file_name().unwrap()))
+                                })
+                            })
+                        };
+                        ret.push(OutputFile {
+                            path,
+                            hardlink,
+                            export_path,
+                            flavor: file_type.flavor,
+                        });
                     }
                     Ok(())
                 };
@@ -381,6 +383,9 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
                     }
                 }
             }
+            CompileMode::Doc { .. } | CompileMode::Doctest | CompileMode::RunCustomBuild => {
+                return Ok(Arc::new(ret));
+            }
         }
         if ret.is_empty() {
             if !unsupported.is_empty() {
@@ -425,6 +430,14 @@ fn compute_metadata<'a, 'cfg>(
     cx: &Context<'a, 'cfg>,
     metas: &mut HashMap<Unit<'a>, Option<Metadata>>,
 ) -> Option<Metadata> {
+    // Check to see if this is a pipelined unit which means that it doesn't
+    // actually do any work, but rather its dependency on an rmeta unit will do
+    // all the work. In that case we'll take the same metadata as our rmeta
+    // compile to ensure that our file names all align.
+    if let Some(rmeta) = cx.rmeta_unit_if_pipelined(unit) {
+        return compute_metadata(&rmeta, cx, metas);
+    }
+
     // No metadata for dylibs because of a couple issues:
     // - macOS encodes the dylib name in the executable,
     // - Windows rustc multiple files of which we can't easily link all of them.

src/cargo/core/compiler/context/mod.rs

@@ -42,6 +42,18 @@ pub struct Context<'a, 'cfg: 'a> {
     unit_dependencies: HashMap<Unit<'a>, Vec<Unit<'a>>>,
     files: Option<CompilationFiles<'a, 'cfg>>,
     package_cache: HashMap<PackageId, &'a Package>,
+
+    // A map from a unit to the dependencies listed for it which are considered
+    // as "order only" dependencies. These dependencies are only used to
+    // sequence compilation correctly and shouldn't inject `--extern` flags, for
+    // example.
+    pub order_only_dependencies: HashMap<Unit<'a>, HashSet<Unit<'a>>>,
+
+    // A set of units which have been "pipelined". These units are in `Build`
+    // mode and depend on a `BuildRmeta` mode unit. The pipelined `Build` units
+    // should do no work because the `BuildRmeta` unit will do all the work for
+    // them.
+    pub pipelined_units: HashSet<Unit<'a>>,
 }
 
 impl<'a, 'cfg> Context<'a, 'cfg> {
@@ -76,6 +88,8 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
             unit_dependencies: HashMap::new(),
             files: None,
             package_cache: HashMap::new(),
+            order_only_dependencies: HashMap::new(),
+            pipelined_units: HashSet::new(),
         })
     }
 
@@ -266,6 +280,8 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
             self.bcx,
             &mut self.unit_dependencies,
             &mut self.package_cache,
+            &mut self.order_only_dependencies,
+            &mut self.pipelined_units,
         )?;
         let files = CompilationFiles::new(
             units,
@@ -453,6 +469,20 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
         }
         Ok(())
     }
+
+    /// Returns the `BuildRmeta` unit which is actually doing compilation if the
+    /// `unit` specified has been pipelined.
+    ///
+    /// If the unit specified has not been pipelined then `None` will be
+    /// returned.
+    pub fn rmeta_unit_if_pipelined(&self, unit: &Unit<'a>) -> Option<Unit<'a>> {
+        if self.pipelined_units.contains(unit) {
+            assert_eq!(unit.mode, CompileMode::Build);
+            Some(unit.with_mode(CompileMode::BuildRmeta, self.bcx.units))
+        } else {
+            None
+        }
+    }
 }
 
 #[derive(Default)]