Auto merge of #67074 - ehuss:extern-options, r=petrochenkov

Add options to --extern flag.

This changes the `--extern` flag so that it can take a series of options that changes its behavior. The general syntax is `[opts ':'] name ['=' path]` where `opts` is a comma separated list of options. Two options are supported, `priv` which replaces `--extern-private` and `noprelude` which avoids adding the crate to the extern prelude.

```text
--extern priv:mylib=/path/to/libmylib.rlib
--extern noprelude:alloc=/path/to/liballoc.rlib
```

`noprelude` is to be used by Cargo's build-std feature in order to use `--extern` to reference standard library crates.

This also includes a second commit which adds the `aux-crate` directive to compiletest. I can split this off into a separate PR if desired, but it helps with defining these kinds of tests. It is based on #54020, and can be used in the future to replace and simplify some of the Makefile tests.

src/librustc_interface/tests.rs

@@ -7,7 +7,7 @@ use rustc::middle::cstore;
 use rustc::session::config::{build_configuration, build_session_options, to_crate_config};
 use rustc::session::config::{LtoCli, LinkerPluginLto, SwitchWithOptPath, ExternEntry};
 use rustc::session::config::{Externs, OutputType, OutputTypes, SymbolManglingVersion};
-use rustc::session::config::{rustc_optgroups, Options, ErrorOutputType, Passes};
+use rustc::session::config::{rustc_optgroups, Options, ErrorOutputType, Passes, ExternLocation};
 use rustc::session::{build_session, Session};
 use rustc::session::search_paths::SearchPath;
 use std::collections::{BTreeMap, BTreeSet};
@@ -38,14 +38,15 @@ fn mk_session(matches: getopts::Matches) -> (Session, CfgSpecs) {
 fn new_public_extern_entry<S, I>(locations: I) -> ExternEntry
 where
     S: Into<String>,
-    I: IntoIterator<Item = Option<S>>,
+    I: IntoIterator<Item = S>,
 {
-    let locations: BTreeSet<_> = locations.into_iter().map(|o| o.map(|s| s.into()))
+    let locations: BTreeSet<_> = locations.into_iter().map(|s| s.into())
         .collect();
 
     ExternEntry {
-        locations,
-        is_private_dep: false
+        location: ExternLocation::ExactPaths(locations),
+        is_private_dep: false,
+        add_prelude: true,
     }
 }
 
@@ -160,33 +161,33 @@ fn test_externs_tracking_hash_different_construction_order() {
     v1.externs = Externs::new(mk_map(vec![
         (
             String::from("a"),
-            new_public_extern_entry(vec![Some("b"), Some("c")])
+            new_public_extern_entry(vec!["b", "c"])
         ),
         (
             String::from("d"),
-            new_public_extern_entry(vec![Some("e"), Some("f")])
+            new_public_extern_entry(vec!["e", "f"])
         ),
     ]));
 
     v2.externs = Externs::new(mk_map(vec![
         (
             String::from("d"),
-            new_public_extern_entry(vec![Some("e"), Some("f")])
+            new_public_extern_entry(vec!["e", "f"])
         ),
         (
             String::from("a"),
-            new_public_extern_entry(vec![Some("b"), Some("c")])
+            new_public_extern_entry(vec!["b", "c"])
         ),
     ]));
 
     v3.externs = Externs::new(mk_map(vec![
         (
             String::from("a"),
-            new_public_extern_entry(vec![Some("b"), Some("c")])
+            new_public_extern_entry(vec!["b", "c"])
         ),
         (
             String::from("d"),
-            new_public_extern_entry(vec![Some("f"), Some("e")])
+            new_public_extern_entry(vec!["f", "e"])
         ),
     ]));
 

src/librustc_metadata/creader.rs

@@ -218,13 +218,14 @@ impl<'a> CrateLoader<'a> {
             let source = self.cstore.get_crate_data(cnum).source();
             if let Some(entry) = self.sess.opts.externs.get(&name.as_str()) {
                 // Only use `--extern crate_name=path` here, not `--extern crate_name`.
-                let found = entry.locations.iter().filter_map(|l| l.as_ref()).any(|l| {
-                    let l = fs::canonicalize(l).ok();
-                    source.dylib.as_ref().map(|p| &p.0) == l.as_ref() ||
-                    source.rlib.as_ref().map(|p| &p.0) == l.as_ref()
-                });
-                if found {
-                    ret = Some(cnum);
+                if let Some(mut files) = entry.files() {
+                    if files.any(|l| {
+                        let l = fs::canonicalize(l).ok();
+                        source.dylib.as_ref().map(|p| &p.0) == l.as_ref() ||
+                        source.rlib.as_ref().map(|p| &p.0) == l.as_ref()
+                    }) {
+                        ret = Some(cnum);
+                    }
                 }
                 return
             }

src/librustc_metadata/locator.rs

@@ -328,8 +328,9 @@ impl<'a> CrateLocator<'a> {
             crate_name,
             exact_paths: if hash.is_none() {
                 sess.opts.externs.get(&crate_name.as_str()).into_iter()
-                    .flat_map(|entry| entry.locations.iter())
-                    .filter_map(|location| location.clone().map(PathBuf::from)).collect()
+                    .filter_map(|entry| entry.files())
+                    .flatten()
+                    .map(|location| PathBuf::from(location)).collect()
             } else {
                 // SVH being specified means this is a transitive dependency,
                 // so `--extern` options do not apply.

src/librustc_resolve/lib.rs

@@ -1127,8 +1127,10 @@ impl<'a> Resolver<'a> {
         definitions.create_root_def(crate_name, session.local_crate_disambiguator());
 
         let mut extern_prelude: FxHashMap<Ident, ExternPreludeEntry<'_>> =
-            session.opts.externs.iter().map(|kv| (Ident::from_str(kv.0), Default::default()))
-                                       .collect();
+            session.opts.externs.iter()
+                .filter(|(_, entry)| entry.add_prelude)
+                .map(|(name, _)| (Ident::from_str(name), Default::default()))
+                .collect();
 
         if !attr::contains_name(&krate.attrs, sym::no_core) {
             extern_prelude.insert(Ident::with_dummy_span(sym::core), Default::default());

src/librustc_session/config.rs

@@ -1,3 +1,5 @@
+// ignore-tidy-filelength
+
 //! Contains infrastructure for configuring the compiler, including parsing
 //! command-line options.
 
@@ -31,7 +33,7 @@ use std::fmt;
 use std::str::{self, FromStr};
 use std::hash::Hasher;
 use std::collections::hash_map::DefaultHasher;
-use std::iter::FromIterator;
+use std::iter::{self, FromIterator};
 use std::path::{Path, PathBuf};
 
 pub struct Config {
@@ -322,10 +324,35 @@ impl OutputTypes {
 #[derive(Clone)]
 pub struct Externs(BTreeMap<String, ExternEntry>);
 
-#[derive(Clone, Debug, Default)]
+#[derive(Clone, Debug)]
 pub struct ExternEntry {
-    pub locations: BTreeSet<Option<String>>,
-    pub is_private_dep: bool
+    pub location: ExternLocation,
+    /// Indicates this is a "private" dependency for the
+    /// `exported_private_dependencies` lint.
+    ///
+    /// This can be set with the `priv` option like
+    /// `--extern priv:name=foo.rlib`.
+    pub is_private_dep: bool,
+    /// Add the extern entry to the extern prelude.
+    ///
+    /// This can be disabled with the `noprelude` option like
+    /// `--extern noprelude:name`.
+    pub add_prelude: bool,
+}
+
+#[derive(Clone, Debug)]
+pub enum ExternLocation {
+    /// Indicates to look for the library in the search paths.
+    ///
+    /// Added via `--extern name`.
+    FoundInLibrarySearchDirectories,
+    /// The locations where this extern entry must be found.
+    ///
+    /// The `CrateLoader` is responsible for loading these and figuring out
+    /// which one to use.
+    ///
+    /// Added via `--extern prelude_name=some_file.rlib`
+    ExactPaths(BTreeSet<String>),
 }
 
 impl Externs {
@@ -342,6 +369,18 @@ impl Externs {
     }
 }
 
+impl ExternEntry {
+    fn new(location: ExternLocation) -> ExternEntry {
+        ExternEntry { location, is_private_dep: false, add_prelude: false }
+    }
+
+    pub fn files(&self) -> Option<impl Iterator<Item = &String>> {
+        match &self.location {
+            ExternLocation::ExactPaths(set) => Some(set.iter()),
+            _ => None,
+        }
+    }
+}
 
 macro_rules! hash_option {
     ($opt_name:ident, $opt_expr:expr, $sub_hashes:expr, [UNTRACKED]) => ({});
@@ -1869,12 +1908,6 @@ pub fn rustc_optgroups() -> Vec<RustcOptGroup> {
             "Specify where an external rust library is located",
             "NAME[=PATH]",
         ),
-        opt::multi_s(
-            "",
-            "extern-private",
-            "Specify where an extern rust library is located, marking it as a private dependency",
-            "NAME=PATH",
-        ),
         opt::opt_s("", "sysroot", "Override the system root", "PATH"),
         opt::multi("Z", "", "Set internal debugging options", "FLAG"),
         opt::opt_s(
@@ -2435,43 +2468,105 @@ fn parse_borrowck_mode(dopts: &DebuggingOptions, error_format: ErrorOutputType)
     }
 }
 
-fn parse_externs(
+pub fn parse_externs(
     matches: &getopts::Matches,
     debugging_opts: &DebuggingOptions,
     error_format: ErrorOutputType,
 ) -> Externs {
-    if matches.opt_present("extern-private") && !debugging_opts.unstable_options {
-        early_error(
-            ErrorOutputType::default(),
-            "'--extern-private' is unstable and only \
-            available for nightly builds of rustc."
-        )
-    }
-
-    // We start out with a `Vec<(Option<String>, bool)>>`,
-    // and later convert it into a `BTreeSet<(Option<String>, bool)>`
-    // This allows to modify entries in-place to set their correct
-    // 'public' value.
+    let is_unstable_enabled = debugging_opts.unstable_options;
     let mut externs: BTreeMap<String, ExternEntry> = BTreeMap::new();
-    for (arg, private) in matches.opt_strs("extern").into_iter().map(|v| (v, false))
-        .chain(matches.opt_strs("extern-private").into_iter().map(|v| (v, true))) {
-
+    for arg in matches.opt_strs("extern") {
         let mut parts = arg.splitn(2, '=');
-        let name = parts.next().unwrap_or_else(||
-            early_error(error_format, "--extern value must not be empty"));
-        let location = parts.next().map(|s| s.to_string());
+        let name = parts
+            .next()
+            .unwrap_or_else(|| early_error(error_format, "--extern value must not be empty"));
+        let path = parts.next().map(|s| s.to_string());
+
+        let mut name_parts = name.splitn(2, ':');
+        let first_part = name_parts.next();
+        let second_part = name_parts.next();
+        let (options, name) = match (first_part, second_part) {
+            (Some(opts), Some(name)) => (Some(opts), name),
+            (Some(name), None) => (None, name),
+            (None, None) => early_error(error_format, "--extern name must not be empty"),
+            _ => unreachable!(),
+        };
+
+        let entry = externs.entry(name.to_owned());
 
-        let entry = externs
-            .entry(name.to_owned())
-            .or_default();
+        use std::collections::btree_map::Entry;
 
+        let entry = if let Some(path) = path {
+            // --extern prelude_name=some_file.rlib
+            match entry {
+                Entry::Vacant(vacant) => {
+                    let files = BTreeSet::from_iter(iter::once(path));
+                    vacant.insert(ExternEntry::new(ExternLocation::ExactPaths(files)))
+                }
+                Entry::Occupied(occupied) => {
+                    let ext_ent = occupied.into_mut();
+                    match ext_ent {
+                        ExternEntry { location: ExternLocation::ExactPaths(files), .. } => {
+                            files.insert(path);
+                        }
+                        ExternEntry {
+                            location: location @ ExternLocation::FoundInLibrarySearchDirectories,
+                            ..
+                        } => {
+                            // Exact paths take precedence over search directories.
+                            let files = BTreeSet::from_iter(iter::once(path));
+                            *location = ExternLocation::ExactPaths(files);
+                        }
+                    }
+                    ext_ent
+                }
+            }
+        } else {
+            // --extern prelude_name
+            match entry {
+                Entry::Vacant(vacant) => {
+                    vacant.insert(ExternEntry::new(ExternLocation::FoundInLibrarySearchDirectories))
+                }
+                Entry::Occupied(occupied) => {
+                    // Ignore if already specified.
+                    occupied.into_mut()
+                }
+            }
+        };
 
-        entry.locations.insert(location.clone());
+        let mut is_private_dep = false;
+        let mut add_prelude = true;
+        if let Some(opts) = options {
+            if !is_unstable_enabled {
+                early_error(
+                    error_format,
+                    "the `-Z unstable-options` flag must also be passed to \
+                     enable `--extern options",
+                );
+            }
+            for opt in opts.split(',') {
+                match opt {
+                    "priv" => is_private_dep = true,
+                    "noprelude" => {
+                        if let ExternLocation::ExactPaths(_) = &entry.location {
+                            add_prelude = false;
+                        } else {
+                            early_error(
+                                error_format,
+                                "the `noprelude` --extern option requires a file path",
+                            );
+                        }
+                    }
+                    _ => early_error(error_format, &format!("unknown --extern option `{}`", opt)),
+                }
+            }
+        }
 
-        // Crates start out being not private,
-        // and go to being private if we see an '--extern-private'
-        // flag
-        entry.is_private_dep |= private;
+        // Crates start out being not private, and go to being private `priv`
+        // is specified.
+        entry.is_private_dep |= is_private_dep;
+        // If any flag is missing `noprelude`, then add to the prelude.
+        entry.add_prelude |= add_prelude;
     }
     Externs(externs)
 }

src/librustdoc/config.rs

@@ -7,10 +7,10 @@ use errors;
 use getopts;
 use rustc::lint::Level;
 use rustc::session;
-use rustc::session::config::{CrateType, parse_crate_types_from_list};
+use rustc::session::config::{CrateType, parse_crate_types_from_list, parse_externs};
 use rustc::session::config::{CodegenOptions, DebuggingOptions, ErrorOutputType, Externs};
 use rustc::session::config::{nightly_options, build_codegen_options, build_debugging_options,
-                             get_cmd_lint_options, host_triple, ExternEntry};
+                             get_cmd_lint_options, host_triple};
 use rustc::session::search_paths::SearchPath;
 use rustc_driver;
 use rustc_target::spec::TargetTriple;
@@ -320,13 +320,7 @@ impl Options {
         let libs = matches.opt_strs("L").iter()
             .map(|s| SearchPath::from_cli_opt(s, error_format))
             .collect();
-        let externs = match parse_externs(&matches) {
-            Ok(ex) => ex,
-            Err(err) => {
-                diag.struct_err(&err).emit();
-                return Err(1);
-            }
-        };
+        let externs = parse_externs(&matches, &debugging_options, error_format);
         let extern_html_root_urls = match parse_extern_html_roots(&matches) {
             Ok(ex) => ex,
             Err(err) => {
@@ -617,24 +611,3 @@ fn parse_extern_html_roots(
 
     Ok(externs)
 }
-
-/// Extracts `--extern CRATE=PATH` arguments from `matches` and
-/// returns a map mapping crate names to their paths or else an
-/// error message.
-/// Also handles `--extern-private` which for the purposes of rustdoc
-/// we can treat as `--extern`
-// FIXME(eddyb) This shouldn't be duplicated with `rustc::session`.
-fn parse_externs(matches: &getopts::Matches) -> Result<Externs, String> {
-    let mut externs: BTreeMap<_, ExternEntry> = BTreeMap::new();
-    for arg in matches.opt_strs("extern").iter().chain(matches.opt_strs("extern-private").iter()) {
-        let mut parts = arg.splitn(2, '=');
-        let name = parts.next().ok_or("--extern value must not be empty".to_string())?;
-        let location = parts.next().map(|s| s.to_string());
-        let name = name.to_string();
-        // For Rustdoc purposes, we can treat all externs as public
-        externs.entry(name)
-            .or_default()
-            .locations.insert(location.clone());
-    }
-    Ok(Externs::new(externs))
-}

src/librustdoc/core.rs

@@ -248,7 +248,9 @@ pub fn run_core(options: RustdocOptions) -> (clean::Crate, RenderInfo, RenderOpt
         ..
     } = options;
 
-    let extern_names: Vec<String> = externs.iter().map(|(s,_)| s).cloned().collect();
+    let extern_names: Vec<String> = externs.iter()
+        .filter(|(_, entry)| entry.add_prelude)
+        .map(|(name, _)| name).cloned().collect();
 
     // Add the doc cfg into the doc build.
     cfgs.push("doc".to_string());