Auto merge of #4568 - Metaswitch:alt-registry-publish, r=withoutboats

Add support for publish to optionally take the index that can be used

This form part of alternative-registries RFC-2141, it allows crates to optionally specify which registries the crate can be be published to.

@carols10cents, one thing that I am unsure about is if there is a plan for publish to still provide index, or for registry to be provided instead. I thought that your general view was that we should move away from the index file. If we do need to map allowed registries to the index then there will be a small amount of extra work required once #4506 is merged.

@withoutboats, happy for this to be merged into your branch if you want, the main reason I did not base it on your branch was due to tests not working on there yet.

src/bin/package.rs

@@ -61,6 +61,7 @@ pub fn execute(options: Options, config: &mut Config) -> CliResult {
         allow_dirty: options.flag_allow_dirty,
         target: options.flag_target.as_ref().map(|t| &t[..]),
         jobs: options.flag_jobs,
+        registry: None,
     })?;
     Ok(())
 }

src/cargo/core/manifest.rs

@@ -29,7 +29,7 @@ pub struct Manifest {
     include: Vec<String>,
     metadata: ManifestMetadata,
     profiles: Profiles,
-    publish: bool,
+    publish: Option<Vec<String>>,
     replace: Vec<(PackageIdSpec, Dependency)>,
     patch: HashMap<Url, Vec<Dependency>>,
     workspace: WorkspaceConfig,
@@ -240,7 +240,7 @@ impl Manifest {
                links: Option<String>,
                metadata: ManifestMetadata,
                profiles: Profiles,
-               publish: bool,
+               publish: Option<Vec<String>>,
                replace: Vec<(PackageIdSpec, Dependency)>,
                patch: HashMap<Url, Vec<Dependency>>,
                workspace: WorkspaceConfig,
@@ -277,7 +277,7 @@ impl Manifest {
     pub fn version(&self) -> &Version { self.package_id().version() }
     pub fn warnings(&self) -> &[DelayedWarning] { &self.warnings }
     pub fn profiles(&self) -> &Profiles { &self.profiles }
-    pub fn publish(&self) -> bool { self.publish }
+    pub fn publish(&self) -> &Option<Vec<String>> { &self.publish }
     pub fn replace(&self) -> &[(PackageIdSpec, Dependency)] { &self.replace }
     pub fn original(&self) -> &TomlManifest { &self.original }
     pub fn patch(&self) -> &HashMap<Url, Vec<Dependency>> { &self.patch }

src/cargo/core/package.rs

@@ -108,7 +108,7 @@ impl Package {
     /// Get the package authors
     pub fn authors(&self) -> &Vec<String> { &self.manifest.metadata().authors }
     /// Whether the package is set to publish
-    pub fn publish(&self) -> bool { self.manifest.publish() }
+    pub fn publish(&self) -> &Option<Vec<String>> { self.manifest.publish() }
 
     /// Whether the package uses a custom build script for any target
     pub fn has_custom_build(&self) -> bool {
@@ -134,10 +134,10 @@ impl Package {
         }
     }
 
-    pub fn to_registry_toml(&self) -> String {
+    pub fn to_registry_toml(&self) -> CargoResult<String> {
         let manifest = self.manifest().original().prepare_for_publish();
-        let toml = toml::to_string(&manifest).unwrap();
-        format!("\
+        let toml = toml::to_string(&manifest)?;
+        Ok(format!("\
             # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO\n\
             #\n\
             # When uploading crates to the registry Cargo will automatically\n\
@@ -151,7 +151,7 @@ impl Package {
             # will likely look very different (and much more reasonable)\n\
             \n\
             {}\
-        ", toml)
+        ", toml))
     }
 }
 

src/cargo/core/source/source_id.rs

@@ -32,6 +32,8 @@ struct SourceIdInner {
     kind: Kind,
     // e.g. the exact git revision of the specified branch for a Git Source
     precise: Option<String>,
+    /// Name of the registry source for alternative registries
+    name: Option<String>,
 }
 
 /// The possible kinds of code source. Along with a URL, this fully defines the
@@ -72,6 +74,7 @@ impl SourceId {
                 canonical_url: git::canonicalize_url(&url)?,
                 url: url,
                 precise: None,
+                name: None,
             }),
         };
         Ok(source_id)
@@ -190,6 +193,7 @@ impl SourceId {
                 canonical_url: git::canonicalize_url(&url)?,
                 url: url,
                 precise: None,
+                name: Some(key.to_string()),
             }),
         })
     }
@@ -211,11 +215,16 @@ impl SourceId {
     /// Is this source from a registry (either local or not)
     pub fn is_registry(&self) -> bool {
         match self.inner.kind {
-            Kind::Registry | Kind::LocalRegistry    => true,
-            _                                       => false,
+            Kind::Registry | Kind::LocalRegistry => true,
+            _                                    => false,
         }
     }
 
+    /// Is this source from an alternative registry
+    pub fn is_alt_registry(&self) -> bool {
+        self.is_registry() && self.inner.name.is_some()
+    }
+
     /// Is this source from a git repository
     pub fn is_git(&self) -> bool {
         match self.inner.kind {

src/cargo/ops/cargo_package.rs

@@ -23,13 +23,17 @@ pub struct PackageOpts<'cfg> {
     pub verify: bool,
     pub jobs: Option<u32>,
     pub target: Option<&'cfg str>,
+    pub registry: Option<String>,
 }
 
 pub fn package(ws: &Workspace,
                opts: &PackageOpts) -> CargoResult<Option<FileLock>> {
     let pkg = ws.current()?;
     let config = ws.config();
-    if !pkg.manifest().features().activated().is_empty() {
+
+    // Allow packaging if a registry has been provided, or if there are no nightly
+    // features enabled.
+    if opts.registry.is_none() && !pkg.manifest().features().activated().is_empty() {
         bail!("cannot package or publish crates which activate nightly-only \
                cargo features")
     }
@@ -251,7 +255,7 @@ fn tar(ws: &Workspace,
             })?;
 
             let mut header = Header::new_ustar();
-            let toml = pkg.to_registry_toml();
+            let toml = pkg.to_registry_toml()?;
             header.set_path(&path)?;
             header.set_entry_type(EntryType::file());
             header.set_mode(0o644);

src/cargo/ops/registry.rs

@@ -42,10 +42,16 @@ pub struct PublishOpts<'cfg> {
 pub fn publish(ws: &Workspace, opts: &PublishOpts) -> CargoResult<()> {
     let pkg = ws.current()?;
 
-    if !pkg.publish() {
-        bail!("some crates cannot be published.\n\
-               `{}` is marked as unpublishable", pkg.name());
+    if let &Some(ref allowed_registries) = pkg.publish() {
+        if !match opts.registry {
+            Some(ref registry) => allowed_registries.contains(registry),
+            None => false,
+        } {
+            bail!("some crates cannot be published.\n\
+                   `{}` is marked as unpublishable", pkg.name());
+        }
     }
+
     if !pkg.manifest().patch().is_empty() {
         bail!("published crates cannot contain [patch] sections");
     }
@@ -66,11 +72,12 @@ pub fn publish(ws: &Workspace, opts: &PublishOpts) -> CargoResult<()> {
         allow_dirty: opts.allow_dirty,
         target: opts.target,
         jobs: opts.jobs,
+        registry: opts.registry.clone(),
     })?.unwrap();
 
     // Upload said tarball to the specified destination
     opts.config.shell().status("Uploading", pkg.package_id().to_string())?;
-    transmit(opts.config, pkg, tarball.file(), &mut registry, opts.dry_run)?;
+    transmit(opts.config, pkg, tarball.file(), &mut registry, &reg_id, opts.dry_run)?;
 
     Ok(())
 }
@@ -86,10 +93,14 @@ fn verify_dependencies(pkg: &Package, registry_src: &SourceId)
             }
         } else if dep.source_id() != registry_src {
             if dep.source_id().is_registry() {
-                bail!("crates cannot be published to crates.io with dependencies sourced from other\n\
-                       registries either publish `{}` on crates.io or pull it into this repository\n\
-                       and specify it with a path and version\n\
-                       (crate `{}` is pulled from {}", dep.name(), dep.name(), dep.source_id());
+                // Block requests to send to a registry if it is not an alternative
+                // registry
+                if !registry_src.is_alt_registry() {
+                    bail!("crates cannot be published to crates.io with dependencies sourced from other\n\
+                           registries either publish `{}` on crates.io or pull it into this repository\n\
+                           and specify it with a path and version\n\
+                           (crate `{}` is pulled from {})", dep.name(), dep.name(), dep.source_id());
+                }
             } else {
                 bail!("crates cannot be published to crates.io with dependencies sourced from \
                        a repository\neither publish `{}` as its own crate on crates.io and \
@@ -106,8 +117,19 @@ fn transmit(config: &Config,
             pkg: &Package,
             tarball: &File,
             registry: &mut Registry,
+            registry_id: &SourceId,
             dry_run: bool) -> CargoResult<()> {
+
     let deps = pkg.dependencies().iter().map(|dep| {
+
+        // If the dependency is from a different registry, then include the
+        // registry in the dependency.
+        let dep_registry = if dep.source_id() != registry_id {
+            Some(dep.source_id().url().to_string())
+        } else {
+            None
+        };
+
         NewCrateDependency {
             optional: dep.is_optional(),
             default_features: dep.uses_default_features(),
@@ -120,6 +142,7 @@ fn transmit(config: &Config,
                 Kind::Build => "build",
                 Kind::Development => "dev",
             }.to_string(),
+            registry: dep_registry,
         }
     }).collect::<Vec<NewCrateDependency>>();
     let manifest = pkg.manifest();
@@ -222,9 +245,10 @@ pub fn registry(config: &Config,
         index: index_config,
     } = registry_configuration(config, registry.clone())?;
     let token = token.or(token_config);
-    let sid = match (index_config, index) {
-        (Some(index), _) | (None, Some(index)) => SourceId::for_registry(&index.to_url()?)?,
-        (None, None) => SourceId::crates_io(config)?,
+    let sid = match (index_config, index, registry) {
+        (_, _, Some(registry)) => SourceId::alt_registry(config, &registry)?,
+        (Some(index), _, _) | (None, Some(index), _) => SourceId::for_registry(&index.to_url()?)?,
+        (None, None, _) => SourceId::crates_io(config)?,
     };
     let api_host = {
         let mut src = RegistrySource::remote(&sid, config);

src/cargo/util/toml/mod.rs

@@ -200,6 +200,8 @@ pub struct DetailedTomlDependency {
 
 #[derive(Debug, Deserialize, Serialize)]
 pub struct TomlManifest {
+    #[serde(rename = "cargo-features")]
+    cargo_features: Option<Vec<String>>,
     package: Option<Box<TomlProject>>,
     project: Option<Box<TomlProject>>,
     profile: Option<TomlProfiles>,
@@ -223,8 +225,6 @@ pub struct TomlManifest {
     patch: Option<BTreeMap<String, BTreeMap<String, TomlDependency>>>,
     workspace: Option<TomlWorkspace>,
     badges: Option<BTreeMap<String, BTreeMap<String, String>>>,
-    #[serde(rename = "cargo-features")]
-    cargo_features: Option<Vec<String>>,
 }
 
 #[derive(Deserialize, Serialize, Clone, Debug, Default)]
@@ -381,6 +381,44 @@ impl<'de> de::Deserialize<'de> for StringOrBool {
     }
 }
 
+#[derive(Clone, Debug, Serialize)]
+#[serde(untagged)]
+pub enum VecStringOrBool {
+    VecString(Vec<String>),
+    Bool(bool),
+}
+
+impl<'de> de::Deserialize<'de> for VecStringOrBool {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+        where D: de::Deserializer<'de>
+    {
+        struct Visitor;
+
+        impl<'de> de::Visitor<'de> for Visitor {
+            type Value = VecStringOrBool;
+
+            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+                formatter.write_str("a boolean or vector of strings")
+            }
+
+            fn visit_seq<V>(self, v: V) -> Result<Self::Value, V::Error>
+                where V: de::SeqAccess<'de>
+            {
+                let seq = de::value::SeqAccessDeserializer::new(v);
+                Vec::deserialize(seq).map(VecStringOrBool::VecString)
+            }
+
+            fn visit_bool<E>(self, b: bool) -> Result<Self::Value, E>
+                where E: de::Error,
+            {
+                Ok(VecStringOrBool::Bool(b))
+            }
+        }
+
+        deserializer.deserialize_any(Visitor)
+    }
+}
+
 #[derive(Deserialize, Serialize, Clone, Debug)]
 pub struct TomlProject {
     name: String,
@@ -390,7 +428,7 @@ pub struct TomlProject {
     links: Option<String>,
     exclude: Option<Vec<String>>,
     include: Option<Vec<String>>,
-    publish: Option<bool>,
+    publish: Option<VecStringOrBool>,
     workspace: Option<String>,
     #[serde(rename = "im-a-teapot")]
     im_a_teapot: Option<bool>,
@@ -655,7 +693,16 @@ impl TomlManifest {
             }
         };
         let profiles = build_profiles(&me.profile);
-        let publish = project.publish.unwrap_or(true);
+        let publish = match project.publish {
+            Some(VecStringOrBool::VecString(ref vecstring)) => {
+                features.require(Feature::alternative_registries()).chain_err(|| {
+                    "the `publish` manifest key is unstable for anything other than a value of true or false"
+                })?;
+                Some(vecstring.clone())
+            },
+            Some(VecStringOrBool::Bool(false)) => Some(vec![]),
+            None | Some(VecStringOrBool::Bool(true)) => None,
+        };
         let mut manifest = Manifest::new(summary,
                                          targets,
                                          exclude,

src/crates-io/lib.rs

@@ -100,6 +100,8 @@ pub struct NewCrateDependency {
     pub version_req: String,
     pub target: Option<String>,
     pub kind: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub registry: Option<String>,
 }
 
 #[derive(Deserialize)]