feat(release): add support for sparse registry URLs (#863)

Co-authored-by: Marco Ieni <11428655+MarcoIeni@users.noreply.github.com>

Cargo.toml

@@ -9,13 +9,14 @@ chrono = { version = "0.4.26", default-features = false }
 clap = "4.3.19"
 clap_complete = "4.3.2"
 conventional_commit_parser = "0.9.4"
-crates-index = "0.19.13"
+crates-index = { version = "0.19.13", features = ["sparse-http"] }
 dirs = "5.0.1"
 dunce = "1.0.4"
 expect-test = "1.4.1"
 fake = "2.6.1"
 git-cliff-core = { version = "1.2.0", default-features = false }
 git-url-parse = "0.4.4"
+http = "0.2"
 ignore = "0.4.20"
 lazy_static = "1.4.0"
 once_cell = "1.18.0"

crates/release_plz_core/Cargo.toml

@@ -28,7 +28,8 @@ lazy_static.workspace = true
 parse-changelog.workspace = true
 rayon.workspace = true
 regex.workspace = true
-reqwest = { workspace = true, features = ["json"] }
+# native-tls-alpn is needed for http2 support. https://doc.rust-lang.org/cargo/reference/registry-index.html#sparse-protocol
+reqwest = { workspace = true, features = ["json", "gzip", "native-tls-alpn"] }
 reqwest-middleware.workspace = true
 reqwest-retry.workspace = true
 secrecy.workspace = true
@@ -41,6 +42,8 @@ walkdir.workspace = true
 toml_edit.workspace = true
 serde_json.workspace = true
 strip-ansi-escapes.workspace = true
+tokio.workspace = true
+http.workspace = true
 
 [dev-dependencies]
 git_cmd = { path = "../git_cmd", features = ["test_fixture"] }

crates/release_plz_core/src/cargo.rs

@@ -1,17 +1,21 @@
 use anyhow::Context;
 use cargo_metadata::Package;
-use crates_index::Index;
+use crates_index::{Crate, Index, SparseIndex};
 use tracing::{debug, info};
 
 use std::{
     env,
     io::{BufRead, BufReader},
     path::Path,
     process::{Command, Stdio},
-    thread::sleep,
     time::{Duration, Instant},
 };
 
+pub enum CargoIndex {
+    Git(Index),
+    Sparse(SparseIndex),
+}
+
 fn cargo_cmd() -> Command {
     let cargo = env::var("CARGO").unwrap_or_else(|_| "cargo".to_owned());
     Command::new(cargo)
@@ -55,40 +59,91 @@ pub fn run_cargo(root: &Path, args: &[&str]) -> anyhow::Result<(String, String)>
     ))
 }
 
-pub fn is_published(index: &mut Index, package: &Package) -> anyhow::Result<bool> {
+pub async fn is_published(index: &mut CargoIndex, package: &Package) -> anyhow::Result<bool> {
+    match index {
+        CargoIndex::Git(index) => is_published_git(index, package),
+        CargoIndex::Sparse(index) => is_in_cache_sparse(index, package).await,
+    }
+}
+
+pub fn is_published_git(index: &mut Index, package: &Package) -> anyhow::Result<bool> {
     // See if we already have the package in cache.
-    if is_in_cache(index, package) {
+    if is_in_cache_git(index, package) {
         return Ok(true);
     }
 
     // The package is not in the cache, so we update the cache.
     index.update()?;
 
     // Try again with updated index.
-    Ok(is_in_cache(index, package))
+    Ok(is_in_cache_git(index, package))
 }
 
-fn is_in_cache(index: &Index, package: &Package) -> bool {
-    if let Some(crate_data) = index.crate_(&package.name) {
-        if crate_data
-            .versions()
-            .iter()
-            .any(|v| v.version() == package.version.to_string())
-        {
+fn is_in_cache_git(index: &Index, package: &Package) -> bool {
+    let crate_data = index.crate_(&package.name);
+    let version = &package.version.to_string();
+    is_in_cache(crate_data.as_ref(), version)
+}
+
+async fn is_in_cache_sparse(index: &SparseIndex, package: &Package) -> anyhow::Result<bool> {
+    let crate_data = fetch_sparse_metadata(index, &package.name).await?;
+    let version = &package.version.to_string();
+    Ok(is_in_cache(crate_data.as_ref(), version))
+}
+
+fn is_in_cache(crate_data: Option<&Crate>, version: &str) -> bool {
+    if let Some(crate_data) = crate_data {
+        if is_version_present(version, crate_data) {
             return true;
         }
     }
     false
 }
 
-pub fn wait_until_published(index: &mut Index, package: &Package) -> anyhow::Result<()> {
+fn is_version_present(version: &str, crate_data: &Crate) -> bool {
+    crate_data.versions().iter().any(|v| v.version() == version)
+}
+
+async fn fetch_sparse_metadata(
+    index: &SparseIndex,
+    crate_name: &str,
+) -> anyhow::Result<Option<Crate>> {
+    let req = index.make_cache_request(crate_name)?;
+    let (parts, _) = req.into_parts();
+    let req = http::Request::from_parts(parts, vec![]);
+
+    let req: reqwest::Request = req.try_into()?;
+
+    let client = reqwest::ClientBuilder::new()
+        .gzip(true)
+        .http2_prior_knowledge()
+        .build()?;
+    let res = client.execute(req).await?;
+
+    let mut builder = http::Response::builder()
+        .status(res.status())
+        .version(res.version());
+
+    if let Some(headers) = builder.headers_mut() {
+        headers.extend(res.headers().iter().map(|(k, v)| (k.clone(), v.clone())));
+    }
+
+    let body = res.bytes().await?;
+    let res = builder.body(body.to_vec())?;
+
+    let crate_data = index.parse_cache_response(crate_name, res, true)?;
+
+    Ok(crate_data)
+}
+
+pub async fn wait_until_published(index: &mut CargoIndex, package: &Package) -> anyhow::Result<()> {
     let now = Instant::now();
     let sleep_time = Duration::from_secs(2);
     let timeout = Duration::from_secs(300);
     let mut logged = false;
 
     loop {
-        if is_published(index, package)? {
+        if is_published(index, package).await? {
             break;
         } else if timeout < now.elapsed() {
             anyhow::bail!("timeout while publishing {}", package.name)
@@ -102,7 +157,7 @@ pub fn wait_until_published(index: &mut Index, package: &Package) -> anyhow::Res
             logged = true;
         }
 
-        sleep(sleep_time);
+        tokio::time::sleep(sleep_time).await;
     }
 
     Ok(())

crates/release_plz_core/src/command/release.rs

@@ -5,14 +5,14 @@ use std::{
 
 use anyhow::Context;
 use cargo_metadata::Package;
-use crates_index::Index;
+use crates_index::{Index, SparseIndex};
 use git_cmd::Repo;
 use secrecy::{ExposeSecret, SecretString};
 use tracing::{info, instrument, warn};
 use url::Url;
 
 use crate::{
-    cargo::{is_published, run_cargo, wait_until_published},
+    cargo::{is_published, run_cargo, wait_until_published, CargoIndex},
     changelog_parser,
     git::backend::GitClient,
     release_order::release_order,
@@ -286,7 +286,7 @@ pub async fn release(input: &ReleaseRequest) -> anyhow::Result<()> {
         }
         let registry_indexes = registry_indexes(package, input.registry.clone())?;
         for mut index in registry_indexes {
-            if is_published(&mut index, package)? {
+            if is_published(&mut index, package).await? {
                 info!("{} {}: already published", package.name, package.version);
                 continue;
             }
@@ -299,7 +299,10 @@ pub async fn release(input: &ReleaseRequest) -> anyhow::Result<()> {
 /// Get the indexes where the package should be published.
 /// If `registry` is specified, it takes precedence over the `publish` field
 /// of the package manifest.
-fn registry_indexes(package: &Package, registry: Option<String>) -> anyhow::Result<Vec<Index>> {
+fn registry_indexes(
+    package: &Package,
+    registry: Option<String>,
+) -> anyhow::Result<Vec<CargoIndex>> {
     let registries = registry
         .map(|r| vec![r])
         .unwrap_or_else(|| package.publish.clone().unwrap_or_default());
@@ -310,18 +313,25 @@ fn registry_indexes(package: &Package, registry: Option<String>) -> anyhow::Resu
                 .context("failed to retrieve registry url")
         })
         .collect::<anyhow::Result<Vec<Url>>>()?;
+
     let mut registry_indexes = registry_urls
         .iter()
-        .map(|u| Index::from_url(&format!("registry+{u}")))
-        .collect::<Result<Vec<Index>, crates_index::Error>>()?;
+        .map(|u| {
+            if u.to_string().starts_with("sparse+") {
+                SparseIndex::from_url(u.as_str()).map(CargoIndex::Sparse)
+            } else {
+                Index::from_url(&format!("registry+{u}")).map(CargoIndex::Git)
+            }
+        })
+        .collect::<Result<Vec<CargoIndex>, crates_index::Error>>()?;
     if registry_indexes.is_empty() {
-        registry_indexes.push(Index::new_cargo_default()?)
+        registry_indexes.push(CargoIndex::Git(Index::new_cargo_default()?))
     }
     Ok(registry_indexes)
 }
 
 async fn release_package(
-    index: &mut Index,
+    index: &mut CargoIndex,
     package: &Package,
     input: &ReleaseRequest,
     git_tag: String,
@@ -364,7 +374,7 @@ async fn release_package(
         );
     } else {
         if publish {
-            wait_until_published(index, package)?;
+            wait_until_published(index, package).await?;
         }
 
         repo.tag(&git_tag)?;