From bb7053cce69cee38a4a73bf995ac524a2e1de141 Mon Sep 17 00:00:00 2001 From: Kelwing Date: Tue, 25 Jul 2023 13:50:51 -0400 Subject: [PATCH] feat(release): add support for sparse registry URLs (#863) Co-authored-by: Marco Ieni <11428655+MarcoIeni@users.noreply.github.com> --- Cargo.lock | 17 ++++ Cargo.toml | 3 +- crates/release_plz_core/Cargo.toml | 5 +- crates/release_plz_core/src/cargo.rs | 85 +++++++++++++++---- .../release_plz_core/src/command/release.rs | 28 ++++-- 5 files changed, 112 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1cbcf4b543..3b5f0503d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,6 +165,19 @@ dependencies = [ "futures-core", ] +[[package]] +name = "async-compression" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b74f44609f0f91493e3082d3734d98497e094777144380ea4db9f9905dd5b6" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-trait" version = "0.1.68" @@ -640,6 +653,7 @@ dependencies = [ "git2", "hex", "home", + "http", "memchr", "rayon", "rustc-hash", @@ -3324,6 +3338,7 @@ dependencies = [ "git-cliff-core", "git-url-parse", "git_cmd", + "http", "ignore", "lazy_static", "next_version", @@ -3354,6 +3369,7 @@ version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ + "async-compression", "base64 0.21.2", "bytes", "encoding_rs", @@ -3378,6 +3394,7 @@ dependencies = [ "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-util", "tower-service", "url", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 21f523b8e1..12065339e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,13 +9,14 @@ chrono = { version = "0.4.26", default-features = false } clap = "4.3.19" clap_complete = "4.3.2" conventional_commit_parser = "0.9.4" -crates-index = "0.19.13" +crates-index = { version = "0.19.13", features = ["sparse-http"] } dirs = "5.0.1" dunce = "1.0.4" expect-test = "1.4.1" fake = "2.6.1" git-cliff-core = { version = "1.2.0", default-features = false } git-url-parse = "0.4.4" +http = "0.2" ignore = "0.4.20" lazy_static = "1.4.0" once_cell = "1.18.0" diff --git a/crates/release_plz_core/Cargo.toml b/crates/release_plz_core/Cargo.toml index 4979e06471..b7c956ec47 100644 --- a/crates/release_plz_core/Cargo.toml +++ b/crates/release_plz_core/Cargo.toml @@ -28,7 +28,8 @@ lazy_static.workspace = true parse-changelog.workspace = true rayon.workspace = true regex.workspace = true -reqwest = { workspace = true, features = ["json"] } +# native-tls-alpn is needed for http2 support. https://doc.rust-lang.org/cargo/reference/registry-index.html#sparse-protocol +reqwest = { workspace = true, features = ["json", "gzip", "native-tls-alpn"] } reqwest-middleware.workspace = true reqwest-retry.workspace = true secrecy.workspace = true @@ -41,6 +42,8 @@ walkdir.workspace = true toml_edit.workspace = true serde_json.workspace = true strip-ansi-escapes.workspace = true +tokio.workspace = true +http.workspace = true [dev-dependencies] git_cmd = { path = "../git_cmd", features = ["test_fixture"] } diff --git a/crates/release_plz_core/src/cargo.rs b/crates/release_plz_core/src/cargo.rs index e4123d4683..81fd6bf930 100644 --- a/crates/release_plz_core/src/cargo.rs +++ b/crates/release_plz_core/src/cargo.rs @@ -1,6 +1,6 @@ use anyhow::Context; use cargo_metadata::Package; -use crates_index::Index; +use crates_index::{Crate, Index, SparseIndex}; use tracing::{debug, info}; use std::{ @@ -8,10 +8,14 @@ use std::{ io::{BufRead, BufReader}, path::Path, process::{Command, Stdio}, - thread::sleep, time::{Duration, Instant}, }; +pub enum CargoIndex { + Git(Index), + Sparse(SparseIndex), +} + fn cargo_cmd() -> Command { let cargo = env::var("CARGO").unwrap_or_else(|_| "cargo".to_owned()); Command::new(cargo) @@ -55,9 +59,16 @@ pub fn run_cargo(root: &Path, args: &[&str]) -> anyhow::Result<(String, String)> )) } -pub fn is_published(index: &mut Index, package: &Package) -> anyhow::Result { +pub async fn is_published(index: &mut CargoIndex, package: &Package) -> anyhow::Result { + match index { + CargoIndex::Git(index) => is_published_git(index, package), + CargoIndex::Sparse(index) => is_in_cache_sparse(index, package).await, + } +} + +pub fn is_published_git(index: &mut Index, package: &Package) -> anyhow::Result { // See if we already have the package in cache. - if is_in_cache(index, package) { + if is_in_cache_git(index, package) { return Ok(true); } @@ -65,30 +76,74 @@ pub fn is_published(index: &mut Index, package: &Package) -> anyhow::Result bool { - if let Some(crate_data) = index.crate_(&package.name) { - if crate_data - .versions() - .iter() - .any(|v| v.version() == package.version.to_string()) - { +fn is_in_cache_git(index: &Index, package: &Package) -> bool { + let crate_data = index.crate_(&package.name); + let version = &package.version.to_string(); + is_in_cache(crate_data.as_ref(), version) +} + +async fn is_in_cache_sparse(index: &SparseIndex, package: &Package) -> anyhow::Result { + let crate_data = fetch_sparse_metadata(index, &package.name).await?; + let version = &package.version.to_string(); + Ok(is_in_cache(crate_data.as_ref(), version)) +} + +fn is_in_cache(crate_data: Option<&Crate>, version: &str) -> bool { + if let Some(crate_data) = crate_data { + if is_version_present(version, crate_data) { return true; } } false } -pub fn wait_until_published(index: &mut Index, package: &Package) -> anyhow::Result<()> { +fn is_version_present(version: &str, crate_data: &Crate) -> bool { + crate_data.versions().iter().any(|v| v.version() == version) +} + +async fn fetch_sparse_metadata( + index: &SparseIndex, + crate_name: &str, +) -> anyhow::Result> { + let req = index.make_cache_request(crate_name)?; + let (parts, _) = req.into_parts(); + let req = http::Request::from_parts(parts, vec![]); + + let req: reqwest::Request = req.try_into()?; + + let client = reqwest::ClientBuilder::new() + .gzip(true) + .http2_prior_knowledge() + .build()?; + let res = client.execute(req).await?; + + let mut builder = http::Response::builder() + .status(res.status()) + .version(res.version()); + + if let Some(headers) = builder.headers_mut() { + headers.extend(res.headers().iter().map(|(k, v)| (k.clone(), v.clone()))); + } + + let body = res.bytes().await?; + let res = builder.body(body.to_vec())?; + + let crate_data = index.parse_cache_response(crate_name, res, true)?; + + Ok(crate_data) +} + +pub async fn wait_until_published(index: &mut CargoIndex, package: &Package) -> anyhow::Result<()> { let now = Instant::now(); let sleep_time = Duration::from_secs(2); let timeout = Duration::from_secs(300); let mut logged = false; loop { - if is_published(index, package)? { + if is_published(index, package).await? { break; } else if timeout < now.elapsed() { anyhow::bail!("timeout while publishing {}", package.name) @@ -102,7 +157,7 @@ pub fn wait_until_published(index: &mut Index, package: &Package) -> anyhow::Res logged = true; } - sleep(sleep_time); + tokio::time::sleep(sleep_time).await; } Ok(()) diff --git a/crates/release_plz_core/src/command/release.rs b/crates/release_plz_core/src/command/release.rs index 5d52c07023..fcfdb68095 100644 --- a/crates/release_plz_core/src/command/release.rs +++ b/crates/release_plz_core/src/command/release.rs @@ -5,14 +5,14 @@ use std::{ use anyhow::Context; use cargo_metadata::Package; -use crates_index::Index; +use crates_index::{Index, SparseIndex}; use git_cmd::Repo; use secrecy::{ExposeSecret, SecretString}; use tracing::{info, instrument, warn}; use url::Url; use crate::{ - cargo::{is_published, run_cargo, wait_until_published}, + cargo::{is_published, run_cargo, wait_until_published, CargoIndex}, changelog_parser, git::backend::GitClient, release_order::release_order, @@ -286,7 +286,7 @@ pub async fn release(input: &ReleaseRequest) -> anyhow::Result<()> { } let registry_indexes = registry_indexes(package, input.registry.clone())?; for mut index in registry_indexes { - if is_published(&mut index, package)? { + if is_published(&mut index, package).await? { info!("{} {}: already published", package.name, package.version); continue; } @@ -299,7 +299,10 @@ pub async fn release(input: &ReleaseRequest) -> anyhow::Result<()> { /// Get the indexes where the package should be published. /// If `registry` is specified, it takes precedence over the `publish` field /// of the package manifest. -fn registry_indexes(package: &Package, registry: Option) -> anyhow::Result> { +fn registry_indexes( + package: &Package, + registry: Option, +) -> anyhow::Result> { let registries = registry .map(|r| vec![r]) .unwrap_or_else(|| package.publish.clone().unwrap_or_default()); @@ -310,18 +313,25 @@ fn registry_indexes(package: &Package, registry: Option) -> anyhow::Resu .context("failed to retrieve registry url") }) .collect::>>()?; + let mut registry_indexes = registry_urls .iter() - .map(|u| Index::from_url(&format!("registry+{u}"))) - .collect::, crates_index::Error>>()?; + .map(|u| { + if u.to_string().starts_with("sparse+") { + SparseIndex::from_url(u.as_str()).map(CargoIndex::Sparse) + } else { + Index::from_url(&format!("registry+{u}")).map(CargoIndex::Git) + } + }) + .collect::, crates_index::Error>>()?; if registry_indexes.is_empty() { - registry_indexes.push(Index::new_cargo_default()?) + registry_indexes.push(CargoIndex::Git(Index::new_cargo_default()?)) } Ok(registry_indexes) } async fn release_package( - index: &mut Index, + index: &mut CargoIndex, package: &Package, input: &ReleaseRequest, git_tag: String, @@ -364,7 +374,7 @@ async fn release_package( ); } else { if publish { - wait_until_published(index, package)?; + wait_until_published(index, package).await?; } repo.tag(&git_tag)?;