From 4d16e29371efdedb2b060bbabc8fed0fad33dc59 Mon Sep 17 00:00:00 2001 From: samypr100 <3933065+samypr100@users.noreply.github.com> Date: Mon, 4 Mar 2024 21:02:22 -0500 Subject: [PATCH 1/6] feat: add linehaul info to uv-client --- Cargo.lock | 114 +++++++++++++++ Cargo.toml | 2 + crates/uv-client/Cargo.toml | 4 + crates/uv-client/src/base_client.rs | 20 ++- crates/uv-client/src/lib.rs | 3 + crates/uv-client/src/linehaul.rs | 111 +++++++++++++++ crates/uv-client/src/mac_version.rs | 36 +++++ crates/uv-client/src/registry_client.rs | 13 ++ crates/uv-client/tests/user_agent_version.rs | 141 ++++++++++++++++++- crates/uv/src/commands/pip_compile.rs | 1 + crates/uv/src/commands/pip_install.rs | 1 + crates/uv/src/commands/pip_sync.rs | 1 + crates/uv/src/commands/venv.rs | 1 + 13 files changed, 446 insertions(+), 2 deletions(-) create mode 100644 crates/uv-client/src/linehaul.rs create mode 100644 crates/uv-client/src/mac_version.rs diff --git a/Cargo.lock b/Cargo.lock index 064164b7900d..1afb21f54ef6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -885,6 +885,15 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63dfa964fe2a66f3fde91fc70b267fe193d822c7e603e2a675a49a7f46ad3f49" +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "derivative" version = "2.2.0" @@ -1956,6 +1965,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "line-wrap" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9" +dependencies = [ + "safemem", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2168,6 +2186,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" version = "0.2.18" @@ -2256,6 +2280,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "os_info" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "006e42d5b888366f1880eda20371fedde764ed2213dc8496f49622fa0c99cd5e" +dependencies = [ + "log", + "winapi", +] + [[package]] name = "overload" version = "0.1.1" @@ -2459,6 +2493,20 @@ dependencies = [ "thiserror", ] +[[package]] +name = "plist" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5699cc8a63d1aa2b1ee8e12b9ad70ac790d65788cd36101fa37f87ea46c4cef" +dependencies = [ + "base64 0.21.7", + "indexmap 2.2.5", + "line-wrap", + "quick-xml", + "serde", + "time", +] + [[package]] name = "png" version = "0.17.13" @@ -2487,6 +2535,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2684,6 +2738,15 @@ dependencies = [ "toml", ] +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.35" @@ -3210,6 +3273,12 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +[[package]] +name = "safemem" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + [[package]] name = "same-file" version = "1.0.6" @@ -3547,6 +3616,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sys-info" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b3a0d0aba8bf96a0e1ddfdc352fc53b3df7f39318c71854910c3c4b024ae52c" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -3726,6 +3805,37 @@ dependencies = [ "tikv-jemalloc-sys", ] +[[package]] +name = "time" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-skia" version = "0.8.4" @@ -4361,9 +4471,11 @@ dependencies = [ "hyper 0.14.28", "insta", "install-wheel-rs", + "os_info", "pep440_rs", "pep508_rs", "platform-tags", + "plist", "pypi-types", "reqwest", "reqwest-middleware", @@ -4376,6 +4488,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "sys-info", "task-local-extensions", "tempfile", "thiserror", @@ -4388,6 +4501,7 @@ dependencies = [ "uv-auth", "uv-cache", "uv-fs", + "uv-interpreter", "uv-normalize", "uv-version", "uv-warnings", diff --git a/Cargo.toml b/Cargo.toml index e9f06c4d0aac..e2d9206292f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ owo-colors = { version = "4.0.0" } pathdiff = { version = "0.2.1" } petgraph = { version = "0.6.4" } platform-info = { version = "2.0.2" } +plist = { version = "1.6.0" } pubgrub = { git = "https://github.com/astral-sh/pubgrub", rev = "addbaf184891d66a2dfd93d241a66d13bfe5de86" } pyo3 = { version = "0.20.3" } pyo3-log = { version = "0.9.0" } @@ -89,6 +90,7 @@ serde = { version = "1.0.197" } serde_json = { version = "1.0.114" } sha1 = { version = "0.10.6" } sha2 = { version = "0.10.8" } +sys-info = { version = "0.9.1" } target-lexicon = { version = "0.12.14" } task-local-extensions = { version = "0.1.4" } tempfile = { version = "3.9.0" } diff --git a/crates/uv-client/Cargo.toml b/crates/uv-client/Cargo.toml index c4ed0ca07f1f..2481c0d6f8db 100644 --- a/crates/uv-client/Cargo.toml +++ b/crates/uv-client/Cargo.toml @@ -14,6 +14,7 @@ platform-tags = { path = "../platform-tags" } uv-auth = { path = "../uv-auth" } uv-cache = { path = "../uv-cache" } uv-fs = { path = "../uv-fs", features = ["tokio"] } +uv-interpreter = { path = "../uv-interpreter" } uv-normalize = { path = "../uv-normalize" } uv-version = { path = "../uv-version" } uv-warnings = { path = "../uv-warnings" } @@ -28,6 +29,7 @@ fs-err = { workspace = true, features = ["tokio"] } futures = { workspace = true } html-escape = { workspace = true } http = { workspace = true } +plist = { workspace = true } reqwest = { workspace = true } reqwest-middleware = { workspace = true } reqwest-retry = { workspace = true } @@ -37,6 +39,7 @@ rustc-hash = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sha2 = { workspace = true } +sys-info = { workspace = true } task-local-extensions = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } @@ -56,4 +59,5 @@ webpki-roots = { version = "0.25.4" } anyhow = { workspace = true } hyper = { version = "0.14.28", features = ["server", "http1"] } insta = { version = "1.36.1" } +os_info = { version = "3.7.0", default-features = false } tokio = { workspace = true, features = ["fs", "macros"] } diff --git a/crates/uv-client/src/base_client.rs b/crates/uv-client/src/base_client.rs index b0fab037f42e..f7e9787ae491 100644 --- a/crates/uv-client/src/base_client.rs +++ b/crates/uv-client/src/base_client.rs @@ -1,3 +1,4 @@ +use pep508_rs::MarkerEnvironment; use reqwest::{Client, ClientBuilder}; use reqwest_middleware::ClientWithMiddleware; use reqwest_retry::policies::ExponentialBackoff; @@ -12,6 +13,7 @@ use uv_fs::Simplified; use uv_version::version; use uv_warnings::warn_user_once; +use crate::linehaul::LineHaul; use crate::middleware::OfflineMiddleware; use crate::tls::Roots; use crate::{tls, Connectivity}; @@ -24,6 +26,7 @@ pub struct BaseClientBuilder { retries: u32, connectivity: Connectivity, client: Option, + markers: Option, } impl BaseClientBuilder { @@ -34,6 +37,7 @@ impl BaseClientBuilder { connectivity: Connectivity::Online, retries: 3, client: None, + markers: None, } } } @@ -69,9 +73,23 @@ impl BaseClientBuilder { self } + #[must_use] + pub fn markers(mut self, markers: MarkerEnvironment) -> Self { + self.markers = Some(markers); + self + } + pub fn build(self) -> BaseClient { // Create user agent. - let user_agent_string = format!("uv/{}", version()); + let mut user_agent_string = format!("uv/{}", version()); + + // Add linehaul metadata. + if let Some(markers) = self.markers { + let linehaul = LineHaul::new(markers); + if let Ok(output) = serde_json::to_string(&linehaul) { + user_agent_string += &format!(" {}", output); + } + } // Timeout options, matching https://doc.rust-lang.org/nightly/cargo/reference/config.html#httptimeout // `UV_REQUEST_TIMEOUT` is provided for backwards compatibility with v0.1.6 diff --git a/crates/uv-client/src/lib.rs b/crates/uv-client/src/lib.rs index c74b92ac57db..41c2bc92e31f 100644 --- a/crates/uv-client/src/lib.rs +++ b/crates/uv-client/src/lib.rs @@ -2,6 +2,7 @@ pub use base_client::BaseClient; pub use cached_client::{CacheControl, CachedClient, CachedClientError, DataWithCachePolicy}; pub use error::{BetterReqwestError, Error, ErrorKind}; pub use flat_index::{FlatDistributions, FlatIndex, FlatIndexClient, FlatIndexError}; +pub use linehaul::LineHaul; pub use registry_client::{ Connectivity, RegistryClient, RegistryClientBuilder, SimpleMetadata, SimpleMetadatum, VersionFiles, @@ -14,6 +15,8 @@ mod error; mod flat_index; mod html; mod httpcache; +mod linehaul; +mod mac_version; mod middleware; mod registry_client; mod remote_metadata; diff --git a/crates/uv-client/src/linehaul.rs b/crates/uv-client/src/linehaul.rs new file mode 100644 index 000000000000..2c7bb009f9ac --- /dev/null +++ b/crates/uv-client/src/linehaul.rs @@ -0,0 +1,111 @@ +use std::env; + +use serde::{Deserialize, Serialize}; + +use pep508_rs::MarkerEnvironment; +use uv_version::version; + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct Installer { + pub name: Option, + pub version: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct Implementation { + pub name: Option, + pub version: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct Libc { + pub lib: Option, + pub version: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct Distro { + pub name: Option, + pub version: Option, + pub id: Option, + pub libc: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct System { + pub name: Option, + pub release: Option, +} + +/// Linehaul structs were derived from +/// . +/// For the sake of parity, the nullability of all the values was kept intact. +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct LineHaul { + pub installer: Option, + pub python: Option, + pub implementation: Option, + pub distro: Option, + pub system: Option, + pub cpu: Option, + pub openssl_version: Option, + pub setuptools_version: Option, + pub rustc_version: Option, + pub ci: Option, +} + +/// Implements Linehaul information format as defined by +/// . +/// This metadata is added to the user agent to enrich PyPI statistics. +impl LineHaul { + /// Initializes Linehaul information based on PEP 508 markers. + pub fn new(markers: MarkerEnvironment) -> Self { + // https://github.com/pypa/pip/blob/24.0/src/pip/_internal/network/session.py#L87 + let looks_like_ci = ["BUILD_BUILDID", "BUILD_ID", "CI", "PIP_IS_CI"] + .iter() + .find_map(|&var_name| env::var(var_name).ok().map(|_| true)); + + // Build Distro as Linehaul expects. + let distro: Option = if cfg!(target_os = "linux") { + // Gather distribution info from /etc/os-release. + sys_info::linux_os_release().ok().map(|info| Distro { + id: info.version_codename, // e.g., Jammy, Focal, etc. + name: info.name, // e.g., Ubuntu, Fedora, etc. + version: info.version_id, // e.g., 22.04, etc. + libc: None, // Skip in uv, likely way too slow. + }) + } else if cfg!(target_os = "macos") { + Some(Distro { + id: None, // N/A + name: Some("macOS".to_string()), // pip hardcodes distro name to macOS. + version: crate::mac_version::get_mac_os_version().ok(), // Same as python's platform.mac_ver[0]. + libc: None, // N/A + }) + } else { + // Always empty on Windows. + None + }; + + Self { + installer: Option::from(Installer { + name: Some("uv".to_string()), + version: Some(version().to_string()), + }), + python: Some(markers.python_full_version.version.to_string()), + implementation: Option::from(Implementation { + name: Some(markers.platform_python_implementation.to_string()), + version: Some(markers.python_full_version.version.to_string()), + }), + distro, + system: Option::from(System { + name: Some(markers.platform_system.to_string()), + release: Some(markers.platform_release.to_string()), + }), + cpu: Some(markers.platform_machine.to_string()), + openssl_version: None, // Should probably always be None in uv. + setuptools_version: None, // Should probably always be None in uv. + rustc_version: None, // Calling rustc --version is likely too slow. + ci: looks_like_ci, + } + } +} diff --git a/crates/uv-client/src/mac_version.rs b/crates/uv-client/src/mac_version.rs new file mode 100644 index 000000000000..33558b730537 --- /dev/null +++ b/crates/uv-client/src/mac_version.rs @@ -0,0 +1,36 @@ +use platform_tags::PlatformError; +use serde::Deserialize; + +/// Get the macOS version from the SystemVersion.plist file. +pub(crate) fn get_mac_os_version() -> Result { + // This is actually what python does + // https://github.com/python/cpython/blob/cb2b3c8d3566ae46b3b8d0718019e1c98484589e/Lib/platform.py#L409-L428 + #[derive(Deserialize)] + #[serde(rename_all = "PascalCase")] + struct SystemVersion { + product_version: String, + } + let system_version: SystemVersion = + plist::from_file("/System/Library/CoreServices/SystemVersion.plist") + .map_err(|err| PlatformError::OsVersionDetectionError(err.to_string()))?; + + let invalid_mac_os_version = || { + PlatformError::OsVersionDetectionError(format!( + "Invalid macOS version {}", + system_version.product_version + )) + }; + match system_version + .product_version + .split('.') + .collect::>() + .as_slice() + { + [major, minor] | [major, minor, _] => { + let _major = major.parse::().map_err(|_| invalid_mac_os_version())?; + let _minor = minor.parse::().map_err(|_| invalid_mac_os_version())?; + Ok(system_version.product_version) + } + _ => Err(invalid_mac_os_version()), + } +} diff --git a/crates/uv-client/src/registry_client.rs b/crates/uv-client/src/registry_client.rs index 23e500bb381c..ee4c02428814 100644 --- a/crates/uv-client/src/registry_client.rs +++ b/crates/uv-client/src/registry_client.rs @@ -17,6 +17,7 @@ use distribution_filename::{DistFilename, SourceDistFilename, WheelFilename}; use distribution_types::{BuiltDist, File, FileLocation, IndexUrl, IndexUrls, Name}; use install_wheel_rs::metadata::{find_archive_dist_info, is_metadata_entry}; use pep440_rs::Version; +use pep508_rs::MarkerEnvironment; use pypi_types::{Metadata23, SimpleJson}; use uv_auth::KeyringProvider; use uv_cache::{Cache, CacheBucket, WheelCache}; @@ -39,6 +40,7 @@ pub struct RegistryClientBuilder { connectivity: Connectivity, cache: Cache, client: Option, + markers: Option, } impl RegistryClientBuilder { @@ -51,6 +53,7 @@ impl RegistryClientBuilder { connectivity: Connectivity::Online, retries: 3, client: None, + markers: None, } } } @@ -98,6 +101,12 @@ impl RegistryClientBuilder { self } + #[must_use] + pub fn markers(mut self, markers: MarkerEnvironment) -> Self { + self.markers = Some(markers); + self + } + pub fn build(self) -> RegistryClient { // Build a base client let mut builder = BaseClientBuilder::new(); @@ -106,6 +115,10 @@ impl RegistryClientBuilder { builder = builder.client(client) } + if let Some(markers) = self.markers { + builder = builder.markers(markers) + } + let client = builder .retries(self.retries) .connectivity(self.connectivity) diff --git a/crates/uv-client/tests/user_agent_version.rs b/crates/uv-client/tests/user_agent_version.rs index 68d7eefe6adf..0c2e790edf97 100644 --- a/crates/uv-client/tests/user_agent_version.rs +++ b/crates/uv-client/tests/user_agent_version.rs @@ -4,9 +4,10 @@ use hyper::header::USER_AGENT; use hyper::server::conn::Http; use hyper::service::service_fn; use hyper::{Body, Request, Response}; +use pep508_rs::{MarkerEnvironment, StringVersion}; use tokio::net::TcpListener; - use uv_cache::Cache; +use uv_client::LineHaul; use uv_client::RegistryClientBuilder; use uv_version::version; @@ -60,3 +61,141 @@ async fn test_user_agent_has_version() -> Result<()> { Ok(()) } + +#[tokio::test] +async fn test_user_agent_has_linehaul() -> Result<()> { + // Set up the TCP listener on a random available port + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + + // Spawn the server loop in a background task + tokio::spawn(async move { + let svc = service_fn(move |req: Request| { + // Get User Agent Header and send it back in the response + let user_agent = req + .headers() + .get(USER_AGENT) + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()) + .unwrap_or_default(); // Empty Default + future::ok::<_, hyper::Error>(Response::new(Body::from(user_agent))) + }); + // Start Hyper Server + let (socket, _) = listener.accept().await.unwrap(); + Http::new() + .http1_keep_alive(false) + .serve_connection(socket, svc) + .with_upgrades() + .await + .expect("Server Started"); + }); + + // Add some representative markers for an Ubuntu CI runner + let markers = MarkerEnvironment { + implementation_name: "cpython".to_string(), + implementation_version: StringVersion { + string: "3.12.2".to_string(), + version: "3.12.2".parse()?, + }, + os_name: "posix".to_string(), + platform_machine: "x86_64".to_string(), + platform_python_implementation: "CPython".to_string(), + platform_release: "6.5.0-1016-azure".to_string(), + platform_system: "Linux".to_string(), + platform_version: "#16~22.04.1-Ubuntu SMP Fri Feb 16 15:42:02 UTC 2024".to_string(), + python_full_version: StringVersion { + string: "3.12.2".to_string(), + version: "3.12.2".parse()?, + }, + python_version: StringVersion { + string: "3.12".to_string(), + version: "3.12".parse()?, + }, + sys_platform: "linux".to_string(), + }; + + // Initialize uv-client + let cache = Cache::temp()?; + let client = RegistryClientBuilder::new(cache) + .markers(markers.clone()) + .build(); + + // Send request to our dummy server + let res = client + .uncached_client() + .get(format!("http://{addr}")) + .send() + .await?; + + // Check the HTTP status + assert!(res.status().is_success()); + + // Check User Agent + let body = res.text().await?; + + // Unpack User-Agent with linehaul + let (uv_version, uv_linehaul) = body.split_once(' ').expect("Failed to split User-Agent."); + + // Deserializing Linehaul + let linehaul: LineHaul = serde_json::from_str(uv_linehaul)?; + + // Assert uv version + assert_eq!(uv_version, format!("uv/{}", version())); + + // Assert linehaul + let installer_info = linehaul.installer.unwrap(); + let system_info = linehaul.system.unwrap(); + let impl_info = linehaul.implementation.unwrap(); + + assert_eq!(installer_info.name.unwrap(), "uv".to_string()); + assert_eq!(installer_info.version.unwrap(), version()); + + assert_eq!(system_info.name.unwrap(), markers.platform_system); + assert_eq!(system_info.release.unwrap(), markers.platform_release); + + assert_eq!( + impl_info.name.unwrap(), + markers.platform_python_implementation + ); + assert_eq!( + impl_info.version.unwrap(), + markers.python_full_version.version.to_string() + ); + + assert_eq!( + linehaul.python.unwrap(), + markers.python_full_version.version.to_string() + ); + assert_eq!(linehaul.cpu.unwrap(), markers.platform_machine); + + assert_eq!(linehaul.openssl_version, None); + assert_eq!(linehaul.setuptools_version, None); + assert_eq!(linehaul.rustc_version, None); + + #[cfg(windows)] + assert_eq!(linehaul.distro, None); + + // Using os_info as to confirm our values are as expected in both Linux and OSX. + #[cfg(target_os = "linux")] + { + let info = os_info::get(); + let distro_info = linehaul.distro.unwrap(); + assert_eq!(distro_info.id.unwrap(), info.codename().unwrap()); + assert_eq!(distro_info.name.unwrap(), info.os_type().to_string()); + assert_eq!(distro_info.version.unwrap(), info.version().to_string()); + assert_eq!(distro_info.libc, None); + } + + // Using os_info as sys-info yields Darwin version, and not mac release version. + #[cfg(target_os = "macos")] + { + let info = os_info::get(); + let distro_info = linehaul.distro.unwrap(); + assert_eq!(distro_info.id, None); + assert_eq!(distro_info.name.unwrap(), "macOS"); + assert_eq!(distro_info.version.unwrap(), info.version().to_string()); + assert_eq!(distro_info.libc, None); + } + + Ok(()) +} diff --git a/crates/uv/src/commands/pip_compile.rs b/crates/uv/src/commands/pip_compile.rs index d30e2b2bc514..00d89e590305 100644 --- a/crates/uv/src/commands/pip_compile.rs +++ b/crates/uv/src/commands/pip_compile.rs @@ -198,6 +198,7 @@ pub(crate) async fn pip_compile( .connectivity(connectivity) .index_urls(index_locations.index_urls()) .keyring_provider(keyring_provider) + .markers(interpreter.markers().clone()) .build(); // Resolve the flat indexes from `--find-links`. diff --git a/crates/uv/src/commands/pip_install.rs b/crates/uv/src/commands/pip_install.rs index 0e7ed86009e6..2a5f2f424f23 100644 --- a/crates/uv/src/commands/pip_install.rs +++ b/crates/uv/src/commands/pip_install.rs @@ -192,6 +192,7 @@ pub(crate) async fn pip_install( .connectivity(connectivity) .index_urls(index_locations.index_urls()) .keyring_provider(keyring_provider) + .markers(interpreter.markers().clone()) .build(); // Resolve the flat indexes from `--find-links`. diff --git a/crates/uv/src/commands/pip_sync.rs b/crates/uv/src/commands/pip_sync.rs index 6a4b5d2bea4e..b2d87e81205a 100644 --- a/crates/uv/src/commands/pip_sync.rs +++ b/crates/uv/src/commands/pip_sync.rs @@ -126,6 +126,7 @@ pub(crate) async fn pip_sync( .connectivity(connectivity) .index_urls(index_locations.index_urls()) .keyring_provider(keyring_provider) + .markers(venv.interpreter().markers().clone()) .build(); // Resolve the flat indexes from `--find-links`. diff --git a/crates/uv/src/commands/venv.rs b/crates/uv/src/commands/venv.rs index caa3e63d860c..e515a9b99455 100644 --- a/crates/uv/src/commands/venv.rs +++ b/crates/uv/src/commands/venv.rs @@ -151,6 +151,7 @@ async fn venv_impl( .index_urls(index_locations.index_urls()) .keyring_provider(keyring_provider) .connectivity(connectivity) + .markers(interpreter.markers().clone()) .build(); // Resolve the flat indexes from `--find-links`. From 16597768067067cd151a5a1d6d722979495566b5 Mon Sep 17 00:00:00 2001 From: samypr100 <3933065+samypr100@users.noreply.github.com> Date: Sun, 17 Mar 2024 13:51:10 -0400 Subject: [PATCH 2/6] chore: re-trigger ci --- crates/uv-client/tests/user_agent_version.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/uv-client/tests/user_agent_version.rs b/crates/uv-client/tests/user_agent_version.rs index 0c2e790edf97..82ed8ef36d88 100644 --- a/crates/uv-client/tests/user_agent_version.rs +++ b/crates/uv-client/tests/user_agent_version.rs @@ -134,7 +134,9 @@ async fn test_user_agent_has_linehaul() -> Result<()> { let body = res.text().await?; // Unpack User-Agent with linehaul - let (uv_version, uv_linehaul) = body.split_once(' ').expect("Failed to split User-Agent."); + let (uv_version, uv_linehaul) = body + .split_once(' ') + .expect("Failed to split User-Agent header."); // Deserializing Linehaul let linehaul: LineHaul = serde_json::from_str(uv_linehaul)?; From 097876d40510de242729a6e7b124e8cc76a9fbc8 Mon Sep 17 00:00:00 2001 From: samypr100 <3933065+samypr100@users.noreply.github.com> Date: Sun, 17 Mar 2024 22:00:28 -0400 Subject: [PATCH 3/6] nit: avoid clone on markers --- crates/uv-client/src/base_client.rs | 10 +++++----- crates/uv-client/src/linehaul.rs | 2 +- crates/uv-client/src/registry_client.rs | 10 +++++----- crates/uv-client/tests/user_agent_version.rs | 4 +--- crates/uv/src/commands/pip_compile.rs | 2 +- crates/uv/src/commands/pip_install.rs | 2 +- crates/uv/src/commands/pip_sync.rs | 2 +- crates/uv/src/commands/venv.rs | 2 +- 8 files changed, 16 insertions(+), 18 deletions(-) diff --git a/crates/uv-client/src/base_client.rs b/crates/uv-client/src/base_client.rs index f7e9787ae491..19ca3e3ff041 100644 --- a/crates/uv-client/src/base_client.rs +++ b/crates/uv-client/src/base_client.rs @@ -20,16 +20,16 @@ use crate::{tls, Connectivity}; /// A builder for an [`RegistryClient`]. #[derive(Debug, Clone)] -pub struct BaseClientBuilder { +pub struct BaseClientBuilder<'a> { keyring_provider: KeyringProvider, native_tls: bool, retries: u32, connectivity: Connectivity, client: Option, - markers: Option, + markers: Option<&'a MarkerEnvironment>, } -impl BaseClientBuilder { +impl BaseClientBuilder<'_> { pub fn new() -> Self { Self { keyring_provider: KeyringProvider::default(), @@ -42,7 +42,7 @@ impl BaseClientBuilder { } } -impl BaseClientBuilder { +impl<'a> BaseClientBuilder<'a> { #[must_use] pub fn keyring_provider(mut self, keyring_provider: KeyringProvider) -> Self { self.keyring_provider = keyring_provider; @@ -74,7 +74,7 @@ impl BaseClientBuilder { } #[must_use] - pub fn markers(mut self, markers: MarkerEnvironment) -> Self { + pub fn markers(mut self, markers: &'a MarkerEnvironment) -> Self { self.markers = Some(markers); self } diff --git a/crates/uv-client/src/linehaul.rs b/crates/uv-client/src/linehaul.rs index 2c7bb009f9ac..8543b1ce70d8 100644 --- a/crates/uv-client/src/linehaul.rs +++ b/crates/uv-client/src/linehaul.rs @@ -59,7 +59,7 @@ pub struct LineHaul { /// This metadata is added to the user agent to enrich PyPI statistics. impl LineHaul { /// Initializes Linehaul information based on PEP 508 markers. - pub fn new(markers: MarkerEnvironment) -> Self { + pub fn new(markers: &MarkerEnvironment) -> Self { // https://github.com/pypa/pip/blob/24.0/src/pip/_internal/network/session.py#L87 let looks_like_ci = ["BUILD_BUILDID", "BUILD_ID", "CI", "PIP_IS_CI"] .iter() diff --git a/crates/uv-client/src/registry_client.rs b/crates/uv-client/src/registry_client.rs index ee4c02428814..1ef56b394bb3 100644 --- a/crates/uv-client/src/registry_client.rs +++ b/crates/uv-client/src/registry_client.rs @@ -32,7 +32,7 @@ use crate::{CachedClient, CachedClientError, Error, ErrorKind}; /// A builder for an [`RegistryClient`]. #[derive(Debug, Clone)] -pub struct RegistryClientBuilder { +pub struct RegistryClientBuilder<'a> { index_urls: IndexUrls, keyring_provider: KeyringProvider, native_tls: bool, @@ -40,10 +40,10 @@ pub struct RegistryClientBuilder { connectivity: Connectivity, cache: Cache, client: Option, - markers: Option, + markers: Option<&'a MarkerEnvironment>, } -impl RegistryClientBuilder { +impl RegistryClientBuilder<'_> { pub fn new(cache: Cache) -> Self { Self { index_urls: IndexUrls::default(), @@ -58,7 +58,7 @@ impl RegistryClientBuilder { } } -impl RegistryClientBuilder { +impl<'a> RegistryClientBuilder<'a> { #[must_use] pub fn index_urls(mut self, index_urls: IndexUrls) -> Self { self.index_urls = index_urls; @@ -102,7 +102,7 @@ impl RegistryClientBuilder { } #[must_use] - pub fn markers(mut self, markers: MarkerEnvironment) -> Self { + pub fn markers(mut self, markers: &'a MarkerEnvironment) -> Self { self.markers = Some(markers); self } diff --git a/crates/uv-client/tests/user_agent_version.rs b/crates/uv-client/tests/user_agent_version.rs index 82ed8ef36d88..38e1b4d13391 100644 --- a/crates/uv-client/tests/user_agent_version.rs +++ b/crates/uv-client/tests/user_agent_version.rs @@ -116,9 +116,7 @@ async fn test_user_agent_has_linehaul() -> Result<()> { // Initialize uv-client let cache = Cache::temp()?; - let client = RegistryClientBuilder::new(cache) - .markers(markers.clone()) - .build(); + let client = RegistryClientBuilder::new(cache).markers(&markers).build(); // Send request to our dummy server let res = client diff --git a/crates/uv/src/commands/pip_compile.rs b/crates/uv/src/commands/pip_compile.rs index 00d89e590305..0b0e9c7b764e 100644 --- a/crates/uv/src/commands/pip_compile.rs +++ b/crates/uv/src/commands/pip_compile.rs @@ -198,7 +198,7 @@ pub(crate) async fn pip_compile( .connectivity(connectivity) .index_urls(index_locations.index_urls()) .keyring_provider(keyring_provider) - .markers(interpreter.markers().clone()) + .markers(&markers) .build(); // Resolve the flat indexes from `--find-links`. diff --git a/crates/uv/src/commands/pip_install.rs b/crates/uv/src/commands/pip_install.rs index 2a5f2f424f23..5effd07a00cf 100644 --- a/crates/uv/src/commands/pip_install.rs +++ b/crates/uv/src/commands/pip_install.rs @@ -192,7 +192,7 @@ pub(crate) async fn pip_install( .connectivity(connectivity) .index_urls(index_locations.index_urls()) .keyring_provider(keyring_provider) - .markers(interpreter.markers().clone()) + .markers(markers) .build(); // Resolve the flat indexes from `--find-links`. diff --git a/crates/uv/src/commands/pip_sync.rs b/crates/uv/src/commands/pip_sync.rs index b2d87e81205a..714ea20d4c51 100644 --- a/crates/uv/src/commands/pip_sync.rs +++ b/crates/uv/src/commands/pip_sync.rs @@ -126,7 +126,7 @@ pub(crate) async fn pip_sync( .connectivity(connectivity) .index_urls(index_locations.index_urls()) .keyring_provider(keyring_provider) - .markers(venv.interpreter().markers().clone()) + .markers(venv.interpreter().markers()) .build(); // Resolve the flat indexes from `--find-links`. diff --git a/crates/uv/src/commands/venv.rs b/crates/uv/src/commands/venv.rs index e515a9b99455..c1045e50fbac 100644 --- a/crates/uv/src/commands/venv.rs +++ b/crates/uv/src/commands/venv.rs @@ -151,7 +151,7 @@ async fn venv_impl( .index_urls(index_locations.index_urls()) .keyring_provider(keyring_provider) .connectivity(connectivity) - .markers(interpreter.markers().clone()) + .markers(interpreter.markers()) .build(); // Resolve the flat indexes from `--find-links`. From ac49eda97848728ac55ca452853b2bbd6ac689d0 Mon Sep 17 00:00:00 2001 From: konstin Date: Mon, 18 Mar 2024 11:15:41 +0100 Subject: [PATCH 4/6] Formatting --- crates/uv-client/src/linehaul.rs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/crates/uv-client/src/linehaul.rs b/crates/uv-client/src/linehaul.rs index 8543b1ce70d8..f475801e2610 100644 --- a/crates/uv-client/src/linehaul.rs +++ b/crates/uv-client/src/linehaul.rs @@ -69,17 +69,25 @@ impl LineHaul { let distro: Option = if cfg!(target_os = "linux") { // Gather distribution info from /etc/os-release. sys_info::linux_os_release().ok().map(|info| Distro { - id: info.version_codename, // e.g., Jammy, Focal, etc. - name: info.name, // e.g., Ubuntu, Fedora, etc. - version: info.version_id, // e.g., 22.04, etc. - libc: None, // Skip in uv, likely way too slow. + // e.g., Jammy, Focal, etc. + id: info.version_codename, + // e.g., Ubuntu, Fedora, etc. + name: info.name, + // e.g., 22.04, etc. + version: info.version_id, + // Skip in uv, likely way too slow. + libc: None, }) } else if cfg!(target_os = "macos") { Some(Distro { - id: None, // N/A - name: Some("macOS".to_string()), // pip hardcodes distro name to macOS. - version: crate::mac_version::get_mac_os_version().ok(), // Same as python's platform.mac_ver[0]. - libc: None, // N/A + // N/A + id: None, + // pip hardcodes distro name to macOS. + name: Some("macOS".to_string()), + // Same as python's platform.mac_ver[0]. + version: crate::mac_version::get_mac_os_version().ok(), + // N/A + libc: None, }) } else { // Always empty on Windows. From b8ceaed6ef617a881961294321b5b6c399ca2860 Mon Sep 17 00:00:00 2001 From: konstin Date: Mon, 18 Mar 2024 11:26:21 +0100 Subject: [PATCH 5/6] Add libc support --- crates/uv-client/src/base_client.rs | 11 ++++++++++- crates/uv-client/src/linehaul.rs | 19 ++++++++++++++++--- crates/uv-client/src/registry_client.rs | 13 +++++++++++++ crates/uv-client/tests/user_agent_version.rs | 18 ++++++++++++++++-- crates/uv/src/commands/pip_compile.rs | 1 + 5 files changed, 56 insertions(+), 6 deletions(-) diff --git a/crates/uv-client/src/base_client.rs b/crates/uv-client/src/base_client.rs index 19ca3e3ff041..1420626dea8e 100644 --- a/crates/uv-client/src/base_client.rs +++ b/crates/uv-client/src/base_client.rs @@ -1,4 +1,5 @@ use pep508_rs::MarkerEnvironment; +use platform_tags::Platform; use reqwest::{Client, ClientBuilder}; use reqwest_middleware::ClientWithMiddleware; use reqwest_retry::policies::ExponentialBackoff; @@ -27,6 +28,7 @@ pub struct BaseClientBuilder<'a> { connectivity: Connectivity, client: Option, markers: Option<&'a MarkerEnvironment>, + platform: Option<&'a Platform>, } impl BaseClientBuilder<'_> { @@ -38,6 +40,7 @@ impl BaseClientBuilder<'_> { retries: 3, client: None, markers: None, + platform: None, } } } @@ -79,13 +82,19 @@ impl<'a> BaseClientBuilder<'a> { self } + #[must_use] + pub fn platform(mut self, platform: &'a Platform) -> Self { + self.platform = Some(platform); + self + } + pub fn build(self) -> BaseClient { // Create user agent. let mut user_agent_string = format!("uv/{}", version()); // Add linehaul metadata. if let Some(markers) = self.markers { - let linehaul = LineHaul::new(markers); + let linehaul = LineHaul::new(markers, self.platform); if let Ok(output) = serde_json::to_string(&linehaul) { user_agent_string += &format!(" {}", output); } diff --git a/crates/uv-client/src/linehaul.rs b/crates/uv-client/src/linehaul.rs index f475801e2610..58b8b32f20c1 100644 --- a/crates/uv-client/src/linehaul.rs +++ b/crates/uv-client/src/linehaul.rs @@ -3,6 +3,7 @@ use std::env; use serde::{Deserialize, Serialize}; use pep508_rs::MarkerEnvironment; +use platform_tags::{Os, Platform}; use uv_version::version; #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] @@ -59,12 +60,24 @@ pub struct LineHaul { /// This metadata is added to the user agent to enrich PyPI statistics. impl LineHaul { /// Initializes Linehaul information based on PEP 508 markers. - pub fn new(markers: &MarkerEnvironment) -> Self { + pub fn new(markers: &MarkerEnvironment, platform: Option<&Platform>) -> Self { // https://github.com/pypa/pip/blob/24.0/src/pip/_internal/network/session.py#L87 let looks_like_ci = ["BUILD_BUILDID", "BUILD_ID", "CI", "PIP_IS_CI"] .iter() .find_map(|&var_name| env::var(var_name).ok().map(|_| true)); + let libc = match platform.map(|platform| platform.os()) { + Some(Os::Manylinux { major, minor }) => Some(Libc { + lib: Some("glibc".to_string()), + version: Some(format!("{major}.{minor}")), + }), + Some(Os::Musllinux { major, minor }) => Some(Libc { + lib: Some("musl".to_string()), + version: Some(format!("{major}.{minor}")), + }), + _ => None, + }; + // Build Distro as Linehaul expects. let distro: Option = if cfg!(target_os = "linux") { // Gather distribution info from /etc/os-release. @@ -75,8 +88,8 @@ impl LineHaul { name: info.name, // e.g., 22.04, etc. version: info.version_id, - // Skip in uv, likely way too slow. - libc: None, + // e.g., glibc 2.38, musl 1.2 + libc, }) } else if cfg!(target_os = "macos") { Some(Distro { diff --git a/crates/uv-client/src/registry_client.rs b/crates/uv-client/src/registry_client.rs index 1ef56b394bb3..e448f4031d84 100644 --- a/crates/uv-client/src/registry_client.rs +++ b/crates/uv-client/src/registry_client.rs @@ -18,6 +18,7 @@ use distribution_types::{BuiltDist, File, FileLocation, IndexUrl, IndexUrls, Nam use install_wheel_rs::metadata::{find_archive_dist_info, is_metadata_entry}; use pep440_rs::Version; use pep508_rs::MarkerEnvironment; +use platform_tags::Platform; use pypi_types::{Metadata23, SimpleJson}; use uv_auth::KeyringProvider; use uv_cache::{Cache, CacheBucket, WheelCache}; @@ -41,6 +42,7 @@ pub struct RegistryClientBuilder<'a> { cache: Cache, client: Option, markers: Option<&'a MarkerEnvironment>, + platform: Option<&'a Platform>, } impl RegistryClientBuilder<'_> { @@ -54,6 +56,7 @@ impl RegistryClientBuilder<'_> { retries: 3, client: None, markers: None, + platform: None, } } } @@ -107,6 +110,12 @@ impl<'a> RegistryClientBuilder<'a> { self } + #[must_use] + pub fn platform(mut self, platform: &'a Platform) -> Self { + self.platform = Some(platform); + self + } + pub fn build(self) -> RegistryClient { // Build a base client let mut builder = BaseClientBuilder::new(); @@ -119,6 +128,10 @@ impl<'a> RegistryClientBuilder<'a> { builder = builder.markers(markers) } + if let Some(platform) = self.platform { + builder = builder.platform(platform) + } + let client = builder .retries(self.retries) .connectivity(self.connectivity) diff --git a/crates/uv-client/tests/user_agent_version.rs b/crates/uv-client/tests/user_agent_version.rs index 38e1b4d13391..ea4fb7d9581d 100644 --- a/crates/uv-client/tests/user_agent_version.rs +++ b/crates/uv-client/tests/user_agent_version.rs @@ -5,6 +5,7 @@ use hyper::server::conn::Http; use hyper::service::service_fn; use hyper::{Body, Request, Response}; use pep508_rs::{MarkerEnvironment, StringVersion}; +use platform_tags::{Arch, Os, Platform}; use tokio::net::TcpListener; use uv_cache::Cache; use uv_client::LineHaul; @@ -113,10 +114,23 @@ async fn test_user_agent_has_linehaul() -> Result<()> { }, sys_platform: "linux".to_string(), }; + // Linux only + let platform = Platform::new( + Os::Manylinux { + major: 2, + minor: 38, + }, + Arch::X86_64, + ); // Initialize uv-client let cache = Cache::temp()?; - let client = RegistryClientBuilder::new(cache).markers(&markers).build(); + let mut builder = RegistryClientBuilder::new(cache).markers(&markers); + + if cfg!(target_os = "linux") { + builder = builder.platform(&platform); + } + let client = builder.build(); // Send request to our dummy server let res = client @@ -183,7 +197,7 @@ async fn test_user_agent_has_linehaul() -> Result<()> { assert_eq!(distro_info.id.unwrap(), info.codename().unwrap()); assert_eq!(distro_info.name.unwrap(), info.os_type().to_string()); assert_eq!(distro_info.version.unwrap(), info.version().to_string()); - assert_eq!(distro_info.libc, None); + assert!(distro_info.libc.is_some()); } // Using os_info as sys-info yields Darwin version, and not mac release version. diff --git a/crates/uv/src/commands/pip_compile.rs b/crates/uv/src/commands/pip_compile.rs index 0b0e9c7b764e..8afd94c855d2 100644 --- a/crates/uv/src/commands/pip_compile.rs +++ b/crates/uv/src/commands/pip_compile.rs @@ -199,6 +199,7 @@ pub(crate) async fn pip_compile( .index_urls(index_locations.index_urls()) .keyring_provider(keyring_provider) .markers(&markers) + .platform(interpreter.platform()) .build(); // Resolve the flat indexes from `--find-links`. From c2d7b2ef49692f25ca65282f531733a9691dffa5 Mon Sep 17 00:00:00 2001 From: konstin Date: Mon, 18 Mar 2024 11:33:16 +0100 Subject: [PATCH 6/6] Add instrumentation --- crates/uv-client/src/linehaul.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/uv-client/src/linehaul.rs b/crates/uv-client/src/linehaul.rs index 58b8b32f20c1..415333ffd09c 100644 --- a/crates/uv-client/src/linehaul.rs +++ b/crates/uv-client/src/linehaul.rs @@ -1,6 +1,7 @@ use std::env; use serde::{Deserialize, Serialize}; +use tracing::instrument; use pep508_rs::MarkerEnvironment; use platform_tags::{Os, Platform}; @@ -60,6 +61,7 @@ pub struct LineHaul { /// This metadata is added to the user agent to enrich PyPI statistics. impl LineHaul { /// Initializes Linehaul information based on PEP 508 markers. + #[instrument(name = "linehaul", skip_all)] pub fn new(markers: &MarkerEnvironment, platform: Option<&Platform>) -> Self { // https://github.com/pypa/pip/blob/24.0/src/pip/_internal/network/session.py#L87 let looks_like_ci = ["BUILD_BUILDID", "BUILD_ID", "CI", "PIP_IS_CI"]