diff --git a/crates/distribution-types/src/index_url.rs b/crates/distribution-types/src/index_url.rs index edfdfea3dd74..5de44451e61e 100644 --- a/crates/distribution-types/src/index_url.rs +++ b/crates/distribution-types/src/index_url.rs @@ -24,6 +24,7 @@ static DEFAULT_INDEX_URL: Lazy = pub enum IndexUrl { Pypi(VerbatimUrl), Url(VerbatimUrl), + Path(VerbatimUrl), } impl IndexUrl { @@ -32,6 +33,7 @@ impl IndexUrl { match self { Self::Pypi(url) => url.raw(), Self::Url(url) => url.raw(), + Self::Path(url) => url.raw(), } } } @@ -41,6 +43,7 @@ impl Display for IndexUrl { match self { Self::Pypi(url) => Display::fmt(url, f), Self::Url(url) => Display::fmt(url, f), + Self::Path(url) => Display::fmt(url, f), } } } @@ -50,6 +53,7 @@ impl Verbatim for IndexUrl { match self { Self::Pypi(url) => url.verbatim(), Self::Url(url) => url.verbatim(), + Self::Path(url) => url.verbatim(), } } } @@ -83,6 +87,7 @@ impl From for Url { match index { IndexUrl::Pypi(url) => url.to_url(), IndexUrl::Url(url) => url.to_url(), + IndexUrl::Path(url) => url.to_url(), } } } @@ -94,6 +99,7 @@ impl Deref for IndexUrl { match &self { Self::Pypi(url) => url, Self::Url(url) => url, + Self::Path(url) => url, } } } diff --git a/crates/uv-client/src/flat_index.rs b/crates/uv-client/src/flat_index.rs index 623e9b26ec99..a468cc64099c 100644 --- a/crates/uv-client/src/flat_index.rs +++ b/crates/uv-client/src/flat_index.rs @@ -205,7 +205,7 @@ impl<'a> FlatIndexClient<'a> { fn read_from_directory(path: &PathBuf) -> Result { // Absolute paths are required for the URL conversion. let path = fs_err::canonicalize(path)?; - let index_url = IndexUrl::Url(VerbatimUrl::from_path(&path)); + let index_url = IndexUrl::Path(VerbatimUrl::from_path(&path)); let mut dists = Vec::new(); for entry in fs_err::read_dir(path)? { diff --git a/crates/uv-client/src/registry_client.rs b/crates/uv-client/src/registry_client.rs index 6fd5b50203e2..5873058e07ec 100644 --- a/crates/uv-client/src/registry_client.rs +++ b/crates/uv-client/src/registry_client.rs @@ -285,6 +285,7 @@ impl RegistryClient { Path::new(&match index { IndexUrl::Pypi(_) => "pypi".to_string(), IndexUrl::Url(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)), + IndexUrl::Path(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)), }), format!("{package_name}.rkyv"), ); diff --git a/crates/uv-distribution/src/index/cached_wheel.rs b/crates/uv-distribution/src/index/cached_wheel.rs index eea658c8f207..5efbc817b7d0 100644 --- a/crates/uv-distribution/src/index/cached_wheel.rs +++ b/crates/uv-distribution/src/index/cached_wheel.rs @@ -20,9 +20,14 @@ pub struct CachedWheel { impl CachedWheel { /// Try to parse a distribution from a cached directory name (like `typing-extensions-4.8.0-py3-none-any`). - pub fn from_built_source(path: &Path) -> Option { + pub fn from_built_source(path: impl AsRef) -> Option { + let path = path.as_ref(); + + // Determine the wheel filename. let filename = path.file_name()?.to_str()?; let filename = WheelFilename::from_stem(filename).ok()?; + + // Convert to a cached wheel. let archive = path.canonicalize().ok()?; let entry = CacheEntry::from_path(archive); let hashes = Vec::new(); @@ -54,7 +59,9 @@ impl CachedWheel { } /// Read a cached wheel from a `.http` pointer (e.g., `anyio-4.0.0-py3-none-any.http`). - pub fn from_http_pointer(path: &Path, cache: &Cache) -> Option { + pub fn from_http_pointer(path: impl AsRef, cache: &Cache) -> Option { + let path = path.as_ref(); + // Determine the wheel filename. let filename = path.file_name()?.to_str()?; let filename = WheelFilename::from_stem(filename).ok()?; @@ -73,7 +80,9 @@ impl CachedWheel { } /// Read a cached wheel from a `.rev` pointer (e.g., `anyio-4.0.0-py3-none-any.rev`). - pub fn from_local_pointer(path: &Path, cache: &Cache) -> Option { + pub fn from_local_pointer(path: impl AsRef, cache: &Cache) -> Option { + let path = path.as_ref(); + // Determine the wheel filename. let filename = path.file_name()?.to_str()?; let filename = WheelFilename::from_stem(filename).ok()?; diff --git a/crates/uv-distribution/src/index/registry_wheel_index.rs b/crates/uv-distribution/src/index/registry_wheel_index.rs index 4cf9c05942bc..29b815d1a7c5 100644 --- a/crates/uv-distribution/src/index/registry_wheel_index.rs +++ b/crates/uv-distribution/src/index/registry_wheel_index.rs @@ -13,7 +13,7 @@ use uv_normalize::PackageName; use uv_types::HashStrategy; use crate::index::cached_wheel::CachedWheel; -use crate::source::{HttpRevisionPointer, HTTP_REVISION}; +use crate::source::{HttpRevisionPointer, LocalRevisionPointer, HTTP_REVISION, LOCAL_REVISION}; /// A local index of distributions that originate from a registry, like `PyPI`. #[derive(Debug)] @@ -88,13 +88,13 @@ impl<'a> RegistryWheelIndex<'a> { ) -> BTreeMap { let mut versions = BTreeMap::new(); - // Collect into owned `IndexUrl` + // Collect into owned `IndexUrl`. let flat_index_urls: Vec = index_locations .flat_index() .filter_map(|flat_index| match flat_index { FlatIndexLocation::Path(path) => { let path = fs_err::canonicalize(path).ok()?; - Some(IndexUrl::Url(VerbatimUrl::from_path(path))) + Some(IndexUrl::Path(VerbatimUrl::from_path(path))) } FlatIndexLocation::Url(url) => { Some(IndexUrl::Url(VerbatimUrl::unknown(url.clone()))) @@ -112,30 +112,37 @@ impl<'a> RegistryWheelIndex<'a> { // For registry wheels, the cache structure is: `//.http` // or `///.rev`. for file in files(&wheel_dir) { - if file - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("http")) - { - if let Some(wheel) = - CachedWheel::from_http_pointer(&wheel_dir.join(&file), cache) - { - // Enforce hash-checking based on the built distribution. - if wheel.satisfies(hasher.get(package)) { - Self::add_wheel(wheel, tags, &mut versions); + match index_url { + // Add files from remote registries. + IndexUrl::Pypi(_) | IndexUrl::Url(_) => { + if file + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("http")) + { + if let Some(wheel) = + CachedWheel::from_http_pointer(wheel_dir.join(file), cache) + { + // Enforce hash-checking based on the built distribution. + if wheel.satisfies(hasher.get(package)) { + Self::add_wheel(wheel, tags, &mut versions); + } + } } } - } - - if file - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("rev")) - { - if let Some(wheel) = - CachedWheel::from_local_pointer(&wheel_dir.join(&file), cache) - { - // Enforce hash-checking based on the built distribution. - if wheel.satisfies(hasher.get(package)) { - Self::add_wheel(wheel, tags, &mut versions); + // Add files from local registries (e.g., `--find-links`). + IndexUrl::Path(_) => { + if file + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("rev")) + { + if let Some(wheel) = + CachedWheel::from_local_pointer(wheel_dir.join(file), cache) + { + // Enforce hash-checking based on the built distribution. + if wheel.satisfies(hasher.get(package)) { + Self::add_wheel(wheel, tags, &mut versions); + } + } } } } @@ -152,18 +159,39 @@ impl<'a> RegistryWheelIndex<'a> { for shard in directories(&cache_shard) { // Read the existing metadata from the cache, if it exists. let cache_shard = cache_shard.shard(shard); - let revision_entry = cache_shard.entry(HTTP_REVISION); - if let Ok(Some(pointer)) = HttpRevisionPointer::read_from(&revision_entry) { + + // Read the revision from the cache. + let revision = match index_url { + // Add files from remote registries. + IndexUrl::Pypi(_) | IndexUrl::Url(_) => { + let revision_entry = cache_shard.entry(HTTP_REVISION); + if let Ok(Some(pointer)) = HttpRevisionPointer::read_from(revision_entry) { + Some(pointer.into_revision()) + } else { + None + } + } + // Add files from local registries (e.g., `--find-links`). + IndexUrl::Path(_) => { + let revision_entry = cache_shard.entry(LOCAL_REVISION); + if let Ok(Some(pointer)) = LocalRevisionPointer::read_from(revision_entry) { + Some(pointer.into_revision()) + } else { + None + } + } + }; + + if let Some(revision) = revision { // Enforce hash-checking based on the source distribution. - let revision = pointer.into_revision(); if revision.satisfies(hasher.get(package)) { for wheel_dir in symlinks(cache_shard.join(revision.id())) { - if let Some(wheel) = CachedWheel::from_built_source(&wheel_dir) { + if let Some(wheel) = CachedWheel::from_built_source(wheel_dir) { Self::add_wheel(wheel, tags, &mut versions); } } } - }; + } } } diff --git a/crates/uv-distribution/src/source/mod.rs b/crates/uv-distribution/src/source/mod.rs index 78ffb00c4b24..474211f4c429 100644 --- a/crates/uv-distribution/src/source/mod.rs +++ b/crates/uv-distribution/src/source/mod.rs @@ -87,6 +87,15 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { ) -> Result { let built_wheel_metadata = match &source { BuildableSource::Dist(SourceDist::Registry(dist)) => { + // For registry source distributions, shard by package, then version, for + // convenience in debugging. + let cache_shard = self.build_context.cache().shard( + CacheBucket::BuiltWheels, + WheelCache::Index(&dist.index) + .wheel_dir(dist.filename.name.as_ref()) + .join(dist.filename.version.to_string()), + ); + let url = match &dist.file.url { FileLocation::RelativeUrl(base, url) => { pypi_types::base_url_join_relative(base, url)? @@ -103,6 +112,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { url: &url, path: Cow::Borrowed(path), }, + &cache_shard, tags, hashes, ) @@ -111,15 +121,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { } }; - // For registry source distributions, shard by package, then version, for - // convenience in debugging. - let cache_shard = self.build_context.cache().shard( - CacheBucket::BuiltWheels, - WheelCache::Index(&dist.index) - .wheel_dir(dist.filename.name.as_ref()) - .join(dist.filename.version.to_string()), - ); - self.url( source, &dist.file.filename, @@ -165,9 +166,19 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { .boxed() .await? } else { - self.archive(source, &PathSourceUrl::from(dist), tags, hashes) - .boxed() - .await? + let cache_shard = self + .build_context + .cache() + .shard(CacheBucket::BuiltWheels, WheelCache::Path(&dist.url).root()); + self.archive( + source, + &PathSourceUrl::from(dist), + &cache_shard, + tags, + hashes, + ) + .boxed() + .await? } } BuildableSource::Url(SourceUrl::Direct(resource)) => { @@ -204,7 +215,13 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { .boxed() .await? } else { - self.archive(source, resource, tags, hashes).boxed().await? + let cache_shard = self.build_context.cache().shard( + CacheBucket::BuiltWheels, + WheelCache::Path(resource.url).root(), + ); + self.archive(source, resource, &cache_shard, tags, hashes) + .boxed() + .await? } } }; @@ -222,6 +239,14 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { ) -> Result { let metadata = match &source { BuildableSource::Dist(SourceDist::Registry(dist)) => { + // For registry source distributions, shard by package, then version. + let cache_shard = self.build_context.cache().shard( + CacheBucket::BuiltWheels, + WheelCache::Index(&dist.index) + .wheel_dir(dist.filename.name.as_ref()) + .join(dist.filename.version.to_string()), + ); + let url = match &dist.file.url { FileLocation::RelativeUrl(base, url) => { pypi_types::base_url_join_relative(base, url)? @@ -238,6 +263,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { url: &url, path: Cow::Borrowed(path), }, + &cache_shard, hashes, ) .boxed() @@ -245,14 +271,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { } }; - // For registry source distributions, shard by package, then version. - let cache_shard = self.build_context.cache().shard( - CacheBucket::BuiltWheels, - WheelCache::Index(&dist.index) - .wheel_dir(dist.filename.name.as_ref()) - .join(dist.filename.version.to_string()), - ); - self.url_metadata( source, &dist.file.filename, @@ -296,7 +314,11 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { .boxed() .await? } else { - self.archive_metadata(source, &PathSourceUrl::from(dist), hashes) + let cache_shard = self + .build_context + .cache() + .shard(CacheBucket::BuiltWheels, WheelCache::Path(&dist.url).root()); + self.archive_metadata(source, &PathSourceUrl::from(dist), &cache_shard, hashes) .boxed() .await? } @@ -334,7 +356,11 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { .boxed() .await? } else { - self.archive_metadata(source, resource, hashes) + let cache_shard = self.build_context.cache().shard( + CacheBucket::BuiltWheels, + WheelCache::Path(resource.url).root(), + ); + self.archive_metadata(source, resource, &cache_shard, hashes) .boxed() .await? } @@ -573,17 +599,13 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { &self, source: &BuildableSource<'_>, resource: &PathSourceUrl<'_>, + cache_shard: &CacheShard, tags: &Tags, hashes: HashPolicy<'_>, ) -> Result { - let cache_shard = self.build_context.cache().shard( - CacheBucket::BuiltWheels, - WheelCache::Path(resource.url).root(), - ); - // Fetch the revision for the source distribution. let revision = self - .archive_revision(source, resource, &cache_shard, hashes) + .archive_revision(source, resource, cache_shard, hashes) .await?; // Before running the build, check that the hashes match. @@ -644,16 +666,12 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { &self, source: &BuildableSource<'_>, resource: &PathSourceUrl<'_>, + cache_shard: &CacheShard, hashes: HashPolicy<'_>, ) -> Result { - let cache_shard = self.build_context.cache().shard( - CacheBucket::BuiltWheels, - WheelCache::Path(resource.url).root(), - ); - // Fetch the revision for the source distribution. let revision = self - .archive_revision(source, resource, &cache_shard, hashes) + .archive_revision(source, resource, cache_shard, hashes) .await?; // Before running the build, check that the hashes match. diff --git a/crates/uv/tests/pip_sync.rs b/crates/uv/tests/pip_sync.rs index 37365b8a29e6..d39c103e26c2 100644 --- a/crates/uv/tests/pip_sync.rs +++ b/crates/uv/tests/pip_sync.rs @@ -2539,12 +2539,12 @@ fn find_links_offline_no_match() -> Result<()> { /// Sync using `--find-links` with a local directory. Ensure that cached wheels are reused. #[test] -fn find_links_cache() -> Result<()> { +fn find_links_wheel_cache() -> Result<()> { let context = TestContext::new("3.12"); let requirements_txt = context.temp_dir.child("requirements.txt"); requirements_txt.write_str(indoc! {r" - tqdm + tqdm==1000.0.0 "})?; // Install `tqdm`. @@ -2585,6 +2585,55 @@ fn find_links_cache() -> Result<()> { Ok(()) } +/// Sync using `--find-links` with a local directory. Ensure that cached source distributions are +/// reused. +#[test] +fn find_links_source_cache() -> Result<()> { + let context = TestContext::new("3.12"); + + let requirements_txt = context.temp_dir.child("requirements.txt"); + requirements_txt.write_str(indoc! {r" + tqdm==999.0.0 + "})?; + + // Install `tqdm`. + uv_snapshot!(context.filters(), command(&context) + .arg("requirements.txt") + .arg("--find-links") + .arg(context.workspace_root.join("scripts/links/")), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Downloaded 1 package in [TIME] + Installed 1 package in [TIME] + + tqdm==999.0.0 + "### + ); + + // Reinstall `tqdm` with `--reinstall`. Ensure that the wheel is reused. + uv_snapshot!(context.filters(), command(&context) + .arg("requirements.txt") + .arg("--reinstall") + .arg("--find-links") + .arg(context.workspace_root.join("scripts/links/")), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Uninstalled 1 package in [TIME] + Installed 1 package in [TIME] + - tqdm==999.0.0 + + tqdm==999.0.0 + "### + ); + + Ok(()) +} + /// Install without network access via the `--offline` flag. #[test] fn offline() -> Result<()> {