Skip to content

Commit

Permalink
Add --find-links source dists to the cache
Browse files Browse the repository at this point in the history
  • Loading branch information
@charliermarsh
charliermarsh committed Apr 11, 2024
1 parent 32f129c commit 2bd3b84
Show file tree
Hide file tree
Showing 7 changed files with 182 additions and 71 deletions.
6 changes: 6 additions & 0 deletions crates/distribution-types/src/index_url.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ static DEFAULT_INDEX_URL: Lazy<IndexUrl> =
pub enum IndexUrl {
Pypi(VerbatimUrl),
Url(VerbatimUrl),
Path(VerbatimUrl),
}

impl IndexUrl {
Expand All @@ -32,6 +33,7 @@ impl IndexUrl {
match self {
Self::Pypi(url) => url.raw(),
Self::Url(url) => url.raw(),
Self::Path(url) => url.raw(),
}
}
}
Expand All @@ -41,6 +43,7 @@ impl Display for IndexUrl {
match self {
Self::Pypi(url) => Display::fmt(url, f),
Self::Url(url) => Display::fmt(url, f),
Self::Path(url) => Display::fmt(url, f),
}
}
}
Expand All @@ -50,6 +53,7 @@ impl Verbatim for IndexUrl {
match self {
Self::Pypi(url) => url.verbatim(),
Self::Url(url) => url.verbatim(),
Self::Path(url) => url.verbatim(),
}
}
}
Expand Down Expand Up @@ -83,6 +87,7 @@ impl From<IndexUrl> for Url {
match index {
IndexUrl::Pypi(url) => url.to_url(),
IndexUrl::Url(url) => url.to_url(),
IndexUrl::Path(url) => url.to_url(),
}
}
}
Expand All @@ -94,6 +99,7 @@ impl Deref for IndexUrl {
match &self {
Self::Pypi(url) => url,
Self::Url(url) => url,
Self::Path(url) => url,
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion crates/uv-client/src/flat_index.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ impl<'a> FlatIndexClient<'a> {
fn read_from_directory(path: &PathBuf) -> Result<FlatIndexEntries, std::io::Error> {
// Absolute paths are required for the URL conversion.
let path = fs_err::canonicalize(path)?;
let index_url = IndexUrl::Url(VerbatimUrl::from_path(&path));
let index_url = IndexUrl::Path(VerbatimUrl::from_path(&path));

let mut dists = Vec::new();
for entry in fs_err::read_dir(path)? {
Expand Down
1 change: 1 addition & 0 deletions crates/uv-client/src/registry_client.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,7 @@ impl RegistryClient {
Path::new(&match index {
IndexUrl::Pypi(_) => "pypi".to_string(),
IndexUrl::Url(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)),
IndexUrl::Path(url) => cache_key::digest(&cache_key::CanonicalUrl::new(url)),
}),
format!("{package_name}.rkyv"),
);
Expand Down
15 changes: 12 additions & 3 deletions crates/uv-distribution/src/index/cached_wheel.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,14 @@ pub struct CachedWheel {

impl CachedWheel {
/// Try to parse a distribution from a cached directory name (like `typing-extensions-4.8.0-py3-none-any`).
pub fn from_built_source(path: &Path) -> Option<Self> {
pub fn from_built_source(path: impl AsRef<Path>) -> Option<Self> {
let path = path.as_ref();

// Determine the wheel filename.
let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?;

// Convert to a cached wheel.
let archive = path.canonicalize().ok()?;
let entry = CacheEntry::from_path(archive);
let hashes = Vec::new();
Expand Down Expand Up @@ -54,7 +59,9 @@ impl CachedWheel {
}

/// Read a cached wheel from a `.http` pointer (e.g., `anyio-4.0.0-py3-none-any.http`).
pub fn from_http_pointer(path: &Path, cache: &Cache) -> Option<Self> {
pub fn from_http_pointer(path: impl AsRef<Path>, cache: &Cache) -> Option<Self> {
let path = path.as_ref();

// Determine the wheel filename.
let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?;
Expand All @@ -73,7 +80,9 @@ impl CachedWheel {
}

/// Read a cached wheel from a `.rev` pointer (e.g., `anyio-4.0.0-py3-none-any.rev`).
pub fn from_local_pointer(path: &Path, cache: &Cache) -> Option<Self> {
pub fn from_local_pointer(path: impl AsRef<Path>, cache: &Cache) -> Option<Self> {
let path = path.as_ref();

// Determine the wheel filename.
let filename = path.file_name()?.to_str()?;
let filename = WheelFilename::from_stem(filename).ok()?;
Expand Down
88 changes: 58 additions & 30 deletions crates/uv-distribution/src/index/registry_wheel_index.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use uv_normalize::PackageName;
use uv_types::HashStrategy;

use crate::index::cached_wheel::CachedWheel;
use crate::source::{HttpRevisionPointer, HTTP_REVISION};
use crate::source::{HttpRevisionPointer, LocalRevisionPointer, HTTP_REVISION, LOCAL_REVISION};

/// A local index of distributions that originate from a registry, like `PyPI`.
#[derive(Debug)]
Expand Down Expand Up @@ -88,13 +88,13 @@ impl<'a> RegistryWheelIndex<'a> {
) -> BTreeMap<Version, CachedRegistryDist> {
let mut versions = BTreeMap::new();

// Collect into owned `IndexUrl`
// Collect into owned `IndexUrl`.
let flat_index_urls: Vec<IndexUrl> = index_locations
.flat_index()
.filter_map(|flat_index| match flat_index {
FlatIndexLocation::Path(path) => {
let path = fs_err::canonicalize(path).ok()?;
Some(IndexUrl::Url(VerbatimUrl::from_path(path)))
Some(IndexUrl::Path(VerbatimUrl::from_path(path)))
}
FlatIndexLocation::Url(url) => {
Some(IndexUrl::Url(VerbatimUrl::unknown(url.clone())))
Expand All @@ -112,30 +112,37 @@ impl<'a> RegistryWheelIndex<'a> {
// For registry wheels, the cache structure is: `<index>/<package-name>/<wheel>.http`
// or `<index>/<package-name>/<version>/<wheel>.rev`.
for file in files(&wheel_dir) {
if file
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("http"))
{
if let Some(wheel) =
CachedWheel::from_http_pointer(&wheel_dir.join(&file), cache)
{
// Enforce hash-checking based on the built distribution.
if wheel.satisfies(hasher.get(package)) {
Self::add_wheel(wheel, tags, &mut versions);
match index_url {
// Add files from remote registries.
IndexUrl::Pypi(_) | IndexUrl::Url(_) => {
if file
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("http"))
{
if let Some(wheel) =
CachedWheel::from_http_pointer(wheel_dir.join(file), cache)
{
// Enforce hash-checking based on the built distribution.
if wheel.satisfies(hasher.get(package)) {
Self::add_wheel(wheel, tags, &mut versions);
}
}
}
}
}

if file
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("rev"))
{
if let Some(wheel) =
CachedWheel::from_local_pointer(&wheel_dir.join(&file), cache)
{
// Enforce hash-checking based on the built distribution.
if wheel.satisfies(hasher.get(package)) {
Self::add_wheel(wheel, tags, &mut versions);
// Add files from local registries (e.g., `--find-links`).
IndexUrl::Path(_) => {
if file
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("rev"))
{
if let Some(wheel) =
CachedWheel::from_local_pointer(wheel_dir.join(file), cache)
{
// Enforce hash-checking based on the built distribution.
if wheel.satisfies(hasher.get(package)) {
Self::add_wheel(wheel, tags, &mut versions);
}
}
}
}
}
Expand All @@ -152,18 +159,39 @@ impl<'a> RegistryWheelIndex<'a> {
for shard in directories(&cache_shard) {
// Read the existing metadata from the cache, if it exists.
let cache_shard = cache_shard.shard(shard);
let revision_entry = cache_shard.entry(HTTP_REVISION);
if let Ok(Some(pointer)) = HttpRevisionPointer::read_from(&revision_entry) {

// Read the revision from the cache.
let revision = match index_url {
// Add files from remote registries.
IndexUrl::Pypi(_) | IndexUrl::Url(_) => {
let revision_entry = cache_shard.entry(HTTP_REVISION);
if let Ok(Some(pointer)) = HttpRevisionPointer::read_from(revision_entry) {
Some(pointer.into_revision())
} else {
None
}
}
// Add files from local registries (e.g., `--find-links`).
IndexUrl::Path(_) => {
let revision_entry = cache_shard.entry(LOCAL_REVISION);
if let Ok(Some(pointer)) = LocalRevisionPointer::read_from(revision_entry) {
Some(pointer.into_revision())
} else {
None
}
}
};

if let Some(revision) = revision {
// Enforce hash-checking based on the source distribution.
let revision = pointer.into_revision();
if revision.satisfies(hasher.get(package)) {
for wheel_dir in symlinks(cache_shard.join(revision.id())) {
if let Some(wheel) = CachedWheel::from_built_source(&wheel_dir) {
if let Some(wheel) = CachedWheel::from_built_source(wheel_dir) {
Self::add_wheel(wheel, tags, &mut versions);
}
}
}
};
}
}
}

Expand Down
Loading

0 comments on commit 2bd3b84

Please sign in to comment.