diff --git a/.gitignore b/.gitignore index da5e94a6c326..79d27d5bf09f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ -.venv - # Generated by Cargo # will have compiled files and executables debug/ @@ -14,5 +12,16 @@ target/ # Use e.g. `--cache-dir cache-docker` to keep a cache across container invocations cache-* -# python tmp files +# Python tmp files __pycache__ + +# Maturin builds, and other native editable builds +*.so +*.pyd +*.dll + +# Profiling +flamegraph.svg +perf.data +perf.data.old +profile.json diff --git a/Cargo.lock b/Cargo.lock index 58adb429ff7b..8897089493dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -855,6 +855,7 @@ dependencies = [ "puffin-git", "puffin-normalize", "pypi-types", + "requirements-txt", "rustc-hash", "serde", "serde_json", @@ -2309,11 +2310,13 @@ dependencies = [ "chrono", "clap", "colored", + "distribution-filename", "distribution-types", "fs-err", "futures", "gourgeist", "indicatif", + "indoc", "insta", "insta-cmd", "install-wheel-rs", @@ -2326,6 +2329,7 @@ dependencies = [ "platform-tags", "predicates", "pubgrub", + "puffin-build", "puffin-cache", "puffin-client", "puffin-dispatch", @@ -2558,6 +2562,7 @@ dependencies = [ "puffin-traits", "pypi-types", "rayon", + "requirements-txt", "rustc-hash", "tempfile", "thiserror", @@ -2629,6 +2634,7 @@ dependencies = [ "puffin-traits", "puffin-warnings", "pypi-types", + "requirements-txt", "reqwest", "rustc-hash", "serde_json", diff --git a/crates/distribution-types/Cargo.toml b/crates/distribution-types/Cargo.toml index 36653c23eeda..62fcaa6c6fde 100644 --- a/crates/distribution-types/Cargo.toml +++ b/crates/distribution-types/Cargo.toml @@ -20,6 +20,7 @@ puffin-cache = { path = "../puffin-cache" } puffin-git = { path = "../puffin-git" } puffin-normalize = { path = "../puffin-normalize" } pypi-types = { path = "../pypi-types" } +requirements-txt = { path = "../requirements-txt" } anyhow = { workspace = true } fs-err = { workspace = true } diff --git a/crates/distribution-types/src/cached.rs b/crates/distribution-types/src/cached.rs index 86855c75d86a..958ad8fa29a0 100644 --- a/crates/distribution-types/src/cached.rs +++ b/crates/distribution-types/src/cached.rs @@ -6,7 +6,7 @@ use distribution_filename::WheelFilename; use pep508_rs::VerbatimUrl; use puffin_normalize::PackageName; -use crate::direct_url::DirectUrl; +use crate::direct_url::{DirectUrl, LocalFileUrl}; use crate::traits::Metadata; use crate::{BuiltDist, Dist, SourceDist, VersionOrUrl}; @@ -30,6 +30,7 @@ pub struct CachedDirectUrlDist { pub filename: WheelFilename, pub url: VerbatimUrl, pub path: PathBuf, + pub editable: bool, } impl Metadata for CachedRegistryDist { @@ -79,11 +80,13 @@ impl CachedDist { filename, url: dist.url, path, + editable: false, }), Dist::Built(BuiltDist::Path(dist)) => Self::Url(CachedDirectUrlDist { filename, url: dist.url, path, + editable: false, }), Dist::Source(SourceDist::Registry(_dist)) => { Self::Registry(CachedRegistryDist { filename, path }) @@ -92,16 +95,19 @@ impl CachedDist { filename, url: dist.url, path, + editable: false, }), Dist::Source(SourceDist::Git(dist)) => Self::Url(CachedDirectUrlDist { filename, url: dist.url, path, + editable: false, }), Dist::Source(SourceDist::Path(dist)) => Self::Url(CachedDirectUrlDist { filename, url: dist.url, path, + editable: dist.editable, }), } } @@ -118,7 +124,24 @@ impl CachedDist { pub fn direct_url(&self) -> Result> { match self { CachedDist::Registry(_) => Ok(None), - CachedDist::Url(dist) => DirectUrl::try_from(dist.url.raw()).map(Some), + CachedDist::Url(dist) => { + if dist.editable { + assert_eq!(dist.url.scheme(), "file", "{}", dist.url); + Ok(Some(DirectUrl::LocalFile(LocalFileUrl { + url: dist.url.raw().clone(), + editable: dist.editable, + }))) + } else { + DirectUrl::try_from(dist.url.raw()).map(Some) + } + } + } + } + + pub fn editable(&self) -> bool { + match self { + CachedDist::Registry(_) => false, + CachedDist::Url(dist) => dist.editable, } } } @@ -130,6 +153,7 @@ impl CachedDirectUrlDist { filename, url, path, + editable: false, } } } @@ -177,6 +201,7 @@ impl CachedWheel { filename: self.filename, url, path: self.path, + editable: false, } } } diff --git a/crates/distribution-types/src/direct_url.rs b/crates/distribution-types/src/direct_url.rs index 75e29c31f99c..c3e60c224a95 100644 --- a/crates/distribution-types/src/direct_url.rs +++ b/crates/distribution-types/src/direct_url.rs @@ -15,16 +15,21 @@ pub enum DirectUrl { Archive(DirectArchiveUrl), } -/// A git repository url +/// A local path url /// /// Examples: -/// * `git+https://git.example.com/MyProject.git` -/// * `git+https://git.example.com/MyProject.git@v1.0#egg=pkg&subdirectory=pkg_dir` +/// * `file:///home/ferris/my_project` #[derive(Debug)] pub struct LocalFileUrl { pub url: Url, + pub editable: bool, } +/// A git repository url +/// +/// Examples: +/// * `git+https://git.example.com/MyProject.git` +/// * `git+https://git.example.com/MyProject.git@v1.0#egg=pkg&subdirectory=pkg_dir` #[derive(Debug)] pub struct DirectGitUrl { pub url: GitUrl, @@ -43,12 +48,6 @@ pub struct DirectArchiveUrl { pub subdirectory: Option, } -impl From<&Url> for LocalFileUrl { - fn from(url: &Url) -> Self { - Self { url: url.clone() } - } -} - impl TryFrom<&Url> for DirectGitUrl { type Error = Error; @@ -106,7 +105,10 @@ impl TryFrom<&Url> for DirectUrl { ))), } } else if url.scheme().eq_ignore_ascii_case("file") { - Ok(Self::LocalFile(LocalFileUrl::from(url))) + Ok(Self::LocalFile(LocalFileUrl { + url: url.clone(), + editable: false, + })) } else { Ok(Self::Archive(DirectArchiveUrl::from(url))) } @@ -131,7 +133,9 @@ impl TryFrom<&LocalFileUrl> for pypi_types::DirectUrl { fn try_from(value: &LocalFileUrl) -> Result { Ok(pypi_types::DirectUrl::LocalDirectory { url: value.url.clone(), - dir_info: pypi_types::DirInfo { editable: None }, + dir_info: pypi_types::DirInfo { + editable: value.editable.then_some(true), + }, }) } } diff --git a/crates/distribution-types/src/editable.rs b/crates/distribution-types/src/editable.rs new file mode 100644 index 000000000000..f4d6c32df899 --- /dev/null +++ b/crates/distribution-types/src/editable.rs @@ -0,0 +1,33 @@ +use std::fmt::{Display, Formatter}; +use std::path::{Path, PathBuf}; +use url::Url; + +use pep508_rs::VerbatimUrl; +use requirements_txt::EditableRequirement; + +#[derive(Debug, Clone)] +pub struct LocalEditable { + pub requirement: EditableRequirement, + /// Either the path to the editable or its checkout + pub path: PathBuf, +} + +impl LocalEditable { + pub fn url(&self) -> &VerbatimUrl { + self.requirement.url() + } + + pub fn raw(&self) -> &Url { + self.requirement.raw() + } + + pub fn path(&self) -> &Path { + &self.path + } +} + +impl Display for LocalEditable { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.requirement.fmt(f) + } +} diff --git a/crates/distribution-types/src/lib.rs b/crates/distribution-types/src/lib.rs index 9dc513d152d7..2c0168cda70c 100644 --- a/crates/distribution-types/src/lib.rs +++ b/crates/distribution-types/src/lib.rs @@ -50,6 +50,7 @@ use pypi_types::{File, IndexUrl}; pub use crate::any::*; pub use crate::cached::*; +pub use crate::editable::LocalEditable; pub use crate::error::*; pub use crate::id::*; pub use crate::installed::*; @@ -59,6 +60,7 @@ pub use crate::traits::*; mod any; mod cached; pub mod direct_url; +mod editable; mod error; mod id; mod installed; @@ -167,6 +169,7 @@ pub struct PathSourceDist { pub name: PackageName, pub url: VerbatimUrl, pub path: PathBuf, + pub editable: bool, } impl Dist { @@ -219,6 +222,7 @@ impl Dist { name, url, path, + editable: false, }))) }; } diff --git a/crates/install-wheel-rs/src/lib.rs b/crates/install-wheel-rs/src/lib.rs index 181fac067a68..3ad2053ad242 100644 --- a/crates/install-wheel-rs/src/lib.rs +++ b/crates/install-wheel-rs/src/lib.rs @@ -92,6 +92,8 @@ pub enum Error { /// the metadata name and the dist info name are lowercase, while the wheel name is uppercase. /// Either way, we just search the wheel for the name. /// +/// Returns the dist info dir prefix without the `.dist-info` extension. +/// /// Reference implementation: pub fn find_dist_info<'a, T: Copy>( filename: &WheelFilename, @@ -106,13 +108,13 @@ pub fn find_dist_info<'a, T: Copy>( && Version::from_str(version).ok()? == filename.version && file == "METADATA" { - Some((payload, dist_info_dir)) + Some((payload, dir_stem)) } else { None } }) .collect(); - let (payload, dist_info_dir) = match metadatas[..] { + let (payload, dist_info_prefix) = match metadatas[..] { [] => { return Err(Error::MissingDistInfo); } @@ -127,7 +129,7 @@ pub fn find_dist_info<'a, T: Copy>( )); } }; - Ok((payload, dist_info_dir)) + Ok((payload, dist_info_prefix)) } /// Given an archive, read the `dist-info` metadata into a buffer. @@ -135,10 +137,11 @@ pub fn read_dist_info( filename: &WheelFilename, archive: &mut ZipArchive, ) -> Result, Error> { - let dist_info_dir = find_dist_info(filename, archive.file_names().map(|name| (name, name)))?.1; + let dist_info_prefix = + find_dist_info(filename, archive.file_names().map(|name| (name, name)))?.1; let mut file = archive - .by_name(&format!("{dist_info_dir}/METADATA")) + .by_name(&format!("{dist_info_prefix}.dist-info/METADATA")) .map_err(|err| Error::Zip(filename.to_string(), err))?; #[allow(clippy::cast_possible_truncation)] @@ -170,8 +173,8 @@ mod test { "Mastodon.py-1.5.1.dist-info/RECORD", ]; let filename = WheelFilename::from_str("Mastodon.py-1.5.1-py2.py3-none-any.whl").unwrap(); - let (_, dist_info_dir) = + let (_, dist_info_prefix) = find_dist_info(&filename, files.into_iter().map(|file| (file, file))).unwrap(); - assert_eq!(dist_info_dir, "Mastodon.py-1.5.1.dist-info"); + assert_eq!(dist_info_prefix, "Mastodon.py-1.5.1"); } } diff --git a/crates/install-wheel-rs/src/wheel.rs b/crates/install-wheel-rs/src/wheel.rs index b7a664486021..8d4173f842f5 100644 --- a/crates/install-wheel-rs/src/wheel.rs +++ b/crates/install-wheel-rs/src/wheel.rs @@ -82,10 +82,10 @@ pub(crate) fn read_scripts_from_section( /// Extras are supposed to be ignored, which happens if you pass None for extras fn parse_scripts( archive: &mut ZipArchive, - dist_info_prefix: &str, + dist_info_dir: &str, extras: Option<&[String]>, ) -> Result<(Vec