Support editable in pip-sync and pip-compile

.gitignore

@@ -1,5 +1,3 @@
-.venv
-
 # Generated by Cargo
 # will have compiled files and executables
 debug/
@@ -14,5 +12,16 @@ target/
 # Use e.g. `--cache-dir cache-docker` to keep a cache across container invocations
 cache-*
 
-# python tmp files
+# Python tmp files
 __pycache__
+
+# Maturin builds, and other native editable builds
+*.so
+*.pyd
+*.dll
+
+# Profiling
+flamegraph.svg
+perf.data
+perf.data.old
+profile.json

crates/distribution-types/Cargo.toml

@@ -20,6 +20,7 @@ puffin-cache = { path = "../puffin-cache" }
 puffin-git = { path = "../puffin-git" }
 puffin-normalize = { path = "../puffin-normalize" }
 pypi-types = { path = "../pypi-types" }
+requirements-txt = { path = "../requirements-txt" }
 
 anyhow = { workspace = true }
 fs-err = { workspace = true }

crates/distribution-types/src/cached.rs

@@ -6,7 +6,7 @@ use distribution_filename::WheelFilename;
 use pep508_rs::VerbatimUrl;
 use puffin_normalize::PackageName;
 
-use crate::direct_url::DirectUrl;
+use crate::direct_url::{DirectUrl, LocalFileUrl};
 use crate::traits::Metadata;
 use crate::{BuiltDist, Dist, SourceDist, VersionOrUrl};
 
@@ -30,6 +30,7 @@ pub struct CachedDirectUrlDist {
     pub filename: WheelFilename,
     pub url: VerbatimUrl,
     pub path: PathBuf,
+    pub editable: bool,
 }
 
 impl Metadata for CachedRegistryDist {
@@ -79,11 +80,13 @@ impl CachedDist {
                 filename,
                 url: dist.url,
                 path,
+                editable: false,
             }),
             Dist::Built(BuiltDist::Path(dist)) => Self::Url(CachedDirectUrlDist {
                 filename,
                 url: dist.url,
                 path,
+                editable: false,
             }),
             Dist::Source(SourceDist::Registry(_dist)) => {
                 Self::Registry(CachedRegistryDist { filename, path })
@@ -92,16 +95,19 @@ impl CachedDist {
                 filename,
                 url: dist.url,
                 path,
+                editable: false,
             }),
             Dist::Source(SourceDist::Git(dist)) => Self::Url(CachedDirectUrlDist {
                 filename,
                 url: dist.url,
                 path,
+                editable: false,
             }),
             Dist::Source(SourceDist::Path(dist)) => Self::Url(CachedDirectUrlDist {
                 filename,
                 url: dist.url,
                 path,
+                editable: dist.editable,
             }),
         }
     }
@@ -118,7 +124,25 @@ impl CachedDist {
     pub fn direct_url(&self) -> Result<Option<DirectUrl>> {
         match self {
             CachedDist::Registry(_) => Ok(None),
-            CachedDist::Url(dist) => DirectUrl::try_from(dist.url.raw()).map(Some),
+            CachedDist::Url(dist) => {
+                if dist.editable {
+                    // TODO(konstin): Do this in the type system
+                    assert_eq!(dist.url.scheme(), "file", "{}", dist.url);
+                    Ok(Some(DirectUrl::LocalFile(LocalFileUrl {
+                        url: dist.url.raw().clone(),
+                        editable: dist.editable,
+                    })))
+                } else {
+                    DirectUrl::try_from(dist.url.raw()).map(Some)
+                }
+            }
+        }
+    }
+
+    pub fn editable(&self) -> bool {
+        match self {
+            CachedDist::Registry(_) => false,
+            CachedDist::Url(dist) => dist.editable,
         }
     }
 }
@@ -130,6 +154,7 @@ impl CachedDirectUrlDist {
             filename,
             url,
             path,
+            editable: false,
         }
     }
 }
@@ -177,6 +202,7 @@ impl CachedWheel {
             filename: self.filename,
             url,
             path: self.path,
+            editable: false,
         }
     }
 }

crates/distribution-types/src/direct_url.rs

@@ -15,16 +15,21 @@ pub enum DirectUrl {
     Archive(DirectArchiveUrl),
 }
 
-/// A git repository url
+/// A local path url
 ///
 /// Examples:
-/// * `git+https://git.example.com/MyProject.git`
-/// * `git+https://git.example.com/MyProject.git@v1.0#egg=pkg&subdirectory=pkg_dir`
+/// * `file:///home/ferris/my_project`
 #[derive(Debug)]
 pub struct LocalFileUrl {
     pub url: Url,
+    pub editable: bool,
 }
 
+/// A git repository url
+///
+/// Examples:
+/// * `git+https://git.example.com/MyProject.git`
+/// * `git+https://git.example.com/MyProject.git@v1.0#egg=pkg&subdirectory=pkg_dir`
 #[derive(Debug)]
 pub struct DirectGitUrl {
     pub url: GitUrl,
@@ -43,12 +48,6 @@ pub struct DirectArchiveUrl {
     pub subdirectory: Option<PathBuf>,
 }
 
-impl From<&Url> for LocalFileUrl {
-    fn from(url: &Url) -> Self {
-        Self { url: url.clone() }
-    }
-}
-
 impl TryFrom<&Url> for DirectGitUrl {
     type Error = Error;
 
@@ -106,7 +105,10 @@ impl TryFrom<&Url> for DirectUrl {
                 ))),
             }
         } else if url.scheme().eq_ignore_ascii_case("file") {
-            Ok(Self::LocalFile(LocalFileUrl::from(url)))
+            Ok(Self::LocalFile(LocalFileUrl {
+                url: url.clone(),
+                editable: false,
+            }))
         } else {
             Ok(Self::Archive(DirectArchiveUrl::from(url)))
         }
@@ -131,7 +133,9 @@ impl TryFrom<&LocalFileUrl> for pypi_types::DirectUrl {
     fn try_from(value: &LocalFileUrl) -> Result<Self, Self::Error> {
         Ok(pypi_types::DirectUrl::LocalDirectory {
             url: value.url.clone(),
-            dir_info: pypi_types::DirInfo { editable: None },
+            dir_info: pypi_types::DirInfo {
+                editable: value.editable.then_some(true),
+            },
         })
     }
 }

crates/distribution-types/src/editable.rs

@@ -0,0 +1,28 @@
+use std::fmt::{Display, Formatter};
+use std::path::{Path, PathBuf};
+
+use pep508_rs::VerbatimUrl;
+use requirements_txt::EditableRequirement;
+
+#[derive(Debug, Clone)]
+pub struct LocalEditable {
+    pub requirement: EditableRequirement,
+    /// Either the path to the editable or its checkout
+    pub path: PathBuf,
+}
+
+impl LocalEditable {
+    pub fn url(&self) -> VerbatimUrl {
+        self.requirement.url()
+    }
+
+    pub fn path(&self) -> &Path {
+        &self.path
+    }
+}
+
+impl Display for LocalEditable {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        self.requirement.fmt(f)
+    }
+}

crates/distribution-types/src/lib.rs

@@ -51,6 +51,7 @@ use pypi_types::{File, IndexUrl};
 
 pub use crate::any::*;
 pub use crate::cached::*;
+pub use crate::editable::LocalEditable;
 pub use crate::error::*;
 pub use crate::id::*;
 pub use crate::installed::*;
@@ -59,6 +60,7 @@ pub use crate::traits::*;
 mod any;
 mod cached;
 pub mod direct_url;
+mod editable;
 mod error;
 mod id;
 mod installed;
@@ -166,6 +168,7 @@ pub struct PathSourceDist {
     pub name: PackageName,
     pub url: VerbatimUrl,
     pub path: PathBuf,
+    pub editable: bool,
 }
 
 impl Dist {
@@ -218,6 +221,7 @@ impl Dist {
                     name,
                     url,
                     path,
+                    editable: false,
                 })))
             };
         }

crates/install-wheel-rs/src/lib.rs

@@ -92,6 +92,8 @@ pub enum Error {
 /// the metadata name and the dist info name are lowercase, while the wheel name is uppercase.
 /// Either way, we just search the wheel for the name.
 ///
+/// Returns the dist info dir prefix without the `.dist-info` extension.
+///
 /// Reference implementation: <https://github.com/pypa/packaging/blob/2f83540272e79e3fe1f5d42abae8df0c14ddf4c2/src/packaging/utils.py#L146-L172>
 pub fn find_dist_info<'a, T: Copy>(
     filename: &WheelFilename,
@@ -106,13 +108,13 @@ pub fn find_dist_info<'a, T: Copy>(
                 && Version::from_str(version).ok()? == filename.version
                 && file == "METADATA"
             {
-                Some((payload, dist_info_dir))
+                Some((payload, dir_stem))
             } else {
                 None
             }
         })
         .collect();
-    let (payload, dist_info_dir) = match metadatas[..] {
+    let (payload, dist_info_prefix) = match metadatas[..] {
         [] => {
             return Err(Error::MissingDistInfo);
         }
@@ -127,18 +129,19 @@ pub fn find_dist_info<'a, T: Copy>(
             ));
         }
     };
-    Ok((payload, dist_info_dir))
+    Ok((payload, dist_info_prefix))
 }
 
 /// Given an archive, read the `dist-info` metadata into a buffer.
 pub fn read_dist_info(
     filename: &WheelFilename,
     archive: &mut ZipArchive<impl Read + Seek + Sized>,
 ) -> Result<Vec<u8>, Error> {
-    let dist_info_dir = find_dist_info(filename, archive.file_names().map(|name| (name, name)))?.1;
+    let dist_info_prefix =
+        find_dist_info(filename, archive.file_names().map(|name| (name, name)))?.1;
 
     let mut file = archive
-        .by_name(&format!("{dist_info_dir}/METADATA"))
+        .by_name(&format!("{dist_info_prefix}.dist-info/METADATA"))
         .map_err(|err| Error::Zip(filename.to_string(), err))?;
 
     #[allow(clippy::cast_possible_truncation)]
@@ -170,8 +173,8 @@ mod test {
             "Mastodon.py-1.5.1.dist-info/RECORD",
         ];
         let filename = WheelFilename::from_str("Mastodon.py-1.5.1-py2.py3-none-any.whl").unwrap();
-        let (_, dist_info_dir) =
+        let (_, dist_info_prefix) =
             find_dist_info(&filename, files.into_iter().map(|file| (file, file))).unwrap();
-        assert_eq!(dist_info_dir, "Mastodon.py-1.5.1.dist-info");
+        assert_eq!(dist_info_prefix, "Mastodon.py-1.5.1");
     }
 }

crates/install-wheel-rs/src/wheel.rs

@@ -82,10 +82,10 @@ pub(crate) fn read_scripts_from_section(
 /// Extras are supposed to be ignored, which happens if you pass None for extras
 fn parse_scripts<R: Read + Seek>(
     archive: &mut ZipArchive<R>,
-    dist_info_prefix: &str,
+    dist_info_dir: &str,
     extras: Option<&[String]>,
 ) -> Result<(Vec<Script>, Vec<Script>), Error> {
-    let entry_points_path = format!("{dist_info_prefix}.dist-info/entry_points.txt");
+    let entry_points_path = format!("{dist_info_dir}/entry_points.txt");
     let entry_points_mapping = match archive.by_name(&entry_points_path) {
         Ok(mut file) => {
             let mut ini_text = String::new();
@@ -1043,7 +1043,7 @@ fn dist_info_metadata(
     archive: &mut ZipArchive<impl Read + Seek + Sized>,
 ) -> Result<Vec<u8>, Error> {
     let mut content = Vec::new();
-    let dist_info_file = format!("{dist_info_prefix}.dist-info/DIST-INFO");
+    let dist_info_file = format!("{dist_info_prefix}.dist-info/METADATA");
     archive
         .by_name(&dist_info_file)
         .map_err(|err| Error::Zip(dist_info_file.clone(), err))?

crates/pep508-rs/src/verbatim_url.rs

@@ -1,5 +1,6 @@
 use std::borrow::Cow;
 use std::ops::Deref;
+use std::path::Path;
 
 use once_cell::sync::Lazy;
 use regex::Regex;
@@ -27,6 +28,15 @@ impl VerbatimUrl {
         })
     }
 
+    /// Helper method `LocalEditable`
+    #[allow(clippy::result_unit_err)] // This comes from the `url` crate
+    pub fn from_path(path: &Path, given: String) -> Result<Self, ()> {
+        Ok(Self {
+            url: Url::from_directory_path(path)?,
+            given: Some(given),
+        })
+    }
+
     /// Return the underlying [`Url`].
     pub fn raw(&self) -> &Url {
         &self.url