Merge branch 'main' into konstin/editable-in-pip-sync

crates/README.md

@@ -68,6 +68,10 @@ Implements the traits defined in `puffin-traits`.
 Client for interacting with built distributions (wheels) and source distributions (sdists).
 Capable of fetching metadata, distribution contents, etc.
 
+## [puffin-extract](./puffin-extract)
+
+Utilities for extracting files from archives.
+
 ## [puffin-fs](./puffin-fs)
 
 Utilities for interacting with the filesystem.

crates/puffin-build/Cargo.toml

@@ -17,13 +17,13 @@ workspace = true
 gourgeist = { path = "../gourgeist" }
 pep508_rs = { path = "../pep508-rs" }
 platform-host = { path = "../platform-host" }
+puffin-extract = { path = "../puffin-extract" }
 puffin-fs = { path = "../puffin-fs" }
 puffin-interpreter = { path = "../puffin-interpreter" }
 puffin-traits = { path = "../puffin-traits" }
 pypi-types = { path = "../pypi-types" }
 
 anyhow = { workspace = true }
-flate2 = { workspace = true }
 fs-err = { workspace = true }
 indoc = { workspace = true }
 itertools = { workspace = true }
@@ -32,14 +32,11 @@ pyproject-toml = { workspace = true }
 regex = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
-tar = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["sync", "process"] }
 toml = { workspace = true }
 tracing = { workspace = true }
-which = { workspace = true}
-zip = { workspace = true }
 
 [dev-dependencies]
 insta = { version = "1.34.0" }

crates/puffin-build/src/lib.rs

@@ -13,22 +13,19 @@ use std::process::Output;
 use std::str::FromStr;
 use std::sync::Arc;
 
-use flate2::read::GzDecoder;
 use fs_err as fs;
-use fs_err::{DirEntry, File};
+use fs_err::DirEntry;
 use indoc::formatdoc;
 use itertools::Itertools;
 use once_cell::sync::Lazy;
 use pyproject_toml::{BuildSystem, Project};
 use regex::Regex;
 use serde::{Deserialize, Serialize};
-use tar::Archive;
 use tempfile::{tempdir, tempdir_in, TempDir};
 use thiserror::Error;
 use tokio::process::Command;
 use tokio::sync::Mutex;
 use tracing::{debug, info_span, instrument};
-use zip::ZipArchive;
 
 use pep508_rs::Requirement;
 use puffin_interpreter::{Interpreter, Virtualenv};
@@ -46,8 +43,8 @@ static MISSING_HEADER_RE: Lazy<Regex> = Lazy::new(|| {
 pub enum Error {
     #[error(transparent)]
     IO(#[from] io::Error),
-    #[error("Failed to read zip file")]
-    Zip(#[from] zip::result::ZipError),
+    #[error("Failed to extract archive: {0}")]
+    Extraction(PathBuf, #[source] puffin_extract::Error),
     #[error("Unsupported archive format (extension not recognized): {0}")]
     UnsupportedArchiveType(String),
     #[error("Invalid source distribution: {0}")]
@@ -653,40 +650,12 @@ async fn create_pep517_build_environment(
 /// Returns the directory with the `pyproject.toml`/`setup.py`
 #[instrument(skip_all, fields(sdist = ? sdist.file_name().unwrap_or(sdist.as_os_str())))]
 fn extract_archive(sdist: &Path, extracted: &PathBuf) -> Result<PathBuf, Error> {
-    if sdist
-        .extension()
-        .is_some_and(|ext| ext.eq_ignore_ascii_case("zip"))
-    {
-        // .zip
-        let mut archive = ZipArchive::new(File::open(sdist)?)?;
-        archive.extract(extracted)?;
-    } else if sdist
-        .extension()
-        .is_some_and(|ext| ext.eq_ignore_ascii_case("gz"))
-        && sdist.file_stem().is_some_and(|stem| {
-            Path::new(stem)
-                .extension()
-                .is_some_and(|ext| ext.eq_ignore_ascii_case("tar"))
-        })
-    {
-        // .tar.gz
-        let mut archive = Archive::new(GzDecoder::new(File::open(sdist)?));
-        // https://github.com/alexcrichton/tar-rs/issues/349
-        archive.set_preserve_mtime(false);
-        archive.unpack(extracted)?;
-    } else {
-        return Err(Error::UnsupportedArchiveType(
-            sdist
-                .file_name()
-                .unwrap_or(sdist.as_os_str())
-                .to_string_lossy()
-                .to_string(),
-        ));
-    }
+    puffin_extract::extract_archive(sdist, extracted)
+        .map_err(|err| Error::Extraction(sdist.to_path_buf(), err))?;
 
     // > A .tar.gz source distribution (sdist) contains a single top-level directory called
     // > `{name}-{version}` (e.g. foo-1.0), containing the source files of the package.
-    // TODO(konstin): Verify the name of the directory
+    // TODO(konstin): Verify the name of the directory.
     let top_level = fs::read_dir(extracted)?.collect::<io::Result<Vec<DirEntry>>>()?;
     let [root] = top_level.as_slice() else {
         return Err(Error::InvalidSourceDist(format!(

crates/puffin-cli/src/commands/pip_compile.rs

@@ -207,7 +207,6 @@ pub(crate) async fn pip_compile(
     };
 
     // Create a manifest of the requirements.
-
     let manifest = Manifest::new(
         requirements,
         constraints,

crates/puffin-distribution/Cargo.toml

@@ -20,6 +20,7 @@ pep440_rs = { path = "../pep440-rs" }
 platform-tags = { path = "../platform-tags" }
 puffin-cache = { path = "../puffin-cache" }
 puffin-client = { path = "../puffin-client" }
+puffin-extract = { path = "../puffin-extract" }
 puffin-fs = { path = "../puffin-fs" }
 puffin-git = { path = "../puffin-git" }
 puffin-normalize = { path = "../puffin-normalize" }
@@ -31,12 +32,10 @@ bytesize = { workspace = true }
 fs-err = { workspace = true }
 fs2 = { workspace = true }
 futures = { workspace = true }
-rayon = { workspace = true }
 reqwest = { workspace = true }
 rustc-hash = { workspace = true }
 serde = { workspace = true , features = ["derive"] }
 serde_json = { workspace = true }
-sha2 = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }

crates/puffin-distribution/src/download.rs

@@ -4,7 +4,7 @@ use anyhow::Result;
 use zip::ZipArchive;
 
 use distribution_filename::WheelFilename;
-use distribution_types::{Dist, SourceDist};
+use distribution_types::Dist;
 use install_wheel_rs::read_dist_info;
 use pypi_types::Metadata21;
 
@@ -86,57 +86,6 @@ impl LocalWheel {
     }
 }
 
-/// A downloaded source distribution.
-#[derive(Debug, Clone)]
-pub struct SourceDistDownload {
-    /// The remote distribution from which this source distribution was downloaded.
-    pub(crate) dist: SourceDist,
-    /// The path to the downloaded archive or directory.
-    pub(crate) sdist_file: PathBuf,
-    /// The subdirectory within the archive or directory.
-    pub(crate) subdirectory: Option<PathBuf>,
-}
-
-/// A downloaded distribution, either a wheel or a source distribution.
-#[derive(Debug)]
-#[allow(clippy::large_enum_variant)]
-pub enum Download {
-    Wheel(LocalWheel),
-    SourceDist(SourceDistDownload),
-}
-
-impl std::fmt::Display for Download {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Download::Wheel(wheel) => write!(f, "{wheel}"),
-            Download::SourceDist(sdist) => write!(f, "{sdist}"),
-        }
-    }
-}
-
-impl std::fmt::Display for LocalWheel {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.remote())
-    }
-}
-
-impl std::fmt::Display for SourceDistDownload {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.remote())
-    }
-}
-
-impl BuiltWheel {
-    /// Read the [`Metadata21`] from a wheel.
-    pub fn read_dist_info(&self) -> Result<Metadata21, Error> {
-        let mut archive = ZipArchive::new(fs_err::File::open(&self.path)?)?;
-        let dist_info = read_dist_info(&self.filename, &mut archive).map_err(|err| {
-            Error::DistInfo(Box::new(self.filename.clone()), self.dist.to_string(), err)
-        })?;
-        Ok(Metadata21::parse(&dist_info)?)
-    }
-}
-
 impl DiskWheel {
     /// Return the [`Dist`] from which this wheel was downloaded.
     pub fn remote(&self) -> &Dist {
@@ -159,9 +108,19 @@ impl BuiltWheel {
     }
 }
 
-impl SourceDistDownload {
-    /// Return the [`Dist`] from which this source distribution was downloaded.
-    pub fn remote(&self) -> &SourceDist {
-        &self.dist
+impl std::fmt::Display for LocalWheel {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.remote())
+    }
+}
+
+impl BuiltWheel {
+    /// Read the [`Metadata21`] from a wheel.
+    pub fn read_dist_info(&self) -> Result<Metadata21, Error> {
+        let mut archive = ZipArchive::new(fs_err::File::open(&self.path)?)?;
+        let dist_info = read_dist_info(&self.filename, &mut archive).map_err(|err| {
+            Error::DistInfo(Box::new(self.filename.clone()), self.dist.to_string(), err)
+        })?;
+        Ok(Metadata21::parse(&dist_info)?)
     }
 }

crates/puffin-distribution/src/lib.rs

@@ -1,11 +1,9 @@
 pub use distribution_database::{DistributionDatabase, DistributionDatabaseError};
-pub use download::{
-    BuiltWheel, DiskWheel, Download, InMemoryWheel, LocalWheel, SourceDistDownload,
-};
+pub use download::{DiskWheel, InMemoryWheel, LocalWheel};
 pub use index::{BuiltWheelIndex, RegistryWheelIndex};
 pub use reporter::Reporter;
 pub use source_dist::{SourceDistCachedBuilder, SourceDistError};
-pub use unzip::{Unzip, UnzipError};
+pub use unzip::Unzip;
 
 mod distribution_database;
 mod download;
@@ -15,4 +13,3 @@ mod locks;
 mod reporter;
 mod source_dist;
 mod unzip;
-mod vendor;

crates/puffin-distribution/src/source_dist.rs

@@ -37,7 +37,7 @@ use puffin_traits::{BuildContext, BuildKind, SourceBuildTrait};
 use pypi_types::Metadata21;
 
 use crate::locks::LockedFile;
-use crate::{Reporter, SourceDistDownload};
+use crate::Reporter;
 
 /// The caller is responsible for adding the source dist information to the error chain
 #[derive(Debug, Error)]
@@ -239,18 +239,12 @@ impl<'a, T: BuildContext> SourceDistCachedBuilder<'a, T> {
             let (temp_dir, sdist_file) = self.download_source_dist_url(response, filename).await?;
             drop(span);
 
-            let download = SourceDistDownload {
-                dist: source_dist.clone(),
-                sdist_file: sdist_file.clone(),
-                subdirectory: subdirectory.map(Path::to_path_buf),
-            };
-
             let (disk_filename, wheel_filename, metadata) = self
                 .build_source_dist(
-                    &download.dist,
+                    source_dist,
                     temp_dir,
-                    &download.sdist_file,
-                    download.subdirectory.as_deref(),
+                    &sdist_file,
+                    subdirectory,
                     &cache_entry,
                 )
                 .await?;