From e5b0cf7f8946bd921d78c20284e65643b9ea3656 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Wed, 20 Mar 2024 23:46:11 -0400 Subject: [PATCH] Add support for unnamed local directory requirements (#2571) ## Summary For example: `cargo run pip install .` The strategy taken here is to attempt to extract the package name from the distribution without executing the PEP 517 build steps. We could choose to do that in the future if this proves lacking, but it adds complexity. Part of: https://github.com/astral-sh/uv/issues/313. --- Cargo.lock | 2 + crates/pypi-types/src/metadata.rs | 24 +++ crates/uv/Cargo.toml | 5 +- crates/uv/src/requirements.rs | 163 +++++++++++++++++- crates/uv/tests/pip_compile.rs | 69 ++++++++ .../setup_cfg_editable/.gitignore | 2 + .../setup_cfg_editable/setup.cfg | 3 + .../setup_cfg_editable/setup.py | 8 + .../setup_cfg_editable/__init__.py | 0 .../setup_py_editable/.gitignore | 2 + .../setup_py_editable/setup.py | 9 + .../setup_py_editable/__init__.py | 0 12 files changed, 284 insertions(+), 3 deletions(-) create mode 100644 scripts/editable-installs/setup_cfg_editable/.gitignore create mode 100644 scripts/editable-installs/setup_cfg_editable/setup.cfg create mode 100644 scripts/editable-installs/setup_cfg_editable/setup.py create mode 100644 scripts/editable-installs/setup_cfg_editable/setup_cfg_editable/__init__.py create mode 100644 scripts/editable-installs/setup_py_editable/.gitignore create mode 100644 scripts/editable-installs/setup_py_editable/setup.py create mode 100644 scripts/editable-installs/setup_py_editable/setup_py_editable/__init__.py diff --git a/Cargo.lock b/Cargo.lock index 8b16e2fb5818..bad56abcdfa0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4355,6 +4355,7 @@ dependencies = [ "chrono", "clap", "clap_complete_command", + "configparser", "console", "ctrlc", "distribution-filename", @@ -4370,6 +4371,7 @@ dependencies = [ "itertools 0.12.1", "miette 6.0.1", "mimalloc", + "once_cell", "owo-colors 4.0.0", "pep508_rs", "platform-tags", diff --git a/crates/pypi-types/src/metadata.rs b/crates/pypi-types/src/metadata.rs index 4636f50b6cb1..93e508ed1207 100644 --- a/crates/pypi-types/src/metadata.rs +++ b/crates/pypi-types/src/metadata.rs @@ -210,6 +210,30 @@ impl Metadata23 { } } +/// Python Package Metadata 1.0 and later as specified in +/// . +/// +/// This is a subset of the full metadata specification, and only includes the +/// fields that have been consistent across all versions of the specification. +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] +#[serde(rename_all = "kebab-case")] +pub struct Metadata10 { + pub name: PackageName, +} + +impl Metadata10 { + /// Parse the [`Metadata10`] from a `PKG-INFO` file, as included in a source distribution. + pub fn parse_pkg_info(content: &[u8]) -> Result { + let headers = Headers::parse(content)?; + let name = PackageName::new( + headers + .get_first_value("Name") + .ok_or(Error::FieldNotFound("Name"))?, + )?; + Ok(Self { name }) + } +} + /// Parse a `Metadata-Version` field into a (major, minor) tuple. fn parse_version(metadata_version: &str) -> Result<(u8, u8), Error> { let (major, minor) = metadata_version diff --git a/crates/uv/Cargo.toml b/crates/uv/Cargo.toml index c68b44f10f3b..87a7066e146d 100644 --- a/crates/uv/Cargo.toml +++ b/crates/uv/Cargo.toml @@ -14,6 +14,7 @@ default-run = "uv" workspace = true [dependencies] +distribution-filename = { workspace = true } distribution-types = { workspace = true } install-wheel-rs = { workspace = true, features = ["clap"], default-features = false } pep508_rs = { workspace = true } @@ -40,6 +41,7 @@ base64 = { workspace = true } chrono = { workspace = true } clap = { workspace = true, features = ["derive", "string"] } clap_complete_command = { workspace = true } +configparser = { workspace = true } console = { workspace = true } ctrlc = { workspace = true } flate2 = { workspace = true, default-features = false } @@ -48,8 +50,10 @@ indexmap = { workspace = true } indicatif = { workspace = true } itertools = { workspace = true } miette = { workspace = true, features = ["fancy"] } +once_cell = { workspace = true } owo-colors = { workspace = true } pyproject-toml = { workspace = true } +regex = { workspace = true } rustc-hash = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -64,7 +68,6 @@ tracing-subscriber = { workspace = true, features = ["json"] } tracing-tree = { workspace = true } unicode-width = { workspace = true } url = { workspace = true } -distribution-filename = { version = "0.0.1", path = "../distribution-filename" } [target.'cfg(target_os = "windows")'.dependencies] mimalloc = { version = "0.1.39" } diff --git a/crates/uv/src/requirements.rs b/crates/uv/src/requirements.rs index 1c2a5ba422d1..272b929b99f6 100644 --- a/crates/uv/src/requirements.rs +++ b/crates/uv/src/requirements.rs @@ -4,14 +4,21 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; use anyhow::{Context, Result}; +use configparser::ini::Ini; use console::Term; use distribution_filename::{SourceDistFilename, WheelFilename}; use indexmap::IndexMap; +use once_cell::sync::Lazy; +use regex::Regex; use rustc_hash::FxHashSet; -use tracing::{instrument, Level}; +use serde::Deserialize; +use tracing::{debug, instrument, Level}; use distribution_types::{FlatIndexLocation, IndexUrl, RemoteSource}; -use pep508_rs::{Requirement, RequirementsTxtRequirement, UnnamedRequirement, VersionOrUrl}; +use pep508_rs::{ + Requirement, RequirementsTxtRequirement, Scheme, UnnamedRequirement, VersionOrUrl, +}; +use pypi_types::Metadata10; use requirements_txt::{EditableRequirement, FindLink, RequirementsTxt}; use uv_client::Connectivity; use uv_fs::Simplified; @@ -576,8 +583,160 @@ impl NamedRequirements { }); } + // Otherwise, download and/or extract the source archive. + if Scheme::parse(requirement.url.scheme()) == Some(Scheme::File) { + let path = requirement.url.to_file_path().map_err(|()| { + anyhow::anyhow!("Unable to convert file URL to path: {requirement}") + })?; + + if !path.exists() { + return Err(anyhow::anyhow!( + "Unnamed requirement at {path} not found", + path = path.simplified_display() + )); + } + + // Attempt to read a `PKG-INFO` from the directory. + if let Some(metadata) = fs_err::read(path.join("PKG-INFO")) + .ok() + .and_then(|contents| Metadata10::parse_pkg_info(&contents).ok()) + { + debug!( + "Found PKG-INFO metadata for {path} ({name})", + path = path.display(), + name = metadata.name + ); + return Ok(Requirement { + name: metadata.name, + extras: requirement.extras, + version_or_url: Some(VersionOrUrl::Url(requirement.url)), + marker: requirement.marker, + }); + } + + // Attempt to read a `pyproject.toml` file. + if let Some(pyproject) = fs_err::read_to_string(path.join("pyproject.toml")) + .ok() + .and_then(|contents| toml::from_str::(&contents).ok()) + { + // Read PEP 621 metadata from the `pyproject.toml`. + if let Some(project) = pyproject.project { + debug!( + "Found PEP 621 metadata for {path} in `pyproject.toml` ({name})", + path = path.display(), + name = project.name + ); + return Ok(Requirement { + name: project.name, + extras: requirement.extras, + version_or_url: Some(VersionOrUrl::Url(requirement.url)), + marker: requirement.marker, + }); + } + + // Read Poetry-specific metadata from the `pyproject.toml`. + if let Some(tool) = pyproject.tool { + if let Some(poetry) = tool.poetry { + if let Some(name) = poetry.name { + debug!( + "Found Poetry metadata for {path} in `pyproject.toml` ({name})", + path = path.display(), + name = name + ); + return Ok(Requirement { + name, + extras: requirement.extras, + version_or_url: Some(VersionOrUrl::Url(requirement.url)), + marker: requirement.marker, + }); + } + } + } + } + + // Attempt to read a `setup.cfg` from the directory. + if let Some(setup_cfg) = fs_err::read_to_string(path.join("setup.cfg")) + .ok() + .and_then(|contents| { + let mut ini = Ini::new_cs(); + ini.set_multiline(true); + ini.read(contents).ok() + }) + { + if let Some(section) = setup_cfg.get("metadata") { + if let Some(Some(name)) = section.get("name") { + if let Ok(name) = PackageName::from_str(name) { + debug!( + "Found setuptools metadata for {path} in `setup.cfg` ({name})", + path = path.display(), + name = name + ); + return Ok(Requirement { + name, + extras: requirement.extras, + version_or_url: Some(VersionOrUrl::Url(requirement.url)), + marker: requirement.marker, + }); + } + } + } + } + + // Attempt to read a `setup.py` from the directory. + if let Ok(setup_py) = fs_err::read_to_string(path.join("setup.py")) { + static SETUP_PY_NAME: Lazy = + Lazy::new(|| Regex::new(r#"name\s*[=:]\s*['"](?P[^'"]+)['"]"#).unwrap()); + + if let Some(name) = SETUP_PY_NAME + .captures(&setup_py) + .and_then(|captures| captures.name("name")) + .map(|name| name.as_str()) + { + if let Ok(name) = PackageName::from_str(name) { + debug!( + "Found setuptools metadata for {path} in `setup.py` ({name})", + path = path.display(), + name = name + ); + return Ok(Requirement { + name, + extras: requirement.extras, + version_or_url: Some(VersionOrUrl::Url(requirement.url)), + marker: requirement.marker, + }); + } + } + } + } + Err(anyhow::anyhow!( "Unable to infer package name for the unnamed requirement: {requirement}" )) } } + +/// A pyproject.toml as specified in PEP 517. +#[derive(Deserialize, Debug)] +#[serde(rename_all = "kebab-case")] +struct PyProjectToml { + project: Option, + tool: Option, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "kebab-case")] +struct Project { + name: PackageName, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "kebab-case")] +struct Tool { + poetry: Option, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "kebab-case")] +struct ToolPoetry { + name: Option, +} diff --git a/crates/uv/tests/pip_compile.rs b/crates/uv/tests/pip_compile.rs index 980b4078b79f..58d5d2325aa0 100644 --- a/crates/uv/tests/pip_compile.rs +++ b/crates/uv/tests/pip_compile.rs @@ -5746,3 +5746,72 @@ fn preserve_hashes_newer_version() -> Result<()> { Ok(()) } + +/// Detect the package name from metadata sources from local directories. +#[test] +fn detect_package_name() -> Result<()> { + let context = TestContext::new("3.12"); + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str(indoc! {r" + ../../scripts/editable-installs/poetry_editable + ../../scripts/editable-installs/black_editable + ../../scripts/editable-installs/setup_py_editable + ../../scripts/editable-installs/setup_cfg_editable + " + })?; + + let filter_path = regex::escape(&requirements_in.user_display().to_string()); + let filters: Vec<_> = [(filter_path.as_str(), "requirements.in")] + .into_iter() + .chain(INSTA_FILTERS.to_vec()) + .collect(); + + uv_snapshot!(filters, context.compile() + .arg(requirements_in.path()) + .current_dir(current_dir()?), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] --exclude-newer 2023-11-18T12:00:00Z requirements.in + anyio==4.0.0 + # via + # httpx + # poetry-editable + black @ ../../scripts/editable-installs/black_editable + certifi==2023.11.17 + # via + # httpcore + # httpx + # requests + charset-normalizer==3.3.2 + # via requests + h11==0.14.0 + # via httpcore + httpcore==1.0.2 + # via httpx + httpx==0.25.1 + # via setup-py-editable + idna==3.4 + # via + # anyio + # httpx + # requests + poetry-editable @ ../../scripts/editable-installs/poetry_editable + requests==2.31.0 + # via setup-cfg-editable + setup-cfg-editable @ ../../scripts/editable-installs/setup_cfg_editable + setup-py-editable @ ../../scripts/editable-installs/setup_py_editable + sniffio==1.3.0 + # via + # anyio + # httpx + urllib3==2.1.0 + # via requests + + ----- stderr ----- + Resolved 14 packages in [TIME] + "###); + + Ok(()) +} diff --git a/scripts/editable-installs/setup_cfg_editable/.gitignore b/scripts/editable-installs/setup_cfg_editable/.gitignore new file mode 100644 index 000000000000..eaa9f051bc40 --- /dev/null +++ b/scripts/editable-installs/setup_cfg_editable/.gitignore @@ -0,0 +1,2 @@ +# Artifacts from the build process. +*.egg-info/ diff --git a/scripts/editable-installs/setup_cfg_editable/setup.cfg b/scripts/editable-installs/setup_cfg_editable/setup.cfg new file mode 100644 index 000000000000..a907ed6354ca --- /dev/null +++ b/scripts/editable-installs/setup_cfg_editable/setup.cfg @@ -0,0 +1,3 @@ +[metadata] +name = setup-cfg-editable +version = 0.0.1 diff --git a/scripts/editable-installs/setup_cfg_editable/setup.py b/scripts/editable-installs/setup_cfg_editable/setup.py new file mode 100644 index 000000000000..7e49ce652982 --- /dev/null +++ b/scripts/editable-installs/setup_cfg_editable/setup.py @@ -0,0 +1,8 @@ +from setuptools import setup + +setup( + install_requires=[ + "requests", + 'importlib-metadata; python_version<"3.10"', + ], +) diff --git a/scripts/editable-installs/setup_cfg_editable/setup_cfg_editable/__init__.py b/scripts/editable-installs/setup_cfg_editable/setup_cfg_editable/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/scripts/editable-installs/setup_py_editable/.gitignore b/scripts/editable-installs/setup_py_editable/.gitignore new file mode 100644 index 000000000000..eaa9f051bc40 --- /dev/null +++ b/scripts/editable-installs/setup_py_editable/.gitignore @@ -0,0 +1,2 @@ +# Artifacts from the build process. +*.egg-info/ diff --git a/scripts/editable-installs/setup_py_editable/setup.py b/scripts/editable-installs/setup_py_editable/setup.py new file mode 100644 index 000000000000..6308361a3434 --- /dev/null +++ b/scripts/editable-installs/setup_py_editable/setup.py @@ -0,0 +1,9 @@ +from setuptools import setup + +setup( + name="setup-py-editable", + version="0.0.1", + install_requires=[ + "httpx", + ], +) diff --git a/scripts/editable-installs/setup_py_editable/setup_py_editable/__init__.py b/scripts/editable-installs/setup_py_editable/setup_py_editable/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1