Support recursive extras in direct pyproject.toml files (#1990)

## Summary

When a `pyproject.toml` is provided directly to `uv pip compile`, we
were failing to resolve recursive extras. The solution I settled on here
is to flatten them recursively when determining the requirements
upfront.

Closes #1987.

## Test Plan

`cargo test`

Cargo.toml

@@ -38,7 +38,6 @@ dashmap = { version = "5.5.3" }
 data-encoding = { version = "2.5.0" }
 derivative = { version = "2.2.0" }
 directories = { version = "5.0.1" }
-dirs = { version = "5.0.1" }
 dunce = { version = "1.0.4" }
 either = { version = "1.9.0" }
 flate2 = { version = "1.0.28", default-features = false }
@@ -106,7 +105,6 @@ unicode-width = { version = "0.1.11" }
 unscanny = { version = "0.1.0" }
 url = { version = "2.5.0" }
 urlencoding = { version = "2.1.3" }
-uuid = { version = "1.7.0", default-features = false }
 walkdir = { version = "2.4.0" }
 which = { version = "6.0.0" }
 zip = { version = "0.6.6", default-features = false, features = ["deflate"] }

crates/uv/Cargo.toml

@@ -49,6 +49,7 @@ dunce = { workspace = true }
 flate2 = { workspace = true, default-features = false }
 fs-err = { workspace = true, features = ["tokio"] }
 futures = { workspace = true }
+indexmap = { workspace = true }
 indicatif = { workspace = true }
 itertools = { workspace = true }
 miette = { workspace = true, features = ["fancy"] }
@@ -67,9 +68,9 @@ tracing = { workspace = true }
 tracing-durations-export = { workspace = true, features = ["plot"], optional = true }
 tracing-subscriber = { workspace = true }
 tracing-tree = { workspace = true }
+unicode-width = { workspace = true }
 url = { workspace = true }
 which = { workspace = true }
-unicode-width = { workspace = true }
 
 [target.'cfg(target_os = "windows")'.dependencies]
 mimalloc = "0.1.39"

crates/uv/src/requirements.rs

@@ -1,9 +1,11 @@
 //! A standard interface for working with heterogeneous sources of requirements.
 
 use std::path::{Path, PathBuf};
+use std::str::FromStr;
 
 use anyhow::{Context, Result};
 use console::Term;
+use indexmap::IndexMap;
 use rustc_hash::FxHashSet;
 
 use distribution_types::{FlatIndexLocation, IndexUrl};
@@ -210,26 +212,37 @@ impl RequirementsSpecification {
                 let mut used_extras = FxHashSet::default();
                 let mut requirements = Vec::new();
                 let mut project_name = None;
+
                 if let Some(project) = pyproject_toml.project {
+                    // Parse the project name.
+                    let parsed_project_name =
+                        PackageName::new(project.name).with_context(|| {
+                            format!("Invalid `project.name` in {}", path.normalized_display())
+                        })?;
+
+                    // Include the default dependencies.
                     requirements.extend(project.dependencies.unwrap_or_default());
-                    // Include any optional dependencies specified in `extras`
+
+                    // Include any optional dependencies specified in `extras`.
                     if !matches!(extras, ExtrasSpecification::None) {
-                        for (name, optional_requirements) in
-                            project.optional_dependencies.unwrap_or_default()
-                        {
-                            // TODO(konstin): It's not ideal that pyproject-toml doesn't use
-                            // `ExtraName`
-                            let normalized_name = ExtraName::new(name)?;
-                            if extras.contains(&normalized_name) {
-                                used_extras.insert(normalized_name);
-                                requirements.extend(optional_requirements);
+                        if let Some(optional_dependencies) = project.optional_dependencies {
+                            for (extra_name, optional_requirements) in &optional_dependencies {
+                                // TODO(konstin): It's not ideal that pyproject-toml doesn't use
+                                // `ExtraName`
+                                let normalized_name = ExtraName::from_str(extra_name)?;
+                                if extras.contains(&normalized_name) {
+                                    used_extras.insert(normalized_name);
+                                    requirements.extend(flatten_extra(
+                                        &parsed_project_name,
+                                        optional_requirements,
+                                        &optional_dependencies,
+                                    )?);
+                                }
                             }
                         }
                     }
-                    // Parse the project name
-                    project_name = Some(PackageName::new(project.name).with_context(|| {
-                        format!("Invalid `project.name` in {}", path.normalized_display())
-                    })?);
+
+                    project_name = Some(parsed_project_name);
                 }
 
                 if requirements.is_empty()
@@ -345,3 +358,71 @@ impl RequirementsSpecification {
         Self::from_sources(requirements, &[], &[], &ExtrasSpecification::None)
     }
 }
+
+/// Given an extra in a project that may contain references to the project
+/// itself, flatten it into a list of requirements.
+///
+/// For example:
+/// ```toml
+/// [project]
+/// name = "my-project"
+/// version = "0.0.1"
+/// dependencies = [
+///     "tomli",
+/// ]
+///
+/// [project.optional-dependencies]
+/// test = [
+///     "pep517",
+/// ]
+/// dev = [
+///     "my-project[test]",
+/// ]
+/// ```
+fn flatten_extra(
+    project_name: &PackageName,
+    requirements: &[Requirement],
+    extras: &IndexMap<String, Vec<Requirement>>,
+) -> Result<Vec<Requirement>> {
+    fn inner(
+        project_name: &PackageName,
+        requirements: &[Requirement],
+        extras: &IndexMap<String, Vec<Requirement>>,
+        seen: &mut FxHashSet<ExtraName>,
+    ) -> Result<Vec<Requirement>> {
+        let mut flattened = Vec::with_capacity(requirements.len());
+        for requirement in requirements {
+            if requirement.name == *project_name {
+                for extra in &requirement.extras {
+                    // Avoid infinite recursion on mutually recursive extras.
+                    if !seen.insert(extra.clone()) {
+                        continue;
+                    }
+
+                    // Flatten the extra requirements.
+                    for (name, extra_requirements) in extras {
+                        let normalized_name = ExtraName::from_str(name)?;
+                        if normalized_name == *extra {
+                            flattened.extend(inner(
+                                project_name,
+                                extra_requirements,
+                                extras,
+                                seen,
+                            )?);
+                        }
+                    }
+                }
+            } else {
+                flattened.push(requirement.clone());
+            }
+        }
+        Ok(flattened)
+    }
+
+    inner(
+        project_name,
+        requirements,
+        extras,
+        &mut FxHashSet::default(),
+    )
+}

crates/uv/tests/pip_compile.rs

@@ -4343,3 +4343,48 @@ fn pre_release_upper_bound_include() -> Result<()> {
 
     Ok(())
 }
+
+/// Resolve from a `pyproject.toml` file with a recursive extra.
+#[test]
+fn compile_pyproject_toml_recursive_extra() -> Result<()> {
+    let context = TestContext::new("3.12");
+    let pyproject_toml = context.temp_dir.child("pyproject.toml");
+    pyproject_toml.write_str(
+        r#"
+[project]
+name = "my-project"
+version = "0.0.1"
+dependencies = [
+    "tomli",
+]
+
+[project.optional-dependencies]
+test = [
+    "pep517",
+    "my-project[dev]"
+]
+dev = [
+    "my-project[test]",
+]
+"#,
+    )?;
+
+    uv_snapshot!(context.compile()
+            .arg("pyproject.toml")
+            .arg("--extra")
+            .arg("dev"), @r###"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    # This file was autogenerated by uv via the following command:
+    #    uv pip compile --cache-dir [CACHE_DIR] --exclude-newer 2023-11-18T12:00:00Z pyproject.toml --extra dev
+    pep517==0.13.1
+    tomli==2.0.1
+
+    ----- stderr -----
+    Resolved 2 packages in [TIME]
+    "###
+    );
+
+    Ok(())
+}