diff --git a/crates/uv-git/src/credentials.rs b/crates/uv-git/src/credentials.rs index b64ec2e2fc49..20ee12c6a67f 100644 --- a/crates/uv-git/src/credentials.rs +++ b/crates/uv-git/src/credentials.rs @@ -1,9 +1,15 @@ -use std::collections::HashMap; -use std::sync::{Arc, RwLock}; - use cache_key::RepositoryUrl; +use std::collections::HashMap; +use std::sync::{Arc, LazyLock, RwLock}; +use tracing::trace; +use url::Url; use uv_auth::Credentials; +/// Global authentication cache for a uv invocation. +/// +/// This is used to share Git credentials within a single process. +pub static GIT_STORE: LazyLock = LazyLock::new(GitStore::default); + /// A store for Git credentials. #[derive(Debug, Default)] pub struct GitStore(RwLock>>); @@ -19,3 +25,16 @@ impl GitStore { self.0.read().unwrap().get(url).cloned() } } + +/// Populate the global authentication store with credentials on a Git URL, if there are any. +/// +/// Returns `true` if the store was updated. +pub fn store_credentials_from_url(url: &Url) -> bool { + if let Some(credentials) = Credentials::from_url(url) { + trace!("Caching credentials for {url}"); + GIT_STORE.insert(RepositoryUrl::new(url), credentials); + true + } else { + false + } +} diff --git a/crates/uv-git/src/lib.rs b/crates/uv-git/src/lib.rs index 10284ff787fb..2ba6bca83b2c 100644 --- a/crates/uv-git/src/lib.rs +++ b/crates/uv-git/src/lib.rs @@ -1,8 +1,6 @@ -use std::sync::LazyLock; - use url::Url; -use crate::credentials::GitStore; +pub use crate::credentials::{store_credentials_from_url, GIT_STORE}; pub use crate::git::GitReference; pub use crate::resolver::{ GitResolver, GitResolverError, RepositoryReference, ResolvedRepositoryReference, @@ -16,11 +14,6 @@ mod resolver; mod sha; mod source; -/// Global authentication cache for a uv invocation. -/// -/// This is used to share Git credentials within a single process. -pub static GIT_STORE: LazyLock = LazyLock::new(GitStore::default); - /// A URL reference to a Git repository. #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Hash, Ord)] pub struct GitUrl { diff --git a/crates/uv-workspace/src/pyproject.rs b/crates/uv-workspace/src/pyproject.rs index af1a5717fd04..faf8a65ce877 100644 --- a/crates/uv-workspace/src/pyproject.rs +++ b/crates/uv-workspace/src/pyproject.rs @@ -120,6 +120,8 @@ pub struct Project { pub version: Option, /// The Python versions this project is compatible with. pub requires_python: Option, + /// The dependencies of the project. + pub dependencies: Option>, /// The optional dependencies of the project. pub optional_dependencies: Option>>, diff --git a/crates/uv-workspace/src/workspace.rs b/crates/uv-workspace/src/workspace.rs index 62a802e635b2..22d811c4c639 100644 --- a/crates/uv-workspace/src/workspace.rs +++ b/crates/uv-workspace/src/workspace.rs @@ -531,6 +531,22 @@ impl Workspace { &self.sources } + /// Returns an iterator over all sources in the workspace. + pub fn iter_sources(&self) -> impl Iterator { + self.packages + .values() + .filter_map(|member| { + member.pyproject_toml().tool.as_ref().and_then(|tool| { + tool.uv + .as_ref() + .and_then(|uv| uv.sources.as_ref()) + .map(ToolUvSources::inner) + .map(|sources| sources.values()) + }) + }) + .flatten() + } + /// The `pyproject.toml` of the workspace. pub fn pyproject_toml(&self) -> &PyProjectToml { &self.pyproject_toml @@ -1608,6 +1624,9 @@ mod tests { "name": "bird-feeder", "version": "1.0.0", "requires-python": ">=3.12", + "dependencies": [ + "anyio>=4.3.0,<5" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -1619,6 +1638,9 @@ mod tests { "name": "bird-feeder", "version": "1.0.0", "requires-python": ">=3.12", + "dependencies": [ + "anyio>=4.3.0,<5" + ], "optional-dependencies": null }, "tool": null @@ -1653,6 +1675,9 @@ mod tests { "name": "bird-feeder", "version": "1.0.0", "requires-python": ">=3.12", + "dependencies": [ + "anyio>=4.3.0,<5" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -1664,6 +1689,9 @@ mod tests { "name": "bird-feeder", "version": "1.0.0", "requires-python": ">=3.12", + "dependencies": [ + "anyio>=4.3.0,<5" + ], "optional-dependencies": null }, "tool": null @@ -1697,6 +1725,10 @@ mod tests { "name": "albatross", "version": "0.1.0", "requires-python": ">=3.12", + "dependencies": [ + "bird-feeder", + "tqdm>=4,<5" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -1707,6 +1739,10 @@ mod tests { "name": "bird-feeder", "version": "1.0.0", "requires-python": ">=3.8", + "dependencies": [ + "anyio>=4.3.0,<5", + "seeds" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -1717,6 +1753,9 @@ mod tests { "name": "seeds", "version": "1.0.0", "requires-python": ">=3.12", + "dependencies": [ + "idna==3.6" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -1732,6 +1771,10 @@ mod tests { "name": "albatross", "version": "0.1.0", "requires-python": ">=3.12", + "dependencies": [ + "bird-feeder", + "tqdm>=4,<5" + ], "optional-dependencies": null }, "tool": { @@ -1786,6 +1829,10 @@ mod tests { "name": "albatross", "version": "0.1.0", "requires-python": ">=3.12", + "dependencies": [ + "bird-feeder", + "tqdm>=4,<5" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -1796,6 +1843,10 @@ mod tests { "name": "bird-feeder", "version": "1.0.0", "requires-python": ">=3.12", + "dependencies": [ + "anyio>=4.3.0,<5", + "seeds" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -1806,6 +1857,9 @@ mod tests { "name": "seeds", "version": "1.0.0", "requires-python": ">=3.12", + "dependencies": [ + "idna==3.6" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -1861,6 +1915,9 @@ mod tests { "name": "albatross", "version": "0.1.0", "requires-python": ">=3.12", + "dependencies": [ + "tqdm>=4,<5" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -1872,6 +1929,9 @@ mod tests { "name": "albatross", "version": "0.1.0", "requires-python": ">=3.12", + "dependencies": [ + "tqdm>=4,<5" + ], "optional-dependencies": null }, "tool": null @@ -1973,6 +2033,9 @@ mod tests { "name": "albatross", "version": "0.1.0", "requires-python": ">=3.12", + "dependencies": [ + "tqdm>=4,<5" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -1983,6 +2046,9 @@ mod tests { "name": "seeds", "version": "1.0.0", "requires-python": ">=3.12", + "dependencies": [ + "idna==3.6" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -1994,6 +2060,9 @@ mod tests { "name": "albatross", "version": "0.1.0", "requires-python": ">=3.12", + "dependencies": [ + "tqdm>=4,<5" + ], "optional-dependencies": null }, "tool": { @@ -2062,6 +2131,9 @@ mod tests { "name": "albatross", "version": "0.1.0", "requires-python": ">=3.12", + "dependencies": [ + "tqdm>=4,<5" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -2072,6 +2144,9 @@ mod tests { "name": "seeds", "version": "1.0.0", "requires-python": ">=3.12", + "dependencies": [ + "idna==3.6" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -2083,6 +2158,9 @@ mod tests { "name": "albatross", "version": "0.1.0", "requires-python": ">=3.12", + "dependencies": [ + "tqdm>=4,<5" + ], "optional-dependencies": null }, "tool": { @@ -2152,6 +2230,9 @@ mod tests { "name": "albatross", "version": "0.1.0", "requires-python": ">=3.12", + "dependencies": [ + "tqdm>=4,<5" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -2162,6 +2243,9 @@ mod tests { "name": "bird-feeder", "version": "1.0.0", "requires-python": ">=3.12", + "dependencies": [ + "anyio>=4.3.0,<5" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -2172,6 +2256,9 @@ mod tests { "name": "seeds", "version": "1.0.0", "requires-python": ">=3.12", + "dependencies": [ + "idna==3.6" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -2183,6 +2270,9 @@ mod tests { "name": "albatross", "version": "0.1.0", "requires-python": ">=3.12", + "dependencies": [ + "tqdm>=4,<5" + ], "optional-dependencies": null }, "tool": { @@ -2252,6 +2342,9 @@ mod tests { "name": "albatross", "version": "0.1.0", "requires-python": ">=3.12", + "dependencies": [ + "tqdm>=4,<5" + ], "optional-dependencies": null }, "pyproject_toml": "[PYPROJECT_TOML]" @@ -2263,6 +2356,9 @@ mod tests { "name": "albatross", "version": "0.1.0", "requires-python": ">=3.12", + "dependencies": [ + "tqdm>=4,<5" + ], "optional-dependencies": null }, "tool": { diff --git a/crates/uv/src/commands/project/sync.rs b/crates/uv/src/commands/project/sync.rs index 980e0eab1022..97d3c025f898 100644 --- a/crates/uv/src/commands/project/sync.rs +++ b/crates/uv/src/commands/project/sync.rs @@ -1,9 +1,19 @@ +use crate::commands::pip::loggers::{DefaultInstallLogger, DefaultResolveLogger, InstallLogger}; +use crate::commands::pip::operations::Modifications; +use crate::commands::project::lock::do_safe_lock; +use crate::commands::project::{ProjectError, SharedState}; +use crate::commands::{pip, project, ExitStatus}; +use crate::printer::Printer; +use crate::settings::{InstallerSettingsRef, ResolverInstallerSettings}; use anyhow::{Context, Result}; -use itertools::Itertools; - use distribution_types::{DirectorySourceDist, Dist, ResolvedDist, SourceDist}; -use pep508_rs::MarkerTree; -use uv_auth::store_credentials_from_url; +use itertools::Itertools; +use pep508_rs::{MarkerTree, Requirement, VersionOrUrl}; +use pypi_types::{ + LenientRequirement, ParsedArchiveUrl, ParsedGitUrl, ParsedUrl, VerbatimParsedUrl, +}; +use std::borrow::Cow; +use std::str::FromStr; use uv_cache::Cache; use uv_client::{Connectivity, FlatIndexClient, RegistryClientBuilder}; use uv_configuration::{ @@ -18,16 +28,9 @@ use uv_python::{PythonDownloads, PythonEnvironment, PythonPreference, PythonRequ use uv_resolver::{FlatIndex, Lock}; use uv_types::{BuildIsolation, HashStrategy}; use uv_warnings::warn_user; +use uv_workspace::pyproject::{Source, ToolUvSources}; use uv_workspace::{DiscoveryOptions, InstallTarget, MemberDiscovery, VirtualProject, Workspace}; -use crate::commands::pip::loggers::{DefaultInstallLogger, DefaultResolveLogger, InstallLogger}; -use crate::commands::pip::operations::Modifications; -use crate::commands::project::lock::do_safe_lock; -use crate::commands::project::{ProjectError, SharedState}; -use crate::commands::{pip, project, ExitStatus}; -use crate::printer::Printer; -use crate::settings::{InstallerSettingsRef, ResolverInstallerSettings}; - /// Sync the project environment. #[allow(clippy::fn_params_excessive_bools)] pub(crate) async fn sync( @@ -250,9 +253,12 @@ pub(super) async fn do_sync( // Add all authenticated sources to the cache. for url in index_locations.urls() { - store_credentials_from_url(url); + uv_auth::store_credentials_from_url(url); } + // Populate credentials from the workspace. + store_credentials_from_workspace(target.workspace()); + // Initialize the registry client. let client = RegistryClientBuilder::new(cache.clone()) .native_tls(native_tls) @@ -399,3 +405,78 @@ fn apply_editable_mode( }), } } + +fn store_credentials_from_workspace(workspace: &Workspace) { + for member in workspace.packages().values() { + // Iterate over the `tool.uv.sources`. + for source in member + .pyproject_toml() + .tool + .as_ref() + .and_then(|tool| tool.uv.as_ref()) + .and_then(|uv| uv.sources.as_ref()) + .map(ToolUvSources::inner) + .iter() + .flat_map(|sources| sources.values()) + { + match source { + Source::Git { git, .. } => { + uv_git::store_credentials_from_url(git); + } + Source::Url { url, .. } => { + uv_auth::store_credentials_from_url(url); + } + _ => {} + } + } + + // Iterate over all dependencies. + let dependencies = member + .pyproject_toml() + .project + .as_ref() + .and_then(|project| project.dependencies.as_ref()) + .into_iter() + .flatten(); + let optional_dependencies = member + .pyproject_toml() + .project + .as_ref() + .and_then(|project| project.optional_dependencies.as_ref()) + .into_iter() + .flat_map(|optional| optional.values()) + .flatten(); + let dev_dependencies = member + .pyproject_toml() + .tool + .as_ref() + .and_then(|tool| tool.uv.as_ref()) + .and_then(|uv| uv.dev_dependencies.as_ref()) + .into_iter() + .flatten(); + + for requirement in dependencies + .chain(optional_dependencies) + .filter_map(|requires_dist| { + LenientRequirement::::from_str(requires_dist) + .map(Requirement::from) + .map(Cow::Owned) + .ok() + }) + .chain(dev_dependencies.map(Cow::Borrowed)) + { + let Some(VersionOrUrl::Url(url)) = &requirement.version_or_url else { + continue; + }; + match &url.parsed_url { + ParsedUrl::Git(ParsedGitUrl { url, .. }) => { + uv_git::store_credentials_from_url(url.repository()); + } + ParsedUrl::Archive(ParsedArchiveUrl { url, .. }) => { + uv_auth::store_credentials_from_url(url); + } + _ => {} + } + } + } +} diff --git a/crates/uv/tests/common/mod.rs b/crates/uv/tests/common/mod.rs index 54d9840ccb4d..c1e273cc1807 100644 --- a/crates/uv/tests/common/mod.rs +++ b/crates/uv/tests/common/mod.rs @@ -442,7 +442,7 @@ impl TestContext { if cfg!(all(windows, debug_assertions)) { // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the // default windows stack of 1MB - command.env("UV_STACK_SIZE", (2 * 1024 * 1024).to_string()); + command.env("UV_STACK_SIZE", (4 * 1024 * 1024).to_string()); } } @@ -533,7 +533,7 @@ impl TestContext { if cfg!(all(windows, debug_assertions)) { // TODO(konstin): Reduce stack usage in debug mode enough that the tests pass with the // default windows stack of 1MB - command.env("UV_STACK_SIZE", (2 * 1024 * 1024).to_string()); + command.env("UV_STACK_SIZE", (4 * 1024 * 1024).to_string()); } command diff --git a/crates/uv/tests/lock.rs b/crates/uv/tests/lock.rs index dee7508218e4..c9fc1d4fdcc1 100644 --- a/crates/uv/tests/lock.rs +++ b/crates/uv/tests/lock.rs @@ -6034,7 +6034,7 @@ fn lock_redact_https() -> Result<()> { /// However, we don't currently avoid persisting Git credentials in `uv.lock`. #[test] -fn lock_redact_git() -> Result<()> { +fn lock_redact_git_pep508() -> Result<()> { let context = TestContext::new("3.12"); let token = decode_token(common::READ_ONLY_GITHUB_TOKEN); @@ -6111,7 +6111,104 @@ fn lock_redact_git() -> Result<()> { "###); // Install from the lockfile. - uv_snapshot!(&filters, context.sync().arg("--frozen"), @r###" + uv_snapshot!(&filters, context.sync().arg("--frozen").arg("--reinstall").arg("--no-cache"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Prepared 2 packages in [TIME] + Installed 2 packages in [TIME] + + foo==0.1.0 (from file://[TEMP_DIR]/) + + uv-private-pypackage==0.1.0 (from git+https://github.com/astral-test/uv-private-pypackage@d780faf0ac91257d4d5a4f0c5a0e4509608c0071) + "###); + + Ok(()) +} + +/// However, we don't currently avoid persisting Git credentials in `uv.lock`. +#[test] +fn lock_redact_git_sources() -> Result<()> { + let context = TestContext::new("3.12"); + let token = decode_token(common::READ_ONLY_GITHUB_TOKEN); + + let filters: Vec<_> = [(token.as_str(), "***")] + .into_iter() + .chain(context.filters()) + .collect(); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str(&formatdoc! { + r#" + [project] + name = "foo" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["uv-private-pypackage"] + + [build-system] + requires = ["setuptools>=42"] + build-backend = "setuptools.build_meta" + + [tool.uv.sources] + uv-private-pypackage = {{ git = "https://{token}@github.com/astral-test/uv-private-pypackage" }} + "#, + token = token, + })?; + + uv_snapshot!(&filters, context.lock(), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + "###); + + let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock")).unwrap(); + + insta::with_settings!({ + filters => filters.clone(), + }, { + assert_snapshot!( + lock, @r###" + version = 1 + requires-python = ">=3.12" + + [options] + exclude-newer = "2024-03-25T00:00:00Z" + + [[package]] + name = "foo" + version = "0.1.0" + source = { editable = "." } + dependencies = [ + { name = "uv-private-pypackage" }, + ] + + [package.metadata] + requires-dist = [{ name = "uv-private-pypackage", git = "https://github.com/astral-test/uv-private-pypackage" }] + + [[package]] + name = "uv-private-pypackage" + version = "0.1.0" + source = { git = "https://github.com/astral-test/uv-private-pypackage#d780faf0ac91257d4d5a4f0c5a0e4509608c0071" } + "### + ); + }); + + // Re-run with `--locked`. + uv_snapshot!(&filters, context.lock().arg("--locked"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + "###); + + // Install from the lockfile. + uv_snapshot!(&filters, context.sync().arg("--frozen").arg("--reinstall").arg("--no-cache"), @r###" success: true exit_code: 0 ----- stdout ----- diff --git a/crates/uv/tests/pip_compile.rs b/crates/uv/tests/pip_compile.rs index f4b0fff8975b..099d2d43956c 100644 --- a/crates/uv/tests/pip_compile.rs +++ b/crates/uv/tests/pip_compile.rs @@ -686,7 +686,7 @@ build-backend = "poetry.core.masonry.api" ----- stdout ----- ----- stderr ----- - error: Failed to extract static metadata from `pyproject.toml` + error: Failed to parse: `pyproject.toml` Caused by: TOML parse error at line 13, column 1 | 13 | [project.dependencies]