From c39f1b6b523a0cd19aca14c7719a7fb41935616e Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Fri, 1 Nov 2024 10:58:37 -0400 Subject: [PATCH] Add frozen targets Use lock Move onto target Validate early --- Cargo.lock | 1 - crates/uv-resolver/src/lib.rs | 4 +- crates/uv-resolver/src/lock/mod.rs | 18 +- .../uv-resolver/src/lock/requirements_txt.rs | 20 +- crates/uv-resolver/src/lock/target.rs | 278 ++++++++++++++++++ crates/uv-workspace/Cargo.toml | 1 - crates/uv-workspace/src/lib.rs | 4 +- crates/uv-workspace/src/workspace.rs | 127 +------- crates/uv/src/commands/project/add.rs | 20 +- crates/uv/src/commands/project/export.rs | 48 ++- crates/uv/src/commands/project/mod.rs | 66 +++-- crates/uv/src/commands/project/remove.rs | 19 +- crates/uv/src/commands/project/run.rs | 51 +++- crates/uv/src/commands/project/sync.rs | 93 +++--- crates/uv/src/commands/project/tree.rs | 11 +- crates/uv/tests/it/export.rs | 83 ++++++ crates/uv/tests/it/sync.rs | 13 + 17 files changed, 614 insertions(+), 243 deletions(-) create mode 100644 crates/uv-resolver/src/lock/target.rs diff --git a/Cargo.lock b/Cargo.lock index 416935969267d..a1aa322c9cc90 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5360,7 +5360,6 @@ version = "0.0.1" dependencies = [ "anyhow", "assert_fs", - "either", "fs-err", "glob", "insta", diff --git a/crates/uv-resolver/src/lib.rs b/crates/uv-resolver/src/lib.rs index 38b4ec140983a..664ca7c3b5143 100644 --- a/crates/uv-resolver/src/lib.rs +++ b/crates/uv-resolver/src/lib.rs @@ -4,8 +4,8 @@ pub use exclude_newer::ExcludeNewer; pub use exclusions::Exclusions; pub use flat_index::{FlatDistributions, FlatIndex}; pub use lock::{ - Lock, LockError, LockVersion, RequirementsTxtExport, ResolverManifest, SatisfiesResult, - TreeDisplay, VERSION, + InstallTarget, Lock, LockError, LockVersion, RequirementsTxtExport, ResolverManifest, + SatisfiesResult, TreeDisplay, VERSION, }; pub use manifest::Manifest; pub use options::{Flexibility, Options, OptionsBuilder}; diff --git a/crates/uv-resolver/src/lock/mod.rs b/crates/uv-resolver/src/lock/mod.rs index 20c4015e7803a..073f6aba06a9c 100644 --- a/crates/uv-resolver/src/lock/mod.rs +++ b/crates/uv-resolver/src/lock/mod.rs @@ -15,6 +15,7 @@ use toml_edit::{value, Array, ArrayOfTables, InlineTable, Item, Table, Value}; use url::Url; pub use crate::lock::requirements_txt::RequirementsTxtExport; +pub use crate::lock::target::InstallTarget; pub use crate::lock::tree::TreeDisplay; use crate::requires_python::SimplifiedMarkerTree; use crate::resolution::{AnnotatedDist, ResolutionGraphNode}; @@ -44,9 +45,10 @@ use uv_pypi_types::{ }; use uv_types::{BuildContext, HashStrategy}; use uv_workspace::dependency_groups::DependencyGroupError; -use uv_workspace::{InstallTarget, Workspace}; +use uv_workspace::Workspace; mod requirements_txt; +mod target; mod tree; /// The current version of the lockfile format. @@ -544,6 +546,20 @@ impl Lock { &self.manifest.members } + /// Return the workspace root used to generate this lock. + pub fn root(&self) -> Option<&PackageName> { + self.packages.iter().find_map(|package| { + let (Source::Editable(path) | Source::Virtual(path)) = &package.id.source else { + return None; + }; + if path == Path::new("") { + Some(&package.id.name) + } else { + None + } + }) + } + /// Returns the supported environments that were used to generate this /// lock. /// diff --git a/crates/uv-resolver/src/lock/requirements_txt.rs b/crates/uv-resolver/src/lock/requirements_txt.rs index 967beaa5c3ef7..df7f900c63d0c 100644 --- a/crates/uv-resolver/src/lock/requirements_txt.rs +++ b/crates/uv-resolver/src/lock/requirements_txt.rs @@ -10,9 +10,6 @@ use petgraph::{Directed, Graph}; use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet}; use url::Url; -use crate::graph_ops::marker_reachability; -use crate::lock::{Package, PackageId, Source}; -use crate::{Lock, LockError}; use uv_configuration::{DevGroupsManifest, EditableMode, ExtrasSpecification, InstallOptions}; use uv_distribution_filename::{DistExtension, SourceDistExtension}; use uv_fs::Simplified; @@ -20,7 +17,10 @@ use uv_git::GitReference; use uv_normalize::ExtraName; use uv_pep508::MarkerTree; use uv_pypi_types::{ParsedArchiveUrl, ParsedGitUrl}; -use uv_workspace::InstallTarget; + +use crate::graph_ops::marker_reachability; +use crate::lock::{Package, PackageId, Source}; +use crate::{InstallTarget, LockError}; type LockGraph<'lock> = Graph, Edge, Directed>; @@ -34,7 +34,6 @@ pub struct RequirementsTxtExport<'lock> { impl<'lock> RequirementsTxtExport<'lock> { pub fn from_lock( - lock: &'lock Lock, target: InstallTarget<'lock>, extras: &ExtrasSpecification, dev: &DevGroupsManifest, @@ -42,7 +41,7 @@ impl<'lock> RequirementsTxtExport<'lock> { hashes: bool, install_options: &'lock InstallOptions, ) -> Result { - let size_guess = lock.packages.len(); + let size_guess = target.lock().packages.len(); let mut petgraph = LockGraph::with_capacity(size_guess, size_guess); let mut inverse = FxHashMap::with_capacity_and_hasher(size_guess, FxBuildHasher); @@ -53,7 +52,8 @@ impl<'lock> RequirementsTxtExport<'lock> { // Add the workspace package to the queue. for root_name in target.packages() { - let dist = lock + let dist = target + .lock() .find_by_name(root_name) .expect("found too many packages matching root") .expect("could not find root"); @@ -88,7 +88,7 @@ impl<'lock> RequirementsTxtExport<'lock> { // Add any development dependencies. for group in dev.iter() { for dep in dist.dependency_groups.get(group).into_iter().flatten() { - let dep_dist = lock.find_by_id(&dep.package_id); + let dep_dist = target.lock().find_by_id(&dep.package_id); // Add the dependency to the graph. if let Entry::Vacant(entry) = inverse.entry(&dep.package_id) { @@ -135,7 +135,7 @@ impl<'lock> RequirementsTxtExport<'lock> { }; for dep in deps { - let dep_dist = lock.find_by_id(&dep.package_id); + let dep_dist = target.lock().find_by_id(&dep.package_id); // Add the dependency to the graph. if let Entry::Vacant(entry) = inverse.entry(&dep.package_id) { @@ -175,7 +175,7 @@ impl<'lock> RequirementsTxtExport<'lock> { install_options.include_package( &package.id.name, target.project_name(), - lock.members(), + target.lock().members(), ) }) .map(|(index, package)| Requirement { diff --git a/crates/uv-resolver/src/lock/target.rs b/crates/uv-resolver/src/lock/target.rs new file mode 100644 index 0000000000000..272fa6c094459 --- /dev/null +++ b/crates/uv-resolver/src/lock/target.rs @@ -0,0 +1,278 @@ +use std::collections::{BTreeMap, VecDeque}; + +use either::Either; +use rustc_hash::FxHashSet; +use uv_configuration::{BuildOptions, DevGroupsManifest, ExtrasSpecification, InstallOptions}; +use uv_distribution_types::{Resolution, ResolvedDist}; +use uv_normalize::{ExtraName, GroupName, PackageName, DEV_DEPENDENCIES}; +use uv_platform_tags::Tags; +use uv_pypi_types::{ResolverMarkerEnvironment, VerbatimParsedUrl}; +use uv_workspace::dependency_groups::{DependencyGroupError, FlatDependencyGroups}; +use uv_workspace::Workspace; + +use crate::lock::{LockErrorKind, Package, TagPolicy}; +use crate::{Lock, LockError}; + +/// A target that can be installed from a lockfile. +#[derive(Debug, Copy, Clone)] +pub enum InstallTarget<'env> { + /// A project (which could be a workspace root or member). + Project { + workspace: &'env Workspace, + name: &'env PackageName, + lock: &'env Lock, + }, + /// An entire workspace. + Workspace { + workspace: &'env Workspace, + lock: &'env Lock, + }, + /// An entire workspace with a (legacy) non-project root. + NonProjectWorkspace { + workspace: &'env Workspace, + lock: &'env Lock, + }, +} + +impl<'env> InstallTarget<'env> { + /// Return the [`Workspace`] of the target. + pub fn workspace(&self) -> &'env Workspace { + match self { + Self::Project { workspace, .. } => workspace, + Self::Workspace { workspace, .. } => workspace, + Self::NonProjectWorkspace { workspace, .. } => workspace, + } + } + + /// Return the [`Lock`] of the target. + pub fn lock(&self) -> &'env Lock { + match self { + Self::Project { lock, .. } => lock, + Self::Workspace { lock, .. } => lock, + Self::NonProjectWorkspace { lock, .. } => lock, + } + } + + /// Return the [`PackageName`] of the target. + pub fn packages(&self) -> impl Iterator { + match self { + Self::Project { name, .. } => Either::Right(Either::Left(std::iter::once(*name))), + Self::NonProjectWorkspace { lock, .. } => Either::Left(lock.members().iter()), + Self::Workspace { lock, .. } => { + // Identify the workspace members. + // + // The members are encoded directly in the lockfile, unless the workspace contains a + // single member at the root, in which case, we identify it by its source. + if lock.members().is_empty() { + Either::Right(Either::Right(lock.root().into_iter())) + } else { + Either::Left(lock.members().iter()) + } + } + } + } + + /// Return the [`InstallTarget`] dependency groups. + /// + /// Returns dependencies that apply to the workspace root, but not any of its members. As such, + /// only returns a non-empty iterator for virtual workspaces, which can include dev dependencies + /// on the virtual root. + pub fn groups( + &self, + ) -> Result< + BTreeMap>>, + DependencyGroupError, + > { + match self { + Self::Project { .. } => Ok(BTreeMap::default()), + Self::Workspace { .. } => Ok(BTreeMap::default()), + Self::NonProjectWorkspace { workspace, .. } => { + // For non-projects, we might have `dependency-groups` or `tool.uv.dev-dependencies` + // that are attached to the workspace root (which isn't a member). + + // First, collect `tool.uv.dev_dependencies` + let dev_dependencies = workspace + .pyproject_toml() + .tool + .as_ref() + .and_then(|tool| tool.uv.as_ref()) + .and_then(|uv| uv.dev_dependencies.as_ref()); + + // Then, collect `dependency-groups` + let dependency_groups = workspace + .pyproject_toml() + .dependency_groups + .iter() + .flatten() + .collect::>(); + + // Merge any overlapping groups. + let mut map = BTreeMap::new(); + for (name, dependencies) in + FlatDependencyGroups::from_dependency_groups(&dependency_groups)? + .into_iter() + .chain( + // Only add the `dev` group if `dev-dependencies` is defined. + dev_dependencies.into_iter().map(|requirements| { + (DEV_DEPENDENCIES.clone(), requirements.clone()) + }), + ) + { + match map.entry(name) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(dependencies); + } + std::collections::btree_map::Entry::Occupied(mut entry) => { + entry.get_mut().extend(dependencies); + } + } + } + + Ok(map) + } + } + } + + /// Return the [`PackageName`] of the target, if available. + pub fn project_name(&self) -> Option<&PackageName> { + match self { + Self::Project { name, .. } => Some(name), + Self::Workspace { .. } => None, + Self::NonProjectWorkspace { .. } => None, + } + } + + /// Convert the [`Lock`] to a [`Resolution`] using the given marker environment, tags, and root. + pub fn to_resolution( + &self, + marker_env: &ResolverMarkerEnvironment, + tags: &Tags, + extras: &ExtrasSpecification, + dev: &DevGroupsManifest, + build_options: &BuildOptions, + install_options: &InstallOptions, + ) -> Result { + let mut queue: VecDeque<(&Package, Option<&ExtraName>)> = VecDeque::new(); + let mut seen = FxHashSet::default(); + + // Add the workspace packages to the queue. + for root_name in self.packages() { + let root = self + .lock() + .find_by_name(root_name) + .map_err(|_| LockErrorKind::MultipleRootPackages { + name: root_name.clone(), + })? + .ok_or_else(|| LockErrorKind::MissingRootPackage { + name: root_name.clone(), + })?; + + if dev.prod() { + // Add the base package. + queue.push_back((root, None)); + + // Add any extras. + match extras { + ExtrasSpecification::None => {} + ExtrasSpecification::All => { + for extra in root.optional_dependencies.keys() { + queue.push_back((root, Some(extra))); + } + } + ExtrasSpecification::Some(extras) => { + for extra in extras { + queue.push_back((root, Some(extra))); + } + } + } + } + + // Add any dev dependencies. + for group in dev.iter() { + for dep in root.dependency_groups.get(group).into_iter().flatten() { + if dep.complexified_marker.evaluate(marker_env, &[]) { + let dep_dist = self.lock().find_by_id(&dep.package_id); + if seen.insert((&dep.package_id, None)) { + queue.push_back((dep_dist, None)); + } + for extra in &dep.extra { + if seen.insert((&dep.package_id, Some(extra))) { + queue.push_back((dep_dist, Some(extra))); + } + } + } + } + } + } + + // Add any dependency groups that are exclusive to the workspace root (e.g., dev + // dependencies in (legacy) non-project workspace roots). + let groups = self + .groups() + .map_err(|err| LockErrorKind::DependencyGroup { err })?; + for group in dev.iter() { + for dependency in groups.get(group).into_iter().flatten() { + if dependency.marker.evaluate(marker_env, &[]) { + let root_name = &dependency.name; + let root = self + .lock() + .find_by_markers(root_name, marker_env) + .map_err(|_| LockErrorKind::MultipleRootPackages { + name: root_name.clone(), + })? + .ok_or_else(|| LockErrorKind::MissingRootPackage { + name: root_name.clone(), + })?; + + // Add the base package. + queue.push_back((root, None)); + + // Add any extras. + for extra in &dependency.extras { + queue.push_back((root, Some(extra))); + } + } + } + } + + let mut map = BTreeMap::default(); + let mut hashes = BTreeMap::default(); + while let Some((dist, extra)) = queue.pop_front() { + let deps = if let Some(extra) = extra { + Either::Left(dist.optional_dependencies.get(extra).into_iter().flatten()) + } else { + Either::Right(dist.dependencies.iter()) + }; + for dep in deps { + if dep.complexified_marker.evaluate(marker_env, &[]) { + let dep_dist = self.lock().find_by_id(&dep.package_id); + if seen.insert((&dep.package_id, None)) { + queue.push_back((dep_dist, None)); + } + for extra in &dep.extra { + if seen.insert((&dep.package_id, Some(extra))) { + queue.push_back((dep_dist, Some(extra))); + } + } + } + } + if install_options.include_package( + &dist.id.name, + self.project_name(), + self.lock().members(), + ) { + map.insert( + dist.id.name.clone(), + ResolvedDist::Installable(dist.to_dist( + self.workspace().install_path(), + TagPolicy::Required(tags), + build_options, + )?), + ); + hashes.insert(dist.id.name.clone(), dist.hashes()); + } + } + let diagnostics = vec![]; + Ok(Resolution::new(map, hashes, diagnostics)) + } +} diff --git a/crates/uv-workspace/Cargo.toml b/crates/uv-workspace/Cargo.toml index d77d08fcd21c1..bcff16606f133 100644 --- a/crates/uv-workspace/Cargo.toml +++ b/crates/uv-workspace/Cargo.toml @@ -29,7 +29,6 @@ uv-pypi-types = { workspace = true } uv-static = { workspace = true } uv-warnings = { workspace = true } -either = { workspace = true } fs-err = { workspace = true } glob = { workspace = true } itertools = { workspace = true } diff --git a/crates/uv-workspace/src/lib.rs b/crates/uv-workspace/src/lib.rs index 74bc631d90403..1fde70b4a4c84 100644 --- a/crates/uv-workspace/src/lib.rs +++ b/crates/uv-workspace/src/lib.rs @@ -1,6 +1,6 @@ pub use workspace::{ - check_nested_workspaces, DiscoveryOptions, InstallTarget, MemberDiscovery, ProjectWorkspace, - VirtualProject, Workspace, WorkspaceError, WorkspaceMember, + check_nested_workspaces, DiscoveryOptions, MemberDiscovery, ProjectWorkspace, VirtualProject, + Workspace, WorkspaceError, WorkspaceMember, }; pub mod dependency_groups; diff --git a/crates/uv-workspace/src/workspace.rs b/crates/uv-workspace/src/workspace.rs index 8dec546cf9d71..f66f771dcbcca 100644 --- a/crates/uv-workspace/src/workspace.rs +++ b/crates/uv-workspace/src/workspace.rs @@ -5,7 +5,6 @@ use crate::pyproject::{ DependencyGroups, Project, PyProjectToml, PyprojectTomlError, Sources, ToolUvSources, ToolUvWorkspace, }; -use either::Either; use glob::{glob, GlobError, PatternError}; use rustc_hash::FxHashSet; use std::collections::{BTreeMap, BTreeSet}; @@ -15,7 +14,7 @@ use uv_distribution_types::Index; use uv_fs::{Simplified, CWD}; use uv_normalize::{GroupName, PackageName, DEV_DEPENDENCIES}; use uv_pep508::{MarkerTree, RequirementOrigin, VerbatimUrl}; -use uv_pypi_types::{Requirement, RequirementSource, SupportedEnvironments, VerbatimParsedUrl}; +use uv_pypi_types::{Requirement, RequirementSource, SupportedEnvironments}; use uv_static::EnvVars; use uv_warnings::{warn_user, warn_user_once}; @@ -1493,130 +1492,6 @@ impl VirtualProject { } } -/// A target that can be installed. -#[derive(Debug, Copy, Clone)] -pub enum InstallTarget<'env> { - /// An entire workspace. - Workspace(&'env Workspace), - /// A (legacy) non-project workspace root. - NonProjectWorkspace(&'env Workspace), - /// A project (which could be a workspace root or member). - Project(&'env ProjectWorkspace), - /// A frozen member within a [`Workspace`]. - FrozenProject(&'env Workspace, &'env PackageName), -} - -impl<'env> InstallTarget<'env> { - /// Create an [`InstallTarget`] for a frozen member within a workspace. - pub fn frozen(project: &'env VirtualProject, package_name: &'env PackageName) -> Self { - Self::FrozenProject(project.workspace(), package_name) - } - - /// Return the [`Workspace`] of the target. - pub fn workspace(&self) -> &Workspace { - match self { - Self::Workspace(workspace) => workspace, - Self::Project(project) => project.workspace(), - Self::NonProjectWorkspace(workspace) => workspace, - Self::FrozenProject(workspace, _) => workspace, - } - } - - /// Return the [`PackageName`] of the target. - pub fn packages(&self) -> impl Iterator { - match self { - Self::Workspace(workspace) => Either::Right(workspace.packages().keys()), - Self::Project(project) => Either::Left(std::iter::once(project.project_name())), - Self::NonProjectWorkspace(workspace) => Either::Right(workspace.packages().keys()), - Self::FrozenProject(_, package_name) => Either::Left(std::iter::once(*package_name)), - } - } - - /// Return the [`InstallTarget`] dependency groups. - /// - /// Returns dependencies that apply to the workspace root, but not any of its members. As such, - /// only returns a non-empty iterator for virtual workspaces, which can include dev dependencies - /// on the virtual root. - pub fn groups( - &self, - ) -> Result< - BTreeMap>>, - DependencyGroupError, - > { - match self { - Self::Workspace(_) | Self::Project(_) | Self::FrozenProject(..) => Ok(BTreeMap::new()), - Self::NonProjectWorkspace(workspace) => { - // For non-projects, we might have `dependency-groups` or `tool.uv.dev-dependencies` - // that are attached to the workspace root (which isn't a member). - - // First, collect `tool.uv.dev_dependencies` - let dev_dependencies = workspace - .pyproject_toml() - .tool - .as_ref() - .and_then(|tool| tool.uv.as_ref()) - .and_then(|uv| uv.dev_dependencies.as_ref()); - - // Then, collect `dependency-groups` - let dependency_groups = workspace - .pyproject_toml() - .dependency_groups - .iter() - .flatten() - .collect::>(); - - // Merge any overlapping groups. - let mut map = BTreeMap::new(); - for (name, dependencies) in - FlatDependencyGroups::from_dependency_groups(&dependency_groups)? - .into_iter() - .chain( - // Only add the `dev` group if `dev-dependencies` is defined. - dev_dependencies.into_iter().map(|requirements| { - (DEV_DEPENDENCIES.clone(), requirements.clone()) - }), - ) - { - match map.entry(name) { - std::collections::btree_map::Entry::Vacant(entry) => { - entry.insert(dependencies); - } - std::collections::btree_map::Entry::Occupied(mut entry) => { - entry.get_mut().extend(dependencies); - } - } - } - - Ok(map) - } - } - } - - /// Return the [`PackageName`] of the target, if available. - pub fn project_name(&self) -> Option<&PackageName> { - match self { - Self::Workspace(_) => None, - Self::Project(project) => Some(project.project_name()), - Self::NonProjectWorkspace(_) => None, - Self::FrozenProject(_, package_name) => Some(package_name), - } - } - - pub fn from_workspace(workspace: &'env VirtualProject) -> Self { - match workspace { - VirtualProject::Project(project) => Self::Workspace(project.workspace()), - VirtualProject::NonProject(workspace) => Self::NonProjectWorkspace(workspace), - } - } - - pub fn from_project(project: &'env VirtualProject) -> Self { - match project { - VirtualProject::Project(project) => Self::Project(project), - VirtualProject::NonProject(workspace) => Self::NonProjectWorkspace(workspace), - } - } -} - #[cfg(test)] #[cfg(unix)] // Avoid path escaping for the unit tests mod tests; diff --git a/crates/uv/src/commands/project/add.rs b/crates/uv/src/commands/project/add.rs index bf25a05dddfde..3a5dfbf6b3d34 100644 --- a/crates/uv/src/commands/project/add.rs +++ b/crates/uv/src/commands/project/add.rs @@ -29,13 +29,13 @@ use uv_python::{ PythonPreference, PythonRequest, PythonVariant, PythonVersionFile, VersionRequest, }; use uv_requirements::{NamedRequirementsResolver, RequirementsSource, RequirementsSpecification}; -use uv_resolver::FlatIndex; +use uv_resolver::{FlatIndex, InstallTarget}; use uv_scripts::Pep723Script; use uv_types::{BuildIsolation, HashStrategy}; use uv_warnings::warn_user_once; use uv_workspace::pyproject::{DependencyType, Source, SourceError}; use uv_workspace::pyproject_mut::{ArrayEdit, DependencyTarget, PyProjectTomlMut}; -use uv_workspace::{DiscoveryOptions, InstallTarget, VirtualProject, Workspace}; +use uv_workspace::{DiscoveryOptions, VirtualProject, Workspace}; use crate::commands::pip::loggers::{ DefaultInstallLogger, DefaultResolveLogger, SummaryResolveLogger, @@ -869,10 +869,22 @@ async fn lock_and_sync( } }; + // Identify the installation target. + let target = match &project { + VirtualProject::Project(project) => InstallTarget::Project { + workspace: project.workspace(), + name: project.project_name(), + lock: &lock, + }, + VirtualProject::NonProject(workspace) => InstallTarget::NonProjectWorkspace { + workspace, + lock: &lock, + }, + }; + project::sync::do_sync( - InstallTarget::from_project(&project), + target, venv, - &lock, &extras, &DevGroupsManifest::from_spec(dev), EditableMode::Editable, diff --git a/crates/uv/src/commands/project/export.rs b/crates/uv/src/commands/project/export.rs index 43c56b41f55a3..e00efc7b00b6a 100644 --- a/crates/uv/src/commands/project/export.rs +++ b/crates/uv/src/commands/project/export.rs @@ -13,13 +13,13 @@ use uv_configuration::{ }; use uv_normalize::PackageName; use uv_python::{PythonDownloads, PythonPreference, PythonRequest}; -use uv_resolver::RequirementsTxtExport; -use uv_workspace::{DiscoveryOptions, InstallTarget, MemberDiscovery, VirtualProject, Workspace}; +use uv_resolver::{InstallTarget, RequirementsTxtExport}; +use uv_workspace::{DiscoveryOptions, MemberDiscovery, VirtualProject, Workspace}; use crate::commands::pip::loggers::DefaultResolveLogger; use crate::commands::project::lock::{do_safe_lock, LockMode}; use crate::commands::project::{ - default_dependency_groups, validate_dependency_groups, ProjectError, ProjectInterpreter, + default_dependency_groups, DependencyGroupsTarget, ProjectError, ProjectInterpreter, }; use crate::commands::{diagnostics, pip, ExitStatus, OutputWriter, SharedState}; use crate::printer::Printer; @@ -53,7 +53,7 @@ pub(crate) async fn export( printer: Printer, ) -> Result { // Identify the project. - let project = if frozen && !all_packages { + let project = if frozen { VirtualProject::discover( project_dir, &DiscoveryOptions { @@ -73,10 +73,23 @@ pub(crate) async fn export( VirtualProject::discover(project_dir, &DiscoveryOptions::default()).await? }; - if project.is_non_project() { + let VirtualProject::Project(project) = &project else { return Err(anyhow::anyhow!("Legacy non-project roots are not supported in `uv export`; add a `[project]` table to your `pyproject.toml` to enable exports")); }; + // Validate that any referenced dependency groups are defined in the workspace. + if !frozen { + let target = if all_packages { + DependencyGroupsTarget::Workspace(project.workspace()) + } else { + DependencyGroupsTarget::Project(project) + }; + target.validate(&dev)?; + } + + // Determine the default groups to include. + let defaults = default_dependency_groups(project.current_project().pyproject_toml())?; + // Determine the lock mode. let interpreter; let mode = if frozen { @@ -144,19 +157,23 @@ pub(crate) async fn export( Err(err) => return Err(err.into()), }; - // Identify the target. - let target = if let Some(package) = package.as_ref().filter(|_| frozen) { - InstallTarget::frozen(&project, package) - } else if all_packages { - InstallTarget::from_workspace(&project) + // Identify the installation target. + let target = if all_packages { + InstallTarget::Workspace { + workspace: project.workspace(), + lock: &lock, + } } else { - InstallTarget::from_project(&project) + InstallTarget::Project { + workspace: project.workspace(), + // If `--frozen --package` is specified, and only the root `pyproject.toml` was + // discovered, the child won't be present in the workspace; but we _know_ that + // we want to install it, so we override the package name. + name: package.as_ref().unwrap_or(project.project_name()), + lock: &lock, + } }; - // Determine the default groups to include. - validate_dependency_groups(target, &dev)?; - let defaults = default_dependency_groups(project.pyproject_toml())?; - // Write the resolved dependencies to the output channel. let mut writer = OutputWriter::new(!quiet || output_file.is_none(), output_file.as_deref()); @@ -164,7 +181,6 @@ pub(crate) async fn export( match format { ExportFormat::RequirementsTxt => { let export = RequirementsTxtExport::from_lock( - &lock, target, &extras, &dev.with_defaults(defaults), diff --git a/crates/uv/src/commands/project/mod.rs b/crates/uv/src/commands/project/mod.rs index dadeb22f9acfe..ca9c6edee0865 100644 --- a/crates/uv/src/commands/project/mod.rs +++ b/crates/uv/src/commands/project/mod.rs @@ -38,7 +38,7 @@ use uv_types::{BuildIsolation, EmptyInstalledPackages, HashStrategy}; use uv_warnings::{warn_user, warn_user_once}; use uv_workspace::dependency_groups::DependencyGroupError; use uv_workspace::pyproject::PyProjectToml; -use uv_workspace::{InstallTarget, Workspace}; +use uv_workspace::{ProjectWorkspace, Workspace}; use crate::commands::pip::loggers::{InstallLogger, ResolveLogger}; use crate::commands::pip::operations::{Changelog, Modifications}; @@ -1366,40 +1366,46 @@ pub(crate) async fn script_python_requirement( )) } -/// Validate the dependency groups requested by the [`DevGroupsSpecification`]. -#[allow(clippy::result_large_err)] -pub(crate) fn validate_dependency_groups( - target: InstallTarget<'_>, - dev: &DevGroupsSpecification, -) -> Result<(), ProjectError> { - for group in dev - .groups() - .into_iter() - .flat_map(GroupsSpecification::names) - { - match target { - InstallTarget::Workspace(workspace) | InstallTarget::NonProjectWorkspace(workspace) => { - // The group must be defined in the workspace. - if !workspace.groups().contains(group) { - return Err(ProjectError::MissingGroupWorkspace(group.clone())); +#[derive(Debug, Copy, Clone)] +pub(crate) enum DependencyGroupsTarget<'env> { + /// The dependency groups can be defined in any workspace member. + Workspace(&'env Workspace), + /// The dependency groups must be defined in the target project. + Project(&'env ProjectWorkspace), +} + +impl DependencyGroupsTarget<'_> { + /// Validate the dependency groups requested by the [`DevGroupsSpecification`]. + #[allow(clippy::result_large_err)] + pub(crate) fn validate(self, dev: &DevGroupsSpecification) -> Result<(), ProjectError> { + for group in dev + .groups() + .into_iter() + .flat_map(GroupsSpecification::names) + { + match self { + Self::Workspace(workspace) => { + // The group must be defined in the workspace. + if !workspace.groups().contains(group) { + return Err(ProjectError::MissingGroupWorkspace(group.clone())); + } } - } - InstallTarget::Project(project) => { - // The group must be defined in the target project. - if !project - .current_project() - .pyproject_toml() - .dependency_groups - .as_ref() - .is_some_and(|groups| groups.contains_key(group)) - { - return Err(ProjectError::MissingGroupProject(group.clone())); + Self::Project(project) => { + // The group must be defined in the target project. + if !project + .current_project() + .pyproject_toml() + .dependency_groups + .as_ref() + .is_some_and(|groups| groups.contains_key(group)) + { + return Err(ProjectError::MissingGroupProject(group.clone())); + } } } - InstallTarget::FrozenProject(_, _) => {} } + Ok(()) } - Ok(()) } /// Returns the default dependency groups from the [`PyProjectToml`]. diff --git a/crates/uv/src/commands/project/remove.rs b/crates/uv/src/commands/project/remove.rs index de6dbbd000f6c..7cbf0cf15e16c 100644 --- a/crates/uv/src/commands/project/remove.rs +++ b/crates/uv/src/commands/project/remove.rs @@ -12,11 +12,12 @@ use uv_fs::Simplified; use uv_normalize::DEV_DEPENDENCIES; use uv_pep508::PackageName; use uv_python::{PythonDownloads, PythonPreference, PythonRequest}; +use uv_resolver::InstallTarget; use uv_scripts::Pep723Script; use uv_warnings::{warn_user, warn_user_once}; use uv_workspace::pyproject::DependencyType; use uv_workspace::pyproject_mut::{DependencyTarget, PyProjectTomlMut}; -use uv_workspace::{DiscoveryOptions, InstallTarget, VirtualProject, Workspace}; +use uv_workspace::{DiscoveryOptions, VirtualProject, Workspace}; use crate::commands::pip::loggers::{DefaultInstallLogger, DefaultResolveLogger}; use crate::commands::pip::operations::Modifications; @@ -235,10 +236,22 @@ pub(crate) async fn remove( // Determine the default groups to include. let defaults = default_dependency_groups(project.pyproject_toml())?; + // Identify the installation target. + let target = match &project { + VirtualProject::Project(project) => InstallTarget::Project { + workspace: project.workspace(), + name: project.project_name(), + lock: &lock, + }, + VirtualProject::NonProject(workspace) => InstallTarget::NonProjectWorkspace { + workspace, + lock: &lock, + }, + }; + project::sync::do_sync( - InstallTarget::from_project(&project), + target, &venv, - &lock, &extras, &DevGroupsManifest::from_defaults(defaults), EditableMode::Editable, diff --git a/crates/uv/src/commands/project/run.rs b/crates/uv/src/commands/project/run.rs index fe50b73f57461..8362e9cdbc24a 100644 --- a/crates/uv/src/commands/project/run.rs +++ b/crates/uv/src/commands/project/run.rs @@ -31,11 +31,11 @@ use uv_python::{ PythonPreference, PythonRequest, PythonVariant, PythonVersionFile, VersionRequest, }; use uv_requirements::{RequirementsSource, RequirementsSpecification}; -use uv_resolver::Lock; +use uv_resolver::{InstallTarget, Lock}; use uv_scripts::Pep723Item; use uv_static::EnvVars; use uv_warnings::warn_user; -use uv_workspace::{DiscoveryOptions, InstallTarget, VirtualProject, Workspace, WorkspaceError}; +use uv_workspace::{DiscoveryOptions, VirtualProject, Workspace, WorkspaceError}; use crate::commands::pip::loggers::{ DefaultInstallLogger, DefaultResolveLogger, SummaryInstallLogger, SummaryResolveLogger, @@ -45,7 +45,7 @@ use crate::commands::pip::operations::Modifications; use crate::commands::project::environment::CachedEnvironment; use crate::commands::project::lock::LockMode; use crate::commands::project::{ - default_dependency_groups, validate_dependency_groups, validate_requires_python, + default_dependency_groups, validate_requires_python, DependencyGroupsTarget, EnvironmentSpecification, ProjectError, PythonRequestSource, WorkspacePython, }; use crate::commands::reporters::PythonDownloadReporter; @@ -556,14 +556,24 @@ pub(crate) async fn run( .flatten(); } } else { - let target = if all_packages { - InstallTarget::from_workspace(&project) - } else { - InstallTarget::from_project(&project) - }; + // Validate that any referenced dependency groups are defined in the workspace. + if !frozen { + let target = match &project { + VirtualProject::Project(project) => { + if all_packages { + DependencyGroupsTarget::Workspace(project.workspace()) + } else { + DependencyGroupsTarget::Project(project) + } + } + VirtualProject::NonProject(workspace) => { + DependencyGroupsTarget::Workspace(workspace) + } + }; + target.validate(&dev)?; + } // Determine the default groups to include. - validate_dependency_groups(target, &dev)?; let defaults = default_dependency_groups(project.pyproject_toml())?; // Determine the lock mode. @@ -616,12 +626,33 @@ pub(crate) async fn run( Err(err) => return Err(err.into()), }; + // Identify the installation target. + let target = match &project { + VirtualProject::Project(project) => { + if all_packages { + InstallTarget::Workspace { + workspace: project.workspace(), + lock: result.lock(), + } + } else { + InstallTarget::Project { + workspace: project.workspace(), + name: project.project_name(), + lock: result.lock(), + } + } + } + VirtualProject::NonProject(workspace) => InstallTarget::NonProjectWorkspace { + workspace, + lock: result.lock(), + }, + }; + let install_options = InstallOptions::default(); project::sync::do_sync( target, &venv, - result.lock(), &extras, &dev.with_defaults(defaults), editable, diff --git a/crates/uv/src/commands/project/sync.rs b/crates/uv/src/commands/project/sync.rs index 24c4f4683d656..3f8e230ce1ca9 100644 --- a/crates/uv/src/commands/project/sync.rs +++ b/crates/uv/src/commands/project/sync.rs @@ -4,6 +4,7 @@ use std::str::FromStr; use anyhow::{Context, Result}; use itertools::Itertools; + use uv_auth::store_credentials; use uv_cache::Cache; use uv_client::{Connectivity, FlatIndexClient, RegistryClientBuilder}; @@ -20,18 +21,18 @@ use uv_pypi_types::{ LenientRequirement, ParsedArchiveUrl, ParsedGitUrl, ParsedUrl, VerbatimParsedUrl, }; use uv_python::{PythonDownloads, PythonEnvironment, PythonPreference, PythonRequest}; -use uv_resolver::{FlatIndex, Lock}; +use uv_resolver::{FlatIndex, InstallTarget}; use uv_types::{BuildIsolation, HashStrategy}; use uv_warnings::warn_user; use uv_workspace::pyproject::{DependencyGroupSpecifier, Source, Sources, ToolUvSources}; -use uv_workspace::{DiscoveryOptions, InstallTarget, MemberDiscovery, VirtualProject, Workspace}; +use uv_workspace::{DiscoveryOptions, MemberDiscovery, VirtualProject, Workspace}; use crate::commands::pip::loggers::{DefaultInstallLogger, DefaultResolveLogger, InstallLogger}; use crate::commands::pip::operations; use crate::commands::pip::operations::Modifications; use crate::commands::project::lock::{do_safe_lock, LockMode}; use crate::commands::project::{ - default_dependency_groups, validate_dependency_groups, ProjectError, SharedState, + default_dependency_groups, DependencyGroupsTarget, ProjectError, SharedState, }; use crate::commands::{diagnostics, project, ExitStatus}; use crate::printer::Printer; @@ -61,7 +62,7 @@ pub(crate) async fn sync( printer: Printer, ) -> Result { // Identify the project. - let project = if frozen && !all_packages { + let project = if frozen { VirtualProject::discover( project_dir, &DiscoveryOptions { @@ -81,6 +82,24 @@ pub(crate) async fn sync( VirtualProject::discover(project_dir, &DiscoveryOptions::default()).await? }; + // Validate that any referenced dependency groups are defined in the workspace. + if !frozen { + let target = match &project { + VirtualProject::Project(project) => { + if all_packages { + DependencyGroupsTarget::Workspace(project.workspace()) + } else { + DependencyGroupsTarget::Project(project) + } + } + VirtualProject::NonProject(workspace) => DependencyGroupsTarget::Workspace(workspace), + }; + target.validate(&dev)?; + } + + // Determine the default groups to include. + let defaults = default_dependency_groups(project.pyproject_toml())?; + // TODO(lucab): improve warning content // if project.workspace().pyproject_toml().has_scripts() @@ -89,22 +108,9 @@ pub(crate) async fn sync( warn_user!("Skipping installation of entry points (`project.scripts`) because this project is not packaged; to install entry points, set `tool.uv.package = true` or define a `build-system`"); } - // Identify the target. - let target = if let Some(package) = package.as_ref().filter(|_| frozen) { - InstallTarget::frozen(&project, package) - } else if all_packages { - InstallTarget::from_workspace(&project) - } else { - InstallTarget::from_project(&project) - }; - - // Determine the default groups to include. - validate_dependency_groups(target, &dev)?; - let defaults = default_dependency_groups(project.pyproject_toml())?; - // Discover or create the virtual environment. let venv = project::get_or_init_environment( - target.workspace(), + project.workspace(), python.as_deref().map(PythonRequest::parse), python_preference, python_downloads, @@ -129,7 +135,7 @@ pub(crate) async fn sync( let lock = match do_safe_lock( mode, - target.workspace(), + project.workspace(), settings.as_ref().into(), LowerBound::Warn, &state, @@ -164,11 +170,35 @@ pub(crate) async fn sync( Err(err) => return Err(err.into()), }; + // Identify the installation target. + let target = match &project { + VirtualProject::Project(project) => { + if all_packages { + InstallTarget::Workspace { + workspace: project.workspace(), + lock: &lock, + } + } else { + InstallTarget::Project { + workspace: project.workspace(), + // If `--frozen --package` is specified, and only the root `pyproject.toml` was + // discovered, the child won't be present in the workspace; but we _know_ that + // we want to install it, so we override the package name. + name: package.as_ref().unwrap_or(project.project_name()), + lock: &lock, + } + } + } + VirtualProject::NonProject(workspace) => InstallTarget::NonProjectWorkspace { + workspace, + lock: &lock, + }, + }; + // Perform the sync operation. do_sync( target, &venv, - &lock, &extras, &dev.with_defaults(defaults), editable, @@ -192,7 +222,6 @@ pub(crate) async fn sync( pub(super) async fn do_sync( target: InstallTarget<'_>, venv: &PythonEnvironment, - lock: &Lock, extras: &ExtrasSpecification, dev: &DevGroupsManifest, editable: EditableMode, @@ -236,13 +265,14 @@ pub(super) async fn do_sync( } = settings; // Validate that the Python version is supported by the lockfile. - if !lock + if !target + .lock() .requires_python() .contains(venv.interpreter().python_version()) { return Err(ProjectError::LockedPythonIncompatibility( venv.interpreter().python_version().clone(), - lock.requires_python().clone(), + target.lock().requires_python().clone(), )); } @@ -250,7 +280,7 @@ pub(super) async fn do_sync( let markers = venv.interpreter().resolver_markers(); // Validate that the platform is supported by the lockfile. - let environments = lock.supported_environments(); + let environments = target.lock().supported_environments(); if !environments.is_empty() { if !environments.iter().any(|env| env.evaluate(&markers, &[])) { return Err(ProjectError::LockedPlatformIncompatibility( @@ -259,7 +289,9 @@ pub(super) async fn do_sync( // what the end user actually wrote. The non-simplified // environments, by contrast, are explicitly // constrained by `requires-python`. - lock.simplified_supported_environments() + target + .lock() + .simplified_supported_environments() .iter() .filter_map(MarkerTree::contents) .map(|env| format!("`{env}`")) @@ -272,15 +304,8 @@ pub(super) async fn do_sync( let tags = venv.interpreter().tags()?; // Read the lockfile. - let resolution = lock.to_resolution( - target, - &markers, - tags, - extras, - dev, - build_options, - &install_options, - )?; + let resolution = + target.to_resolution(&markers, tags, extras, dev, build_options, &install_options)?; // Always skip virtual projects, which shouldn't be built or installed. let resolution = apply_no_virtual_project(resolution); diff --git a/crates/uv/src/commands/project/tree.rs b/crates/uv/src/commands/project/tree.rs index 3b209e3f2684e..0a9a22ff8ea4b 100644 --- a/crates/uv/src/commands/project/tree.rs +++ b/crates/uv/src/commands/project/tree.rs @@ -9,13 +9,13 @@ use uv_configuration::{Concurrency, DevGroupsSpecification, LowerBound, TargetTr use uv_pep508::PackageName; use uv_python::{PythonDownloads, PythonPreference, PythonRequest, PythonVersion}; use uv_resolver::TreeDisplay; -use uv_workspace::{DiscoveryOptions, InstallTarget, Workspace}; +use uv_workspace::{DiscoveryOptions, Workspace}; use crate::commands::pip::loggers::DefaultResolveLogger; use crate::commands::pip::resolution_markers; use crate::commands::project::lock::LockMode; use crate::commands::project::{ - default_dependency_groups, validate_dependency_groups, ProjectInterpreter, + default_dependency_groups, DependencyGroupsTarget, ProjectInterpreter, }; use crate::commands::{project, ExitStatus, SharedState}; use crate::printer::Printer; @@ -49,8 +49,13 @@ pub(crate) async fn tree( // Find the project requirements. let workspace = Workspace::discover(project_dir, &DiscoveryOptions::default()).await?; + // Validate that any referenced dependency groups are defined in the workspace. + if !frozen { + let target = DependencyGroupsTarget::Workspace(&workspace); + target.validate(&dev)?; + } + // Determine the default groups to include. - validate_dependency_groups(InstallTarget::Workspace(&workspace), &dev)?; let defaults = default_dependency_groups(workspace.pyproject_toml())?; // Find an interpreter for the project, unless `--frozen` and `--universal` are both set. diff --git a/crates/uv/tests/it/export.rs b/crates/uv/tests/it/export.rs index 2adcc3e086607..01d49ca19f217 100644 --- a/crates/uv/tests/it/export.rs +++ b/crates/uv/tests/it/export.rs @@ -573,6 +573,89 @@ fn all() -> Result<()> { Ok(()) } +#[test] +fn frozen() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["anyio==3.7.0", "child"] + + [tool.uv.workspace] + members = ["child"] + + [tool.uv.sources] + child = { workspace = true } + + [build-system] + requires = ["setuptools>=42"] + build-backend = "setuptools.build_meta" + "#, + )?; + + let child = context.temp_dir.child("child"); + child.child("pyproject.toml").write_str( + r#" + [project] + name = "child" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["iniconfig>=2"] + + [build-system] + requires = ["setuptools>=42"] + build-backend = "setuptools.build_meta" + "#, + )?; + + context.lock().assert().success(); + + // Remove the child `pyproject.toml`. + fs_err::remove_dir_all(child.path())?; + + uv_snapshot!(context.filters(), context.export().arg("--all"), @r###" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: Failed to build: `project @ file://[TEMP_DIR]/` + Caused by: Failed to parse entry: `child` + Caused by: Package is not included as workspace package in `tool.uv.workspace` + "###); + + uv_snapshot!(context.filters(), context.export().arg("--all").arg("--frozen"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv export --cache-dir [CACHE_DIR] --all --frozen + -e . + -e ./child + anyio==3.7.0 \ + --hash=sha256:275d9973793619a5374e1c89a4f4ad3f4b0a5510a2b5b939444bee8f4c4d37ce \ + --hash=sha256:eddca883c4175f14df8aedce21054bfca3adb70ffe76a9f607aef9d7fa2ea7f0 + idna==3.6 \ + --hash=sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca \ + --hash=sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f + iniconfig==2.0.0 \ + --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ + --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 + sniffio==1.3.1 \ + --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc \ + --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 + + ----- stderr ----- + "###); + + Ok(()) +} + #[test] fn non_project() -> Result<()> { let context = TestContext::new("3.12"); diff --git a/crates/uv/tests/it/sync.rs b/crates/uv/tests/it/sync.rs index d44729ab61d9a..104f159ba84a9 100644 --- a/crates/uv/tests/it/sync.rs +++ b/crates/uv/tests/it/sync.rs @@ -1882,6 +1882,19 @@ fn no_install_workspace() -> Result<()> { error: Could not find root package `fake` "###); + // Even if `--all` is used. + uv_snapshot!(context.filters(), context.sync().arg("--all").arg("--no-install-workspace").arg("--frozen"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Installed 3 packages in [TIME] + + anyio==3.7.0 + + idna==3.6 + + sniffio==1.3.1 + "###); + // But we do require the root `pyproject.toml`. fs_err::remove_file(context.temp_dir.join("pyproject.toml"))?;