diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index 1ba34bbe1..b88057452 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -53,7 +53,8 @@ determine_conda_executable, is_micromamba, ) -from conda_lock.lockfile import ( +from conda_lock.lockfile import parse_conda_lock_file, write_conda_lock_file +from conda_lock.lockfile.v2prelim.models import ( GitMeta, InputMeta, LockedDependency, @@ -62,8 +63,6 @@ MetadataOption, TimeMeta, UpdateSpecification, - parse_conda_lock_file, - write_conda_lock_file, ) from conda_lock.lookup import set_lookup_location from conda_lock.models.channel import Channel diff --git a/conda_lock/conda_solver.py b/conda_lock/conda_solver.py index 9036dcc03..56b57d95a 100644 --- a/conda_lock/conda_solver.py +++ b/conda_lock/conda_solver.py @@ -24,7 +24,8 @@ conda_pkgs_dir, is_micromamba, ) -from conda_lock.lockfile import HashModel, LockedDependency, apply_categories +from conda_lock.lockfile import apply_categories +from conda_lock.lockfile.v2prelim.models import HashModel, LockedDependency from conda_lock.models.channel import Channel from conda_lock.models.lock_spec import Dependency, VersionedDependency diff --git a/conda_lock/lockfile/__init__.py b/conda_lock/lockfile/__init__.py index 760bd0380..3d69db1d2 100644 --- a/conda_lock/lockfile/__init__.py +++ b/conda_lock/lockfile/__init__.py @@ -1,26 +1,21 @@ -import json import pathlib from collections import defaultdict from textwrap import dedent -from typing import Any, Collection, Dict, List, Mapping, Optional, Sequence, Set, Union +from typing import Collection, Dict, List, Mapping, Optional, Sequence, Set, Union import yaml +from conda_lock.lockfile.v1.models import Lockfile as LockfileV1 +from conda_lock.lockfile.v2prelim.models import ( + LockedDependency, + Lockfile, + MetadataOption, + lockfile_v1_to_v2, +) from conda_lock.lookup import conda_name_to_pypi_name from conda_lock.models.lock_spec import Dependency -from .models import DependencySource as DependencySource -from .models import GitMeta as GitMeta -from .models import HashModel as HashModel -from .models import InputMeta as InputMeta -from .models import LockedDependency, Lockfile -from .models import LockKey as LockKey -from .models import LockMeta as LockMeta -from .models import MetadataOption -from .models import TimeMeta as TimeMeta -from .models import UpdateSpecification as UpdateSpecification - def _seperator_munge_get( d: Mapping[str, Union[List[LockedDependency], LockedDependency]], key: str @@ -138,14 +133,11 @@ def parse_conda_lock_file(path: pathlib.Path) -> Lockfile: with path.open() as f: content = yaml.safe_load(f) version = content.pop("version", None) - if not (isinstance(version, int) and version <= Lockfile.version): + if version == 1: + return lockfile_v1_to_v2(LockfileV1.parse_obj(content)) + else: raise ValueError(f"{path} has unknown version {version}") - for p in content["package"]: - del p["optional"] - - return Lockfile.parse_obj(content) - def write_conda_lock_file( content: Lockfile, @@ -206,23 +198,5 @@ def write_section(text: str) -> None: conda-lock {metadata_flags}{' '.join('-f '+path for path in content.metadata.sources)} --lockfile {path.name} """ ) - - output: Dict[str, Any] = { - "version": Lockfile.version, - "metadata": json.loads( - content.metadata.json( - by_alias=True, exclude_unset=True, exclude_none=True - ) - ), - "package": [ - { - **package.dict( - by_alias=True, exclude_unset=True, exclude_none=True - ), - "optional": (package.category != "main"), - } - for package in content.package - ], - } - + output = content.to_v1().dict_for_output() yaml.dump(output, stream=f, sort_keys=False) diff --git a/conda_lock/lockfile/models.py b/conda_lock/lockfile/v1/models.py similarity index 74% rename from conda_lock/lockfile/models.py rename to conda_lock/lockfile/v1/models.py index 296a98023..83ba9c440 100644 --- a/conda_lock/lockfile/models.py +++ b/conda_lock/lockfile/v1/models.py @@ -1,12 +1,22 @@ import datetime import enum import hashlib +import json import logging import pathlib import typing -from collections import defaultdict, namedtuple -from typing import TYPE_CHECKING, AbstractSet, ClassVar, Dict, List, Optional, Union +from collections import namedtuple +from typing import ( + TYPE_CHECKING, + AbstractSet, + Any, + ClassVar, + Dict, + List, + Optional, + Union, +) if TYPE_CHECKING: @@ -36,7 +46,7 @@ class HashModel(StrictModel): sha256: Optional[str] = None -class LockedDependency(StrictModel): +class BaseLockedDependency(StrictModel): name: str version: str manager: Literal["conda", "pip"] @@ -58,6 +68,10 @@ def validate_hash(cls, v: HashModel, values: Dict[str, typing.Any]) -> HashModel return v +class LockedDependency(BaseLockedDependency): + optional: bool + + class MetadataOption(enum.Enum): TimeStamp = "timestamp" GitSha = "git_sha" @@ -283,89 +297,15 @@ class Lockfile(StrictModel): package: List[LockedDependency] metadata: LockMeta - def __or__(self, other: "Lockfile") -> "Lockfile": - return other.__ror__(self) - - def __ror__(self, other: "Optional[Lockfile]") -> "Lockfile": - """ - merge self into other - """ - if other is None: - return self - elif not isinstance(other, Lockfile): - raise TypeError - - assert self.metadata.channels == other.metadata.channels - - ours = {d.key(): d for d in self.package} - theirs = {d.key(): d for d in other.package} - - # Pick ours preferentially - package: List[LockedDependency] = [] - for key in sorted(set(ours.keys()).union(theirs.keys())): - if key not in ours or key[-1] not in self.metadata.platforms: - package.append(theirs[key]) - else: - package.append(ours[key]) - - # Resort the conda packages topologically - final_package = self._toposort(package) - return Lockfile(package=final_package, metadata=other.metadata | self.metadata) - - def toposort_inplace(self) -> None: - self.package = self._toposort(self.package) - - @staticmethod - def _toposort( - package: List[LockedDependency], update: bool = False - ) -> List[LockedDependency]: - platforms = {d.platform for d in package} - - # Resort the conda packages topologically - final_package: List[LockedDependency] = [] - for platform in sorted(platforms): - from .._vendor.conda.common.toposort import toposort - - # Add the remaining non-conda packages in the order in which they appeared. - # Order the pip packages topologically ordered (might be not 100% perfect if they depend on - # other conda packages, but good enough - for manager in ["conda", "pip"]: - lookup = defaultdict(set) - packages: Dict[str, LockedDependency] = {} - - for d in package: - if d.platform != platform: - continue - - if d.manager != manager: - continue - - lookup[d.name] = set(d.dependencies) - packages[d.name] = d - - ordered = toposort(lookup) - for package_name in ordered: - # since we could have a pure dep in here, that does not have a package - # eg a pip package that depends on a conda package (the conda package will not be in this list) - dep = packages.get(package_name) - if dep is None: - continue - if dep.manager != manager: - continue - # skip virtual packages - if dep.manager == "conda" and dep.name.startswith("__"): - continue - - final_package.append(dep) - - return final_package - - -class UpdateSpecification: - def __init__( - self, - locked: Optional[List[LockedDependency]] = None, - update: Optional[List[str]] = None, - ): - self.locked = locked or [] - self.update = update or [] + def dict_for_output(self) -> Dict[str, Any]: + """Convert the lockfile to a dictionary that can be written to a file.""" + return { + "version": Lockfile.version, + "metadata": json.loads( + self.metadata.json(by_alias=True, exclude_unset=True, exclude_none=True) + ), + "package": [ + package.dict(by_alias=True, exclude_unset=True, exclude_none=True) + for package in self.package + ], + } diff --git a/conda_lock/lockfile/v2prelim/models.py b/conda_lock/lockfile/v2prelim/models.py new file mode 100644 index 000000000..2e846c987 --- /dev/null +++ b/conda_lock/lockfile/v2prelim/models.py @@ -0,0 +1,173 @@ +from collections import defaultdict +from typing import ClassVar, Dict, List, Optional + +from conda_lock.lockfile.v1.models import ( + BaseLockedDependency, + DependencySource, + GitMeta, + HashModel, + InputMeta, +) +from conda_lock.lockfile.v1.models import LockedDependency as LockedDependencyV1 +from conda_lock.lockfile.v1.models import Lockfile as LockfileV1 +from conda_lock.lockfile.v1.models import LockMeta, MetadataOption, TimeMeta +from conda_lock.models import StrictModel + + +class LockedDependency(BaseLockedDependency): + def to_v1(self) -> LockedDependencyV1: + return LockedDependencyV1( + name=self.name, + version=self.version, + manager=self.manager, + platform=self.platform, + dependencies=self.dependencies, + url=self.url, + hash=self.hash, + category=self.category, + source=self.source, + build=self.build, + optional=self.category != "main", + ) + + +class Lockfile(StrictModel): + version: ClassVar[int] = 2 + + package: List[LockedDependency] + metadata: LockMeta + + def __or__(self, other: "Lockfile") -> "Lockfile": + return other.__ror__(self) + + def __ror__(self, other: "Optional[Lockfile]") -> "Lockfile": + """ + merge self into other + """ + if other is None: + return self + elif not isinstance(other, Lockfile): + raise TypeError + + assert self.metadata.channels == other.metadata.channels + + ours = {d.key(): d for d in self.package} + theirs = {d.key(): d for d in other.package} + + # Pick ours preferentially + package: List[LockedDependency] = [] + for key in sorted(set(ours.keys()).union(theirs.keys())): + if key not in ours or key[-1] not in self.metadata.platforms: + package.append(theirs[key]) + else: + package.append(ours[key]) + + # Resort the conda packages topologically + final_package = self._toposort(package) + return Lockfile(package=final_package, metadata=other.metadata | self.metadata) + + def toposort_inplace(self) -> None: + self.package = self._toposort(self.package) + + @staticmethod + def _toposort( + package: List[LockedDependency], update: bool = False + ) -> List[LockedDependency]: + platforms = {d.platform for d in package} + + # Resort the conda packages topologically + final_package: List[LockedDependency] = [] + for platform in sorted(platforms): + from conda_lock._vendor.conda.common.toposort import toposort + + # Add the remaining non-conda packages in the order in which they appeared. + # Order the pip packages topologically ordered (might be not 100% perfect if they depend on + # other conda packages, but good enough + for manager in ["conda", "pip"]: + lookup = defaultdict(set) + packages: Dict[str, LockedDependency] = {} + + for d in package: + if d.platform != platform: + continue + + if d.manager != manager: + continue + + lookup[d.name] = set(d.dependencies) + packages[d.name] = d + + ordered = toposort(lookup) + for package_name in ordered: + # since we could have a pure dep in here, that does not have a package + # eg a pip package that depends on a conda package (the conda package will not be in this list) + dep = packages.get(package_name) + if dep is None: + continue + if dep.manager != manager: + continue + # skip virtual packages + if dep.manager == "conda" and dep.name.startswith("__"): + continue + + final_package.append(dep) + + return final_package + + def to_v1(self) -> LockfileV1: + return LockfileV1( + package=[p.to_v1() for p in self.package], + metadata=self.metadata, + ) + + +def _locked_dependency_v1_to_v2(dep: LockedDependencyV1) -> LockedDependency: + """Convert a LockedDependency from v1 to v2. + + * Remove the optional field (it is always equal to category != "main") + """ + return LockedDependency( + name=dep.name, + version=dep.version, + manager=dep.manager, + platform=dep.platform, + dependencies=dep.dependencies, + url=dep.url, + hash=dep.hash, + category=dep.category, + source=dep.source, + build=dep.build, + ) + + +def lockfile_v1_to_v2(lockfile_v1: LockfileV1) -> Lockfile: + """Convert a Lockfile from v1 to v2.""" + return Lockfile( + package=[_locked_dependency_v1_to_v2(p) for p in lockfile_v1.package], + metadata=lockfile_v1.metadata, + ) + + +class UpdateSpecification: + def __init__( + self, + locked: Optional[List[LockedDependency]] = None, + update: Optional[List[str]] = None, + ): + self.locked = locked or [] + self.update = update or [] + + +__all__ = [ + "DependencySource", + "GitMeta", + "HashModel", + "InputMeta", + "LockedDependency", + "Lockfile", + "LockMeta", + "MetadataOption", + "TimeMeta", + "UpdateSpecification", + "lockfile_v1_to_v2", +] diff --git a/conda_lock/pypi_solver.py b/conda_lock/pypi_solver.py index 65d0e3cfc..e9bc1e018 100644 --- a/conda_lock/pypi_solver.py +++ b/conda_lock/pypi_solver.py @@ -9,7 +9,6 @@ from clikit.io import ConsoleIO, NullIO from packaging.tags import compatible_tags, cpython_tags -from conda_lock import lockfile from conda_lock._vendor.poetry.core.packages import Dependency as PoetryDependency from conda_lock._vendor.poetry.core.packages import Package as PoetryPackage from conda_lock._vendor.poetry.core.packages import ( @@ -24,6 +23,12 @@ from conda_lock._vendor.poetry.repositories.pypi_repository import PyPiRepository from conda_lock._vendor.poetry.repositories.repository import Repository from conda_lock._vendor.poetry.utils.env import Env +from conda_lock.lockfile import apply_categories +from conda_lock.lockfile.v2prelim.models import ( + DependencySource, + HashModel, + LockedDependency, +) from conda_lock.lookup import conda_name_to_pypi_name from conda_lock.models import lock_spec @@ -164,7 +169,7 @@ def get_dependency(dep: lock_spec.Dependency) -> PoetryDependency: raise ValueError(f"Unknown requirement {dep}") -def get_package(locked: lockfile.LockedDependency) -> PoetryPackage: +def get_package(locked: LockedDependency) -> PoetryPackage: if locked.source is not None: return PoetryPackage( locked.name, @@ -179,13 +184,13 @@ def get_package(locked: lockfile.LockedDependency) -> PoetryPackage: def solve_pypi( pip_specs: Dict[str, lock_spec.Dependency], use_latest: List[str], - pip_locked: Dict[str, lockfile.LockedDependency], - conda_locked: Dict[str, lockfile.LockedDependency], + pip_locked: Dict[str, LockedDependency], + conda_locked: Dict[str, LockedDependency], python_version: str, platform: str, allow_pypi_requests: bool = True, verbose: bool = False, -) -> Dict[str, lockfile.LockedDependency]: +) -> Dict[str, LockedDependency]: """ Solve pip dependencies for the given platform @@ -224,7 +229,7 @@ def solve_pypi( locked = Repository() python_packages = dict() - locked_dep: lockfile.LockedDependency + locked_dep: LockedDependency for locked_dep in conda_locked.values(): if locked_dep.name.startswith("__"): continue @@ -274,18 +279,16 @@ def solve_pypi( # Extract distributions from Poetry package plan, ignoring uninstalls # (usually: conda package with no pypi equivalent) and skipped ops # (already installed) - requirements: List[lockfile.LockedDependency] = [] + requirements: List[LockedDependency] = [] for op in result: if not isinstance(op, Uninstall) and not op.skipped: # Take direct references verbatim - source: Optional[lockfile.DependencySource] = None + source: Optional[DependencySource] = None if op.package.source_type == "url": url, fragment = urldefrag(op.package.source_url) hash_type, hash = fragment.split("=") - hash = lockfile.HashModel(**{hash_type: hash}) - source = lockfile.DependencySource( - type="url", url=op.package.source_url - ) + hash = HashModel(**{hash_type: hash}) + source = DependencySource(type="url", url=op.package.source_url) # Choose the most specific distribution for the target else: link = chooser.choose_for(op.package) @@ -293,10 +296,10 @@ def solve_pypi( hashes: Dict[str, str] = {} if link.hash_name is not None and link.hash is not None: hashes[link.hash_name] = link.hash - hash = lockfile.HashModel.parse_obj(hashes) + hash = HashModel.parse_obj(hashes) requirements.append( - lockfile.LockedDependency( + LockedDependency( name=op.package.name, version=str(op.package.version), manager="pip", @@ -330,9 +333,7 @@ def solve_pypi( else: planned[pypi_name] = [locked_dep] - lockfile.apply_categories( - requested=pip_specs, planned=planned, convert_to_pip_names=True - ) + apply_categories(requested=pip_specs, planned=planned, convert_to_pip_names=True) return {dep.name: dep for dep in requirements} diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index cf6c42c91..352c27097 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -55,11 +55,11 @@ is_micromamba, reset_conda_pkgs_dir, ) -from conda_lock.lockfile import ( +from conda_lock.lockfile import parse_conda_lock_file +from conda_lock.lockfile.v2prelim.models import ( HashModel, LockedDependency, MetadataOption, - parse_conda_lock_file, ) from conda_lock.models.channel import Channel from conda_lock.models.lock_spec import VersionedDependency