From c299482e84efa92ecd4b130218a1475594d0bdef Mon Sep 17 00:00:00 2001 From: David Hotham Date: Mon, 19 Sep 2022 12:42:01 +0100 Subject: [PATCH] use tomli and tomliw to read and write lockfile --- poetry.lock | 16 ++++- pyproject.toml | 2 + src/poetry/console/commands/add.py | 2 +- src/poetry/console/commands/remove.py | 2 +- src/poetry/packages/locker.py | 79 +++++++++------------ tests/conftest.py | 2 +- tests/helpers.py | 2 +- tests/packages/test_locker.py | 99 +++++++++++++++++++-------- 8 files changed, 126 insertions(+), 78 deletions(-) diff --git a/poetry.lock b/poetry.lock index 69ea29c1b13..797a63baf00 100644 --- a/poetry.lock +++ b/poetry.lock @@ -815,7 +815,15 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "tomli-w" +version = "1.0.0" +description = "A lil' TOML writer" +category = "main" optional = false python-versions = ">=3.7" @@ -951,7 +959,7 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>= [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "1ebadc410b20420b0b570d94dc78220ae54ad814498b1c2fa3fb3cbd183a1413" +content-hash = "35c6abbc027cbb24d7e46c1e4c6f01b4b18b73d4411b614928739803301847e2" [metadata.files] attrs = [ @@ -1554,6 +1562,10 @@ tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +tomli-w = [ + {file = "tomli_w-1.0.0-py3-none-any.whl", hash = "sha256:9f2a07e8be30a0729e533ec968016807069991ae2fd921a78d42f429ae5f4463"}, + {file = "tomli_w-1.0.0.tar.gz", hash = "sha256:f463434305e0336248cac9c2dc8076b707d8a12d019dd349f5c1e382dd1ae1b9"}, +] tomlkit = [ {file = "tomlkit-0.11.5-py3-none-any.whl", hash = "sha256:f2ef9da9cef846ee027947dc99a45d6b68a63b0ebc21944649505bf2e8bc5fe7"}, {file = "tomlkit-0.11.5.tar.gz", hash = "sha256:571854ebbb5eac89abcb4a2e47d7ea27b89bf29e09c35395da6f03dd4ae23d1c"}, diff --git a/pyproject.toml b/pyproject.toml index c66038743a9..52583922735 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,8 @@ platformdirs = "^2.5.2" requests = "^2.18" requests-toolbelt = "^0.9.1" shellingham = "^1.5" +tomli = "^2.0.1" +tomli-w = "^1.0.0" # exclude 0.11.2 and 0.11.3 due to https://github.com/sdispater/tomlkit/issues/225 tomlkit = ">=0.11.1,<1.0.0,!=0.11.2,!=0.11.3" trove-classifiers = "^2022.5.19" diff --git a/src/poetry/console/commands/add.py b/src/poetry/console/commands/add.py index 6a0c38f9f23..d4a85bc6d90 100644 --- a/src/poetry/console/commands/add.py +++ b/src/poetry/console/commands/add.py @@ -233,7 +233,7 @@ def handle(self) -> int: # Refresh the locker self.poetry.set_locker( - self.poetry.locker.__class__(self.poetry.locker.lock.path, poetry_content) + self.poetry.locker.__class__(self.poetry.locker.lock, poetry_content) ) self.installer.set_locker(self.poetry.locker) diff --git a/src/poetry/console/commands/remove.py b/src/poetry/console/commands/remove.py index 452d8f60a9a..29e3021684b 100644 --- a/src/poetry/console/commands/remove.py +++ b/src/poetry/console/commands/remove.py @@ -103,7 +103,7 @@ def handle(self) -> int: # Refresh the locker self.poetry.set_locker( - self.poetry.locker.__class__(self.poetry.locker.lock.path, poetry_content) + self.poetry.locker.__class__(self.poetry.locker.lock, poetry_content) ) self.installer.set_locker(self.poetry.locker) diff --git a/src/poetry/packages/locker.py b/src/poetry/packages/locker.py index 8a6c62e75f7..cc31be7c6fc 100644 --- a/src/poetry/packages/locker.py +++ b/src/poetry/packages/locker.py @@ -11,20 +11,16 @@ from typing import Any from typing import cast +import tomli +import tomli_w + from packaging.utils import canonicalize_name from poetry.core.constraints.version import Version from poetry.core.constraints.version import parse_constraint from poetry.core.packages.dependency import Dependency from poetry.core.packages.package import Package -from poetry.core.toml.file import TOMLFile from poetry.core.version.markers import parse_marker from poetry.core.version.requirements import InvalidRequirement -from tomlkit import array -from tomlkit import comment -from tomlkit import document -from tomlkit import inline_table -from tomlkit import table -from tomlkit.exceptions import TOMLKitError if TYPE_CHECKING: @@ -32,8 +28,6 @@ from poetry.core.packages.file_dependency import FileDependency from poetry.core.packages.url_dependency import URLDependency from poetry.core.packages.vcs_dependency import VCSDependency - from tomlkit.items import Table - from tomlkit.toml_document import TOMLDocument from poetry.repositories.lockfile_repository import LockfileRepository @@ -53,17 +47,17 @@ class Locker: _relevant_keys = [*_legacy_keys, "group"] def __init__(self, lock: str | Path, local_config: dict[str, Any]) -> None: - self._lock = TOMLFile(lock) + self._lock = lock if isinstance(lock, Path) else Path(lock) self._local_config = local_config - self._lock_data: TOMLDocument | None = None + self._lock_data: dict[str, Any] | None = None self._content_hash = self._get_content_hash() @property - def lock(self) -> TOMLFile: + def lock(self) -> Path: return self._lock @property - def lock_data(self) -> TOMLDocument: + def lock_data(self) -> dict[str, Any]: if self._lock_data is None: self._lock_data = self._get_lock_data() @@ -73,7 +67,7 @@ def is_locked(self) -> bool: """ Checks whether the locker has been locked (lockfile found). """ - if not self._lock.exists(): + if not self.lock.exists(): return False return "package" in self.lock_data @@ -82,7 +76,8 @@ def is_fresh(self) -> bool: """ Checks whether the lock file is still up to date with the current hash. """ - lock = self._lock.read() + with self.lock.open("rb") as f: + lock = tomli.load(f) metadata = lock.get("metadata", {}) if "content-hash" in metadata: @@ -114,7 +109,7 @@ def locked_repository(self) -> LockfileRepository: source_type = source.get("type") url = source.get("url") if source_type in ["directory", "file"]: - url = self._lock.path.parent.joinpath(url).resolve().as_posix() + url = self.lock.parent.joinpath(url).resolve().as_posix() name = info["name"] package = Package( @@ -199,7 +194,7 @@ def locked_repository(self) -> LockfileRepository: package.marker = parse_marker(split_dep[1].strip()) for dep_name, constraint in info.get("dependencies", {}).items(): - root_dir = self._lock.path.parent + root_dir = self.lock.parent if package.source_type == "directory": # root dir should be the source of the package relative to the lock # path @@ -229,19 +224,18 @@ def set_lock_data(self, root: Package, packages: list[Package]) -> bool: package_specs = self._lock_packages(packages) # Retrieving hashes for package in package_specs: - files = array() + files = [] for f in package["files"]: - file_metadata = inline_table() + file_metadata = {} for k, v in sorted(f.items()): file_metadata[k] = v files.append(file_metadata) - package["files"] = files.multiline(True) + package["files"] = files - lock = document() - lock.add(comment(GENERATED_COMMENT)) + lock: dict[str, Any] = {} lock["package"] = package_specs if root.extras: @@ -263,12 +257,10 @@ def set_lock_data(self, root: Package, packages: list[Package]) -> bool: return False - def _write_lock_data(self, data: TOMLDocument) -> None: - self.lock.write(data) - - # Checking lock file data consistency - if data != self.lock.read(): - raise RuntimeError("Inconsistent lock file data.") + def _write_lock_data(self, data: dict[str, Any]) -> None: + with self.lock.open("wb") as f: + f.write(f"# {GENERATED_COMMENT}\n\n".encode()) + tomli_w.dump(data, f) self._lock_data = None @@ -289,16 +281,17 @@ def _get_content_hash(self) -> str: return sha256(json.dumps(relevant_content, sort_keys=True).encode()).hexdigest() - def _get_lock_data(self) -> TOMLDocument: - if not self._lock.exists(): + def _get_lock_data(self) -> dict[str, Any]: + if not self.lock.exists(): raise RuntimeError("No lockfile found. Unable to read locked packages") - try: - lock_data: TOMLDocument = self._lock.read() - except TOMLKitError as e: - raise RuntimeError(f"Unable to read the lock file ({e}).") + with self.lock.open("rb") as f: + try: + lock_data = tomli.load(f) + except tomli.TOMLDecodeError as e: + raise RuntimeError(f"Unable to read the lock file ({e}).") - metadata = cast("Table", lock_data["metadata"]) + metadata = lock_data["metadata"] lock_version = Version.parse(metadata.get("lock-version", "1.0")) current_version = Version.parse(self._VERSION) accepted_versions = parse_constraint(self._READ_VERSION_RANGE) @@ -349,7 +342,7 @@ def _dump_package(self, package: Package) -> dict[str, Any]: if dependency.pretty_name not in dependencies: dependencies[dependency.pretty_name] = [] - constraint = inline_table() + constraint: dict[str, Any] = {} if dependency.is_directory(): dependency = cast("DirectoryDependency", dependency) @@ -415,14 +408,10 @@ def _dump_package(self, package: Package) -> dict[str, Any]: } if dependencies: - data["dependencies"] = table() - for k, constraints in dependencies.items(): - if len(constraints) == 1: - data["dependencies"][k] = constraints[0] - else: - data["dependencies"][k] = array().multiline(True) - for constraint in constraints: - data["dependencies"][k].append(constraint) + data["dependencies"] = { + name: constraints[0] if len(constraints) == 1 else constraints + for name, constraints in dependencies.items() + } if package.extras: extras = {} @@ -438,7 +427,7 @@ def _dump_package(self, package: Package) -> dict[str, Any]: url = Path( os.path.relpath( Path(url).resolve(), - Path(self._lock.path.parent).resolve(), + Path(self.lock.parent).resolve(), ) ).as_posix() diff --git a/tests/conftest.py b/tests/conftest.py index 5de0afdd013..629c641295e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -412,7 +412,7 @@ def _factory( poetry = Factory().create_poetry(project_dir) locker = TestLocker( - poetry.locker.lock.path, locker_config or poetry.locker._local_config + poetry.locker.lock, locker_config or poetry.locker._local_config ) locker.write() diff --git a/tests/helpers.py b/tests/helpers.py index 2e57f1b2356..9ccb9e7730e 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -181,7 +181,7 @@ def reset_poetry(self) -> None: self._poetry.set_pool(poetry.pool) self._poetry.set_config(poetry.config) self._poetry.set_locker( - TestLocker(poetry.locker.lock.path, self._poetry.local_config) + TestLocker(poetry.locker.lock, self._poetry.local_config) ) diff --git a/tests/packages/test_locker.py b/tests/packages/test_locker.py index 9da6c422f7c..4b5926bc905 100644 --- a/tests/packages/test_locker.py +++ b/tests/packages/test_locker.py @@ -105,8 +105,8 @@ def test_lock_file_data_is_ordered(locker: Locker, root: ProjectPackage): optional = false python-versions = "*" files = [ - {{file = "bar", hash = "123"}}, - {{file = "foo", hash = "456"}}, + {{ file = "bar", hash = "123" }}, + {{ file = "foo", hash = "456" }}, ] [package.dependencies] @@ -120,7 +120,7 @@ def test_lock_file_data_is_ordered(locker: Locker, root: ProjectPackage): optional = false python-versions = "*" files = [ - {{file = "baz", hash = "345"}}, + {{ file = "baz", hash = "345" }}, ] [[package]] @@ -231,7 +231,9 @@ def test_locker_properly_loads_extras(locker: Locker): content-hash = "c3d07fca33fba542ef2b2a4d75bf5b48d892d21a830e2ad9c952ba5123a52f77" """ # noqa: E800 - locker.lock.write(tomlkit.parse(content)) + data = tomlkit.parse(content) + with open(locker.lock, "w", encoding="utf-8") as f: + f.write(data.as_string()) packages = locker.locked_repository().packages @@ -294,7 +296,9 @@ def test_locker_properly_loads_nested_extras(locker: Locker): content-hash = "123456789" """ # noqa: E800 - locker.lock.write(tomlkit.parse(content)) + data = tomlkit.parse(content) + with open(locker.lock, "w", encoding="utf-8") as f: + f.write(data.as_string()) repository = locker.locked_repository() assert len(repository.packages) == 3 @@ -359,7 +363,9 @@ def test_locker_properly_loads_extras_legacy(locker: Locker): content-hash = "123456789" """ # noqa: E800 - locker.lock.write(tomlkit.parse(content)) + data = tomlkit.parse(content) + with open(locker.lock, "w", encoding="utf-8") as f: + f.write(data.as_string()) repository = locker.locked_repository() assert len(repository.packages) == 2 @@ -399,7 +405,9 @@ def test_locker_properly_loads_subdir(locker: Locker) -> None: python-versions = "*" content-hash = "115cf985d932e9bf5f540555bbdd75decbb62cac81e399375fc19f6277f8c1d8" """ - locker.lock.write(tomlkit.parse(content)) + data = tomlkit.parse(content) + with open(locker.lock, "w", encoding="utf-8") as f: + f.write(data.as_string()) repository = locker.locked_repository() assert len(repository.packages) == 1 @@ -495,7 +503,9 @@ def test_locker_properly_assigns_metadata_files(locker: Locker) -> None: {file = "demo-1.0-py3-none-any.whl", hash = "sha256"}, ] """ - locker.lock.write(tomlkit.parse(content)) + data = tomlkit.parse(content) + with open(locker.lock, "w", encoding="utf-8") as f: + f.write(data.as_string()) repository = locker.locked_repository() assert len(repository.packages) == 5 @@ -580,12 +590,14 @@ def test_lock_file_should_not_have_mixed_types(locker: Locker, root: ProjectPack [package.dependencies] B = [ - {{version = "^1.0.0"}}, - {{version = ">=1.0.0", optional = true}}, + {{ version = "^1.0.0" }}, + {{ version = ">=1.0.0", optional = true }}, ] [package.extras] -foo = ["B (>=1.0.0)"] +foo = [ + "B (>=1.0.0)", +] [metadata] lock-version = "2.0" @@ -687,7 +699,9 @@ def test_locker_should_emit_warnings_if_lock_version_is_newer_but_allowed( """ caplog.set_level(logging.WARNING, logger="poetry.packages.locker") - locker.lock.write(tomlkit.parse(content)) + data = tomlkit.parse(content) + with open(locker.lock, "w", encoding="utf-8") as f: + f.write(data.as_string()) _ = locker.lock_data @@ -717,7 +731,9 @@ def test_locker_should_raise_an_error_if_lock_version_is_newer_and_not_allowed( """ # noqa: E800 caplog.set_level(logging.WARNING, logger="poetry.packages.locker") - locker.lock.write(tomlkit.parse(content)) + data = tomlkit.parse(content) + with open(locker.lock, "w", encoding="utf-8") as f: + f.write(data.as_string()) with pytest.raises(RuntimeError, match="^The lock file is not compatible"): _ = locker.lock_data @@ -746,8 +762,14 @@ def test_extras_dependencies_are_ordered(locker: Locker, root: ProjectPackage): python-versions = "*" files = [] -[package.dependencies] -B = {{version = "^1.0.0", extras = ["a", "b", "c"], optional = true}} +[package.dependencies.B] +version = "^1.0.0" +extras = [ + "a", + "b", + "c", +] +optional = true [metadata] lock-version = "2.0" @@ -775,7 +797,9 @@ def test_locker_should_neither_emit_warnings_nor_raise_error_for_lower_compatibl """ caplog.set_level(logging.WARNING, logger="poetry.packages.locker") - locker.lock.write(tomlkit.parse(content)) + data = tomlkit.parse(content) + with open(locker.lock, "w", encoding="utf-8") as f: + f.write(data.as_string()) _ = locker.lock_data @@ -834,12 +858,22 @@ def test_locker_dumps_dependency_information_correctly( python-versions = "*" files = [] -[package.dependencies] -B = {{path = "project_with_extras", develop = true}} -C = {{path = "directory/project_with_transitive_directory_dependencies"}} -D = {{path = "distributions/demo-0.1.0.tar.gz"}} -E = {{url = "https://python-poetry.org/poetry-1.2.0.tar.gz"}} -F = {{git = "https://github.com/python-poetry/poetry.git", branch = "foo"}} +[package.dependencies.B] +path = "project_with_extras" +develop = true + +[package.dependencies.C] +path = "directory/project_with_transitive_directory_dependencies" + +[package.dependencies.D] +path = "distributions/demo-0.1.0.tar.gz" + +[package.dependencies.E] +url = "https://python-poetry.org/poetry-1.2.0.tar.gz" + +[package.dependencies.F] +git = "https://github.com/python-poetry/poetry.git" +branch = "foo" [metadata] lock-version = "2.0" @@ -929,8 +963,16 @@ def test_locker_dumps_dependency_extras_in_correct_order( files = [] [package.extras] -B = ["first (==1.0.0)", "second (==1.0.0)", "third (==1.0.0)"] -C = ["first (==1.0.0)", "second (==1.0.0)", "third (==1.0.0)"] +B = [ + "first (==1.0.0)", + "second (==1.0.0)", + "third (==1.0.0)", +] +C = [ + "first (==1.0.0)", + "second (==1.0.0)", + "third (==1.0.0)", +] [metadata] lock-version = "2.0" @@ -970,7 +1012,10 @@ def test_locked_repository_uses_root_dir_of_package( content-hash = "115cf985d932e9bf5f540555bbdd75decbb62cac81e399375fc19f6277f8c1d8" """ # noqa: E800 - locker.lock.write(tomlkit.parse(content)) + data = tomlkit.parse(content) + with open(locker.lock, "w", encoding="utf-8") as f: + f.write(data.as_string()) + create_dependency_patch = mocker.patch( "poetry.factory.Factory.create_dependency", autospec=True ) @@ -983,7 +1028,7 @@ def test_locked_repository_uses_root_dir_of_package( root_dir = call_kwargs["root_dir"] assert root_dir.match("*/lib/libA") # relative_to raises an exception if not relative - is_relative_to comes in py3.9 - assert root_dir.relative_to(locker.lock.path.parent.resolve()) is not None + assert root_dir.relative_to(locker.lock.parent.resolve()) is not None @pytest.mark.parametrize( @@ -1017,7 +1062,7 @@ def test_content_hash_with_legacy_is_compatible( relevant_content[key] = local_config.get(key) locker = locker.__class__( - lock=locker.lock.path, + lock=locker.lock, local_config=local_config, )