From 4bb1d1daf919b484cf210186f294353842db6f94 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Sat, 11 Nov 2023 01:08:03 +0100 Subject: [PATCH 1/3] Use tomlkit to dump updated dependencies --- ci_cd/tasks/update_deps.py | 69 ++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/ci_cd/tasks/update_deps.py b/ci_cd/tasks/update_deps.py index 31440d04..af4f11a5 100644 --- a/ci_cd/tasks/update_deps.py +++ b/ci_cd/tasks/update_deps.py @@ -29,7 +29,6 @@ parse_ignore_entries, parse_ignore_rules, regenerate_requirement, - update_file, update_specifier_set, warning_msg, ) @@ -44,8 +43,44 @@ LOGGER = logging.getLogger(__name__) +def _update_pyproject( + original_dependency: str, updated_dependency: str, pyproject: tomlkit.TOMLDocument +) -> None: + """Update dependencu in pyproject data structure. + + First, check and update the dependency if it is in the "dependencies" group + Then, check and update if it is in any of the "optional-dependencies" groups. + + Essentially, we allow for the original dependency to be in multiple groups. + """ + LOGGER.debug( + "Updating pyproject data structure for %r to %r", + original_dependency, + updated_dependency, + ) + + if original_dependency in pyproject["project"].get("dependencies", []): + index = pyproject["project"]["dependencies"].index(original_dependency) + pyproject["project"]["dependencies"][index] = updated_dependency.replace( + '"', "'" + ) + + for extra_name, extra_dependencies in ( + pyproject["project"].get("optional-dependencies", {}).items() + ): + if original_dependency in extra_dependencies: + index = pyproject["project"]["optional-dependencies"][extra_name].index( + original_dependency + ) + pyproject["project"]["optional-dependencies"][extra_name][ + index + ] = updated_dependency.replace('"', "'") + + def _format_and_update_dependency( - requirement: Requirement, raw_dependency_line: str, pyproject_path: Path + requirement: Requirement, + raw_dependency_line: str, + pyproject: tomlkit.TOMLDocument = None, ) -> None: """Regenerate dependency without changing anything but the formatting. @@ -59,12 +94,8 @@ def _format_and_update_dependency( ) LOGGER.debug("Regenerated dependency: %r", updated_dependency) if updated_dependency != raw_dependency_line: - # Update pyproject.toml since the dependency formatting has changed - LOGGER.debug("Updating pyproject.toml for %r", requirement.name) - update_file( - pyproject_path, - (re.escape(raw_dependency_line), updated_dependency.replace('"', "'")), - ) + # Update pyproject data structure since the dependency formatting has changed + _update_pyproject(raw_dependency_line, updated_dependency, pyproject) @task( @@ -174,7 +205,8 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s already_handled_packages: set[str] = {project_name} # Build the list of dependencies listed in pyproject.toml - dependencies: list[str] = pyproject.get("project", {}).get("dependencies", []) + dependencies: list[str] = [] + dependencies.extend(pyproject.get("project", {}).get("dependencies", [])) for optional_deps in ( pyproject.get("project", {}).get("optional-dependencies", {}).values() ): @@ -214,9 +246,7 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s LOGGER.info(msg) print(info_msg(msg), flush=True) - _format_and_update_dependency( - parsed_requirement, dependency, pyproject_path - ) + _format_and_update_dependency(parsed_requirement, dependency, pyproject) already_handled_packages.add(parsed_requirement) continue @@ -233,9 +263,7 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s LOGGER.warning(msg) print(warning_msg(msg), flush=True) - _format_and_update_dependency( - parsed_requirement, dependency, pyproject_path - ) + _format_and_update_dependency(parsed_requirement, dependency, pyproject) already_handled_packages.add(parsed_requirement) continue @@ -408,14 +436,8 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s ) LOGGER.debug("Updated dependency: %r", updated_dependency) - pattern_sub_line = re.escape(dependency) - replacement_sub_line = updated_dependency.replace('"', "'") + _update_pyproject(dependency, updated_dependency, pyproject) - LOGGER.debug("pattern_sub_line: %s", pattern_sub_line) - LOGGER.debug("replacement_sub_line: %s", replacement_sub_line) - - # Update pyproject.toml - update_file(pyproject_path, (pattern_sub_line, replacement_sub_line)) already_handled_packages.add(parsed_requirement) updated_packages[parsed_requirement.name] = ",".join( str(_) @@ -431,6 +453,9 @@ def update_deps( # pylint: disable=too-many-branches,too-many-locals,too-many-s f"{Emoji.CROSS_MARK.value} Errors occurred! See printed statements above." ) + # Update pyproject.toml + pyproject_path.write_text(tomlkit.dumps(pyproject), encoding="utf-8") + if updated_packages: print( f"{Emoji.PARTY_POPPER.value} Successfully updated the following " From af295196b31310c10a474a8ff1a6aba1af439a19 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 7 Dec 2023 17:33:03 +0100 Subject: [PATCH 2/3] Update test to take the new writing method into account We only write to the pyproject.toml file once, and only if there were no errors. --- tests/tasks/test_update_deps.py | 52 +++++++++++++++++---------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/tests/tasks/test_update_deps.py b/tests/tasks/test_update_deps.py index 3114daf8..8bd6d237 100644 --- a/tests/tasks/test_update_deps.py +++ b/tests/tasks/test_update_deps.py @@ -1052,16 +1052,8 @@ def test_unresolvable_specifier_set( assert terminal_msg.search(capsys.readouterr().err) is not None, terminal_msg -@pytest.mark.parametrize( - ["skip_unnormalized_python_package_names", "fail_fast"], - [(True, True), (False, False), (False, True), (True, False)], - ids=[ - "skip_unnormalized_python_package_names, fail_fast", - "no skip_unnormalized_python_package_names, no fail_fast", - "no skip_unnormalized_python_package_names, fail_fast", - "skip_unnormalized_python_package_names, no fail_fast", - ], -) +@pytest.mark.parametrize("fail_fast", [True, False]) +@pytest.mark.parametrize("skip_unnormalized_python_package_names", [True, False]) def test_skip_unnormalized_python_package_names( tmp_path: Path, skip_unnormalized_python_package_names: bool, @@ -1128,7 +1120,22 @@ def test_skip_unnormalized_python_package_names( r"statements above\.$" ) + # Due to an atomistic approach, if an error occurs, the pyproject.toml file will + # not be updated. + successful_expected_pyproject_file_data = """[project] +name = "{{ cookiecutter.project_slug }}" +requires-python = ">=3.8" + +dependencies = [] + +[project.optional-dependencies] +dev = ["pytest~=7.4"] +all = ["{{ cookiecutter.project_slug }}[dev]"] +""" + erroneous_expected_pyproject_file_data = pyproject_file_data + if skip_unnormalized_python_package_names: + # This should end in success update_deps( context, root_repo_path=str(tmp_path), @@ -1152,7 +1159,13 @@ def test_skip_unnormalized_python_package_names( terminal_error_msg.search(stdouterr.err) is None ), f"{terminal_error_msg!r} unexpectedly found in {stdouterr.err}" + assert ( + pyproject_file.read_text(encoding="utf8") + == successful_expected_pyproject_file_data + ) + else: + # This should end in failure with pytest.raises(SystemExit, match=raise_msg): update_deps( context, @@ -1185,18 +1198,7 @@ def test_skip_unnormalized_python_package_names( terminal_error_msg.search(stdouterr.err) is not None ), f"{terminal_error_msg!r} not found in {stdouterr.err}" - # In both cases, the pyproject.toml file should be updated for pytest. - # When/if a more atomistic approach is taken, then this should *NOT* be the case - # for runs where an error occurs. - expected_pyproject_file_data = """[project] -name = "{{ cookiecutter.project_slug }}" -requires-python = ">=3.8" - -dependencies = [] - -[project.optional-dependencies] -dev = ["pytest~=7.4"] -all = ["{{ cookiecutter.project_slug }}[dev]"] -""" - - assert pyproject_file.read_text(encoding="utf8") == expected_pyproject_file_data + assert ( + pyproject_file.read_text(encoding="utf8") + == erroneous_expected_pyproject_file_data + ) From c653b52dbf463b6cba975bf0cd93c9fd734abfeb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 10:27:33 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- ci_cd/tasks/update_deps.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci_cd/tasks/update_deps.py b/ci_cd/tasks/update_deps.py index f96a75a8..15b4f5e1 100644 --- a/ci_cd/tasks/update_deps.py +++ b/ci_cd/tasks/update_deps.py @@ -84,9 +84,9 @@ def _update_pyproject( index = pyproject["project"]["optional-dependencies"][extra_name].index( original_dependency ) - pyproject["project"]["optional-dependencies"][extra_name][ - index - ] = updated_dependency.replace('"', "'") + pyproject["project"]["optional-dependencies"][extra_name][index] = ( + updated_dependency.replace('"', "'") + ) def _format_and_update_dependency(