Port update-build-files renamers to fix (#17240)

(Note not deprecating the goal in this PR)

This ports the field/target renamers to be `fix` plugins and updates `update-build-files` to use the new source-of-truth.

Commit useful to review.

build-support/bin/generate_docs.py

@@ -232,6 +232,7 @@ def create_parser() -> argparse.ArgumentParser:
 def run_pants_help_all() -> dict[str, Any]:
     # List all (stable enough) backends here.
     backends = [
+        "pants.backend.build_files.fix.deprecations",
         "pants.backend.build_files.fmt.black",
         "pants.backend.build_files.fmt.buildifier",
         "pants.backend.build_files.fmt.yapf",

pants.toml

@@ -4,6 +4,7 @@ print_stacktrace = true
 # Enable our custom loose-source plugins.
 pythonpath = ["%(buildroot)s/pants-plugins"]
 backend_packages.add = [
+  "pants.backend.build_files.fix.deprecations",
   "pants.backend.build_files.fmt.black",
   "pants.backend.python",
   "pants.backend.experimental.python.packaging.pyoxidizer",

src/python/pants/backend/build_files/fix/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()

src/python/pants/backend/build_files/fix/base.py

@@ -0,0 +1,20 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+from typing import Iterable
+
+from pants.backend.build_files.utils import _get_build_file_partitioner_rules
+from pants.core.goals.fix import FixFilesRequest
+from pants.core.util_rules.partitions import PartitionerType
+
+
+class FixBuildFilesRequest(FixFilesRequest):
+    partitioner_type = PartitionerType.CUSTOM
+
+    @classmethod
+    def _get_rules(cls) -> Iterable:
+        assert cls.partitioner_type is PartitionerType.CUSTOM
+        yield from _get_build_file_partitioner_rules(cls)
+        yield from super()._get_rules()

src/python/pants/backend/build_files/fix/deprecations/BUILD

@@ -0,0 +1,6 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()
+
+python_tests(name="tests")

src/python/pants/backend/build_files/fix/deprecations/base.py

@@ -0,0 +1,32 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import tokenize
+from dataclasses import dataclass
+from io import BytesIO
+
+from pants.engine.internals.parser import ParseError
+
+
+@dataclass(frozen=True)
+class FixBUILDFileRequest:
+    path: str
+    content: bytes
+
+    @property
+    def lines(self) -> list[str]:
+        return self.content.decode("utf-8").splitlines(keepends=True)
+
+    def tokenize(self) -> list[tokenize.TokenInfo]:
+        try:
+            return list(tokenize.tokenize(BytesIO(self.content).readline))
+        except tokenize.TokenError as e:
+            raise ParseError(f"Failed to parse {self.path}: {e}")
+
+
+@dataclass(frozen=True)
+class FixedBUILDFile:
+    path: str
+    content: bytes

src/python/pants/backend/build_files/fix/deprecations/register.py

@@ -0,0 +1,11 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.backend.build_files.fix.deprecations import renamed_fields_rules, renamed_targets_rules
+
+
+def rules():
+    return [
+        *renamed_targets_rules.rules(),
+        *renamed_fields_rules.rules(),
+    ]

src/python/pants/backend/build_files/fix/deprecations/renamed_fields_rules.py

@@ -0,0 +1,178 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import tokenize
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import DefaultDict, Mapping
+
+from pants.backend.build_files.fix.base import FixBuildFilesRequest
+from pants.backend.build_files.fix.deprecations.base import FixBUILDFileRequest, FixedBUILDFile
+from pants.backend.build_files.fix.deprecations.subsystem import BUILDDeprecationsFixer
+from pants.core.goals.fix import FixResult
+from pants.engine.fs import CreateDigest, DigestContents, FileContent
+from pants.engine.internals.native_engine import Digest, Snapshot
+from pants.engine.internals.selectors import Get, MultiGet
+from pants.engine.rules import collect_rules, rule
+from pants.engine.target import RegisteredTargetTypes, TargetGenerator
+from pants.engine.unions import UnionMembership
+from pants.util.frozendict import FrozenDict
+from pants.util.logging import LogLevel
+
+
+class RenameFieldsInFilesRequest(FixBuildFilesRequest):
+    tool_subsystem = BUILDDeprecationsFixer
+
+
+class RenameFieldsInFileRequest(FixBUILDFileRequest):
+    pass
+
+
+@dataclass(frozen=True)
+class RenamedFieldTypes:
+    """Map deprecated field names to their new name, per target."""
+
+    target_field_renames: FrozenDict[str, FrozenDict[str, str]]
+
+    @classmethod
+    def from_dict(cls, data: Mapping[str, Mapping[str, str]]) -> RenamedFieldTypes:
+        return cls(
+            FrozenDict(
+                {
+                    target_name: FrozenDict(
+                        {
+                            deprecated_field_name: new_field_name
+                            for deprecated_field_name, new_field_name in field_renames.items()
+                        }
+                    )
+                    for target_name, field_renames in data.items()
+                }
+            )
+        )
+
+
+@rule
+def determine_renamed_field_types(
+    target_types: RegisteredTargetTypes, union_membership: UnionMembership
+) -> RenamedFieldTypes:
+    target_field_renames: DefaultDict[str, dict[str, str]] = defaultdict(dict)
+    for tgt in target_types.types:
+        field_types = list(tgt.class_field_types(union_membership))
+        if issubclass(tgt, TargetGenerator):
+            field_types.extend(tgt.moved_fields)
+
+        for field_type in field_types:
+            if field_type.deprecated_alias is not None:
+                target_field_renames[tgt.alias][field_type.deprecated_alias] = field_type.alias
+
+        # Make sure we also update deprecated fields in deprecated targets.
+        if tgt.deprecated_alias is not None:
+            target_field_renames[tgt.deprecated_alias] = target_field_renames[tgt.alias]
+
+    return RenamedFieldTypes.from_dict(target_field_renames)
+
+
+@rule
+def fix_single(
+    request: RenameFieldsInFileRequest,
+    renamed_field_types: RenamedFieldTypes,
+) -> FixedBUILDFile:
+    pants_target: str = ""
+    level: int = 0
+    tokens = iter(request.tokenize())
+
+    def parse_level(token: tokenize.TokenInfo) -> bool:
+        """Returns true if token was consumed."""
+        nonlocal level
+
+        if level == 0 or token.type is not tokenize.OP or token.string not in ["(", ")"]:
+            return False
+
+        if token.string == "(":
+            level += 1
+        elif token.string == ")":
+            level -= 1
+
+        return True
+
+    def parse_target(token: tokenize.TokenInfo) -> bool:
+        """Returns true if we're parsing a field name for a top level target."""
+        nonlocal pants_target
+        nonlocal level
+
+        if parse_level(token):
+            # Consumed parenthesis operator.
+            return False
+
+        if token.type is not tokenize.NAME:
+            return False
+
+        if level == 0 and next_token_is("("):
+            level = 1
+            pants_target = token.string
+            # Current token consumed.
+            return False
+
+        return level == 1
+
+    def next_token_is(string: str, token_type=tokenize.OP) -> bool:
+        for next_token in tokens:
+            if next_token.type is tokenize.NL:
+                continue
+            parse_level(next_token)
+            return next_token.type is token_type and next_token.string == string
+        return False
+
+    def should_be_renamed(token: tokenize.TokenInfo) -> bool:
+        nonlocal pants_target
+
+        if not parse_target(token):
+            return False
+
+        if pants_target not in renamed_field_types.target_field_renames:
+            return False
+
+        return (
+            next_token_is("=")
+            and token.string in renamed_field_types.target_field_renames[pants_target]
+        )
+
+    updated_text_lines = list(request.lines)
+    for token in tokens:
+        if not should_be_renamed(token):
+            continue
+        line_index = token.start[0] - 1
+        line = updated_text_lines[line_index]
+        prefix = line[: token.start[1]]
+        suffix = line[token.end[1] :]
+        new_symbol = renamed_field_types.target_field_renames[pants_target][token.string]
+        updated_text_lines[line_index] = f"{prefix}{new_symbol}{suffix}"
+
+    return FixedBUILDFile(request.path, content="".join(updated_text_lines).encode("utf-8"))
+
+
+@rule(desc="Fix deprecated field names", level=LogLevel.DEBUG)
+async def fix(
+    request: RenameFieldsInFilesRequest.Batch,
+) -> FixResult:
+    digest_contents = await Get(DigestContents, Digest, request.snapshot.digest)
+    fixed_contents = await MultiGet(
+        Get(FixedBUILDFile, RenameFieldsInFileRequest(file_content.path, file_content.content))
+        for file_content in digest_contents
+    )
+    snapshot = await Get(
+        Snapshot,
+        CreateDigest(FileContent(content.path, content.content) for content in fixed_contents),
+    )
+    return FixResult(
+        request.snapshot, snapshot, "", "", tool_name=RenameFieldsInFilesRequest.tool_name
+    )
+
+
+def rules():
+    return [
+        *collect_rules(),
+        *RenameFieldsInFilesRequest.rules(),
+    ]

src/python/pants/backend/build_files/fix/deprecations/renamed_fields_rules_test.py

@@ -0,0 +1,84 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import pytest
+
+from pants.backend.build_files.fix.deprecations.renamed_fields_rules import (
+    RenamedFieldTypes,
+    RenameFieldsInFileRequest,
+    determine_renamed_field_types,
+    fix_single,
+)
+from pants.engine.target import RegisteredTargetTypes, StringField, Target, TargetGenerator
+from pants.engine.unions import UnionMembership
+from pants.util.frozendict import FrozenDict
+
+
+def test_determine_renamed_fields() -> None:
+    class DeprecatedField(StringField):
+        alias = "new_name"
+        deprecated_alias = "old_name"
+        deprecated_alias_removal_version = "99.9.0.dev0"
+
+    class OkayField(StringField):
+        alias = "okay"
+
+    class Tgt(Target):
+        alias = "tgt"
+        core_fields = (DeprecatedField, OkayField)
+        deprecated_alias = "deprecated_tgt"
+        deprecated_alias_removal_version = "99.9.0.dev0"
+
+    class TgtGenerator(TargetGenerator):
+        alias = "generator"
+        core_fields = ()
+        moved_fields = (DeprecatedField, OkayField)
+
+    registered_targets = RegisteredTargetTypes.create([Tgt, TgtGenerator])
+    result = determine_renamed_field_types(registered_targets, UnionMembership({}))
+    deprecated_fields = FrozenDict({DeprecatedField.deprecated_alias: DeprecatedField.alias})
+    assert result.target_field_renames == FrozenDict(
+        {k: deprecated_fields for k in (TgtGenerator.alias, Tgt.alias, Tgt.deprecated_alias)}
+    )
+
+
+@pytest.mark.parametrize(
+    "lines",
+    (
+        # Already valid.
+        ["target(new_name='')"],
+        ["target(new_name = 56 ) "],
+        ["target(foo=1, new_name=2)"],
+        ["target(", "new_name", "=3)"],
+        # Unrelated lines.
+        ["", "123", "target()", "name='new_name'"],
+        ["unaffected(deprecated_name='not this target')"],
+        ["target(nested=here(deprecated_name='too deep'))"],
+    ),
+)
+def test_rename_deprecated_field_types_noops(lines: list[str]) -> None:
+    content = "\n".join(lines).encode("utf-8")
+    result = fix_single(
+        RenameFieldsInFileRequest("BUILD", content=content),
+        RenamedFieldTypes.from_dict({"target": {"deprecated_name": "new_name"}}),
+    )
+    assert result.content == content
+
+
+@pytest.mark.parametrize(
+    "lines,expected",
+    (
+        (["tgt1(deprecated_name='')"], ["tgt1(new_name='')"]),
+        (["tgt1 ( deprecated_name = ' ', ", ")"], ["tgt1 ( new_name = ' ', ", ")"]),
+        (["tgt1(deprecated_name='')  # comment"], ["tgt1(new_name='')  # comment"]),
+        (["tgt1(", "deprecated_name", "=", ")"], ["tgt1(", "new_name", "=", ")"]),
+    ),
+)
+def test_rename_deprecated_field_types_rewrite(lines: list[str], expected: list[str]) -> None:
+    result = fix_single(
+        RenameFieldsInFileRequest("BUILD", content="\n".join(lines).encode("utf-8")),
+        RenamedFieldTypes.from_dict({"tgt1": {"deprecated_name": "new_name"}}),
+    )
+    assert result.content == "\n".join(expected).encode("utf-8")