Synthesize targets for Python lockfiles (#17097)

This shows how the synthetic target API from #16998 can be used to synthesize targets for lockfiles, which may then be included as dependencies for the requirements.

The last commit builds on top of #16998.

Fixes #16933 

```bash
$ ./pants peek --exclude-defaults 3rdparty/python:python#setuptools
```
```json
[
  {
    "address": "3rdparty/python#setuptools",
    "target_type": "python_requirement",
    "dependencies": [
      "3rdparty/python/requirements.txt",
      "3rdparty/python/user_reqs.lock:python-default"
    ],
    "dependencies_raw": [
      "3rdparty/python/requirements.txt",
      "3rdparty/python/user_reqs.lock:python-default"
    ],
    "requirements": [
      "setuptools<64.0,>=63.1.0"
    ]
  }
]
```

```bash
$ ./pants peek --exclude-defaults 3rdparty/python/user_reqs.lock
```
```json
[
  {
    "address": "3rdparty/python/user_reqs.lock:python-default",
    "target_type": "_lockfile",
    "dependencies": [],
    "source_raw": "user_reqs.lock",
    "sources": [
      "3rdparty/python/user_reqs.lock"
    ]
  }
]
```

src/python/pants/backend/python/goals/lockfile.py

@@ -3,9 +3,11 @@
 
 from __future__ import annotations
 
-import logging
+import itertools
+import os.path
 from collections import defaultdict
 from dataclasses import dataclass
+from operator import itemgetter
 from typing import Iterable
 
 from pants.backend.python.pip_requirement import PipRequirement
@@ -35,6 +37,8 @@
 )
 from pants.core.util_rules.lockfile_metadata import calculate_invalidation_digest
 from pants.engine.fs import CreateDigest, Digest, DigestContents, FileContent, MergeDigests
+from pants.engine.internals.synthetic_targets import SyntheticAddressMaps, SyntheticTargetsRequest
+from pants.engine.internals.target_adaptor import TargetAdaptor
 from pants.engine.process import ProcessCacheScope, ProcessResult
 from pants.engine.rules import Get, collect_rules, rule, rule_helper
 from pants.engine.target import AllTargets
@@ -43,8 +47,6 @@
 from pants.util.logging import LogLevel
 from pants.util.ordered_set import FrozenOrderedSet
 
-logger = logging.getLogger(__name__)
-
 
 @dataclass(frozen=True)
 class GeneratePythonLockfile(GenerateLockfile):
@@ -268,10 +270,50 @@ async def setup_user_lockfile_requests(
     )
 
 
+@dataclass(frozen=True)
+class PythonSyntheticLockfileTargetsRequest(SyntheticTargetsRequest):
+    """Register the type used to create synthetic targets for Python lockfiles.
+
+    As the paths for all lockfiles are known up-front, we set the `path` field to
+    `SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS` so that we get a single request for all
+    our synthetic targets rather than one request per directory.
+    """
+
+    path: str = SyntheticTargetsRequest.SINGLE_REQUEST_FOR_ALL_TARGETS
+
+
+@rule
+async def python_lockfile_synthetic_targets(
+    request: PythonSyntheticLockfileTargetsRequest,
+    python_setup: PythonSetup,
+) -> SyntheticAddressMaps:
+    if not python_setup.enable_resolves:
+        return SyntheticAddressMaps()
+
+    resolves = [
+        (os.path.dirname(lockfile), os.path.basename(lockfile), name)
+        for name, lockfile in python_setup.resolves.items()
+    ]
+    return SyntheticAddressMaps.for_targets_request(
+        request,
+        [
+            (
+                os.path.join(spec_path, "BUILD.python-lockfiles"),
+                tuple(
+                    TargetAdaptor("_lockfiles", name=name, sources=[lockfile])
+                    for _, lockfile, name in lockfiles
+                ),
+            )
+            for spec_path, lockfiles in itertools.groupby(sorted(resolves), key=itemgetter(0))
+        ],
+    )
+
+
 def rules():
     return (
         *collect_rules(),
         UnionRule(GenerateLockfile, GeneratePythonLockfile),
         UnionRule(KnownUserResolveNamesRequest, KnownPythonUserResolveNamesRequest),
         UnionRule(RequestedUserResolveNames, RequestedPythonUserResolveNames),
+        UnionRule(SyntheticTargetsRequest, PythonSyntheticLockfileTargetsRequest),
     )

src/python/pants/backend/python/macros/pipenv_requirements.py

@@ -13,6 +13,7 @@
     TypeStubsModuleMappingField,
 )
 from pants.backend.python.pip_requirement import PipRequirement
+from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import (
     PythonRequirementModulesField,
     PythonRequirementResolveField,
@@ -70,7 +71,9 @@ class GenerateFromPipenvRequirementsRequest(GenerateTargetsRequest):
 # TODO(#10655): differentiate between Pipfile vs. Pipfile.lock.
 @rule(desc="Generate `python_requirement` targets from Pipfile.lock", level=LogLevel.DEBUG)
 async def generate_from_pipenv_requirement(
-    request: GenerateFromPipenvRequirementsRequest, union_membership: UnionMembership
+    request: GenerateFromPipenvRequirementsRequest,
+    union_membership: UnionMembership,
+    python_setup: PythonSetup,
 ) -> GeneratedTargets:
     generator = request.generator
     lock_rel_path = generator[PipenvSourceField].value
@@ -90,6 +93,15 @@ async def generate_from_pipenv_requirement(
         union_membership,
     )
 
+    req_deps = [file_tgt.address.spec]
+
+    resolve = request.template.get(
+        PythonRequirementResolveField.alias, python_setup.default_resolve
+    )
+    lockfile = python_setup.resolves.get(resolve) if python_setup.enable_resolves else None
+    if lockfile:
+        req_deps.append(f"{lockfile}:{resolve}")
+
     digest_contents = await Get(
         DigestContents,
         PathGlobs(
@@ -106,9 +118,7 @@ def generate_tgt(parsed_req: PipRequirement) -> PythonRequirementTarget:
         normalized_proj_name = canonicalize_project_name(parsed_req.project_name)
         tgt_overrides = overrides.pop(normalized_proj_name, {})
         if Dependencies.alias in tgt_overrides:
-            tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [
-                file_tgt.address.spec
-            ]
+            tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + req_deps
 
         return PythonRequirementTarget(
             {
@@ -120,7 +130,7 @@ def generate_tgt(parsed_req: PipRequirement) -> PythonRequirementTarget:
                 ),
                 # This may get overridden by `tgt_overrides`, which will have already added in
                 # the file tgt.
-                Dependencies.alias: [file_tgt.address.spec],
+                Dependencies.alias: req_deps,
                 **tgt_overrides,
             },
             request.template_address.create_generated(parsed_req.project_name),

src/python/pants/backend/python/macros/pipenv_requirements_test.py

@@ -48,8 +48,8 @@ def assert_pipenv_requirements(
 
 
 def test_pipfile_lock(rule_runner: RuleRunner) -> None:
-    """This tests that we correctly create a new python_requirement_library for each entry in a
-    Pipfile.lock file.
+    """This tests that we correctly create a new python_requirement for each entry in a Pipfile.lock
+    file.
 
     Edge cases:
 
@@ -89,3 +89,45 @@ def test_pipfile_lock(rule_runner: RuleRunner) -> None:
             TargetGeneratorSourcesHelperTarget({"source": "Pipfile.lock"}, file_addr),
         },
     )
+
+
+def test_pipfile_lockfile_dependency(rule_runner: RuleRunner) -> None:
+    """This tests that we adds a dependency on the lockfile for the resolve for each generated
+    python_requirement."""
+    rule_runner.set_options(["--python-enable-resolves"])
+    file_addr = Address("", target_name="reqs", relative_file_path="Pipfile.lock")
+    lock_addr = Address(
+        "3rdparty/python", target_name="python-default", relative_file_path="default.lock"
+    )
+    assert_pipenv_requirements(
+        rule_runner,
+        "pipenv_requirements(name='reqs', module_mapping={'ansicolors': ['colors']})",
+        {
+            "default": {"ansicolors": {"version": ">=1.18.0"}},
+            "develop": {
+                "cachetools": {
+                    "markers": "python_version ~= '3.5'",
+                    "version": "==4.1.1",
+                    "extras": ["ring", "mongo"],
+                }
+            },
+        },
+        expected_targets={
+            PythonRequirementTarget(
+                {
+                    "requirements": ["ansicolors>=1.18.0"],
+                    "modules": ["colors"],
+                    "dependencies": [file_addr.spec, lock_addr.spec],
+                },
+                Address("", target_name="reqs", generated_name="ansicolors"),
+            ),
+            PythonRequirementTarget(
+                {
+                    "requirements": ["cachetools[ring, mongo]==4.1.1;python_version ~= '3.5'"],
+                    "dependencies": [file_addr.spec, lock_addr.spec],
+                },
+                Address("", target_name="reqs", generated_name="cachetools"),
+            ),
+            TargetGeneratorSourcesHelperTarget({"source": file_addr.filename}, file_addr),
+        },
+    )

src/python/pants/backend/python/macros/poetry_requirements.py

@@ -21,6 +21,7 @@
     TypeStubsModuleMappingField,
 )
 from pants.backend.python.pip_requirement import PipRequirement
+from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import (
     PythonRequirementModulesField,
     PythonRequirementResolveField,
@@ -457,6 +458,7 @@ async def generate_from_python_requirement(
     request: GenerateFromPoetryRequirementsRequest,
     build_root: BuildRoot,
     union_membership: UnionMembership,
+    python_setup: PythonSetup,
 ) -> GeneratedTargets:
     generator = request.generator
     pyproject_rel_path = generator[PoetryRequirementsSourceField].value
@@ -476,6 +478,15 @@ async def generate_from_python_requirement(
         union_membership,
     )
 
+    req_deps = [file_tgt.address.spec]
+
+    resolve = request.template.get(
+        PythonRequirementResolveField.alias, python_setup.default_resolve
+    )
+    lockfile = python_setup.resolves.get(resolve) if python_setup.enable_resolves else None
+    if lockfile:
+        req_deps.append(f"{lockfile}:{resolve}")
+
     digest_contents = await Get(
         DigestContents,
         PathGlobs(
@@ -500,9 +511,7 @@ def generate_tgt(parsed_req: PipRequirement) -> PythonRequirementTarget:
         normalized_proj_name = canonicalize_project_name(parsed_req.project_name)
         tgt_overrides = overrides.pop(normalized_proj_name, {})
         if Dependencies.alias in tgt_overrides:
-            tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [
-                file_tgt.address.spec
-            ]
+            tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + req_deps
 
         return PythonRequirementTarget(
             {
@@ -514,7 +523,7 @@ def generate_tgt(parsed_req: PipRequirement) -> PythonRequirementTarget:
                 ),
                 # This may get overridden by `tgt_overrides`, which will have already added in
                 # the file tgt.
-                Dependencies.alias: [file_tgt.address.spec],
+                Dependencies.alias: req_deps,
                 **tgt_overrides,
             },
             request.template_address.create_generated(parsed_req.project_name),

src/python/pants/backend/python/macros/poetry_requirements_test.py

@@ -555,6 +555,35 @@ def test_source_override(rule_runner: RuleRunner) -> None:
     )
 
 
+def test_lockfile_dependency(rule_runner: RuleRunner) -> None:
+    rule_runner.set_options(["--python-enable-resolves"])
+    file_addr = Address("", target_name="reqs", relative_file_path="pyproject.toml")
+    lock_addr = Address(
+        "3rdparty/python", target_name="python-default", relative_file_path="default.lock"
+    )
+    assert_poetry_requirements(
+        rule_runner,
+        "poetry_requirements(name='reqs')",
+        dedent(
+            """\
+            [tool.poetry.dependencies]
+            ansicolors = ">=1.18.0"
+            [tool.poetry.dev-dependencies]
+            """
+        ),
+        expected_targets={
+            PythonRequirementTarget(
+                {
+                    "dependencies": [file_addr.spec, lock_addr.spec],
+                    "requirements": ["ansicolors>=1.18.0"],
+                },
+                address=Address("", target_name="reqs", generated_name="ansicolors"),
+            ),
+            TargetGeneratorSourcesHelperTarget({"source": file_addr.filename}, file_addr),
+        },
+    )
+
+
 def test_non_pep440_error(rule_runner: RuleRunner) -> None:
     with engine_error(contains='Failed to parse requirement foo = "~r62b" in pyproject.toml'):
         assert_poetry_requirements(

src/python/pants/backend/python/macros/python_requirements.py

@@ -14,6 +14,7 @@
     TypeStubsModuleMappingField,
 )
 from pants.backend.python.pip_requirement import PipRequirement
+from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import (
     PythonRequirementModulesField,
     PythonRequirementResolveField,
@@ -82,7 +83,9 @@ class GenerateFromPythonRequirementsRequest(GenerateTargetsRequest):
 
 @rule(desc="Generate `python_requirement` targets from requirements.txt", level=LogLevel.DEBUG)
 async def generate_from_python_requirement(
-    request: GenerateFromPythonRequirementsRequest, union_membership: UnionMembership
+    request: GenerateFromPythonRequirementsRequest,
+    union_membership: UnionMembership,
+    python_setup: PythonSetup,
 ) -> GeneratedTargets:
     generator = request.generator
     requirements_rel_path = generator[PythonRequirementsSourceField].value
@@ -102,6 +105,15 @@ async def generate_from_python_requirement(
         union_membership,
     )
 
+    req_deps = [file_tgt.address.spec]
+
+    resolve = request.template.get(
+        PythonRequirementResolveField.alias, python_setup.default_resolve
+    )
+    lockfile = python_setup.resolves.get(resolve) if python_setup.enable_resolves else None
+    if lockfile:
+        req_deps.append(f"{lockfile}:{resolve}")
+
     digest_contents = await Get(
         DigestContents,
         PathGlobs(
@@ -126,9 +138,7 @@ def generate_tgt(
         normalized_proj_name = canonicalize_project_name(project_name)
         tgt_overrides = overrides.pop(normalized_proj_name, {})
         if Dependencies.alias in tgt_overrides:
-            tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [
-                file_tgt.address.spec
-            ]
+            tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + req_deps
 
         return PythonRequirementTarget(
             {
@@ -140,7 +150,7 @@ def generate_tgt(
                 ),
                 # This may get overridden by `tgt_overrides`, which will have already added in
                 # the file tgt.
-                Dependencies.alias: [file_tgt.address.spec],
+                Dependencies.alias: req_deps,
                 **tgt_overrides,
             },
             request.template_address.create_generated(project_name),

src/python/pants/backend/python/macros/python_requirements_test.py

@@ -191,3 +191,26 @@ def test_source_override(rule_runner: RuleRunner) -> None:
             TargetGeneratorSourcesHelperTarget({"source": "subdir/requirements.txt"}, file_addr),
         },
     )
+
+
+def test_lockfile_dependency(rule_runner: RuleRunner) -> None:
+    rule_runner.set_options(["--python-enable-resolves"])
+    reqs_addr = Address("", target_name="reqs", relative_file_path="requirements.txt")
+    lock_addr = Address(
+        "3rdparty/python", target_name="python-default", relative_file_path="default.lock"
+    )
+    assert_python_requirements(
+        rule_runner,
+        "python_requirements(name='reqs')",
+        "ansicolors>=1.18.0",
+        expected_targets={
+            PythonRequirementTarget(
+                {
+                    "requirements": ["ansicolors>=1.18.0"],
+                    "dependencies": [reqs_addr.spec, lock_addr.spec],
+                },
+                Address("", target_name="reqs", generated_name="ansicolors"),
+            ),
+            TargetGeneratorSourcesHelperTarget({"source": reqs_addr.filename}, reqs_addr),
+        },
+    )

src/python/pants/core/register.py

@@ -29,6 +29,8 @@
     FileTarget,
     GenericTarget,
     HTTPSource,
+    LockfilesGeneratorTarget,
+    LockfileTarget,
     RelocatedFiles,
     ResourcesGeneratorTarget,
     ResourceTarget,
@@ -91,15 +93,17 @@ def rules():
 def target_types():
     return [
         ArchiveTarget,
-        FileTarget,
+        DockerEnvironmentTarget,
         FilesGeneratorTarget,
+        FileTarget,
         GenericTarget,
-        ResourceTarget,
-        ResourcesGeneratorTarget,
-        RelocatedFiles,
         LocalEnvironmentTarget,
-        DockerEnvironmentTarget,
+        LockfilesGeneratorTarget,
+        LockfileTarget,
+        RelocatedFiles,
         RemoteEnvironmentTarget,
+        ResourcesGeneratorTarget,
+        ResourceTarget,
     ]