Fix Pantsd not properly invalidating when CPPFLAGS and LDFLAGS ch…

…ange (#17009)

This adds `PythonNativeCodeEnvironment`, a value created by a rule that encapsulates the environment var fetching needed to make `PythonNativeCodeSubsystem.EnvironmentAware` resolve correctly.

Fixes #14612

src/python/pants/backend/python/subsystems/python_native_code.py

@@ -1,9 +1,13 @@
 # Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-import os
-from typing import Dict
+from __future__ import annotations
 
+from dataclasses import dataclass
+from typing import Dict, Sequence
+
+from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest
+from pants.engine.rules import Get, collect_rules, rule
 from pants.option.option_types import StrListOption
 from pants.option.subsystem import Subsystem
 from pants.util.strutil import safe_shlex_join, safe_shlex_split
@@ -13,22 +17,61 @@ class PythonNativeCodeSubsystem(Subsystem):
     options_scope = "python-native-code"
     help = "Options for building native code using Python, e.g. when resolving distributions."
 
-    class EnvironmentAware:
+    class EnvironmentAware(Subsystem.EnvironmentAware):
         # TODO(#7735): move the --cpp-flags and --ld-flags to a general subprocess support subsystem.
-        cpp_flags = StrListOption(
-            default=safe_shlex_split(os.environ.get("CPPFLAGS", "")),
-            help="Override the `CPPFLAGS` environment variable for any forked subprocesses.",
+        _cpp_flags = StrListOption(
+            default=["<CPPFLAGS>"],
+            help=(
+                "Override the `CPPFLAGS` environment variable for any forked subprocesses. "
+                "Use the value `['<CPPFLAGS>']` to inherit the value of the `CPPFLAGS` "
+                "environment variable from your runtime environment target."
+            ),
             advanced=True,
         )
-        ld_flags = StrListOption(
-            default=safe_shlex_split(os.environ.get("LDFLAGS", "")),
-            help="Override the `LDFLAGS` environment variable for any forked subprocesses.",
+        _ld_flags = StrListOption(
+            default=["<LDFLAGS>"],
+            help=(
+                "Override the `LDFLAGS` environment variable for any forked subprocesses. "
+                "Use the value `['<LDFLAGS>']` to inherit the value of the `LDFLAGS` environment "
+                "variable from your runtime environment target."
+            ),
             advanced=True,
         )
 
-        @property
-        def environment_dict(self) -> Dict[str, str]:
-            return {
-                "CPPFLAGS": safe_shlex_join(self.cpp_flags),
-                "LDFLAGS": safe_shlex_join(self.ld_flags),
-            }
+
+@dataclass(frozen=True)
+class PythonNativeCodeEnvironment:
+
+    cpp_flags: tuple[str, ...]
+    ld_flags: tuple[str, ...]
+
+    @property
+    def environment_dict(self) -> Dict[str, str]:
+        return {
+            "CPPFLAGS": safe_shlex_join(self.cpp_flags),
+            "LDFLAGS": safe_shlex_join(self.ld_flags),
+        }
+
+
+@rule
+async def resolve_python_native_code_environment(
+    env_aware: PythonNativeCodeSubsystem.EnvironmentAware,
+) -> PythonNativeCodeEnvironment:
+
+    env_vars = await Get(EnvironmentVars, EnvironmentVarsRequest(("CPPFLAGS", "LDFLAGS")))
+
+    def iter_values(env_var: str, values: Sequence[str]):
+        for value in values:
+            if value == f"<{env_var}>":
+                yield from safe_shlex_split(env_vars.get(env_var, ""))
+            else:
+                yield value
+
+    return PythonNativeCodeEnvironment(
+        cpp_flags=tuple(iter_values("CPPFLAGS", env_aware._cpp_flags)),
+        ld_flags=tuple(iter_values("LDFLAGS", env_aware._ld_flags)),
+    )
+
+
+def rules():
+    return [*collect_rules()]

src/python/pants/backend/python/util_rules/pex_cli.py

@@ -9,7 +9,8 @@
 from pathlib import Path
 from typing import Iterable, List, Mapping, Optional, Tuple
 
-from pants.backend.python.subsystems.python_native_code import PythonNativeCodeSubsystem
+from pants.backend.python.subsystems import python_native_code
+from pants.backend.python.subsystems.python_native_code import PythonNativeCodeEnvironment
 from pants.backend.python.util_rules import pex_environment
 from pants.backend.python.util_rules.pex_environment import (
     PexEnvironment,
@@ -124,7 +125,7 @@ async def setup_pex_cli_process(
     request: PexCliProcess,
     pex_pex: PexPEX,
     pex_env: PexEnvironment,
-    python_native_code_subsystem: PythonNativeCodeSubsystem.EnvironmentAware,
+    python_native_code_environment: PythonNativeCodeEnvironment,
     global_options: GlobalOptions,
     pex_subsystem: PexSubsystem,
 ) -> Process:
@@ -190,7 +191,7 @@ async def setup_pex_cli_process(
     normalized_argv = complete_pex_env.create_argv(pex_pex.exe, *args, python=request.python)
     env = {
         **complete_pex_env.environment_dict(python_configured=request.python is not None),
-        **python_native_code_subsystem.environment_dict,
+        **python_native_code_environment.environment_dict,
         **(request.extra_env or {}),
         # If a subcommand is used, we need to use the `pex3` console script.
         **({"PEX_SCRIPT": "pex3"} if request.subcommand else {}),
@@ -211,4 +212,9 @@ async def setup_pex_cli_process(
 
 
 def rules():
-    return [*collect_rules(), *external_tool.rules(), *pex_environment.rules()]
+    return [
+        *collect_rules(),
+        *external_tool.rules(),
+        *pex_environment.rules(),
+        *python_native_code.rules(),
+    ]