diff --git a/src/python/pants/backend/codegen/thrift/apache/rules.py b/src/python/pants/backend/codegen/thrift/apache/rules.py index 3cd5ed5e5bb..bb0f1420c0d 100644 --- a/src/python/pants/backend/codegen/thrift/apache/rules.py +++ b/src/python/pants/backend/codegen/thrift/apache/rules.py @@ -121,9 +121,12 @@ async def generate_apache_thrift_sources( @rule -async def setup_thrift_tool(apache_thrift: ApacheThriftSubsystem) -> ApacheThriftSetup: +async def setup_thrift_tool( + apache_thrift: ApacheThriftSubsystem, + apache_thrift_env_aware: ApacheThriftSubsystem.EnvironmentAware, +) -> ApacheThriftSetup: env = await Get(EnvironmentVars, EnvironmentVarsRequest(["PATH"])) - search_paths = apache_thrift.thrift_search_paths(env) + search_paths = apache_thrift_env_aware.thrift_search_paths(env) all_thrift_binary_paths = await Get( BinaryPaths, BinaryPathRequest( diff --git a/src/python/pants/backend/codegen/thrift/apache/subsystem.py b/src/python/pants/backend/codegen/thrift/apache/subsystem.py index a466ede7a18..ca47b62a353 100644 --- a/src/python/pants/backend/codegen/thrift/apache/subsystem.py +++ b/src/python/pants/backend/codegen/thrift/apache/subsystem.py @@ -15,19 +15,6 @@ class ApacheThriftSubsystem(Subsystem): options_scope = "apache-thrift" help = "Apache Thrift IDL compiler (https://thrift.apache.org/)." - _thrift_search_paths = StrListOption( - default=[""], - help=softwrap( - """ - A list of paths to search for Thrift. - - Specify absolute paths to directories with the `thrift` binary, e.g. `/usr/bin`. - Earlier entries will be searched first. - - The special string `""` will expand to the contents of the PATH env var. - """ - ), - ) expected_version = StrOption( default="0.15", help=softwrap( @@ -42,14 +29,29 @@ class ApacheThriftSubsystem(Subsystem): ), ) - def thrift_search_paths(self, env: EnvironmentVars) -> tuple[str, ...]: - def iter_path_entries(): - for entry in self._thrift_search_paths: - if entry == "": - path = env.get("PATH") - if path: - yield from path.split(os.pathsep) - else: - yield entry - - return tuple(OrderedSet(iter_path_entries())) + class EnvironmentAware: + _thrift_search_paths = StrListOption( + default=[""], + help=softwrap( + """ + A list of paths to search for Thrift. + + Specify absolute paths to directories with the `thrift` binary, e.g. `/usr/bin`. + Earlier entries will be searched first. + + The special string `""` will expand to the contents of the PATH env var. + """ + ), + ) + + def thrift_search_paths(self, env: EnvironmentVars) -> tuple[str, ...]: + def iter_path_entries(): + for entry in self._thrift_search_paths: + if entry == "": + path = env.get("PATH") + if path: + yield from path.split(os.pathsep) + else: + yield entry + + return tuple(OrderedSet(iter_path_entries())) diff --git a/src/python/pants/backend/docker/goals/publish.py b/src/python/pants/backend/docker/goals/publish.py index ead5080a550..77fcde5156d 100644 --- a/src/python/pants/backend/docker/goals/publish.py +++ b/src/python/pants/backend/docker/goals/publish.py @@ -51,7 +51,10 @@ def get_output_data(self) -> PublishOutputData: @rule async def push_docker_images( - request: PublishDockerImageRequest, docker: DockerBinary, options: DockerOptions + request: PublishDockerImageRequest, + docker: DockerBinary, + options: DockerOptions, + options_env_aware: DockerOptions.EnvironmentAware, ) -> PublishProcesses: tags = tuple( chain.from_iterable( @@ -71,7 +74,7 @@ async def push_docker_images( ] ) - env = await Get(EnvironmentVars, EnvironmentVarsRequest(options.env_vars)) + env = await Get(EnvironmentVars, EnvironmentVarsRequest(options_env_aware.env_vars)) skip_push = defaultdict(set) jobs: list[PublishPackages] = [] refs: list[str] = [] diff --git a/src/python/pants/backend/docker/goals/run_image.py b/src/python/pants/backend/docker/goals/run_image.py index 6aba96dfe91..840353bf39d 100644 --- a/src/python/pants/backend/docker/goals/run_image.py +++ b/src/python/pants/backend/docker/goals/run_image.py @@ -16,10 +16,13 @@ @rule async def docker_image_run_request( - field_set: DockerFieldSet, docker: DockerBinary, options: DockerOptions + field_set: DockerFieldSet, + docker: DockerBinary, + options: DockerOptions, + options_env_aware: DockerOptions.EnvironmentAware, ) -> RunRequest: env, image = await MultiGet( - Get(EnvironmentVars, EnvironmentVarsRequest(options.env_vars)), + Get(EnvironmentVars, EnvironmentVarsRequest(options_env_aware.env_vars)), Get(BuiltPackage, PackageFieldSet, field_set), ) tag = cast(BuiltDockerImage, image.artifacts[0]).tags[0] diff --git a/src/python/pants/backend/docker/subsystems/docker_options.py b/src/python/pants/backend/docker/subsystems/docker_options.py index bdc789743d7..449821d52a8 100644 --- a/src/python/pants/backend/docker/subsystems/docker_options.py +++ b/src/python/pants/backend/docker/subsystems/docker_options.py @@ -34,6 +34,48 @@ class DockerOptions(Subsystem): options_scope = "docker" help = "Options for interacting with Docker." + class EnvironmentAware: + _env_vars = ShellStrListOption( + help=softwrap( + """ + Environment variables to set for `docker` invocations. + + Entries are either strings in the form `ENV_VAR=value` to set an explicit value; + or just `ENV_VAR` to copy the value from Pants's own environment. + """ + ), + advanced=True, + ) + _executable_search_paths = StrListOption( + default=[""], + help=softwrap( + """ + The PATH value that will be used to find the Docker client and any tools required. + + The special string `""` will expand to the contents of the PATH env var. + """ + ), + advanced=True, + metavar="", + ) + + @property + def env_vars(self) -> tuple[str, ...]: + return tuple(sorted(set(self._env_vars))) + + @memoized_method + def executable_search_path(self, env: EnvironmentVars) -> tuple[str, ...]: + def iter_path_entries(): + for entry in self._executable_search_paths: + if entry == "": + path = env.get("PATH") + if path: + yield from path.split(os.pathsep) + else: + yield entry + + return tuple(OrderedSet(iter_path_entries())) + _registries = DictOption[Any]( help=softwrap( """ @@ -154,17 +196,6 @@ class DockerOptions(Subsystem): default=False, help="Whether to log the Docker output to the console. If false, only the image ID is logged.", ) - _env_vars = ShellStrListOption( - help=softwrap( - """ - Environment variables to set for `docker` invocations. - - Entries are either strings in the form `ENV_VAR=value` to set an explicit value; - or just `ENV_VAR` to copy the value from Pants's own environment. - """ - ), - advanced=True, - ) run_args = ShellStrListOption( default=["--interactive", "--tty"] if sys.stdout.isatty() else [], help=softwrap( @@ -187,18 +218,6 @@ class DockerOptions(Subsystem): """ ), ) - _executable_search_paths = StrListOption( - default=[""], - help=softwrap( - """ - The PATH value that will be used to find the Docker client and any tools required. - - The special string `""` will expand to the contents of the PATH env var. - """ - ), - advanced=True, - metavar="", - ) _tools = StrListOption( default=[], help=softwrap( @@ -221,10 +240,6 @@ class DockerOptions(Subsystem): def build_args(self) -> tuple[str, ...]: return tuple(sorted(set(self._build_args))) - @property - def env_vars(self) -> tuple[str, ...]: - return tuple(sorted(set(self._env_vars))) - @property def tools(self) -> tuple[str, ...]: return tuple(sorted(set(self._tools))) @@ -232,16 +247,3 @@ def tools(self) -> tuple[str, ...]: @memoized_method def registries(self) -> DockerRegistries: return DockerRegistries.from_dict(self._registries) - - @memoized_method - def executable_search_path(self, env: EnvironmentVars) -> tuple[str, ...]: - def iter_path_entries(): - for entry in self._executable_search_paths: - if entry == "": - path = env.get("PATH") - if path: - yield from path.split(os.pathsep) - else: - yield entry - - return tuple(OrderedSet(iter_path_entries())) diff --git a/src/python/pants/backend/docker/util_rules/docker_binary.py b/src/python/pants/backend/docker/util_rules/docker_binary.py index b0e9ac6dd3c..8401d2157ad 100644 --- a/src/python/pants/backend/docker/util_rules/docker_binary.py +++ b/src/python/pants/backend/docker/util_rules/docker_binary.py @@ -126,10 +126,12 @@ class DockerBinaryRequest: @rule(desc="Finding the `docker` binary and related tooling", level=LogLevel.DEBUG) async def find_docker( - docker_request: DockerBinaryRequest, docker_options: DockerOptions + docker_request: DockerBinaryRequest, + docker_options: DockerOptions, + docker_options_env_aware: DockerOptions.EnvironmentAware, ) -> DockerBinary: env = await Get(EnvironmentVars, EnvironmentVarsRequest(["PATH"])) - search_path = docker_options.executable_search_path(env) + search_path = docker_options_env_aware.executable_search_path(env) request = BinaryPathRequest( binary_name="docker", search_path=search_path, diff --git a/src/python/pants/backend/docker/util_rules/docker_build_env.py b/src/python/pants/backend/docker/util_rules/docker_build_env.py index 3a63fa5f491..ea3dc3c5b09 100644 --- a/src/python/pants/backend/docker/util_rules/docker_build_env.py +++ b/src/python/pants/backend/docker/util_rules/docker_build_env.py @@ -60,12 +60,14 @@ class DockerBuildEnvironmentRequest: @rule async def docker_build_environment_vars( - request: DockerBuildEnvironmentRequest, docker_options: DockerOptions + request: DockerBuildEnvironmentRequest, + docker_options: DockerOptions, + docker_env_aware: DockerOptions.EnvironmentAware, ) -> DockerBuildEnvironment: build_args = await Get(DockerBuildArgs, DockerBuildArgsRequest(request.target)) env_vars = KeyValueSequenceUtil.from_strings( *{build_arg for build_arg in build_args if "=" not in build_arg}, - *docker_options.env_vars, + *docker_env_aware.env_vars, ) env = await Get(EnvironmentVars, EnvironmentVarsRequest(tuple(env_vars))) return DockerBuildEnvironment.create(env) diff --git a/src/python/pants/backend/python/goals/publish.py b/src/python/pants/backend/python/goals/publish.py index a42bf73014b..a796e31a960 100644 --- a/src/python/pants/backend/python/goals/publish.py +++ b/src/python/pants/backend/python/goals/publish.py @@ -139,6 +139,7 @@ def twine_env(env: EnvironmentVars, repo: str) -> EnvironmentVars: async def twine_upload( request: PublishPythonPackageRequest, twine_subsystem: TwineSubsystem, + twine_environment_aware: TwineSubsystem.EnvironmentAware, global_options: GlobalOptions, ) -> PublishProcesses: dists = tuple( @@ -176,7 +177,7 @@ async def twine_upload( Get(ConfigFiles, ConfigFilesRequest, twine_subsystem.config_request()), ) - ca_cert_request = twine_subsystem.ca_certs_digest_request(global_options.ca_certs_path) + ca_cert_request = twine_environment_aware.ca_certs_digest_request(global_options.ca_certs_path) ca_cert = await Get(Snapshot, CreateDigest, ca_cert_request) if ca_cert_request else None ca_cert_digest = (ca_cert.digest,) if ca_cert else () diff --git a/src/python/pants/backend/python/subsystems/python_native_code.py b/src/python/pants/backend/python/subsystems/python_native_code.py index 8f695c3e9a1..d07b2e26f8d 100644 --- a/src/python/pants/backend/python/subsystems/python_native_code.py +++ b/src/python/pants/backend/python/subsystems/python_native_code.py @@ -13,21 +13,22 @@ class PythonNativeCodeSubsystem(Subsystem): options_scope = "python-native-code" help = "Options for building native code using Python, e.g. when resolving distributions." - # TODO(#7735): move the --cpp-flags and --ld-flags to a general subprocess support subsystem. - cpp_flags = StrListOption( - default=safe_shlex_split(os.environ.get("CPPFLAGS", "")), - help="Override the `CPPFLAGS` environment variable for any forked subprocesses.", - advanced=True, - ) - ld_flags = StrListOption( - default=safe_shlex_split(os.environ.get("LDFLAGS", "")), - help="Override the `LDFLAGS` environment variable for any forked subprocesses.", - advanced=True, - ) + class EnvironmentAware: + # TODO(#7735): move the --cpp-flags and --ld-flags to a general subprocess support subsystem. + cpp_flags = StrListOption( + default=safe_shlex_split(os.environ.get("CPPFLAGS", "")), + help="Override the `CPPFLAGS` environment variable for any forked subprocesses.", + advanced=True, + ) + ld_flags = StrListOption( + default=safe_shlex_split(os.environ.get("LDFLAGS", "")), + help="Override the `LDFLAGS` environment variable for any forked subprocesses.", + advanced=True, + ) - @property - def environment_dict(self) -> Dict[str, str]: - return { - "CPPFLAGS": safe_shlex_join(self.cpp_flags), - "LDFLAGS": safe_shlex_join(self.ld_flags), - } + @property + def environment_dict(self) -> Dict[str, str]: + return { + "CPPFLAGS": safe_shlex_join(self.cpp_flags), + "LDFLAGS": safe_shlex_join(self.ld_flags), + } diff --git a/src/python/pants/backend/python/subsystems/twine.py b/src/python/pants/backend/python/subsystems/twine.py index bffae6c2e89..2ba9a8cb3a0 100644 --- a/src/python/pants/backend/python/subsystems/twine.py +++ b/src/python/pants/backend/python/subsystems/twine.py @@ -71,19 +71,6 @@ class TwineSubsystem(PythonToolBase): """ ), ) - ca_certs_path = StrOption( - advanced=True, - default="", - help=softwrap( - """ - Path to a file containing PEM-format CA certificates used for verifying secure - connections when publishing python distributions. - - Uses the value from `[GLOBAL].ca_certs_path` by default. Set to `""` to - not use the default CA certificate. - """ - ), - ) def config_request(self) -> ConfigFilesRequest: # Refer to https://twine.readthedocs.io/en/latest/#configuration for how config files are @@ -95,18 +82,33 @@ def config_request(self) -> ConfigFilesRequest: check_existence=[".pypirc"], ) - def ca_certs_digest_request(self, default_ca_certs_path: str | None) -> CreateDigest | None: - ca_certs_path: str | None = self.ca_certs_path - if ca_certs_path == "": - ca_certs_path = default_ca_certs_path - if not ca_certs_path or ca_certs_path == "": - return None - - # The certs file will typically not be in the repo, so we can't digest it via a PathGlobs. - # Instead we manually create a FileContent for it. - ca_certs_content = Path(ca_certs_path).read_bytes() - chrooted_ca_certs_path = os.path.basename(ca_certs_path) - return CreateDigest((FileContent(chrooted_ca_certs_path, ca_certs_content),)) + class EnvironmentAware: + ca_certs_path = StrOption( + advanced=True, + default="", + help=softwrap( + """ + Path to a file containing PEM-format CA certificates used for verifying secure + connections when publishing python distributions. + + Uses the value from `[GLOBAL].ca_certs_path` by default. Set to `""` to + not use the default CA certificate. + """ + ), + ) + + def ca_certs_digest_request(self, default_ca_certs_path: str | None) -> CreateDigest | None: + ca_certs_path: str | None = self.ca_certs_path + if ca_certs_path == "": + ca_certs_path = default_ca_certs_path + if not ca_certs_path or ca_certs_path == "": + return None + + # The certs file will typically not be in the repo, so we can't digest it via a PathGlobs. + # Instead we manually create a FileContent for it. + ca_certs_content = Path(ca_certs_path).read_bytes() + chrooted_ca_certs_path = os.path.basename(ca_certs_path) + return CreateDigest((FileContent(chrooted_ca_certs_path, ca_certs_content),)) class TwineLockfileSentinel(GeneratePythonToolLockfileSentinel): diff --git a/src/python/pants/backend/python/util_rules/pex_cli.py b/src/python/pants/backend/python/util_rules/pex_cli.py index fb665e94dba..e51bdf6d106 100644 --- a/src/python/pants/backend/python/util_rules/pex_cli.py +++ b/src/python/pants/backend/python/util_rules/pex_cli.py @@ -124,7 +124,7 @@ async def setup_pex_cli_process( request: PexCliProcess, pex_pex: PexPEX, pex_env: PexEnvironment, - python_native_code_subsystem: PythonNativeCodeSubsystem, + python_native_code_subsystem: PythonNativeCodeSubsystem.EnvironmentAware, global_options: GlobalOptions, pex_subsystem: PexSubsystem, ) -> Process: diff --git a/src/python/pants/backend/python/util_rules/pex_environment.py b/src/python/pants/backend/python/util_rules/pex_environment.py index adfd428a883..2aaf8bec629 100644 --- a/src/python/pants/backend/python/util_rules/pex_environment.py +++ b/src/python/pants/backend/python/util_rules/pex_environment.py @@ -29,21 +29,36 @@ class PexSubsystem(Subsystem): options_scope = "pex" help = "How Pants uses Pex to run Python subprocesses." - # TODO(#9760): We'll want to deprecate this in favor of a global option which allows for a - # per-process override. - _executable_search_paths = StrListOption( - default=[""], - help=softwrap( - """ - The PATH value that will be used by the PEX subprocess and any subprocesses it - spawns. + class EnvironmentAware: + # TODO(#9760): We'll want to deprecate this in favor of a global option which allows for a + # per-process override. + _executable_search_paths = StrListOption( + default=[""], + help=softwrap( + """ + The PATH value that will be used by the PEX subprocess and any subprocesses it + spawns. + + The special string `""` will expand to the contents of the PATH env var. + """ + ), + advanced=True, + metavar="", + ) + + @memoized_method + def path(self, env: EnvironmentVars) -> tuple[str, ...]: + def iter_path_entries(): + for entry in self._executable_search_paths: + if entry == "": + path = env.get("PATH") + if path: + yield from path.split(os.pathsep) + else: + yield entry + + return tuple(OrderedSet(iter_path_entries())) - The special string `""` will expand to the contents of the PATH env var. - """ - ), - advanced=True, - metavar="", - ) _verbosity = IntOption( default=0, help="Set the verbosity level of PEX logging, from 0 (no logging) up to 9 (max logging).", @@ -64,19 +79,6 @@ class PexSubsystem(Subsystem): advanced=True, ) - @memoized_method - def path(self, env: EnvironmentVars) -> tuple[str, ...]: - def iter_path_entries(): - for entry in self._executable_search_paths: - if entry == "": - path = env.get("PATH") - if path: - yield from path.split(os.pathsep) - else: - yield entry - - return tuple(OrderedSet(iter_path_entries())) - @property def verbosity(self) -> int: level = self._verbosity @@ -161,11 +163,12 @@ async def find_pex_python( python_bootstrap: PythonBootstrap, python_binary: PythonBinary, pex_subsystem: PexSubsystem, + pex_environment_aware: PexSubsystem.EnvironmentAware, subprocess_env_vars: SubprocessEnvironmentVars, named_caches_dir: NamedCachesDirOption, ) -> PexEnvironment: return PexEnvironment( - path=pex_subsystem.path(python_bootstrap.environment), + path=pex_environment_aware.path(python_bootstrap.environment), interpreter_search_paths=tuple(python_bootstrap.interpreter_search_paths()), subprocess_environment_dict=subprocess_env_vars.vars, named_caches_dir=named_caches_dir.val, diff --git a/src/python/pants/backend/shell/shell_command.py b/src/python/pants/backend/shell/shell_command.py index 6334931a2df..9d005da9b57 100644 --- a/src/python/pants/backend/shell/shell_command.py +++ b/src/python/pants/backend/shell/shell_command.py @@ -104,7 +104,7 @@ def _shell_tool_safe_env_name(tool_name: str) -> str: @rule async def prepare_shell_command_process( - request: ShellCommandProcessRequest, shell_setup: ShellSetup, bash: BashBinary + request: ShellCommandProcessRequest, shell_setup: ShellSetup.EnvironmentAware, bash: BashBinary ) -> Process: shell_command = request.target interactive = shell_command.has_field(ShellCommandRunWorkdirField) diff --git a/src/python/pants/backend/shell/shell_setup.py b/src/python/pants/backend/shell/shell_setup.py index 6f9247fe9d6..a92a1535c08 100644 --- a/src/python/pants/backend/shell/shell_setup.py +++ b/src/python/pants/backend/shell/shell_setup.py @@ -17,19 +17,6 @@ class ShellSetup(Subsystem): options_scope = "shell-setup" help = "Options for Pants's Shell support." - _executable_search_path = StrListOption( - default=[""], - help=softwrap( - """ - The PATH value that will be used to find shells and to run certain processes - like the shunit2 test runner. - - The special string `""` will expand to the contents of the PATH env var. - """ - ), - advanced=True, - metavar="", - ) dependency_inference = BoolOption( default=True, help="Infer Shell dependencies on other Shell files by analyzing `source` statements.", @@ -45,15 +32,30 @@ class ShellSetup(Subsystem): advanced=True, ) - @memoized_method - def executable_search_path(self, env: EnvironmentVars) -> tuple[str, ...]: - def iter_path_entries(): - for entry in self._executable_search_path: - if entry == "": - path = env.get("PATH") - if path: - yield from path.split(os.pathsep) - else: - yield entry - - return tuple(OrderedSet(iter_path_entries())) + class EnvironmentAware: + _executable_search_path = StrListOption( + default=[""], + help=softwrap( + """ + The PATH value that will be used to find shells and to run certain processes + like the shunit2 test runner. + + The special string `""` will expand to the contents of the PATH env var. + """ + ), + advanced=True, + metavar="", + ) + + @memoized_method + def executable_search_path(self, env: EnvironmentVars) -> tuple[str, ...]: + def iter_path_entries(): + for entry in self._executable_search_path: + if entry == "": + path = env.get("PATH") + if path: + yield from path.split(os.pathsep) + else: + yield entry + + return tuple(OrderedSet(iter_path_entries())) diff --git a/src/python/pants/backend/shell/shunit2_test_runner.py b/src/python/pants/backend/shell/shunit2_test_runner.py index fab05baf225..5585a39c2c5 100644 --- a/src/python/pants/backend/shell/shunit2_test_runner.py +++ b/src/python/pants/backend/shell/shunit2_test_runner.py @@ -124,7 +124,8 @@ class Shunit2Runner: @rule(desc="Determine shunit2 shell") async def determine_shunit2_shell( - request: Shunit2RunnerRequest, shell_setup: ShellSetup + request: Shunit2RunnerRequest, + shell_setup: ShellSetup.EnvironmentAware, ) -> Shunit2Runner: if request.shell_field.value is not None: tgt_shell = Shunit2Shell(request.shell_field.value) @@ -158,7 +159,7 @@ async def determine_shunit2_shell( @rule(desc="Setup shunit2", level=LogLevel.DEBUG) async def setup_shunit2_for_target( request: TestSetupRequest, - shell_setup: ShellSetup, + shell_setup: ShellSetup.EnvironmentAware, test_subsystem: TestSubsystem, test_extra_env: TestExtraEnv, global_options: GlobalOptions, diff --git a/src/python/pants/core/goals/test.py b/src/python/pants/core/goals/test.py index c1910d5cece..ecf0945de03 100644 --- a/src/python/pants/core/goals/test.py +++ b/src/python/pants/core/goals/test.py @@ -313,6 +313,17 @@ class TestSubsystem(GoalSubsystem): def activated(cls, union_membership: UnionMembership) -> bool: return TestFieldSet in union_membership + class EnvironmentAware: + extra_env_vars = StrListOption( + help=softwrap( + """ + Additional environment variables to include in test processes. + Entries are strings in the form `ENV_VAR=value` to use explicitly; or just + `ENV_VAR` to copy the value of a variable in Pants's own environment. + """ + ), + ) + debug = BoolOption( default=False, help=softwrap( @@ -365,15 +376,6 @@ def activated(cls, union_membership: UnionMembership) -> bool: advanced=True, help="Path to write test reports to. Must be relative to the build root.", ) - extra_env_vars = StrListOption( - help=softwrap( - """ - Additional environment variables to include in test processes. - Entries are strings in the form `ENV_VAR=value` to use explicitly; or just - `ENV_VAR` to copy the value of a variable in Pants's own environment. - """ - ), - ) shard = StrOption( default="", help=softwrap( @@ -678,9 +680,9 @@ class TestExtraEnv: @rule -async def get_filtered_environment(test_subsystem: TestSubsystem) -> TestExtraEnv: +async def get_filtered_environment(test_env_aware: TestSubsystem.EnvironmentAware) -> TestExtraEnv: return TestExtraEnv( - await Get(EnvironmentVars, EnvironmentVarsRequest(test_subsystem.extra_env_vars)) + await Get(EnvironmentVars, EnvironmentVarsRequest(test_env_aware.extra_env_vars)) ) diff --git a/src/python/pants/core/util_rules/environments.py b/src/python/pants/core/util_rules/environments.py index 899c8a93346..3c70f84d7f8 100644 --- a/src/python/pants/core/util_rules/environments.py +++ b/src/python/pants/core/util_rules/environments.py @@ -5,8 +5,9 @@ import dataclasses import logging +import shlex from dataclasses import dataclass -from typing import Any, ClassVar, Iterable, cast +from typing import Any, Callable, ClassVar, Iterable, Optional, Tuple, Type, Union, cast from pants.build_graph.address import Address, AddressInput from pants.engine.engine_aware import EngineAwareParameter @@ -29,6 +30,7 @@ WrappedTargetRequest, ) from pants.engine.unions import UnionRule +from pants.option import custom_types from pants.option.global_options import GlobalOptions from pants.option.option_types import DictOption, OptionsInfo, collect_options_info from pants.option.subsystem import Subsystem @@ -656,15 +658,28 @@ class _EnvironmentSensitiveOptionFieldMixin: option_name: ClassVar[str] +class ShellStringSequenceField(StringSequenceField): + @classmethod + def compute_value( + cls, raw_value: Optional[Iterable[str]], address: Address + ) -> Optional[Tuple[str, ...]]: + """Computes a flattened shlexed arg list from an iterable of strings.""" + if not raw_value: + return () + + return tuple(arg for raw_arg in raw_value for arg in shlex.split(raw_arg)) + + # Maps between non-list option value types and corresponding fields -_SIMPLE_OPTIONS: dict[type, type[Field]] = { +_SIMPLE_OPTIONS: dict[Union[Type, Callable[[str], Any]], Type[Field]] = { str: StringField, } # Maps between the member types for list options. Each element is the # field type, and the `value` type for the field. -_LIST_OPTIONS: dict[type, type[Field]] = { +_LIST_OPTIONS: dict[Union[Type, Callable[[str], Any]], Type[Field]] = { str: StringSequenceField, + custom_types.shell_str: ShellStringSequenceField, } diff --git a/src/python/pants/core/util_rules/subprocess_environment.py b/src/python/pants/core/util_rules/subprocess_environment.py index e3a4a9c79cb..590c5d065df 100644 --- a/src/python/pants/core/util_rules/subprocess_environment.py +++ b/src/python/pants/core/util_rules/subprocess_environment.py @@ -20,25 +20,26 @@ class SubprocessEnvironment(Subsystem): options_scope = "subprocess-environment" help = "Environment settings for forked subprocesses." - _env_vars = StrListOption( - default=["LANG", "LC_CTYPE", "LC_ALL", "SSL_CERT_FILE", "SSL_CERT_DIR"], - help=softwrap( - f""" - Environment variables to set for process invocations. - - Entries are either strings in the form `ENV_VAR=value` to set an explicit value; - or just `ENV_VAR` to copy the value from Pants's own environment. - - See {doc_url('options#addremove-semantics')} for how to add and remove Pants's - default for this option. - """ - ), - advanced=True, - ) + class EnvironmentAware: + _env_vars = StrListOption( + default=["LANG", "LC_CTYPE", "LC_ALL", "SSL_CERT_FILE", "SSL_CERT_DIR"], + help=softwrap( + f""" + Environment variables to set for process invocations. + + Entries are either strings in the form `ENV_VAR=value` to set an explicit value; + or just `ENV_VAR` to copy the value from Pants's own environment. + + See {doc_url('options#addremove-semantics')} for how to add and remove Pants's + default for this option. + """ + ), + advanced=True, + ) - @property - def env_vars_to_pass_to_subprocesses(self) -> Tuple[str, ...]: - return tuple(sorted(set(self._env_vars))) + @property + def env_vars_to_pass_to_subprocesses(self) -> Tuple[str, ...]: + return tuple(sorted(set(self._env_vars))) @dataclass(frozen=True) @@ -48,7 +49,7 @@ class SubprocessEnvironmentVars: @rule async def get_subprocess_environment( - subproc_env: SubprocessEnvironment, + subproc_env: SubprocessEnvironment.EnvironmentAware, ) -> SubprocessEnvironmentVars: return SubprocessEnvironmentVars( await Get( diff --git a/src/python/pants/jvm/jdk_rules.py b/src/python/pants/jvm/jdk_rules.py index b710210ded9..7bb8dd905fe 100644 --- a/src/python/pants/jvm/jdk_rules.py +++ b/src/python/pants/jvm/jdk_rules.py @@ -195,7 +195,12 @@ async def internal_jdk(jvm: JvmSubsystem) -> InternalJdk: @rule async def prepare_jdk_environment( - jvm: JvmSubsystem, coursier: Coursier, nailgun_: Nailgun, bash: BashBinary, request: JdkRequest + jvm: JvmSubsystem, + jvm_env_aware: JvmSubsystem.EnvironmentAware, + coursier: Coursier, + nailgun_: Nailgun, + bash: BashBinary, + request: JdkRequest, ) -> JdkEnvironment: nailgun = nailgun_.classpath_entry @@ -297,7 +302,7 @@ def prefixed(arg: str) -> str: ] ), ), - global_jvm_options=jvm.global_options, + global_jvm_options=jvm_env_aware.global_options, nailgun_jar=os.path.join(JdkEnvironment.bin_dir, nailgun.filenames[0]), coursier=coursier, jre_major_version=jre_major_version, diff --git a/src/python/pants/jvm/subsystems.py b/src/python/pants/jvm/subsystems.py index 1879606b78b..2b0e46e5326 100644 --- a/src/python/pants/jvm/subsystems.py +++ b/src/python/pants/jvm/subsystems.py @@ -23,6 +23,20 @@ class JvmSubsystem(Subsystem): """ ) + class EnvironmentAware: + global_options = StrListOption( + help=softwrap( + """ + List of JVM options to pass to all JVM processes. + + Options set here will be used by any JVM processes required by Pants, with + the exception of heap memory settings like `-Xmx`, which need to be set + using `[GLOBAL].process_total_child_memory_usage` and `[GLOBAL].process_per_child_memory_usage`. + """ + ), + advanced=True, + ) + tool_jdk = StrOption( default="temurin:1.11", help=softwrap( @@ -74,18 +88,6 @@ class JvmSubsystem(Subsystem): """ ), ) - global_options = StrListOption( - help=softwrap( - """ - List of JVM options to pass to all JVM processes. - - Options set here will be used by any JVM processes required by Pants, with - the exception of heap memory settings like `-Xmx`, which need to be set - using `[GLOBAL].process_total_child_memory_usage` and `[GLOBAL].process_per_child_memory_usage`. - """ - ), - advanced=True, - ) reproducible_jars = BoolOption( default=False, help=softwrap(