Skip to content

Commit

Permalink
Add [environment].aliases and compatible_platforms field (#16704)
Browse files Browse the repository at this point in the history
Part of #7735 and the design doc at https://docs.google.com/document/d/1vXRHWK7ZjAIp2BxWLwRYm1QOKDeXx02ONQWvXDloxkg/edit?usp=sharing.

A followup PR will allow individual targets to set the `environment` field, which means that you can run part of your codebase with different environments than others. 

For the `environment` field to work, we decided environments should have an "alias", which is a logical name the user defines, like `[python].resolves`. This is for two reasons:

1. We need the user to statically define which `local_environment` targets are valid. We can't auto-discover via `AllTargets` because it would result in a rule graph cycle.
2. We want to let users override which environment is used via the options system, e.g. redefine what `centos6` points to. They need to be able to do this without rewriting BUILD files.

The `environment` field will default to `__local__`, where Pants auto-detects which environment to use based on the platform. This means adding a `compatible_platforms` field so that you can, for example, have a Linux-only environment and macOS-only environment. In a followup, we also likely want to add `[environments].local_override` to instruct Pants which value to use with `__local__`.

[ci skip-rust]
  • Loading branch information
Eric-Arellano authored Aug 31, 2022
1 parent f61a98c commit fbb402a
Show file tree
Hide file tree
Showing 4 changed files with 158 additions and 78 deletions.
1 change: 1 addition & 0 deletions BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ _local_environment(

_local_environment(
name="macos_local_env",
compatible_platforms=["macos_arm64", "macos_x86_64"],
# Avoid system Python interpreters, which tend to be broken on macOS.
python_interpreter_search_paths=["<PYENV>"],
)
9 changes: 3 additions & 6 deletions pants.toml
Original file line number Diff line number Diff line change
Expand Up @@ -96,14 +96,11 @@ root_patterns = [
"/",
]

[environments-preview.platforms_to_local_environment]
linux_arm64 = "//:default_env"
linux_x86_64 = "//:default_env"
# TODO(#7735): Set to `//:macos_local_env` after figuring out why our Build Wheels Mac job is
[environments-preview.aliases]
default_env = "//:default_env"
# TODO(#7735): Define `//:macos_local_env` after figuring out why our Build Wheels Mac job is
# failing when this is set:
# https://github.com/pantsbuild/pants/runs/8082954359?check_suite_focus=true#step:9:657
macos_arm64 = "//:default_env"
macos_x86_64 = "//:default_env"

[tailor]
build_file_header = """\
Expand Down
153 changes: 102 additions & 51 deletions src/python/pants/core/util_rules/environments.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@
from __future__ import annotations

from dataclasses import dataclass
from typing import cast

from pants.build_graph.address import Address, AddressInput
from pants.engine.platform import Platform
from pants.engine.rules import Get, collect_rules, rule
from pants.engine.rules import Get, MultiGet, collect_rules, rule
from pants.engine.target import (
COMMON_TARGET_FIELDS,
StringSequenceField,
Expand All @@ -17,7 +18,7 @@
)
from pants.option.option_types import DictOption
from pants.option.subsystem import Subsystem
from pants.util.memo import memoized_property
from pants.util.frozendict import FrozenDict
from pants.util.strutil import softwrap


Expand All @@ -30,50 +31,46 @@ class EnvironmentsSubsystem(Subsystem):
"""
)

_platforms_to_local_environment = DictOption[str](
aliases = DictOption[str](
help=softwrap(
"""
A mapping of platform strings to addresses to `_local_environment` targets. For example:
A mapping of logical names to addresses to `_local_environment` targets. For example:
[environments-preview.platforms_to_local_environment]
linux_arm64 = "//:linux_arm64_environment"
linux_x86_64 = "//:linux_x86_environment"
macos_arm64 = "//:macos_environment"
macos_x86_64 = "//:macos_environment"
[environments-preview.aliases]
linux_local = "//:linux_env"
macos_local = "//:macos_env"
linux_ci = "build-support:linux_ci_env"
macos_ci = "build-support:macos_ci_env"
Pants will detect what platform you are currently on and load the specified
environment. If a platform is not defined, then Pants will use legacy options like
`[python-bootstrap].search_path`.
TODO(#7735): explain how aliases are used once they are consumed.
Warning: this feature is experimental and this option may be changed and removed before
the first 2.15 alpha release.
Pants will ignore any environment targets that are not given an alias via this option.
"""
)
)

@memoized_property
def platforms_to_local_environment(self) -> dict[str, str]:
valid_platforms = {plat.value for plat in Platform}
invalid_keys = set(self._platforms_to_local_environment.keys()) - valid_platforms
if invalid_keys:
raise ValueError(
softwrap(
f"""
Unrecognized platforms as the keys to the option
`[environments-preview].platforms_to_local_environment`: {sorted(invalid_keys)}
All valid platforms: {sorted(valid_platforms)}
"""
)
)
return self._platforms_to_local_environment


# -------------------------------------------------------------------------------------------
# Environment targets
# -------------------------------------------------------------------------------------------


class CompatiblePlatformsField(StringSequenceField):
alias = "compatible_platforms"
default = tuple(plat.value for plat in Platform)
valid_choices = Platform
value: tuple[str, ...]
help = softwrap(
"""
Which platforms this environment can be used with.
This is used for Pants to automatically determine which which environment target to use for
the user's machine. Currently, there must be exactly one environment target for the
platform.
"""
)


class PythonInterpreterSearchPathsField(StringSequenceField):
alias = "python_interpreter_search_paths"
default = ("<PYENV>", "<PATH>")
Expand Down Expand Up @@ -123,6 +120,7 @@ class LocalEnvironmentTarget(Target):
alias = "_local_environment"
core_fields = (
*COMMON_TARGET_FIELDS,
CompatiblePlatformsField,
PythonInterpreterSearchPathsField,
PythonBootstrapBinaryNamesField,
)
Expand All @@ -140,40 +138,93 @@ class LocalEnvironmentTarget(Target):
# -------------------------------------------------------------------------------------------


class NoCompatibleEnvironmentError(Exception):
pass


class AmbiguousEnvironmentError(Exception):
pass


class AllEnvironmentTargets(FrozenDict[str, LocalEnvironmentTarget]):
"""A mapping of environment aliases to their corresponding environment target."""


@dataclass(frozen=True)
class ChosenLocalEnvironment:
tgt: LocalEnvironmentTarget | None


@rule
async def determine_all_environments(
environments_subsystem: EnvironmentsSubsystem,
) -> AllEnvironmentTargets:
_description_of_origin = "the option [environments-preview].aliases"
addresses = await MultiGet(
Get(
Address,
AddressInput,
AddressInput.parse(raw_address, description_of_origin=_description_of_origin),
)
for raw_address in environments_subsystem.aliases.values()
)
wrapped_targets = await MultiGet(
Get(
WrappedTarget,
WrappedTargetRequest(address, description_of_origin=_description_of_origin),
)
for address in addresses
)
# TODO(#7735): validate the correct target type is used?
return AllEnvironmentTargets(
(alias, cast(LocalEnvironmentTarget, wrapped_tgt.target))
for alias, wrapped_tgt in zip(environments_subsystem.aliases.keys(), wrapped_targets)
)


@rule
async def choose_local_environment(
platform: Platform, environments_subsystem: EnvironmentsSubsystem
platform: Platform, all_environment_targets: AllEnvironmentTargets
) -> ChosenLocalEnvironment:
raw_address = environments_subsystem.platforms_to_local_environment.get(platform.value)
if not raw_address:
if not all_environment_targets:
return ChosenLocalEnvironment(None)
_description_of_origin = "the option [environments-preview].platforms_to_local_environment"
address = await Get(
Address,
AddressInput,
AddressInput.parse(raw_address, description_of_origin=_description_of_origin),
)
wrapped_target = await Get(
WrappedTarget, WrappedTargetRequest(address, description_of_origin=_description_of_origin)
)
# TODO(#7735): this is not idiomatic to check for the target subclass.
if not isinstance(wrapped_target.target, LocalEnvironmentTarget):
raise ValueError(
compatible_targets = [
tgt
for tgt in all_environment_targets.values()
if platform.value in tgt[CompatiblePlatformsField].value
]
if not compatible_targets:
raise NoCompatibleEnvironmentError(
softwrap(
f"""
No `_local_environment` targets from `[environments-preview].aliases` are
compatible with the current platform: {platform.value}
To fix, either adjust the `{CompatiblePlatformsField.alias}` field from the targets
in `[environments-preview].aliases` to include `{platform.value}`, or define a new
`_local_environment` target with `{platform.value}` included in the
`{CompatiblePlatformsField.alias}` field. (Current targets from
`[environments-preview].aliases`:
{sorted(tgt.address.spec for tgt in all_environment_targets.values())})
"""
)
)
elif len(compatible_targets) > 1:
# TODO(#7735): Allow the user to disambiguate what __local__ means via an option.
raise AmbiguousEnvironmentError(
softwrap(
f"""
Expected to use the address to a `_local_environment` target in the option
`[environments-preview].platforms_to_local_environment`, but the platform
`{platform.value}` was set to the target {address} with the target type
`{wrapped_target.target.alias}`.
Multiple `_local_environment` targets from `[environments-preview].aliases`
are compatible with the current platform `{platform.value}`, so it is ambiguous
which to use: {sorted(tgt.address.spec for tgt in compatible_targets)}
To fix, either adjust the `{CompatiblePlatformsField.alias}` field from those
targets so that only one includes the value `{platform.value}`, or change
`[environments-preview].aliases` so that it does not define some of those targets.
"""
)
)
return ChosenLocalEnvironment(wrapped_target.target)
return ChosenLocalEnvironment(compatible_targets[0])


def rules():
Expand Down
73 changes: 52 additions & 21 deletions src/python/pants/core/util_rules/environments_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,47 +9,78 @@

from pants.build_graph.address import Address
from pants.core.util_rules import environments
from pants.core.util_rules.environments import ChosenLocalEnvironment, LocalEnvironmentTarget
from pants.testutil.rule_runner import QueryRule, RuleRunner
from pants.core.util_rules.environments import (
AllEnvironmentTargets,
AmbiguousEnvironmentError,
ChosenLocalEnvironment,
LocalEnvironmentTarget,
NoCompatibleEnvironmentError,
)
from pants.testutil.rule_runner import QueryRule, RuleRunner, engine_error


@pytest.fixture
def rule_runner() -> RuleRunner:
return RuleRunner(
rules=[
*environments.rules(),
QueryRule(AllEnvironmentTargets, []),
QueryRule(ChosenLocalEnvironment, []),
],
target_types=[LocalEnvironmentTarget],
)


def test_choose_local_environment(rule_runner: RuleRunner) -> None:
def test_all_environments(rule_runner: RuleRunner) -> None:
rule_runner.write_files(
{
"BUILD": dedent(
"""\
_local_environment(name='local_env')
_local_environment(name='e1')
_local_environment(name='e2')
_local_environment(name='no-alias')
"""
)
}
)
# If `--platforms-to-local-environments` is not set, do not choose an environment.
assert rule_runner.request(ChosenLocalEnvironment, []).tgt is None

rule_runner.set_options(
[
# Note that we cannot inject the `Platform` to an arbitrary value: it will use the
# platform of the test executor. So, we use the same environment for all platforms.
(
"--environments-preview-platforms-to-local-environment="
+ "{'macos_arm64': '//:local_env', "
+ "'macos_x86_64': '//:local_env', "
+ "'linux_arm64': '//:local_env', "
+ "'linux_x86_64': '//:local_env'}"
),
]
rule_runner.set_options(["--environments-preview-aliases={'e1': '//:e1', 'e2': '//:e2'}"])
result = rule_runner.request(AllEnvironmentTargets, [])
assert result == AllEnvironmentTargets(
{
"e1": LocalEnvironmentTarget({}, Address("", target_name="e1")),
"e2": LocalEnvironmentTarget({}, Address("", target_name="e2")),
}
)
assert rule_runner.request(ChosenLocalEnvironment, []).tgt == LocalEnvironmentTarget(
{}, Address("", target_name="local_env")


def test_choose_local_environment(rule_runner: RuleRunner) -> None:
rule_runner.write_files(
{
"BUILD": dedent(
"""\
_local_environment(name='e1')
_local_environment(name='e2')
_local_environment(name='not-compatible', compatible_platforms=[])
"""
)
}
)

def get_env() -> ChosenLocalEnvironment:
return rule_runner.request(ChosenLocalEnvironment, [])

# If `--aliases` is not set, do not choose an environment.
assert get_env().tgt is None

rule_runner.set_options(["--environments-preview-aliases={'e': '//:e1'}"])
assert get_env().tgt == LocalEnvironmentTarget({}, Address("", target_name="e1"))

# Error if `--aliases` set, but no compatible platforms
rule_runner.set_options(["--environments-preview-aliases={'e': '//:not-compatible'}"])
with engine_error(NoCompatibleEnvironmentError):
get_env()

# Error if >1 compatible targets.
rule_runner.set_options(["--environments-preview-aliases={'e1': '//:e1', 'e2': '//:e2'}"])
with engine_error(AmbiguousEnvironmentError):
get_env()

0 comments on commit fbb402a

Please sign in to comment.