Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add --docker-execution option #17049

Merged
merged 2 commits into from
Sep 28, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 41 additions & 26 deletions src/python/pants/core/util_rules/environments.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,10 @@ def docker_platform_field_default_factory(
class DockerFallbackEnvironmentField(FallbackEnvironmentField):
help = softwrap(
f"""
The environment to fallback to when this Docker environment cannot be used because the
The environment to fallback to when this Docker environment cannot be used because either
the global option `--docker-execution` is false, or the
field `{DockerPlatformField.alias}` is not compatible with the local host's CPU
architecture. (This is only an issue when the local host is Linux; macOS is fine.)
architecture (this is only an issue when the local host is Linux; macOS is fine).

Must be an environment name from the option `[environments-preview].names`, the
special string `{LOCAL_ENVIRONMENT_MATCHER}` to use the relevant local environment, or the
Expand Down Expand Up @@ -494,30 +495,44 @@ async def resolve_environment_name(

localhost_platform = Platform.create_for_localhost().value

if (
env_tgt.val.has_field(DockerFallbackEnvironmentField)
and localhost_platform in (Platform.linux_x86_64.value, Platform.linux_arm64.value)
and localhost_platform != env_tgt.val[DockerPlatformField].normalized_value.value
):
return await _apply_fallback_environment(
env_tgt.val,
error_msg=softwrap(
f"""
The docker environment `{request.raw_value}` is specified in
{request.description_of_origin}, but it cannot be used because the local host has
the platform `{localhost_platform}` and the Docker environment has the platform
{env_tgt.val[DockerPlatformField].normalized_value}.

Consider setting the field `{FallbackEnvironmentField.alias}` for the target
{env_tgt.val.address}, such as to a `docker_environment` target that sets
`{DockerPlatformField.alias}` to `{localhost_platform}`. Alternatively, consider
not explicitly setting the field `{DockerPlatformField.alias}` for the target
{env_tgt.val.address} because the default behavior is to use the CPU architecture
of the current host for the platform (although this requires the docker image
supports that CPU architecture).
"""
),
)
if env_tgt.val.has_field(DockerFallbackEnvironmentField):
if not global_options.docker_execution:
return await _apply_fallback_environment(
env_tgt.val,
error_msg=softwrap(
f"""
The global option `--docker-execution` is set to false, but the Docker
environment `{request.raw_value}` is used in {request.description_of_origin}.

Either enable the option `--docker-execution`, or set the field
`{FallbackEnvironmentField.alias}` for the target {env_tgt.val.address}.
"""
),
)

if (
localhost_platform in (Platform.linux_x86_64.value, Platform.linux_arm64.value)
and localhost_platform != env_tgt.val[DockerPlatformField].normalized_value.value
):
return await _apply_fallback_environment(
env_tgt.val,
error_msg=softwrap(
f"""
The Docker environment `{request.raw_value}` is specified in
{request.description_of_origin}, but it cannot be used because the local host has
the platform `{localhost_platform}` and the Docker environment has the platform
{env_tgt.val[DockerPlatformField].normalized_value}.

Consider setting the field `{FallbackEnvironmentField.alias}` for the target
{env_tgt.val.address}, such as to a `docker_environment` target that sets
`{DockerPlatformField.alias}` to `{localhost_platform}`. Alternatively, consider
not explicitly setting the field `{DockerPlatformField.alias}` for the target
{env_tgt.val.address} because the default behavior is to use the CPU architecture
of the current host for the platform (although this requires the docker image
Eric-Arellano marked this conversation as resolved.
Show resolved Hide resolved
supports that CPU architecture).
"""
),
)

if (
env_tgt.val.has_field(LocalFallbackEnvironmentField)
Expand Down
16 changes: 14 additions & 2 deletions src/python/pants/core/util_rules/environments_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -255,14 +255,18 @@ def get_name(v: str) -> EnvironmentName:

def test_resolve_environment_name_local_and_docker_fallbacks(monkeypatch) -> None:
# We can't monkeypatch the Platform with RuleRunner, so instead use run_run_with_mocks.
def get_env_name(env_tgt: Target, platform: Platform) -> str | None:
def get_env_name(
env_tgt: Target, platform: Platform, *, docker_execution: bool = True
) -> str | None:
monkeypatch.setattr(Platform, "create_for_localhost", lambda: platform)
result = run_rule_with_mocks(
resolve_environment_name,
rule_args=[
EnvironmentNameRequest("env", description_of_origin="foo"),
create_subsystem(EnvironmentsSubsystem, names={"env": "", "fallback": ""}),
create_subsystem(GlobalOptions, remote_execution=False),
create_subsystem(
GlobalOptions, remote_execution=False, docker_execution=docker_execution
),
],
mock_gets=[
MockGet(
Expand Down Expand Up @@ -327,6 +331,14 @@ def create_docker_tgt(
# the Docker environment should be used.
assert get_env_name(create_docker_tgt(), Platform.linux_arm64) == "env"

# If Docker execution is disabled, though, fallback.
assert (
get_env_name(create_docker_tgt(fallback=True), Platform.linux_arm64, docker_execution=False)
== "fallback"
)
with pytest.raises(NoFallbackEnvironmentError):
get_env_name(create_docker_tgt(), Platform.linux_arm64, docker_execution=False)

# The Docker env can be used if we're on macOS, or on Linux and the CPU arch matches.
for plat in (Platform.macos_arm64, Platform.macos_x86_64, Platform.linux_x86_64):
assert get_env_name(create_docker_tgt(platform=Platform.linux_x86_64), plat) == "env"
Expand Down
16 changes: 16 additions & 0 deletions src/python/pants/option/global_options.py
Original file line number Diff line number Diff line change
Expand Up @@ -1701,6 +1701,22 @@ class GlobalOptions(BootstrapOptions, Subsystem):
advanced=True,
)

docker_execution = BoolOption(
default=True,
advanced=True,
help=softwrap(
"""
If true, `docker_environment` targets can be used to run builds inside a Docker
container.

If false, anytime a `docker_environment` target is used, Pants will instead fallback to
whatever the target's `fallback_environment` field is set to.

This can be useful, for example, if you want to always use Docker locally, but disable
it in CI, or vice versa.
"""
),
)
remote_execution_extra_platform_properties = StrListOption(
advanced=True,
help=softwrap(
Expand Down