Skip to content

Commit

Permalink
Add --docker-execution option (#17049)
Browse files Browse the repository at this point in the history
This gives more flexibility to likely user stories, such as wanting to use Docker in CI but not locally, or vice versa.

Before, the sole way to turn off Docker was on Linux that you used a different platform for the Docker image than the local host. That condition is not possible on macOS, so there was no way to disable Docker there.

[ci skip-rust]
[ci skip-build-wheels]
  • Loading branch information
Eric-Arellano authored Sep 28, 2022
1 parent 26d950d commit d8380aa
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 28 deletions.
67 changes: 41 additions & 26 deletions src/python/pants/core/util_rules/environments.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,10 @@ def docker_platform_field_default_factory(
class DockerFallbackEnvironmentField(FallbackEnvironmentField):
help = softwrap(
f"""
The environment to fallback to when this Docker environment cannot be used because the
The environment to fallback to when this Docker environment cannot be used because either
the global option `--docker-execution` is false, or the
field `{DockerPlatformField.alias}` is not compatible with the local host's CPU
architecture. (This is only an issue when the local host is Linux; macOS is fine.)
architecture (this is only an issue when the local host is Linux; macOS is fine).
Must be an environment name from the option `[environments-preview].names`, the
special string `{LOCAL_ENVIRONMENT_MATCHER}` to use the relevant local environment, or the
Expand Down Expand Up @@ -494,30 +495,44 @@ async def resolve_environment_name(

localhost_platform = Platform.create_for_localhost().value

if (
env_tgt.val.has_field(DockerFallbackEnvironmentField)
and localhost_platform in (Platform.linux_x86_64.value, Platform.linux_arm64.value)
and localhost_platform != env_tgt.val[DockerPlatformField].normalized_value.value
):
return await _apply_fallback_environment(
env_tgt.val,
error_msg=softwrap(
f"""
The docker environment `{request.raw_value}` is specified in
{request.description_of_origin}, but it cannot be used because the local host has
the platform `{localhost_platform}` and the Docker environment has the platform
{env_tgt.val[DockerPlatformField].normalized_value}.
Consider setting the field `{FallbackEnvironmentField.alias}` for the target
{env_tgt.val.address}, such as to a `docker_environment` target that sets
`{DockerPlatformField.alias}` to `{localhost_platform}`. Alternatively, consider
not explicitly setting the field `{DockerPlatformField.alias}` for the target
{env_tgt.val.address} because the default behavior is to use the CPU architecture
of the current host for the platform (although this requires the docker image
supports that CPU architecture).
"""
),
)
if env_tgt.val.has_field(DockerFallbackEnvironmentField):
if not global_options.docker_execution:
return await _apply_fallback_environment(
env_tgt.val,
error_msg=softwrap(
f"""
The global option `--docker-execution` is set to false, but the Docker
environment `{request.raw_value}` is used in {request.description_of_origin}.
Either enable the option `--docker-execution`, or set the field
`{FallbackEnvironmentField.alias}` for the target {env_tgt.val.address}.
"""
),
)

if (
localhost_platform in (Platform.linux_x86_64.value, Platform.linux_arm64.value)
and localhost_platform != env_tgt.val[DockerPlatformField].normalized_value.value
):
return await _apply_fallback_environment(
env_tgt.val,
error_msg=softwrap(
f"""
The Docker environment `{request.raw_value}` is specified in
{request.description_of_origin}, but it cannot be used because the local host has
the platform `{localhost_platform}` and the Docker environment has the platform
{env_tgt.val[DockerPlatformField].normalized_value}.
Consider setting the field `{FallbackEnvironmentField.alias}` for the target
{env_tgt.val.address}, such as to a `docker_environment` target that sets
`{DockerPlatformField.alias}` to `{localhost_platform}`. Alternatively, consider
not explicitly setting the field `{DockerPlatformField.alias}` for the target
{env_tgt.val.address} because the default behavior is to use the CPU architecture
of the current host for the platform (although this requires that the docker image
supports that CPU architecture).
"""
),
)

if (
env_tgt.val.has_field(LocalFallbackEnvironmentField)
Expand Down
16 changes: 14 additions & 2 deletions src/python/pants/core/util_rules/environments_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -255,14 +255,18 @@ def get_name(v: str) -> EnvironmentName:

def test_resolve_environment_name_local_and_docker_fallbacks(monkeypatch) -> None:
# We can't monkeypatch the Platform with RuleRunner, so instead use run_run_with_mocks.
def get_env_name(env_tgt: Target, platform: Platform) -> str | None:
def get_env_name(
env_tgt: Target, platform: Platform, *, docker_execution: bool = True
) -> str | None:
monkeypatch.setattr(Platform, "create_for_localhost", lambda: platform)
result = run_rule_with_mocks(
resolve_environment_name,
rule_args=[
EnvironmentNameRequest("env", description_of_origin="foo"),
create_subsystem(EnvironmentsSubsystem, names={"env": "", "fallback": ""}),
create_subsystem(GlobalOptions, remote_execution=False),
create_subsystem(
GlobalOptions, remote_execution=False, docker_execution=docker_execution
),
],
mock_gets=[
MockGet(
Expand Down Expand Up @@ -327,6 +331,14 @@ def create_docker_tgt(
# the Docker environment should be used.
assert get_env_name(create_docker_tgt(), Platform.linux_arm64) == "env"

# If Docker execution is disabled, though, fallback.
assert (
get_env_name(create_docker_tgt(fallback=True), Platform.linux_arm64, docker_execution=False)
== "fallback"
)
with pytest.raises(NoFallbackEnvironmentError):
get_env_name(create_docker_tgt(), Platform.linux_arm64, docker_execution=False)

# The Docker env can be used if we're on macOS, or on Linux and the CPU arch matches.
for plat in (Platform.macos_arm64, Platform.macos_x86_64, Platform.linux_x86_64):
assert get_env_name(create_docker_tgt(platform=Platform.linux_x86_64), plat) == "env"
Expand Down
16 changes: 16 additions & 0 deletions src/python/pants/option/global_options.py
Original file line number Diff line number Diff line change
Expand Up @@ -1701,6 +1701,22 @@ class GlobalOptions(BootstrapOptions, Subsystem):
advanced=True,
)

docker_execution = BoolOption(
default=True,
advanced=True,
help=softwrap(
"""
If true, `docker_environment` targets can be used to run builds inside a Docker
container.
If false, anytime a `docker_environment` target is used, Pants will instead fallback to
whatever the target's `fallback_environment` field is set to.
This can be useful, for example, if you want to always use Docker locally, but disable
it in CI, or vice versa.
"""
),
)
remote_execution_extra_platform_properties = StrListOption(
advanced=True,
help=softwrap(
Expand Down

0 comments on commit d8380aa

Please sign in to comment.