feat: add runtime_env toolchain suite to replace "autodetecting" toolchain

.bazelci/presubmit.yml

@@ -182,6 +182,7 @@ tasks:
     platform: rbe_ubuntu1604
     test_flags:
       - "--test_tag_filters=-integration-test,-acceptance-test"
+      - "--extra_toolchains=@buildkite_config//config:cc-toolchain"
 
   integration_test_build_file_generation_ubuntu_minimum_supported_workspace:
     <<: *minimum_supported_version

CHANGELOG.md

@@ -31,6 +31,9 @@ A brief description of the categories of changes:
 * (toolchains) The exec tools toolchain now finds its interpreter by reusing
   the regular interpreter toolchain. This avoids having to duplicate specifying
   where the runtime for the exec tools toolchain is.
+* (toolchains) ({obj}`//python:autodetecting_toolchain`) is deprecated. It is
+  replaced by {obj}`//python/runtime_env_toolchains:all`. The old target will be
+  removed in a future release.
 
 ### Fixed
 * (bzlmod): Targets in `all_requirements` now use the same form as targets returned by the `requirement` macro.
@@ -47,6 +50,10 @@ A brief description of the categories of changes:
 * (rules) The first element of the default outputs is now the executable again.
 * (pip) Fixed crash when pypi packages lacked a sha (e.g. yanked packages)
 
+### Added
+* (toolchains) {obj}`//python/runtime_env_toolchains:all`, which is a drop-in
+  replacement for the "autodetecting" toolchain.
+
 ### Removed
 * (pip): Removes the `entrypoint` macro that was replaced by `py_console_script_binary` in 0.26.0.
 

docs/sphinx/api/python/index.md

@@ -26,18 +26,11 @@ provides:
 
 ::::{target} autodetecting_toolchain
 
-A simple toolchain that simply uses `python3` from the runtime environment.
+Legacy toolchain; despite its name, it doesn't autodetect anything.
 
-Note that this toolchain provides no build-time information, which makes it of
-limited utility.
+:::{deprecated} 0.34.0
 
-This is only provided to aid migration off the builtin Bazel toolchain 
-(`@bazel_tools//python:autodetecting_toolchain`), and is largely only applicable
-to WORKSPACE builds.
-
-:::{deprecated} unspecified
-
-Switch to using a hermetic toolchain or manual toolchain configuration instead.
+Use {obj}`@rules_python//python/runtime_env_toolchain:all` instead.
 :::
-
 ::::
+

docs/sphinx/api/python/runtime_env_toolchains/index.md

@@ -0,0 +1,38 @@
+:::{default-domain} bzl
+:::
+:::{bzl:currentfile} //python/runtime_env_toolchain:BUILD.bazel
+:::
+
+# //python/runtime_env_toolchain
+
+::::{target} all
+
+A set of toolchains that invoke `python3` from the runtime environment.
+
+Note that this toolchain provides no build-time information, which makes it of
+limited utility. This is because the invocation of `python3` is done when a
+program is run, not at build time.
+
+This is only provided to aid migration off the builtin Bazel toolchain 
+(`@bazel_tools//python:autodetecting_toolchain`), and is largely only applicable
+to WORKSPACE builds.
+
+To use this target, register it as a toolchain in WORKSPACE or MODULE.bazel:
+
+:::
+register_toolchains("@rules_python//python/runtime_env_toolchains:all")
+:::
+
+The benefit of this target over the legacy targets is this defines additional
+toolchain types that rules_python needs. This prevents toolchain resolution from
+continuing to search elsewhere (e.g. potentially incurring a download of the
+hermetic runtimes when they won't be used).
+
+:::{deprecated} 0.34.0
+
+Switch to using a hermetic toolchain or manual toolchain configuration instead.
+:::
+
+:::{versionadded} 0.34.0
+:::
+::::

docs/sphinx/toolchains.md

@@ -240,5 +240,5 @@ automatically registers a higher-priority toolchain; it won't be used unless
 there is a toolchain misconfiguration somewhere.
 
 To aid migration off the Bazel-builtin toolchain, rules_python provides
-{obj}`@rules_python//python:autodetecting_toolchain`. This is an equivalent
+{obj}`@rules_python//python/runtime_env_toolchains:all`. This is an equivalent
 toolchain, but is implemented using rules_python's objects.

python/BUILD.bazel

@@ -24,7 +24,6 @@ that @rules_python//python is only concerned with the core rules.
 """
 
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-load("//python/private:autodetecting_toolchain.bzl", "define_autodetecting_toolchain")
 load(":current_py_toolchain.bzl", "current_py_toolchain")
 
 package(default_visibility = ["//visibility:public"])
@@ -320,11 +319,16 @@ toolchain_type(
 # safe if you know for a fact that your build is completely compatible with the
 # version of the `python` command installed on the target platform.
 
-define_autodetecting_toolchain(name = "autodetecting_toolchain")
+alias(
+    name = "autodetecting_toolchain",
+    actual = "//python/runtime_env_toolchains:runtime_env_toolchain",
+    deprecation = "Use //python/runtime_env_toolchains:all instead",
+)
 
 alias(
     name = "autodetecting_toolchain_nonstrict",
-    actual = ":autodetecting_toolchain",
+    actual = "//python/runtime_env_toolchains:runtime_env_toolchain",
+    deprecation = "Use //python/runtime_env_toolchains:all instead",
 )
 
 # ========= Packaging rules =========

python/private/BUILD.bazel

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load("@bazel_skylib//rules:common_settings.bzl", "bool_setting")
 load("//python:py_binary.bzl", "py_binary")
 load("//python:py_library.bzl", "py_library")
 load("//python:versions.bzl", "print_toolchains_checksums")
@@ -58,9 +59,10 @@ bzl_library(
 )
 
 bzl_library(
-    name = "autodetecting_toolchain_bzl",
-    srcs = ["autodetecting_toolchain.bzl"],
+    name = "runtime_env_toolchain_bzl",
+    srcs = ["runtime_env_toolchain.bzl"],
     deps = [
+        ":py_exec_tools_toolchain_bzl",
         ":toolchain_types_bzl",
         "//python:py_runtime_bzl",
         "//python:py_runtime_pair_bzl",
@@ -140,6 +142,7 @@ bzl_library(
         ":py_cc_toolchain_info_bzl",
         ":rules_cc_srcs_bzl",
         ":util_bzl",
+        "@bazel_skylib//rules:common_settings",
     ],
 )
 
@@ -164,7 +167,11 @@ bzl_library(
 bzl_library(
     name = "py_exec_tools_toolchain_bzl",
     srcs = ["py_exec_tools_toolchain.bzl"],
-    deps = ["//python/private/common:providers_bzl"],
+    deps = [
+        ":toolchain_types_bzl",
+        "//python/private/common:providers_bzl",
+        "@bazel_skylib//rules:common_settings",
+    ],
 )
 
 bzl_library(
@@ -192,6 +199,7 @@ bzl_library(
     deps = [
         "//python:py_runtime_bzl",
         "//python:py_runtime_info_bzl",
+        "@bazel_skylib//rules:common_settings",
     ],
 )
 
@@ -365,6 +373,15 @@ config_setting(
     },
 )
 
+# This should only be set by analysis tests to expose additional metadata to
+# aid testing, so a setting instead of a flag.
+bool_setting(
+    name = "visible_for_testing",
+    build_setting_default = False,
+    # This is only because it is an implicit dependency by the toolchains.
+    visibility = ["//visibility:public"],
+)
+
 print_toolchains_checksums(name = "print_toolchains_checksums")
 
 # Used for py_console_script_gen rule

python/private/py_cc_toolchain_rule.bzl

@@ -18,6 +18,7 @@ NOTE: This is a beta-quality feature. APIs subject to change until
 https://github.com/bazelbuild/rules_python/issues/824 is considered done.
 """
 
+load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
 load("@rules_cc//cc:defs.bzl", "CcInfo")
 load(":py_cc_toolchain_info.bzl", "PyCcToolchainInfo")
 
@@ -37,8 +38,12 @@ def _py_cc_toolchain_impl(ctx):
         ),
         python_version = ctx.attr.python_version,
     )
+    extra_kwargs = {}
+    if ctx.attr._visible_for_testing[BuildSettingInfo].value:
+        extra_kwargs["toolchain_label"] = ctx.label
     return [platform_common.ToolchainInfo(
         py_cc_toolchain = py_cc_toolchain,
+        **extra_kwargs
     )]
 
 py_cc_toolchain = rule(
@@ -60,6 +65,9 @@ py_cc_toolchain = rule(
             doc = "The Major.minor Python version, e.g. 3.11",
             mandatory = True,
         ),
+        "_visible_for_testing": attr.label(
+            default = "//python/private:visible_for_testing",
+        ),
     },
     doc = """\
 A toolchain for a Python runtime's C/C++ information (e.g. headers)

python/private/py_exec_tools_toolchain.bzl

@@ -14,14 +14,22 @@
 
 """Rule that defines a toolchain for build tools."""
 
+load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
 load("//python/private:toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE")
 load(":py_exec_tools_info.bzl", "PyExecToolsInfo")
 
 def _py_exec_tools_toolchain_impl(ctx):
-    return [platform_common.ToolchainInfo(exec_tools = PyExecToolsInfo(
-        exec_interpreter = ctx.attr.exec_interpreter,
-        precompiler = ctx.attr.precompiler,
-    ))]
+    extra_kwargs = {}
+    if ctx.attr._visible_for_testing[BuildSettingInfo].value:
+        extra_kwargs["toolchain_label"] = ctx.label
+
+    return [platform_common.ToolchainInfo(
+        exec_tools = PyExecToolsInfo(
+            exec_interpreter = ctx.attr.exec_interpreter,
+            precompiler = ctx.attr.precompiler,
+        ),
+        **extra_kwargs
+    )]
 
 py_exec_tools_toolchain = rule(
     implementation = _py_exec_tools_toolchain_impl,
@@ -36,6 +44,9 @@ py_exec_tools_toolchain = rule(
             cfg = "exec",
             doc = "See PyExecToolsInfo.precompiler",
         ),
+        "_visible_for_testing": attr.label(
+            default = "//python/private:visible_for_testing",
+        ),
     },
 )
 

python/private/py_runtime_pair_rule.bzl

@@ -14,6 +14,7 @@
 
 """Implementation of py_runtime_pair."""
 
+load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
 load("//python:py_runtime_info.bzl", "PyRuntimeInfo")
 load("//python/private:reexports.bzl", "BuiltinPyRuntimeInfo")
 load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER")
@@ -40,9 +41,14 @@ def _py_runtime_pair_impl(ctx):
     #     fail("Using Python 2 is not supported and disabled; see " +
     #          "https://github.com/bazelbuild/bazel/issues/15684")
 
+    extra_kwargs = {}
+    if ctx.attr._visible_for_testing[BuildSettingInfo].value:
+        extra_kwargs["toolchain_label"] = ctx.label
+
     return [platform_common.ToolchainInfo(
         py2_runtime = py2_runtime,
         py3_runtime = py3_runtime,
+        **extra_kwargs
     )]
 
 def _get_py_runtime_info(target):
@@ -85,6 +91,9 @@ The runtime to use for Python 3 targets. Must have `python_version` set to
 `PY3`.
 """,
         ),
+        "_visible_for_testing": attr.label(
+            default = "//python/private:visible_for_testing",
+        ),
     },
     fragments = ["py"],
     doc = """\