Implement autodetecting Python toolchain

This replaces the stub default Python toolchain with one that actually locates the target platform's Python interpreter at runtime. Try it out with

    bazel build //some_py_binary --experimental_use_python_toolchains

and note that, unlike before (#4815), the correct Python interpreter gets invoked by default regardless of whether you specify `--python_version=PY2` or `--python_version=PY3`.

This toolchain is only intended as a last resort, if the user doesn't define and register a better toolchain (that satisfies the target platform constraints).

Work toward #7375 and #4815. Follow-up work needed to add a test (#7843) and windows support (#7844).

RELNOTES: None
PiperOrigin-RevId: 240417315

src/test/java/com/google/devtools/build/lib/packages/util/BazelMockPythonSupport.java

@@ -33,6 +33,7 @@ public void setup(MockToolsConfig config) throws IOException {
     addTool(config, "tools/python/python_version.bzl");
     addTool(config, "tools/python/srcs_version.bzl");
     addTool(config, "tools/python/toolchain.bzl");
+    addTool(config, "tools/python/utils.bzl");
 
     config.create(
         TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/python/BUILD",

src/test/shell/bazel/python_version_test.sh

@@ -43,6 +43,9 @@ fi
 source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \
   || { echo "integration_test_setup.sh not found!" >&2; exit 1; }
 
+# TODO(bazelbuild/continuous-integration#578): Enable this test for Mac and
+# Windows.
+
 # `uname` returns the current platform, e.g "MSYS_NT-10.0" or "Linux".
 # `tr` converts all upper case letters to lower case.
 # `case` matches the result if the `uname | tr` expression to string prefixes
@@ -79,13 +82,7 @@ fi
 # Use a py_runtime that invokes either the system's Python 2 or Python 3
 # interpreter based on the Python mode. On Unix this is a workaround for #4815.
 #
-# TODO(brandjon): Get this running on windows by creating .bat wrappers that
-# invoke "py -2" and "py -3". Make sure our windows workers have both Python
-# versions installed.
-#
-# TODO(brandjon): Get this running on mac -- our workers lack a Python 2
-# installation.
-#
+# TODO(brandjon): Replace this with the autodetecting Python toolchain.
 function use_system_python_2_3_runtimes() {
   PYTHON2_BIN=$(which python2 || echo "")
   PYTHON3_BIN=$(which python3 || echo "")

tools/python/BUILD.tools

@@ -1,5 +1,5 @@
 load(":python_version.bzl", "define_python_version_flag")
-load(":toolchain.bzl", "py_runtime_pair")
+load(":toolchain.bzl", "define_autodetecting_toolchain")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -65,17 +65,14 @@ constraint_setting(name = "py2_interpreter_path")
 # system Python 3 interpreter on a platform.
 constraint_setting(name = "py3_interpreter_path")
 
-# A Python toolchain that, at execution time, attempts to detect a platform
-# runtime having the appropriate major Python version.
-
-py_runtime_pair(
-    name = "autodetecting_py_runtime_pair",
-    # TODO(brandjon): Not yet implemented. Currently this provides no runtimes,
-    # so it will just fail at analysis time if you attempt to use it.
-)
+# Definitions for a Python toolchain that, at execution time, attempts to detect
+# a platform runtime having the appropriate major Python version.
+#
+# This is a toolchain of last resort that gets automatically registered in all
+# workspaces. Ideally you should register your own Python toolchain, which will
+# supersede this one so long as its constraints match the target platform.
 
-toolchain(
+define_autodetecting_toolchain(
     name = "autodetecting_toolchain",
-    toolchain = ":autodetecting_py_runtime_pair",
-    toolchain_type = ":toolchain_type",
+    pywrapper_template = "pywrapper_template.txt",
 )

tools/python/python_version.bzl

@@ -50,12 +50,19 @@ _python_version_flag = rule(
 def define_python_version_flag(name):
     """Defines the target to expose the Python version to select().
 
-    For use only by @bazel_tools//python:BUILD; see the documentation comment
-    there.
+    For use only by @bazel_tools//tools/python:BUILD; see the documentation
+    comment there.
 
     Args:
-        name: The name of the target to introduce.
+        name: The name of the target to introduce. Must have value
+            "python_version". This param is present only to make the BUILD file
+            more readable.
     """
+    if native.package_name() != "tools/python":
+        fail("define_python_version_flag() is private to " +
+             "@bazel_tools//tools/python")
+    if name != "python_version":
+        fail("Python version flag must be named 'python_version'")
 
     # Config settings for the underlying native flags we depend on:
     # --force_python, --python_version, and --incompatible_py3_is_default.

tools/python/pywrapper_template.txt

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# TODO(#7843): integration tests for failure to find python / wrong version
+# found / error while trying to print version
+
+set -euo pipefail
+
+GENERAL_FAILURE_MESSAGE="Error: The default python toolchain \
+(@bazel_tools//tools/python:autodetecting_toolchain) was unable to locate a \
+suitable Python interpreter on the target platform at execution time. Please \
+register an appropriate Python toolchain. See the documentation for \
+py_runtime_pair here:
+https://github.com/bazelbuild/bazel/blob/master/tools/python/toolchain.bzl."
+
+# Try the "python%VERSION%" command name first, then fall back on "python".
+PYTHON_BIN=$(which python%VERSION% || echo "")
+USED_FALLBACK=0
+if [[ -z "${PYTHON_BIN:-}" ]]; then
+    PYTHON_BIN=$(which python || echo "")
+    USED_FALLBACK=1
+fi
+if [[ -z "${PYTHON_BIN:-}" ]]; then
+    echo "$GENERAL_FAILURE_MESSAGE"
+    echo "Failure reason: Cannot locate 'python%VERSION%' or 'python' on the \
+target platform's PATH, which is:
+$PATH"
+fi
+
+# Verify that we grabbed an interpreter with the right version.
+VERSION_STR=$("$PYTHON_BIN" -V 2>&1)
+if ! grep -q " %VERSION%\." <<< $VERSION_STR; then
+    echo "$GENERAL_FAILURE_MESSAGE"
+    echo "Failure reason: According to '$PYTHON_BIN -V', version is \
+'$VERSION_STR', but we need version %VERSION%"
+fi
+
+exec "$PYTHON_BIN" "$@"

tools/python/toolchain.bzl

@@ -14,6 +14,8 @@
 
 """Definitions related to the Python toolchain."""
 
+load(":utils.bzl", "expand_pyversion_template")
+
 def _py_runtime_pair_impl(ctx):
     if ctx.attr.py2_runtime != None:
         py2_runtime = ctx.attr.py2_runtime[PyRuntimeInfo]
@@ -62,7 +64,7 @@ rule returning a `PyRuntimeInfo` provider may be used.
 This rule returns a `platform_common.ToolchainInfo` provider with the following
 schema:
 
-```
+```python
 platform_common.ToolchainInfo(
     py2_runtime = <PyRuntimeInfo or None>,
     py3_runtime = <PyRuntimeInfo or None>,
@@ -71,7 +73,9 @@ platform_common.ToolchainInfo(
 
 Example usage:
 
-```
+```python
+# In your BUILD file...
+
 load("@bazel_tools//tools/python/toolchain.bzl", "py_runtime_pair")
 
 py_runtime(
@@ -99,5 +103,76 @@ toolchain(
     toolchain_type = "@bazel_tools//tools/python:toolchain_type",
 )
 ```
+
+```python
+# In your WORKSPACE...
+
+register_toolchains("//my_pkg:my_toolchain")
+```
 """,
 )
+
+# TODO(#7844): Add support for a windows (.bat) version of the autodetecting
+# toolchain, based on the "py" wrapper (e.g. "py -2" and "py -3"). Use select()
+# in the template attr of the _generate_*wrapper targets.
+
+def define_autodetecting_toolchain(name, pywrapper_template):
+    """Defines the autodetecting Python toolchain.
+
+    For use only by @bazel_tools//tools/python:BUILD; see the documentation
+    comment there.
+
+    Args:
+        name: The name of the toolchain to introduce. Must have value
+            "autodetecting_toolchain". This param is present only to make the
+            BUILD file more readable.
+        pywrapper_template: The label of the pywrapper_template.txt file.
+    """
+    if native.package_name() != "tools/python":
+        fail("define_autodetecting_toolchain() is private to " +
+             "@bazel_tools//tools/python")
+    if name != "autodetecting_toolchain":
+        fail("Python autodetecting toolchain must be named " +
+             "'autodetecting_toolchain'")
+
+    expand_pyversion_template(
+        name = "_generate_wrappers",
+        template = pywrapper_template,
+        out2 = ":py2wrapper.sh",
+        out3 = ":py3wrapper.sh",
+        visibility = ["//visibility:private"],
+    )
+
+    # Note that the pywrapper script is a .sh file, not a sh_binary target. If
+    # we needed to make it a proper shell target, e.g. because it needed to
+    # access runfiles and needed to depend on the runfiles library, then we'd
+    # have to use a workaround to allow it to be depended on by py_runtime. See
+    # https://github.com/bazelbuild/bazel/issues/4286#issuecomment-475661317.
+
+    native.py_runtime(
+        name = "_autodetecting_py2_runtime",
+        interpreter = ":py2wrapper.sh",
+        python_version = "PY2",
+        visibility = ["//visibility:private"],
+    )
+
+    native.py_runtime(
+        name = "_autodetecting_py3_runtime",
+        interpreter = ":py3wrapper.sh",
+        python_version = "PY3",
+        visibility = ["//visibility:private"],
+    )
+
+    py_runtime_pair(
+        name = "_autodetecting_py_runtime_pair",
+        py2_runtime = ":_autodetecting_py2_runtime",
+        py3_runtime = ":_autodetecting_py3_runtime",
+        visibility = ["//visibility:public"],
+    )
+
+    native.toolchain(
+        name = name,
+        toolchain = ":_autodetecting_py_runtime_pair",
+        toolchain_type = ":toolchain_type",
+        visibility = ["//visibility:public"],
+    )

tools/python/utils.bzl

@@ -0,0 +1,53 @@
+# Copyright 2019 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for the @bazel_tools//tools/python package.
+
+This file does not access any Python-rules-specific logic, and is therefore
+less likely to be broken by Python-related changes. That in turn means this
+file is less likely to cause bootstrapping issues.
+"""
+
+def _expand_pyversion_template_impl(ctx):
+    if ctx.outputs.out2:
+        ctx.actions.expand_template(
+            template = ctx.file.template,
+            output = ctx.outputs.out2,
+            substitutions = {"%VERSION%": "2"},
+            is_executable = True,
+        )
+    if ctx.outputs.out3:
+        ctx.actions.expand_template(
+            template = ctx.file.template,
+            output = ctx.outputs.out3,
+            substitutions = {"%VERSION%": "3"},
+            is_executable = True,
+        )
+
+expand_pyversion_template = rule(
+    implementation = _expand_pyversion_template_impl,
+    attrs = {
+        "template": attr.label(
+            allow_single_file = True,
+            doc = "The input template file.",
+        ),
+        "out2": attr.output(doc = """\
+The output file produced by substituting "%VERSION%" with "2"."""),
+        "out3": attr.output(doc = """\
+The output file produced by substituting "%VERSION%" with "3"."""),
+    },
+    doc = """\
+Given a template file, generates two expansions by replacing the substring
+"%VERSION%" with "2" and "3".""",
+)