diff --git a/src/test/java/com/google/devtools/build/lib/packages/util/BazelMockPythonSupport.java b/src/test/java/com/google/devtools/build/lib/packages/util/BazelMockPythonSupport.java index 224bcb478dafc2..ae665ffc6871f1 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/util/BazelMockPythonSupport.java +++ b/src/test/java/com/google/devtools/build/lib/packages/util/BazelMockPythonSupport.java @@ -33,6 +33,7 @@ public void setup(MockToolsConfig config) throws IOException { addTool(config, "tools/python/python_version.bzl"); addTool(config, "tools/python/srcs_version.bzl"); addTool(config, "tools/python/toolchain.bzl"); + addTool(config, "tools/python/utils.bzl"); config.create( TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/python/BUILD", diff --git a/src/test/shell/bazel/python_version_test.sh b/src/test/shell/bazel/python_version_test.sh index aadb0e42517f80..4712a6924a5fc6 100755 --- a/src/test/shell/bazel/python_version_test.sh +++ b/src/test/shell/bazel/python_version_test.sh @@ -43,6 +43,9 @@ fi source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \ || { echo "integration_test_setup.sh not found!" >&2; exit 1; } +# TODO(bazelbuild/continuous-integration#578): Enable this test for Mac and +# Windows. + # `uname` returns the current platform, e.g "MSYS_NT-10.0" or "Linux". # `tr` converts all upper case letters to lower case. # `case` matches the result if the `uname | tr` expression to string prefixes @@ -79,13 +82,7 @@ fi # Use a py_runtime that invokes either the system's Python 2 or Python 3 # interpreter based on the Python mode. On Unix this is a workaround for #4815. # -# TODO(brandjon): Get this running on windows by creating .bat wrappers that -# invoke "py -2" and "py -3". Make sure our windows workers have both Python -# versions installed. -# -# TODO(brandjon): Get this running on mac -- our workers lack a Python 2 -# installation. -# +# TODO(brandjon): Replace this with the autodetecting Python toolchain. function use_system_python_2_3_runtimes() { PYTHON2_BIN=$(which python2 || echo "") PYTHON3_BIN=$(which python3 || echo "") diff --git a/tools/python/BUILD.tools b/tools/python/BUILD.tools index 1efd7546452899..35c789ba647e32 100644 --- a/tools/python/BUILD.tools +++ b/tools/python/BUILD.tools @@ -1,5 +1,5 @@ load(":python_version.bzl", "define_python_version_flag") -load(":toolchain.bzl", "py_runtime_pair") +load(":toolchain.bzl", "define_autodetecting_toolchain") package(default_visibility = ["//visibility:public"]) @@ -65,17 +65,14 @@ constraint_setting(name = "py2_interpreter_path") # system Python 3 interpreter on a platform. constraint_setting(name = "py3_interpreter_path") -# A Python toolchain that, at execution time, attempts to detect a platform -# runtime having the appropriate major Python version. - -py_runtime_pair( - name = "autodetecting_py_runtime_pair", - # TODO(brandjon): Not yet implemented. Currently this provides no runtimes, - # so it will just fail at analysis time if you attempt to use it. -) +# Definitions for a Python toolchain that, at execution time, attempts to detect +# a platform runtime having the appropriate major Python version. +# +# This is a toolchain of last resort that gets automatically registered in all +# workspaces. Ideally you should register your own Python toolchain, which will +# supersede this one so long as its constraints match the target platform. -toolchain( +define_autodetecting_toolchain( name = "autodetecting_toolchain", - toolchain = ":autodetecting_py_runtime_pair", - toolchain_type = ":toolchain_type", + pywrapper_template = "pywrapper_template.txt", ) diff --git a/tools/python/python_version.bzl b/tools/python/python_version.bzl index 8bb45b86c0dce4..aecad07e0223c9 100644 --- a/tools/python/python_version.bzl +++ b/tools/python/python_version.bzl @@ -50,12 +50,19 @@ _python_version_flag = rule( def define_python_version_flag(name): """Defines the target to expose the Python version to select(). - For use only by @bazel_tools//python:BUILD; see the documentation comment - there. + For use only by @bazel_tools//tools/python:BUILD; see the documentation + comment there. Args: - name: The name of the target to introduce. + name: The name of the target to introduce. Must have value + "python_version". This param is present only to make the BUILD file + more readable. """ + if native.package_name() != "tools/python": + fail("define_python_version_flag() is private to " + + "@bazel_tools//tools/python") + if name != "python_version": + fail("Python version flag must be named 'python_version'") # Config settings for the underlying native flags we depend on: # --force_python, --python_version, and --incompatible_py3_is_default. diff --git a/tools/python/pywrapper_template.txt b/tools/python/pywrapper_template.txt new file mode 100644 index 00000000000000..c2f5b1882cda9c --- /dev/null +++ b/tools/python/pywrapper_template.txt @@ -0,0 +1,37 @@ +#!/bin/bash + +# TODO(#7843): integration tests for failure to find python / wrong version +# found / error while trying to print version + +set -euo pipefail + +GENERAL_FAILURE_MESSAGE="Error: The default python toolchain \ +(@bazel_tools//tools/python:autodetecting_toolchain) was unable to locate a \ +suitable Python interpreter on the target platform at execution time. Please \ +register an appropriate Python toolchain. See the documentation for \ +py_runtime_pair here: +https://github.com/bazelbuild/bazel/blob/master/tools/python/toolchain.bzl." + +# Try the "python%VERSION%" command name first, then fall back on "python". +PYTHON_BIN=$(which python%VERSION% || echo "") +USED_FALLBACK=0 +if [[ -z "${PYTHON_BIN:-}" ]]; then + PYTHON_BIN=$(which python || echo "") + USED_FALLBACK=1 +fi +if [[ -z "${PYTHON_BIN:-}" ]]; then + echo "$GENERAL_FAILURE_MESSAGE" + echo "Failure reason: Cannot locate 'python%VERSION%' or 'python' on the \ +target platform's PATH, which is: +$PATH" +fi + +# Verify that we grabbed an interpreter with the right version. +VERSION_STR=$("$PYTHON_BIN" -V 2>&1) +if ! grep -q " %VERSION%\." <<< $VERSION_STR; then + echo "$GENERAL_FAILURE_MESSAGE" + echo "Failure reason: According to '$PYTHON_BIN -V', version is \ +'$VERSION_STR', but we need version %VERSION%" +fi + +exec "$PYTHON_BIN" "$@" diff --git a/tools/python/toolchain.bzl b/tools/python/toolchain.bzl index 3fe9475cb0520d..656ab623ddbc13 100644 --- a/tools/python/toolchain.bzl +++ b/tools/python/toolchain.bzl @@ -14,6 +14,8 @@ """Definitions related to the Python toolchain.""" +load(":utils.bzl", "expand_pyversion_template") + def _py_runtime_pair_impl(ctx): if ctx.attr.py2_runtime != None: py2_runtime = ctx.attr.py2_runtime[PyRuntimeInfo] @@ -62,7 +64,7 @@ rule returning a `PyRuntimeInfo` provider may be used. This rule returns a `platform_common.ToolchainInfo` provider with the following schema: -``` +```python platform_common.ToolchainInfo( py2_runtime = , py3_runtime = , @@ -71,7 +73,9 @@ platform_common.ToolchainInfo( Example usage: -``` +```python +# In your BUILD file... + load("@bazel_tools//tools/python/toolchain.bzl", "py_runtime_pair") py_runtime( @@ -99,5 +103,76 @@ toolchain( toolchain_type = "@bazel_tools//tools/python:toolchain_type", ) ``` + +```python +# In your WORKSPACE... + +register_toolchains("//my_pkg:my_toolchain") +``` """, ) + +# TODO(#7844): Add support for a windows (.bat) version of the autodetecting +# toolchain, based on the "py" wrapper (e.g. "py -2" and "py -3"). Use select() +# in the template attr of the _generate_*wrapper targets. + +def define_autodetecting_toolchain(name, pywrapper_template): + """Defines the autodetecting Python toolchain. + + For use only by @bazel_tools//tools/python:BUILD; see the documentation + comment there. + + Args: + name: The name of the toolchain to introduce. Must have value + "autodetecting_toolchain". This param is present only to make the + BUILD file more readable. + pywrapper_template: The label of the pywrapper_template.txt file. + """ + if native.package_name() != "tools/python": + fail("define_autodetecting_toolchain() is private to " + + "@bazel_tools//tools/python") + if name != "autodetecting_toolchain": + fail("Python autodetecting toolchain must be named " + + "'autodetecting_toolchain'") + + expand_pyversion_template( + name = "_generate_wrappers", + template = pywrapper_template, + out2 = ":py2wrapper.sh", + out3 = ":py3wrapper.sh", + visibility = ["//visibility:private"], + ) + + # Note that the pywrapper script is a .sh file, not a sh_binary target. If + # we needed to make it a proper shell target, e.g. because it needed to + # access runfiles and needed to depend on the runfiles library, then we'd + # have to use a workaround to allow it to be depended on by py_runtime. See + # https://github.com/bazelbuild/bazel/issues/4286#issuecomment-475661317. + + native.py_runtime( + name = "_autodetecting_py2_runtime", + interpreter = ":py2wrapper.sh", + python_version = "PY2", + visibility = ["//visibility:private"], + ) + + native.py_runtime( + name = "_autodetecting_py3_runtime", + interpreter = ":py3wrapper.sh", + python_version = "PY3", + visibility = ["//visibility:private"], + ) + + py_runtime_pair( + name = "_autodetecting_py_runtime_pair", + py2_runtime = ":_autodetecting_py2_runtime", + py3_runtime = ":_autodetecting_py3_runtime", + visibility = ["//visibility:public"], + ) + + native.toolchain( + name = name, + toolchain = ":_autodetecting_py_runtime_pair", + toolchain_type = ":toolchain_type", + visibility = ["//visibility:public"], + ) diff --git a/tools/python/utils.bzl b/tools/python/utils.bzl new file mode 100644 index 00000000000000..9278e32efc67fd --- /dev/null +++ b/tools/python/utils.bzl @@ -0,0 +1,53 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for the @bazel_tools//tools/python package. + +This file does not access any Python-rules-specific logic, and is therefore +less likely to be broken by Python-related changes. That in turn means this +file is less likely to cause bootstrapping issues. +""" + +def _expand_pyversion_template_impl(ctx): + if ctx.outputs.out2: + ctx.actions.expand_template( + template = ctx.file.template, + output = ctx.outputs.out2, + substitutions = {"%VERSION%": "2"}, + is_executable = True, + ) + if ctx.outputs.out3: + ctx.actions.expand_template( + template = ctx.file.template, + output = ctx.outputs.out3, + substitutions = {"%VERSION%": "3"}, + is_executable = True, + ) + +expand_pyversion_template = rule( + implementation = _expand_pyversion_template_impl, + attrs = { + "template": attr.label( + allow_single_file = True, + doc = "The input template file.", + ), + "out2": attr.output(doc = """\ +The output file produced by substituting "%VERSION%" with "2"."""), + "out3": attr.output(doc = """\ +The output file produced by substituting "%VERSION%" with "3"."""), + }, + doc = """\ +Given a template file, generates two expansions by replacing the substring +"%VERSION%" with "2" and "3".""", +)