feat(coverage): Register coverage.py to hermetic toolchains (#977)

This allows including the coverage package as part of the toolchain dependencies, which is mixed into a test's dependencies when `bazel coverage` is run (if coverage is not enabled, no extra dependency is added)

For now, it's disabled by default because enabling it poses the risk of having two versions of coverage installed (one from the toolchain, one from the user's dependencies).

The user can turn the coverage_tool setting by passing
`register_coverage_tool=(True|False)` to `python_register_toolchains` or
`python_register_multi_toolchains` call or specifying the
`coverage_tool` label as described in the `versions.bzl` file.

Use coverage.py v6.5.0 because the latest has `types.py` in the package
directory, which imports from Python's stdlib `types` [1]. Somehow the
Python interpreter is thinking that the `from types import FrameType` is
referring to the currently interpreted file and everything breaks. I
would have expected the package to use absolute imports and only attempt
to import from `coverage.types` if we use `coverage.types` and not just
a plain `types` import.

NOTE: Coverage is only for non-windows platforms.

Update tests to:
- ensure that we can still use the toolchain as previously.
- ensure that we are not downloading extra deps if they are not needed.

* Also changes the projects bazelrc to use a remotejdk, which makes it easier for contributors because they don't have to locally install a jdk to get going.

[1]: https://github.com/nedbat/coveragepy/blob/master/coverage/types.py
[3]: bazelbuild/bazel#15835

.bazelci/presubmit.yml

@@ -34,6 +34,20 @@ buildifier:
 .reusable_build_test_all: &reusable_build_test_all
   build_targets: ["..."]
   test_targets: ["..."]
+.coverage_targets_example_bzlmod: &coverage_targets_example_bzlmod
+  coverage_targets: ["//:test"]
+.coverage_targets_example_multi_python: &coverage_targets_example_multi_python
+  coverage_targets:
+    - //tests:my_lib_3_10_test
+    - //tests:my_lib_3_11_test
+    - //tests:my_lib_3_8_test
+    - //tests:my_lib_3_9_test
+    - //tests:my_lib_default_test
+    - //tests:version_3_10_test
+    - //tests:version_3_11_test
+    - //tests:version_3_8_test
+    - //tests:version_3_9_test
+    - //tests:version_default_test
 tasks:
   gazelle_extension:
     name: Test the Gazelle extension
@@ -89,42 +103,50 @@ tasks:
 
   integration_test_bzlmod_ubuntu:
     <<: *reusable_build_test_all
+    <<: *coverage_targets_example_bzlmod
     name: bzlmod integration tests on Ubuntu
     working_directory: examples/bzlmod
     platform: ubuntu2004
   integration_test_bzlmod_debian:
     <<: *reusable_build_test_all
+    <<: *coverage_targets_example_bzlmod
     name: bzlmod integration tests on Debian
     working_directory: examples/bzlmod
     platform: debian11
   integration_test_bzlmod_macos:
     <<: *reusable_build_test_all
+    <<: *coverage_targets_example_bzlmod
     name: bzlmod integration tests on macOS
     working_directory: examples/bzlmod
     platform: macos
   integration_test_bzlmod_windows:
     <<: *reusable_build_test_all
+    # coverage is not supported on Windows
     name: bzlmod integration tests on Windows
     working_directory: examples/bzlmod
     platform: windows
 
   integration_test_multi_python_versions_ubuntu:
     <<: *reusable_build_test_all
+    <<: *coverage_targets_example_multi_python
     name: multi_python_versions integration tests on Ubuntu
     working_directory: examples/multi_python_versions
     platform: ubuntu2004
   integration_test_multi_python_versions_debian:
     <<: *reusable_build_test_all
+    <<: *coverage_targets_example_multi_python
     name: multi_python_versions integration tests on Debian
     working_directory: examples/multi_python_versions
     platform: debian11
   integration_test_multi_python_versions_macos:
     <<: *reusable_build_test_all
+    <<: *coverage_targets_example_multi_python
     name: multi_python_versions integration tests on macOS
     working_directory: examples/multi_python_versions
     platform: macos
   integration_test_multi_python_versions_windows:
     <<: *reusable_build_test_all
+    # coverage is not supported on Windows
     name: multi_python_versions integration tests on Windows
     working_directory: examples/multi_python_versions
     platform: windows

MODULE.bazel

@@ -29,4 +29,20 @@ use_repo(
     "pypi__tomli",
     "pypi__wheel",
     "pypi__zipp",
+    # coverage_deps managed by running ./tools/update_coverage_deps.py <version>
+    "pypi__coverage_cp310_aarch64-apple-darwin",
+    "pypi__coverage_cp310_aarch64-unknown-linux-gnu",
+    "pypi__coverage_cp310_x86_64-apple-darwin",
+    "pypi__coverage_cp310_x86_64-unknown-linux-gnu",
+    "pypi__coverage_cp311_aarch64-unknown-linux-gnu",
+    "pypi__coverage_cp311_x86_64-apple-darwin",
+    "pypi__coverage_cp311_x86_64-unknown-linux-gnu",
+    "pypi__coverage_cp38_aarch64-apple-darwin",
+    "pypi__coverage_cp38_aarch64-unknown-linux-gnu",
+    "pypi__coverage_cp38_x86_64-apple-darwin",
+    "pypi__coverage_cp38_x86_64-unknown-linux-gnu",
+    "pypi__coverage_cp39_aarch64-apple-darwin",
+    "pypi__coverage_cp39_aarch64-unknown-linux-gnu",
+    "pypi__coverage_cp39_x86_64-apple-darwin",
+    "pypi__coverage_cp39_x86_64-unknown-linux-gnu",
 )

examples/bzlmod/.bazelrc

@@ -1 +1,3 @@
 common --experimental_enable_bzlmod
+
+coverage --java_runtime_version=remotejdk_11

examples/bzlmod/BUILD.bazel

@@ -12,7 +12,7 @@ compile_pip_requirements(
 
 py_library(
     name = "lib",
-    srcs = ["__init__.py"],
+    srcs = ["lib.py"],
     deps = [
         requirement("pylint"),
         requirement("tabulate"),

examples/bzlmod/MODULE.bazel

@@ -13,6 +13,7 @@ local_path_override(
 python = use_extension("@rules_python//python:extensions.bzl", "python")
 python.toolchain(
     name = "python3_9",
+    configure_coverage_tool = True,
     python_version = "3.9",
 )
 use_repo(python, "python3_9_toolchains")

examples/bzlmod/__main__.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __init__ import main
+from lib import main
 
 if __name__ == "__main__":
     print(main([["A", 1], ["B", 2]]))

examples/bzlmod/test.py

@@ -14,7 +14,7 @@
 
 import unittest
 
-from __init__ import main
+from lib import main
 
 
 class ExampleTest(unittest.TestCase):

examples/multi_python_versions/.bazelrc

@@ -3,3 +3,5 @@ test --test_output=errors
 # Windows requires these for multi-python support:
 build --enable_runfiles
 startup --windows_enable_symlinks
+
+coverage --java_runtime_version=remotejdk_11

examples/multi_python_versions/WORKSPACE

@@ -22,11 +22,14 @@ python_register_multi_toolchains(
         "3.8",
         "3.9",
         "3.10",
+        "3.11",
     ],
+    register_coverage_tool = True,
 )
 
 load("@python//:pip.bzl", "multi_pip_parse")
 load("@python//3.10:defs.bzl", interpreter_3_10 = "interpreter")
+load("@python//3.11:defs.bzl", interpreter_3_11 = "interpreter")
 load("@python//3.8:defs.bzl", interpreter_3_8 = "interpreter")
 load("@python//3.9:defs.bzl", interpreter_3_9 = "interpreter")
 
@@ -35,11 +38,13 @@ multi_pip_parse(
     default_version = default_python_version,
     python_interpreter_target = {
         "3.10": interpreter_3_10,
+        "3.11": interpreter_3_11,
         "3.8": interpreter_3_8,
         "3.9": interpreter_3_9,
     },
     requirements_lock = {
         "3.10": "//requirements:requirements_lock_3_10.txt",
+        "3.11": "//requirements:requirements_lock_3_11.txt",
         "3.8": "//requirements:requirements_lock_3_8.txt",
         "3.9": "//requirements:requirements_lock_3_9.txt",
     },

examples/multi_python_versions/requirements/BUILD.bazel

@@ -1,4 +1,5 @@
 load("@python//3.10:defs.bzl", compile_pip_requirements_3_10 = "compile_pip_requirements")
+load("@python//3.11:defs.bzl", compile_pip_requirements_3_11 = "compile_pip_requirements")
 load("@python//3.8:defs.bzl", compile_pip_requirements_3_8 = "compile_pip_requirements")
 load("@python//3.9:defs.bzl", compile_pip_requirements_3_9 = "compile_pip_requirements")
 
@@ -22,3 +23,10 @@ compile_pip_requirements_3_10(
     requirements_in = "requirements.in",
     requirements_txt = "requirements_lock_3_10.txt",
 )
+
+compile_pip_requirements_3_11(
+    name = "requirements_3_11",
+    extra_args = ["--allow-unsafe"],
+    requirements_in = "requirements.in",
+    requirements_txt = "requirements_lock_3_11.txt",
+)

examples/multi_python_versions/requirements/requirements_lock_3_11.txt

@@ -0,0 +1,56 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    bazel run //requirements:requirements_3_11.update
+#
+websockets==10.3 \
+    --hash=sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af \
+    --hash=sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c \
+    --hash=sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76 \
+    --hash=sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47 \
+    --hash=sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69 \
+    --hash=sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079 \
+    --hash=sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c \
+    --hash=sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55 \
+    --hash=sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02 \
+    --hash=sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559 \
+    --hash=sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3 \
+    --hash=sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e \
+    --hash=sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978 \
+    --hash=sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98 \
+    --hash=sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae \
+    --hash=sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755 \
+    --hash=sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d \
+    --hash=sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991 \
+    --hash=sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1 \
+    --hash=sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680 \
+    --hash=sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247 \
+    --hash=sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f \
+    --hash=sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2 \
+    --hash=sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7 \
+    --hash=sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4 \
+    --hash=sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667 \
+    --hash=sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb \
+    --hash=sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094 \
+    --hash=sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36 \
+    --hash=sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79 \
+    --hash=sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500 \
+    --hash=sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e \
+    --hash=sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582 \
+    --hash=sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442 \
+    --hash=sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd \
+    --hash=sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6 \
+    --hash=sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731 \
+    --hash=sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4 \
+    --hash=sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d \
+    --hash=sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8 \
+    --hash=sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f \
+    --hash=sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677 \
+    --hash=sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8 \
+    --hash=sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9 \
+    --hash=sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e \
+    --hash=sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b \
+    --hash=sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916 \
+    --hash=sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4
+    # via -r requirements/requirements.in

examples/multi_python_versions/tests/BUILD.bazel

@@ -1,4 +1,5 @@
 load("@python//3.10:defs.bzl", py_binary_3_10 = "py_binary", py_test_3_10 = "py_test")
+load("@python//3.11:defs.bzl", py_binary_3_11 = "py_binary", py_test_3_11 = "py_test")
 load("@python//3.8:defs.bzl", py_binary_3_8 = "py_binary", py_test_3_8 = "py_test")
 load("@python//3.9:defs.bzl", py_binary_3_9 = "py_binary", py_test_3_9 = "py_test")
 load("@rules_python//python:defs.bzl", "py_binary", "py_test")
@@ -27,6 +28,12 @@ py_binary_3_10(
     main = "version.py",
 )
 
+py_binary_3_11(
+    name = "version_3_11",
+    srcs = ["version.py"],
+    main = "version.py",
+)
+
 py_test(
     name = "my_lib_default_test",
     srcs = ["my_lib_test.py"],
@@ -55,6 +62,13 @@ py_test_3_10(
     deps = ["//libs/my_lib"],
 )
 
+py_test_3_11(
+    name = "my_lib_3_11_test",
+    srcs = ["my_lib_test.py"],
+    main = "my_lib_test.py",
+    deps = ["//libs/my_lib"],
+)
+
 py_test(
     name = "version_default_test",
     srcs = ["version_test.py"],
@@ -83,6 +97,13 @@ py_test_3_10(
     main = "version_test.py",
 )
 
+py_test_3_11(
+    name = "version_3_11_test",
+    srcs = ["version_test.py"],
+    env = {"VERSION_CHECK": "3.11"},
+    main = "version_test.py",
+)
+
 py_test(
     name = "version_default_takes_3_10_subprocess_test",
     srcs = ["cross_version_test.py"],

python/extensions.bzl

@@ -19,27 +19,40 @@ load("@rules_python//python:repositories.bzl", "python_register_toolchains")
 load("@rules_python//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_repository_attrs", "use_isolated", "whl_library")
 load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
 load("@rules_python//python/pip_install:requirements_parser.bzl", parse_requirements = "parse")
+load("@rules_python//python/private:coverage_deps.bzl", "install_coverage_deps")
 
 def _python_impl(module_ctx):
     for mod in module_ctx.modules:
         for attr in mod.tags.toolchain:
             python_register_toolchains(
                 name = attr.name,
                 python_version = attr.python_version,
+                bzlmod = True,
                 # Toolchain registration in bzlmod is done in MODULE file
                 register_toolchains = False,
+                register_coverage_tool = attr.configure_coverage_tool,
             )
 
 python = module_extension(
     implementation = _python_impl,
     tag_classes = {
-        "toolchain": tag_class(attrs = dict({"name": attr.string(mandatory = True), "python_version": attr.string(mandatory = True)})),
+        "toolchain": tag_class(
+            attrs = {
+                "configure_coverage_tool": attr.bool(
+                    mandatory = False,
+                    doc = "Whether or not to configure the default coverage tool for the toolchains.",
+                ),
+                "name": attr.string(mandatory = True),
+                "python_version": attr.string(mandatory = True),
+            },
+        ),
     },
 )
 
 # buildifier: disable=unused-variable
 def _internal_deps_impl(module_ctx):
     pip_install_dependencies()
+    install_coverage_deps()
 
 internal_deps = module_extension(
     implementation = _internal_deps_impl,