From b92927d86ca202c72bd981d208149fd788511b6d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Sep 2024 14:11:13 +0900 Subject: [PATCH] feat(bzlmod): add python.override APIs (#2222) Before this PR the users could not override how the hermetic toolchain is downloaded when in `bzlmod`. However, the same APIs would be available to users using `WORKSPACE`. With this we also allow root modules to restrict which toolchain versions the non-root modules, which may be helpful when optimizing the CI runtimes so that we don't waste time downloading multiple `micro` versions of the same `3.X` python version, which most of the times have identical behavior. Whilst at it, tweak the `semver` implementation to allow for testing of absence of values in the original input. Work towards #2081 and this should be one of the last items that are blocking #1361 from the API point of view. Replaces #2151. --------- Co-authored-by: Richard Levasseur --- .bazelrc | 4 +- CHANGELOG.md | 8 + MODULE.bazel | 2 +- docs/toolchains.md | 17 +- examples/bzlmod/MODULE.bazel | 51 +- examples/bzlmod/MODULE.bazel.lock | 4 +- python/extensions/python.bzl | 33 +- python/private/BUILD.bazel | 4 +- python/private/common/py_runtime_rule.bzl | 8 +- python/private/py_repositories.bzl | 4 +- python/private/pypi/BUILD.bazel | 2 +- python/private/python.bzl | 593 +++++++++++++++--- python/private/python_register_toolchains.bzl | 4 + python/private/python_repository.bzl | 20 +- python/private/semver.bzl | 10 +- python/private/toolchains_repo.bzl | 3 - .../ignore_root_user_error/bzlmod_test.py | 12 +- tests/python/python_tests.bzl | 494 ++++++++++++++- tests/semver/semver_test.bzl | 6 +- 19 files changed, 1137 insertions(+), 142 deletions(-) diff --git a/.bazelrc b/.bazelrc index b484751c3c..1ca469cd75 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered test --test_output=errors diff --git a/CHANGELOG.md b/CHANGELOG.md index 88a8378cd1..0fd84923ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,10 @@ A brief description of the categories of changes: * (gazelle): Update error messages when unable to resolve a dependency to be more human-friendly. * (flags) The {obj}`--python_version` flag now also returns {obj}`config_common.FeatureFlagInfo`. +* (toolchain): The toolchain patches now expose the `patch_strip` attribute + that one should use when patching toolchains. Please set it if you are + patching python interpreter. In the next release the default will be set to + `0` which better reflects the defaults used in public `bazel` APIs. * (toolchains) When {obj}`py_runtime.interpreter_version_info` isn't specified, the {obj}`--python_version` flag will determine the value. This allows specifying the build-time Python version for the @@ -65,6 +69,10 @@ A brief description of the categories of changes: ### Added +* (bzlmod): Toolchain overrides can now be done using the new + {bzl:obj}`python.override`, {bzl:obj}`python.single_version_override` and + {bzl:obj}`python.single_version_platform_override` tag classes. + See [#2081](https://github.com/bazelbuild/rules_python/issues/2081). * (rules) Executables provide {obj}`PyExecutableInfo`, which contains executable-specific information useful for packaging an executable or or deriving a new one from the original. diff --git a/MODULE.bazel b/MODULE.bazel index 424b4f84c1..58c7ae229b 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -81,7 +81,7 @@ dev_python = use_extension( "python", dev_dependency = True, ) -dev_python.rules_python_private_testing( +dev_python.override( register_all_versions = True, ) diff --git a/docs/toolchains.md b/docs/toolchains.md index 2ac0099304..6df6f22a2a 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -169,6 +169,21 @@ Variables](https://bazel.build/reference/be/make-variables). See the {gh-path}`test_current_py_toolchain ` target for an example. +### Overriding toolchain defaults and adding more versions + +One can perform various overrides for the registered toolchains from the root +module. For example, the following use cases would be supported using the +existing attributes: + +* Limiting the available toolchains for the entire `bzlmod` transitive graph + via {attr}`python.override.available_python_versions`. +* Setting particular `X.Y.Z` Python versions when modules request `X.Y` version + via {attr}`python.override.minor_mapping`. +* Per-version control of the coverage tool used using + {attr}`python.single_version_platform_override.coverage_tool`. +* Adding additional Python versions via {bzl:obj}`python.single_version_override` or + {bzl:obj}`python.single_version_platform_override`. + ## Workspace configuration To import rules_python in your project, you first need to add it to your @@ -244,7 +259,7 @@ automatically registers a higher-priority toolchain; it won't be used unless there is a toolchain misconfiguration somewhere. To aid migration off the Bazel-builtin toolchain, rules_python provides -{obj}`@rules_python//python/runtime_env_toolchains:all`. This is an equivalent +{bzl:obj}`@rules_python//python/runtime_env_toolchains:all`. This is an equivalent toolchain, but is implemented using rules_python's objects. diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index b7b46b7dba..4ac8191051 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -22,7 +22,7 @@ bazel_dep(name = "protobuf", version = "24.4", repo_name = "com_google_protobuf" python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( configure_coverage_tool = True, - # Only set when you have mulitple toolchain versions. + # Only set when you have multiple toolchain versions. is_default = True, python_version = "3.9", ) @@ -37,6 +37,55 @@ python.toolchain( python_version = "3.10", ) +# One can override the actual toolchain versions that are available, which can be useful +# when optimizing what gets downloaded and when. +python.override( + available_python_versions = [ + "3.10.9", + "3.9.19", + # The following is used by the `other_module` and we need to include it here + # as well. + "3.11.8", + ], + # Also override the `minor_mapping` so that the root module, + # instead of rules_python's defaults, controls what full version + # is used when `3.x` is requested. + minor_mapping = { + "3.10": "3.10.9", + "3.11": "3.11.8", + "3.9": "3.9.19", + }, +) + +# Or the sources that the toolchains come from for all platforms +python.single_version_override( + patch_strip = 1, + # The user can specify patches to be applied to all interpreters. + patches = [], + python_version = "3.10.2", + sha256 = { + "aarch64-apple-darwin": "1409acd9a506e2d1d3b65c1488db4e40d8f19d09a7df099667c87a506f71c0ef", + "aarch64-unknown-linux-gnu": "8f351a8cc348bb45c0f95b8634c8345ec6e749e483384188ad865b7428342703", + "x86_64-apple-darwin": "8146ad4390710ec69b316a5649912df0247d35f4a42e2aa9615bffd87b3e235a", + "x86_64-pc-windows-msvc": "a1d9a594cd3103baa24937ad9150c1a389544b4350e859200b3e5c036ac352bd", + "x86_64-unknown-linux-gnu": "9b64eca2a94f7aff9409ad70bdaa7fbbf8148692662e764401883957943620dd", + }, + urls = ["20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz"], +) + +# Or a single platform. This can be used in combination with the +# `single_version_override` and `single_version_platform_override` will be +# applied after `single_version_override`. Any values present in this override +# will overwrite the values set by the `single_version_override` +python.single_version_platform_override( + patch_strip = 1, + patches = [], + platform = "aarch64-apple-darwin", + python_version = "3.10.2", + sha256 = "1409acd9a506e2d1d3b65c1488db4e40d8f19d09a7df099667c87a506f71c0ef", + urls = ["20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz"], +) + # You only need to load this repositories if you are using multiple Python versions. # See the tests folder for various examples on using multiple Python versions. # The names "python_3_9" and "python_3_10" are autmatically created by the repo diff --git a/examples/bzlmod/MODULE.bazel.lock b/examples/bzlmod/MODULE.bazel.lock index 234dc46cab..cb8fbe287c 100644 --- a/examples/bzlmod/MODULE.bazel.lock +++ b/examples/bzlmod/MODULE.bazel.lock @@ -1231,7 +1231,7 @@ }, "@@rules_python~//python/extensions:pip.bzl%pip": { "general": { - "bzlTransitiveDigest": "Cca+OPIA5xqQl5ND8j44y5jBkjQfz+36h8Hgq0N26I0=", + "bzlTransitiveDigest": "ED3oUrLQz/MTptq8JOZ03sjD7HZ3naUeFS3XFpxz4tg=", "usagesDigest": "MChlcSw99EuW3K7OOoMcXQIdcJnEh6YmfyjJm+9mxIg=", "recordedFileInputs": { "@@other_module~//requirements_lock_3_11.txt": "a7d0061366569043d5efcf80e34a32c732679367cb3c831c4cdc606adc36d314", @@ -6140,7 +6140,7 @@ }, "@@rules_python~//python/private/pypi:pip.bzl%pip_internal": { "general": { - "bzlTransitiveDigest": "wkQNgti5fKONsAUn77ZyTWdfaJfI2cSDrSI0b4JzmI8=", + "bzlTransitiveDigest": "vEOIMpxlh8qbHkABunGFRr+IDbabjCM/hUF0V3GGTus=", "usagesDigest": "Y8ihY+R57BAFhalrVLVdJFrpwlbsiKz9JPJ99ljF7HA=", "recordedFileInputs": { "@@rules_python~//tools/publish/requirements.txt": "031e35d03dde03ae6305fe4b3d1f58ad7bdad857379752deede0f93649991b8a", diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index 4148d90877..0f0da006a7 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -12,7 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -"Python toolchain module extensions for use with bzlmod" +"""Python toolchain module extensions for use with bzlmod. + +:::{topic} Basic usage + +The simplest way to configure the toolchain with `rules_python` is as follows. + +```starlark +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + is_default = True, + python_version = "3.11", +) +use_repo(python, "python_3_11") +``` + +::::{seealso} +For more in-depth documentation see the {obj}`python.toolchain`. +:::: +::: + +:::{topic} Overrides + +Overrides can be done at 3 different levels: +* Overrides affecting all python toolchain versions on all platforms - {obj}`python.override`. +* Overrides affecting a single toolchain versions on all platforms - {obj}`python.single_version_override`. +* Overrides affecting a single toolchain versions on a single platforms - {obj}`python.single_version_platform_override`. + +::::{seealso} +The main documentation page on registering [toolchains](/toolchains). +:::: +::: +""" load("//python/private:python.bzl", _python = "python") diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 5fa551454e..bfe3764296 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -141,11 +141,12 @@ bzl_library( srcs = ["python.bzl"], deps = [ ":full_version_bzl", + ":python_register_toolchains_bzl", ":pythons_hub_bzl", ":repo_utils_bzl", + ":semver_bzl", ":toolchains_repo_bzl", ":util_bzl", - "//python:repositories_bzl", "@bazel_features//:features", ], ) @@ -202,7 +203,6 @@ bzl_library( name = "pythons_hub_bzl", srcs = ["pythons_hub.bzl"], deps = [ - ":full_version_bzl", ":py_toolchain_suite_bzl", ], ) diff --git a/python/private/common/py_runtime_rule.bzl b/python/private/common/py_runtime_rule.bzl index dd40f76a00..b339425099 100644 --- a/python/private/common/py_runtime_rule.bzl +++ b/python/private/common/py_runtime_rule.bzl @@ -202,8 +202,8 @@ See @bazel_tools//tools/python:python_bootstrap_template.txt for more variables. "coverage_tool": attr.label( allow_files = False, doc = """ -This is a target to use for collecting code coverage information from `py_binary` -and `py_test` targets. +This is a target to use for collecting code coverage information from +{rule}`py_binary` and {rule}`py_test` targets. If set, the target must either produce a single file or be an executable target. The path to the single file, or the executable if the target is executable, @@ -212,7 +212,7 @@ runfiles will be added to the runfiles when coverage is enabled. The entry point for the tool must be loadable by a Python interpreter (e.g. a `.py` or `.pyc` file). It must accept the command line arguments -of coverage.py (https://coverage.readthedocs.io), at least including +of [`coverage.py`](https://coverage.readthedocs.io), at least including the `run` and `lcov` subcommands. """, ), @@ -311,7 +311,7 @@ The template to use when two stage bootstrapping is enabled default = DEFAULT_STUB_SHEBANG, doc = """ "Shebang" expression prepended to the bootstrapping Python stub script -used when executing `py_binary` targets. +used when executing {rule}`py_binary` targets. See https://github.com/bazelbuild/bazel/issues/8685 for motivation. diff --git a/python/private/py_repositories.bzl b/python/private/py_repositories.bzl index ace3750a2b..8ddcb5d3a7 100644 --- a/python/private/py_repositories.bzl +++ b/python/private/py_repositories.bzl @@ -25,8 +25,8 @@ def http_archive(**kwargs): def py_repositories(): """Runtime dependencies that users must install. - This function should be loaded and called in the user's WORKSPACE. - With bzlmod enabled, this function is not needed since MODULE.bazel handles transitive deps. + This function should be loaded and called in the user's `WORKSPACE`. + With `bzlmod` enabled, this function is not needed since `MODULE.bazel` handles transitive deps. """ maybe( internal_config_repo, diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 2b25bfbfb4..8cfd3d6525 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -59,7 +59,6 @@ bzl_library( srcs = ["extension.bzl"], deps = [ ":attrs_bzl", - "//python/private:semver_bzl", ":hub_repository_bzl", ":parse_requirements_bzl", ":evaluate_markers_bzl", @@ -71,6 +70,7 @@ bzl_library( "//python/private:full_version_bzl", "//python/private:normalize_name_bzl", "//python/private:version_label_bzl", + "//python/private:semver_bzl", "@bazel_features//:features", ] + [ "@pythons_hub//:interpreters_bzl", diff --git a/python/private/python.bzl b/python/private/python.bzl index 98b089f7ca..cedf39a5c7 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -12,14 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -"Python toolchain module extensions for use with bzlmod" +"Python toolchain module extensions for use with bzlmod." load("@bazel_features//:features.bzl", "bazel_features") -load("//python:versions.bzl", "MINOR_MAPPING", "TOOL_VERSIONS") +load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "PLATFORMS", "TOOL_VERSIONS") +load(":auth.bzl", "AUTH_ATTRS") load(":full_version.bzl", "full_version") load(":python_register_toolchains.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") load(":repo_utils.bzl", "repo_utils") +load(":semver.bzl", "semver") load(":text_util.bzl", "render") load(":toolchains_repo.bzl", "multi_toolchain_aliases") load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER") @@ -29,11 +31,12 @@ load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER") _MAX_NUM_TOOLCHAINS = 9999 _TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS)) -def parse_modules(module_ctx): +def parse_modules(*, module_ctx, _fail = fail): """Parse the modules and return a struct for registrations. Args: module_ctx: {type}`module_ctx` module context. + _fail: {type}`function` the failure function, mainly for testing. Returns: A struct with the following attributes: @@ -61,7 +64,7 @@ def parse_modules(module_ctx): # This is a toolchain_info struct. default_toolchain = None - # Map of version string to the toolchain_info struct + # Map of string Major.Minor or Major.Minor.Patch to the toolchain_info struct global_toolchain_versions = {} ignore_root_user_error = None @@ -73,10 +76,16 @@ def parse_modules(module_ctx): if not module_ctx.modules[0].tags.toolchain: ignore_root_user_error = False + config = _get_toolchain_config(modules = module_ctx.modules, _fail = _fail) + + seen_versions = {} for mod in module_ctx.modules: module_toolchain_versions = [] - - toolchain_attr_structs = _create_toolchain_attr_structs(mod) + toolchain_attr_structs = _create_toolchain_attr_structs( + mod = mod, + seen_versions = seen_versions, + config = config, + ) for toolchain_attr in toolchain_attr_structs: toolchain_version = toolchain_attr.python_version @@ -149,6 +158,7 @@ def parse_modules(module_ctx): if debug_info: debug_info["toolchains_registered"].append({ "ignore_root_user_error": ignore_root_user_error, + "module": {"is_root": mod.is_root, "name": mod.name}, "name": toolchain_name, }) @@ -166,6 +176,8 @@ def parse_modules(module_ctx): elif toolchain_info: toolchains.append(toolchain_info) + config.default.setdefault("ignore_root_user_error", ignore_root_user_error) + # A default toolchain is required so that the non-version-specific rules # are able to match a toolchain. if default_toolchain == None: @@ -185,11 +197,9 @@ def parse_modules(module_ctx): fail("more than {} python versions are not supported".format(_MAX_NUM_TOOLCHAINS)) return struct( + config = config, debug_info = debug_info, default_python_version = toolchains[-1].python_version, - defaults = { - "ignore_root_user_error": ignore_root_user_error, - }, toolchains = [ struct( python_version = t.python_version, @@ -201,16 +211,24 @@ def parse_modules(module_ctx): ) def _python_impl(module_ctx): - py = parse_modules(module_ctx) + py = parse_modules(module_ctx = module_ctx) for toolchain_info in py.toolchains: - python_register_toolchains( - name = toolchain_info.name, - python_version = toolchain_info.python_version, - register_coverage_tool = toolchain_info.register_coverage_tool, - minor_mapping = MINOR_MAPPING, - **py.defaults + # Ensure that we pass the full version here. + full_python_version = full_version( + version = toolchain_info.python_version, + minor_mapping = py.config.minor_mapping, ) + kwargs = { + "python_version": full_python_version, + "register_coverage_tool": toolchain_info.register_coverage_tool, + } + + # Allow overrides per python version + kwargs.update(py.config.kwargs.get(toolchain_info.python_version, {})) + kwargs.update(py.config.kwargs.get(full_python_version, {})) + kwargs.update(py.config.default) + python_register_toolchains(name = toolchain_info.name, **kwargs) # Create the pythons_hub repo for the interpreter meta data and the # the various toolchains. @@ -223,7 +241,7 @@ def _python_impl(module_ctx): for index, toolchain in enumerate(py.toolchains) ], toolchain_python_versions = [ - full_version(version = t.python_version, minor_mapping = MINOR_MAPPING) + full_version(version = t.python_version, minor_mapping = py.config.minor_mapping) for t in py.toolchains ], # The last toolchain is the default; it can't have version constraints @@ -264,6 +282,9 @@ def _fail_duplicate_module_toolchain_version(version, module): )) def _warn_duplicate_global_toolchain_version(version, first, second_toolchain_name, second_module_name, logger): + if not logger: + return + logger.info(lambda: ( "Ignoring toolchain '{second_toolchain}' from module '{second_module}': " + "Toolchain '{first_toolchain}' from module '{first_module}' " + @@ -284,25 +305,234 @@ def _fail_multiple_default_toolchains(first, second): second = second, )) -def _create_toolchain_attr_structs(mod): +def _validate_version(*, version, _fail = fail): + parsed = semver(version) + if parsed.patch == None or parsed.build or parsed.pre_release: + _fail("The 'python_version' attribute needs to specify an 'X.Y.Z' semver-compatible version, got: '{}'".format(version)) + return False + + return True + +def _process_single_version_overrides(*, tag, _fail = fail, default): + if not _validate_version(version = tag.python_version, _fail = _fail): + return + + available_versions = default["tool_versions"] + kwargs = default.setdefault("kwargs", {}) + + if tag.sha256 or tag.urls: + if not (tag.sha256 and tag.urls): + _fail("Both `sha256` and `urls` overrides need to be provided together") + return + + for platform in (tag.sha256 or []): + if platform not in PLATFORMS: + _fail("The platform must be one of {allowed} but got '{got}'".format( + allowed = sorted(PLATFORMS), + got = platform, + )) + return + + sha256 = dict(tag.sha256) or available_versions[tag.python_version]["sha256"] + override = { + "sha256": sha256, + "strip_prefix": { + platform: tag.strip_prefix + for platform in sha256 + }, + "url": { + platform: list(tag.urls) + for platform in tag.sha256 + } or available_versions[tag.python_version]["url"], + } + + if tag.patches: + override["patch_strip"] = { + platform: tag.patch_strip + for platform in sha256 + } + override["patches"] = { + platform: list(tag.patches) + for platform in sha256 + } + + available_versions[tag.python_version] = {k: v for k, v in override.items() if v} + + if tag.distutils_content: + kwargs.setdefault(tag.python_version, {})["distutils_content"] = tag.distutils_content + if tag.distutils: + kwargs.setdefault(tag.python_version, {})["distutils"] = tag.distutils + +def _process_single_version_platform_overrides(*, tag, _fail = fail, default): + if not _validate_version(version = tag.python_version, _fail = _fail): + return + + available_versions = default["tool_versions"] + + if tag.python_version not in available_versions: + if not tag.urls or not tag.sha256 or not tag.strip_prefix: + _fail("When introducing a new python_version '{}', 'sha256', 'strip_prefix' and 'urls' must be specified".format(tag.python_version)) + return + available_versions[tag.python_version] = {} + + if tag.coverage_tool: + available_versions[tag.python_version].setdefault("coverage_tool", {})[tag.platform] = tag.coverage_tool + if tag.patch_strip: + available_versions[tag.python_version].setdefault("patch_strip", {})[tag.platform] = tag.patch_strip + if tag.patches: + available_versions[tag.python_version].setdefault("patches", {})[tag.platform] = list(tag.patches) + if tag.sha256: + available_versions[tag.python_version].setdefault("sha256", {})[tag.platform] = tag.sha256 + if tag.strip_prefix: + available_versions[tag.python_version].setdefault("strip_prefix", {})[tag.platform] = tag.strip_prefix + if tag.urls: + available_versions[tag.python_version].setdefault("url", {})[tag.platform] = tag.urls + +def _process_global_overrides(*, tag, default, _fail = fail): + if tag.available_python_versions: + available_versions = default["tool_versions"] + all_versions = dict(available_versions) + available_versions.clear() + for v in tag.available_python_versions: + if v not in all_versions: + _fail("unknown version '{}', known versions are: {}".format( + v, + sorted(all_versions), + )) + return + + available_versions[v] = all_versions[v] + + if tag.minor_mapping: + for minor_version, full_version in tag.minor_mapping.items(): + parsed = semver(minor_version) + if parsed.patch != None or parsed.build or parsed.pre_release: + fail("Expected the key to be of `X.Y` format but got `{}`".format(minor_version)) + parsed = semver(full_version) + if parsed.patch == None: + fail("Expected the value to at least be of `X.Y.Z` format but got `{}`".format(minor_version)) + + default["minor_mapping"] = tag.minor_mapping + + forwarded_attrs = sorted(AUTH_ATTRS) + [ + "ignore_root_user_error", + "base_url", + "register_all_versions", + ] + for key in forwarded_attrs: + if getattr(tag, key, None): + default[key] = getattr(tag, key) + +def _override_defaults(*overrides, modules, _fail = fail, default): + mod = modules[0] if modules else None + if not mod or not mod.is_root: + return + + overriden_keys = [] + + for override in overrides: + for tag in getattr(mod.tags, override.name): + key = override.key(tag) + if key not in overriden_keys: + overriden_keys.append(key) + elif key: + _fail("Only a single 'python.{}' can be present for '{}'".format(override.name, key)) + return + else: + _fail("Only a single 'python.{}' can be present".format(override.name)) + return + + override.fn(tag = tag, _fail = _fail, default = default) + +def _get_toolchain_config(*, modules, _fail = fail): + # Items that can be overridden + available_versions = { + version: { + # Use a dicts straight away so that we could do URL overrides for a + # single version. + "sha256": dict(item["sha256"]), + "strip_prefix": { + platform: item["strip_prefix"] + for platform in item["sha256"] + }, + "url": { + platform: [item["url"]] + for platform in item["sha256"] + }, + } + for version, item in TOOL_VERSIONS.items() + } + default = { + "base_url": DEFAULT_RELEASE_BASE_URL, + "tool_versions": available_versions, + } + + _override_defaults( + # First override by single version, because the sha256 will replace + # anything that has been there before. + struct( + name = "single_version_override", + key = lambda t: t.python_version, + fn = _process_single_version_overrides, + ), + # Then override particular platform entries if they need to be overridden. + struct( + name = "single_version_platform_override", + key = lambda t: (t.python_version, t.platform), + fn = _process_single_version_platform_overrides, + ), + # Then finally add global args and remove the unnecessary toolchains. + # This ensures that we can do further validations when removing. + struct( + name = "override", + key = lambda t: None, + fn = _process_global_overrides, + ), + modules = modules, + default = default, + _fail = _fail, + ) + + minor_mapping = default.pop("minor_mapping", {}) + register_all_versions = default.pop("register_all_versions", False) + kwargs = default.pop("kwargs", {}) + + if not minor_mapping: + versions = {} + for version_string in available_versions: + v = semver(version_string) + versions.setdefault("{}.{}".format(v.major, v.minor), []).append((int(v.patch), version_string)) + + minor_mapping = { + major_minor: max(subset)[1] + for major_minor, subset in versions.items() + } + + return struct( + kwargs = kwargs, + minor_mapping = minor_mapping, + default = default, + register_all_versions = register_all_versions, + ) + +def _create_toolchain_attr_structs(*, mod, config, seen_versions): arg_structs = [] - seen_versions = {} + for tag in mod.tags.toolchain: - arg_structs.append(_create_toolchain_attrs_struct(tag = tag, toolchain_tag_count = len(mod.tags.toolchain))) + arg_structs.append(_create_toolchain_attrs_struct( + tag = tag, + toolchain_tag_count = len(mod.tags.toolchain), + )) + seen_versions[tag.python_version] = True - if mod.is_root: - register_all = False - for tag in mod.tags.rules_python_private_testing: - if tag.register_all_versions: - register_all = True - break - if register_all: - arg_structs.extend([ - _create_toolchain_attrs_struct(python_version = v) - for v in TOOL_VERSIONS.keys() - if v not in seen_versions - ]) + if config.register_all_versions: + arg_structs.extend([ + _create_toolchain_attrs_struct(python_version = v) + for v in config.default["tool_versions"].keys() + config.minor_mapping.keys() + if v not in seen_versions + ]) + return arg_structs def _create_toolchain_attrs_struct(*, tag = None, python_version = None, toolchain_tag_count = None): @@ -329,78 +559,295 @@ def _get_bazel_version_specific_kwargs(): return kwargs -python = module_extension( - doc = """Bzlmod extension that is used to register Python toolchains. -""", - implementation = _python_impl, - tag_classes = { - "rules_python_private_testing": tag_class( - attrs = { - "register_all_versions": attr.bool(default = False), - }, - ), - "toolchain": tag_class( - doc = """Tag class used to register Python toolchains. +_toolchain = tag_class( + doc = """Tag class used to register Python toolchains. Use this tag class to register one or more Python toolchains. This class is also potentially called by sub modules. The following covers different business rules and use cases. -Toolchains in the Root Module +:::{topic} Toolchains in the Root Module This class registers all toolchains in the root module. +::: -Toolchains in Sub Modules +:::{topic} Toolchains in Sub Modules It will create a toolchain that is in a sub module, if the toolchain of the same name does not exist in the root module. The extension stops name clashing between toolchains in the root module and toolchains in sub modules. You cannot configure more than one toolchain as the default toolchain. +::: -Toolchain set as the default version +:::{topic} Toolchain set as the default version This extension will not create a toolchain that exists in a sub module, if the sub module toolchain is marked as the default version. If you have more than one toolchain in your root module, you need to set one of the toolchains as the default version. If there is only one toolchain it is set as the default toolchain. +::: -Toolchain repository name +:::{topic} Toolchain repository name A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. `python_3_10`. The `major` and `minor` components are `major` and `minor` are the Python version from the `python_version` attribute. + +If a toolchain is registered in `X.Y.Z`, then similarly the toolchain name will +be `python_{major}_{minor}_{patch}`, e.g. `python_3_10_19`. +::: + +:::{topic} Toolchain detection +The definition of the first toolchain wins, which means that the root module +can override settings for any python toolchain available. This relies on the +documented module traversal from the {obj}`module_ctx.modules`. +::: + +:::{tip} +In order to use a different name than the above, you can use the following `MODULE.bazel` +syntax: +```starlark +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + is_default = True, + python_version = "3.11", +) + +use_repo(python, my_python_name = "python_3_11") +``` + +Then the python interpreter will be available as `my_python_name`. +::: """, - attrs = { - "configure_coverage_tool": attr.bool( - mandatory = False, - doc = "Whether or not to configure the default coverage tool for the toolchains.", - ), - "ignore_root_user_error": attr.bool( - default = False, - doc = """\ -If False, the Python runtime installation will be made read only. This improves + attrs = { + "configure_coverage_tool": attr.bool( + mandatory = False, + doc = "Whether or not to configure the default coverage tool provided by `rules_python` for the compatible toolchains.", + ), + "ignore_root_user_error": attr.bool( + default = False, + doc = """\ +If `False`, the Python runtime installation will be made read only. This improves the ability for Bazel to cache it, but prevents the interpreter from creating -pyc files for the standard library dynamically at runtime as they are loaded. +`.pyc` files for the standard library dynamically at runtime as they are loaded. -If True, the Python runtime installation is read-write. This allows the -interpreter to create pyc files for the standard library, but, because they are +If `True`, the Python runtime installation is read-write. This allows the +interpreter to create `.pyc` files for the standard library, but, because they are created as needed, it adversely affects Bazel's ability to cache the runtime and can result in spurious build failures. """, - mandatory = False, - ), - "is_default": attr.bool( - mandatory = False, - doc = "Whether the toolchain is the default version", - ), - "python_version": attr.string( - mandatory = True, - doc = "The Python version, in `major.minor` format, e.g " + - "'3.12', to create a toolchain for. Patch level " + - "granularity (e.g. '3.12.1') is not supported.", - ), - }, + mandatory = False, + ), + "is_default": attr.bool( + mandatory = False, + doc = "Whether the toolchain is the default version", ), + "python_version": attr.string( + mandatory = True, + doc = """\ +The Python version, in `major.minor` or `major.minor.patch` format, e.g +`3.12` (or `3.12.3`), to create a toolchain for. +""", + ), + }, +) + +_override = tag_class( + doc = """Tag class used to override defaults and behaviour of the module extension. + +:::{versionadded} 0.36.0 +::: +""", + attrs = { + "available_python_versions": attr.string_list( + mandatory = False, + doc = """\ +The list of available python tool versions to use. Must be in `X.Y.Z` format. +If the unknown version given the processing of the extension will fail - all of +the versions in the list have to be defined with +{obj}`python.single_version_override` or +{obj}`python.single_version_platform_override` before they are used in this +list. + +This attribute is usually used in order to ensure that no unexpected transitive +dependencies are introduced. +""", + ), + "base_url": attr.string( + mandatory = False, + doc = "The base URL to be used when downloading toolchains.", + default = DEFAULT_RELEASE_BASE_URL, + ), + "ignore_root_user_error": attr.bool( + default = False, + doc = """\ +If `False`, the Python runtime installation will be made read only. This improves +the ability for Bazel to cache it, but prevents the interpreter from creating +`.pyc` files for the standard library dynamically at runtime as they are loaded. + +If `True`, the Python runtime installation is read-write. This allows the +interpreter to create `.pyc` files for the standard library, but, because they are +created as needed, it adversely affects Bazel's ability to cache the runtime and +can result in spurious build failures. +""", + mandatory = False, + ), + "minor_mapping": attr.string_dict( + mandatory = False, + doc = """\ +The mapping between `X.Y` to `X.Y.Z` versions to be used when setting up +toolchains. It defaults to the interpreter with the highest available patch +version for each minor version. For example if one registers `3.10.3`, `3.10.4` +and `3.11.4` then the default for the `minor_mapping` dict will be: +```starlark +{ +"3.10": "3.10.4", +"3.11": "3.11.4", +} +``` +""", + default = {}, + ), + "register_all_versions": attr.bool(default = False, doc = "Add all versions"), + } | AUTH_ATTRS, +) + +_single_version_override = tag_class( + doc = """Override single python version URLs and patches for all platforms. + +:::{note} +This will replace any existing configuration for the given python version. +::: + +:::{tip} +If you would like to modify the configuration for a specific `(version, +platform)`, please use the {obj}`single_version_platform_override` tag +class. +::: + +:::{versionadded} 0.36.0 +::: +""", + attrs = { + # NOTE @aignas 2024-09-01: all of the attributes except for `version` + # can be part of the `python.toolchain` call. That would make it more + # ergonomic to define new toolchains and to override values for old + # toolchains. The same semantics of the `first one wins` would apply, + # so technically there is no need for any overrides? + # + # Although these attributes would override the code that is used by the + # code in non-root modules, so technically this could be thought as + # being overridden. + # + # rules_go has a single download call: + # https://github.com/bazelbuild/rules_go/blob/master/go/private/extensions.bzl#L38 + # + # However, we need to understand how to accommodate the fact that + # {attr}`single_version_override.version` only allows patch versions. + "distutils": attr.label( + allow_single_file = True, + doc = "A distutils.cfg file to be included in the Python installation. " + + "Either {attr}`distutils` or {attr}`distutils_content` can be specified, but not both.", + mandatory = False, + ), + "distutils_content": attr.string( + doc = "A distutils.cfg file content to be included in the Python installation. " + + "Either {attr}`distutils` or {attr}`distutils_content` can be specified, but not both.", + mandatory = False, + ), + "patch_strip": attr.int( + mandatory = False, + doc = "Same as the --strip argument of Unix patch.", + default = 0, + ), + "patches": attr.label_list( + mandatory = False, + doc = "A list of labels pointing to patch files to apply for the interpreter repository. They are applied in the list order and are applied before any platform-specific patches are applied.", + ), + "python_version": attr.string( + mandatory = True, + doc = "The python version to override URLs for. Must be in `X.Y.Z` format.", + ), + "sha256": attr.string_dict( + mandatory = False, + doc = "The python platform to sha256 dict. See {attr}`python.single_version_platform_override.platform` for allowed key values.", + ), + "strip_prefix": attr.string( + mandatory = False, + doc = "The 'strip_prefix' for the archive, defaults to 'python'.", + default = "python", + ), + "urls": attr.string_list( + mandatory = False, + doc = "The URL template to fetch releases for this Python version. See {attr}`python.single_version_platform_override.urls` for documentation.", + ), + }, +) + +_single_version_platform_override = tag_class( + doc = """Override single python version for a single existing platform. + +If the `(version, platform)` is new, we will add it to the existing versions and will +use the same `url` template. + +:::{tip} +If you would like to add or remove platforms to a single python version toolchain +configuration, please use {obj}`single_version_override`. +::: + +:::{versionadded} 0.36.0 +::: +""", + attrs = { + "coverage_tool": attr.label( + doc = """\ +The coverage tool to be used for a particular Python interpreter. This can override +`rules_python` defaults. +""", + ), + "patch_strip": attr.int( + mandatory = False, + doc = "Same as the --strip argument of Unix patch.", + default = 0, + ), + "patches": attr.label_list( + mandatory = False, + doc = "A list of labels pointing to patch files to apply for the interpreter repository. They are applied in the list order and are applied after the common patches are applied.", + ), + "platform": attr.string( + mandatory = True, + values = PLATFORMS.keys(), + doc = "The platform to override the values for, must be one of:\n{}.".format("\n".join(sorted(["* `{}`".format(p) for p in PLATFORMS]))), + ), + "python_version": attr.string( + mandatory = True, + doc = "The python version to override URLs for. Must be in `X.Y.Z` format.", + ), + "sha256": attr.string( + mandatory = False, + doc = "The sha256 for the archive", + ), + "strip_prefix": attr.string( + mandatory = False, + doc = "The 'strip_prefix' for the archive, defaults to 'python'.", + default = "python", + ), + "urls": attr.string_list( + mandatory = False, + doc = "The URL template to fetch releases for this Python version. If the URL template results in a relative fragment, default base URL is going to be used. Occurrences of `{python_version}`, `{platform}` and `{build}` will be interpolated based on the contents in the override and the known {attr}`platform` values.", + ), + }, +) + +python = module_extension( + doc = """Bzlmod extension that is used to register Python toolchains. +""", + implementation = _python_impl, + tag_classes = { + "override": _override, + "single_version_override": _single_version_override, + "single_version_platform_override": _single_version_platform_override, + "toolchain": _toolchain, }, **_get_bazel_version_specific_kwargs() ) diff --git a/python/private/python_register_toolchains.bzl b/python/private/python_register_toolchains.bzl index 7638d76313..d20e049610 100644 --- a/python/private/python_register_toolchains.bzl +++ b/python/private/python_register_toolchains.bzl @@ -46,6 +46,10 @@ def python_register_toolchains( **kwargs): """Convenience macro for users which does typical setup. + With `bzlmod` enabled, this function is not needed since `rules_python` is + handling everything. In order to override the default behaviour from the + root module one can see the docs for the {rule}`python` extension. + - Create a repository for each built-in platform like "python_3_8_linux_amd64" - this repository is lazily fetched when Python is needed for that platform. - Create a repository exposing toolchains for each platform like diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl index 28a2c959dd..2710299b39 100644 --- a/python/private/python_repository.bzl +++ b/python/private/python_repository.bzl @@ -124,7 +124,6 @@ def _python_repository_impl(rctx): patches = rctx.attr.patches if patches: for patch in patches: - # Should take the strip as an attr, but this is fine for the moment rctx.patch(patch, strip = rctx.attr.patch_strip) # Write distutils.cfg to the Python installation. @@ -302,27 +301,14 @@ python_repository = repository_rule( doc = "Override mapping of hostnames to authorization patterns; mirrors the eponymous attribute from http_archive", ), "coverage_tool": attr.string( - # Mirrors the definition at - # https://github.com/bazelbuild/bazel/blob/master/src/main/starlark/builtins_bzl/common/python/py_runtime_rule.bzl doc = """ -This is a target to use for collecting code coverage information from `py_binary` -and `py_test` targets. - -If set, the target must either produce a single file or be an executable target. -The path to the single file, or the executable if the target is executable, -determines the entry point for the python coverage tool. The target and its -runfiles will be added to the runfiles when coverage is enabled. - -The entry point for the tool must be loadable by a Python interpreter (e.g. a -`.py` or `.pyc` file). It must accept the command line arguments -of coverage.py (https://coverage.readthedocs.io), at least including -the `run` and `lcov` subcommands. +This is a target to use for collecting code coverage information from {rule}`py_binary` +and {rule}`py_test` targets. The target is accepted as a string by the python_repository and evaluated within the context of the toolchain repository. -For more information see the official bazel docs -(https://bazel.build/reference/be/python#py_runtime.coverage_tool). +For more information see {attr}`py_runtime.coverage_tool`. """, ), "distutils": attr.label( diff --git a/python/private/semver.bzl b/python/private/semver.bzl index d77249ff9f..73d6b130ae 100644 --- a/python/private/semver.bzl +++ b/python/private/semver.bzl @@ -17,8 +17,8 @@ def _key(version): return ( version.major, - version.minor, - version.patch, + version.minor or 0, + version.patch or 0, # non pre-release versions are higher version.pre_release == "", # then we compare each element of the pre_release tag separately @@ -47,7 +47,7 @@ def semver(version): """Parse the semver version and return the values as a struct. Args: - version: {type}`str` the version string + version: {type}`str` the version string. Returns: A {type}`struct` with `major`, `minor`, `patch` and `build` attributes. @@ -62,9 +62,9 @@ def semver(version): # buildifier: disable=uninitialized self = struct( major = int(major), - minor = int(minor or "0"), + minor = int(minor) if minor.isdigit() else None, # NOTE: this is called `micro` in the Python interpreter versioning scheme - patch = int(patch or "0"), + patch = int(patch) if patch.isdigit() else None, pre_release = pre_release, build = build, # buildifier: disable=uninitialized diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index 528b86fc3b..4fae987c74 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -56,9 +56,6 @@ def python_toolchain_build_file_content( build_content: Text containing toolchain definitions """ - # We create a list of toolchain content from iterating over - # the enumeration of PLATFORMS. We enumerate PLATFORMS in - # order to get us an index to increment the increment. return "\n\n".join([ """\ py_toolchain_suite( diff --git a/tests/integration/ignore_root_user_error/bzlmod_test.py b/tests/integration/ignore_root_user_error/bzlmod_test.py index 98715b32ec..1283415987 100644 --- a/tests/integration/ignore_root_user_error/bzlmod_test.py +++ b/tests/integration/ignore_root_user_error/bzlmod_test.py @@ -28,8 +28,16 @@ def test_toolchains(self): debug_info = json.loads(debug_path.read_bytes()) expected = [ - {"ignore_root_user_error": True, "name": "python_3_11"}, - {"ignore_root_user_error": True, "name": "python_3_10"}, + { + "ignore_root_user_error": True, + "module": {"is_root": False, "name": "submodule"}, + "name": "python_3_10", + }, + { + "ignore_root_user_error": True, + "module": {"is_root": True, "name": "ignore_root_user_error"}, + "name": "python_3_11", + }, ] self.assertCountEqual(debug_info["toolchains_registered"], expected) diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index acbd6676dc..101313da4f 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -15,13 +15,11 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:python.bzl", _parse_modules = "parse_modules") # buildifier: disable=bzl-visibility +load("//python:versions.bzl", "MINOR_MAPPING") +load("//python/private:python.bzl", "parse_modules") # buildifier: disable=bzl-visibility _tests = [] -def parse_modules(*, mctx, **kwargs): - return _parse_modules(module_ctx = mctx, **kwargs) - def _mock_mctx(*modules, environ = {}): return struct( os = struct(environ = environ), @@ -41,12 +39,14 @@ def _mock_mctx(*modules, environ = {}): ], ) -def _mod(*, name, toolchain = [], rules_python_private_testing = [], is_root = True): +def _mod(*, name, toolchain = [], override = [], single_version_override = [], single_version_platform_override = [], is_root = True): return struct( name = name, tags = struct( toolchain = toolchain, - rules_python_private_testing = rules_python_private_testing, + override = override, + single_version_override = single_version_override, + single_version_platform_override = single_version_platform_override, ), is_root = is_root, ) @@ -58,17 +58,88 @@ def _toolchain(python_version, *, is_default = False, **kwargs): **kwargs ) +def _override( + auth_patterns = {}, + available_python_versions = [], + base_url = "", + ignore_root_user_error = False, + minor_mapping = {}, + netrc = "", + register_all_versions = False): + return struct( + auth_patterns = auth_patterns, + available_python_versions = available_python_versions, + base_url = base_url, + ignore_root_user_error = ignore_root_user_error, + minor_mapping = minor_mapping, + netrc = netrc, + register_all_versions = register_all_versions, + ) + +def _single_version_override( + python_version = "", + sha256 = {}, + urls = [], + patch_strip = 0, + patches = [], + strip_prefix = "python", + distutils_content = "", + distutils = None): + if not python_version: + fail("missing mandatory args: python_version ({})".format(python_version)) + + return struct( + python_version = python_version, + sha256 = sha256, + urls = urls, + patch_strip = patch_strip, + patches = patches, + strip_prefix = strip_prefix, + distutils_content = distutils_content, + distutils = distutils, + ) + +def _single_version_platform_override( + coverage_tool = None, + patch_strip = 0, + patches = [], + platform = "", + python_version = "", + sha256 = "", + strip_prefix = "python", + urls = []): + if not platform or not python_version: + fail("missing mandatory args: platform ({}) and python_version ({})".format(platform, python_version)) + + return struct( + sha256 = sha256, + urls = urls, + strip_prefix = strip_prefix, + platform = platform, + coverage_tool = coverage_tool, + python_version = python_version, + patch_strip = patch_strip, + patches = patches, + ) + def _test_default(env): py = parse_modules( - mctx = _mock_mctx( + module_ctx = _mock_mctx( _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), ), ) - env.expect.that_collection(py.defaults.keys()).contains_exactly([ + # The value there should be consistent in bzlmod with the automatically + # calculated value Please update the MINOR_MAPPING in //python:versions.bzl + # when this part starts failing. + env.expect.that_dict(py.config.minor_mapping).contains_exactly(MINOR_MAPPING) + env.expect.that_collection(py.config.kwargs).has_size(0) + env.expect.that_collection(py.config.default.keys()).contains_exactly([ + "base_url", "ignore_root_user_error", + "tool_versions", ]) - env.expect.that_bool(py.defaults["ignore_root_user_error"]).equals(False) + env.expect.that_bool(py.config.default["ignore_root_user_error"]).equals(False) env.expect.that_str(py.default_python_version).equals("3.11") want_toolchain = struct( @@ -82,14 +153,11 @@ _tests.append(_test_default) def _test_default_some_module(env): py = parse_modules( - mctx = _mock_mctx( + module_ctx = _mock_mctx( _mod(name = "rules_python", toolchain = [_toolchain("3.11")], is_root = False), ), ) - env.expect.that_collection(py.defaults.keys()).contains_exactly([ - "ignore_root_user_error", - ]) env.expect.that_str(py.default_python_version).equals("3.11") want_toolchain = struct( @@ -103,7 +171,7 @@ _tests.append(_test_default_some_module) def _test_default_with_patch_version(env): py = parse_modules( - mctx = _mock_mctx( + module_ctx = _mock_mctx( _mod(name = "rules_python", toolchain = [_toolchain("3.11.2")]), ), ) @@ -121,7 +189,7 @@ _tests.append(_test_default_with_patch_version) def _test_default_non_rules_python(env): py = parse_modules( - mctx = _mock_mctx( + module_ctx = _mock_mctx( # NOTE @aignas 2024-09-06: the first item in the module_ctx.modules # could be a non-root module, which is the case if the root module # does not make any calls to the extension. @@ -141,7 +209,7 @@ _tests.append(_test_default_non_rules_python) def _test_default_non_rules_python_ignore_root_user_error(env): py = parse_modules( - mctx = _mock_mctx( + module_ctx = _mock_mctx( _mod( name = "my_module", toolchain = [_toolchain("3.12", ignore_root_user_error = True)], @@ -150,7 +218,7 @@ def _test_default_non_rules_python_ignore_root_user_error(env): ), ) - env.expect.that_bool(py.defaults["ignore_root_user_error"]).equals(True) + env.expect.that_bool(py.config.default["ignore_root_user_error"]).equals(True) env.expect.that_str(py.default_python_version).equals("3.12") my_module_toolchain = struct( @@ -170,9 +238,41 @@ def _test_default_non_rules_python_ignore_root_user_error(env): _tests.append(_test_default_non_rules_python_ignore_root_user_error) +def _test_default_non_rules_python_ignore_root_user_error_override(env): + py = parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.12")], + override = [_override(ignore_root_user_error = True)], + ), + _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), + ), + ) + + env.expect.that_bool(py.config.default["ignore_root_user_error"]).equals(True) + env.expect.that_str(py.default_python_version).equals("3.12") + + my_module_toolchain = struct( + name = "python_3_12", + python_version = "3.12", + register_coverage_tool = False, + ) + rules_python_toolchain = struct( + name = "python_3_11", + python_version = "3.11", + register_coverage_tool = False, + ) + env.expect.that_collection(py.toolchains).contains_exactly([ + rules_python_toolchain, + my_module_toolchain, + ]).in_order() + +_tests.append(_test_default_non_rules_python_ignore_root_user_error_override) + def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): py = parse_modules( - mctx = _mock_mctx( + module_ctx = _mock_mctx( _mod(name = "my_module", toolchain = [_toolchain("3.13")]), _mod(name = "some_module", toolchain = [_toolchain("3.12", ignore_root_user_error = True)]), _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), @@ -180,7 +280,7 @@ def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): ) env.expect.that_str(py.default_python_version).equals("3.13") - env.expect.that_bool(py.defaults["ignore_root_user_error"]).equals(False) + env.expect.that_bool(py.config.default["ignore_root_user_error"]).equals(False) my_module_toolchain = struct( name = "python_3_13", @@ -207,7 +307,7 @@ _tests.append(_test_default_non_rules_python_ignore_root_user_error_non_root_mod def _test_first_occurance_of_the_toolchain_wins(env): py = parse_modules( - mctx = _mock_mctx( + module_ctx = _mock_mctx( _mod(name = "my_module", toolchain = [_toolchain("3.12")]), _mod(name = "some_module", toolchain = [_toolchain("3.12", configure_coverage_tool = True)]), _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), @@ -235,15 +335,365 @@ def _test_first_occurance_of_the_toolchain_wins(env): rules_python_toolchain, my_module_toolchain, # default toolchain is last ]).in_order() + env.expect.that_dict(py.debug_info).contains_exactly({ "toolchains_registered": [ - {"ignore_root_user_error": False, "name": "python_3_12"}, - {"ignore_root_user_error": False, "name": "python_3_11"}, + {"ignore_root_user_error": False, "module": {"is_root": True, "name": "my_module"}, "name": "python_3_12"}, + {"ignore_root_user_error": False, "module": {"is_root": False, "name": "rules_python"}, "name": "python_3_11"}, ], }) _tests.append(_test_first_occurance_of_the_toolchain_wins) +def _test_auth_overrides(env): + py = parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.12")], + override = [ + _override( + netrc = "/my/netrc", + auth_patterns = {"foo": "bar"}, + ), + ], + ), + _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), + ), + ) + + env.expect.that_dict(py.config.default).contains_at_least({ + "auth_patterns": {"foo": "bar"}, + "ignore_root_user_error": False, + "netrc": "/my/netrc", + }) + env.expect.that_str(py.default_python_version).equals("3.12") + + my_module_toolchain = struct( + name = "python_3_12", + python_version = "3.12", + register_coverage_tool = False, + ) + rules_python_toolchain = struct( + name = "python_3_11", + python_version = "3.11", + register_coverage_tool = False, + ) + env.expect.that_collection(py.toolchains).contains_exactly([ + rules_python_toolchain, + my_module_toolchain, + ]).in_order() + +_tests.append(_test_auth_overrides) + +def _test_add_new_version(env): + py = parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + single_version_override = [ + _single_version_override( + python_version = "3.13.0", + sha256 = { + "aarch64-unknown-linux-gnu": "deadbeef", + }, + urls = ["example.org"], + patch_strip = 0, + patches = [], + strip_prefix = "prefix", + distutils_content = "", + distutils = None, + ), + ], + single_version_platform_override = [ + _single_version_platform_override( + sha256 = "deadb00f", + urls = ["something.org", "else.org"], + strip_prefix = "python", + platform = "aarch64-unknown-linux-gnu", + coverage_tool = "specific_cov_tool", + python_version = "3.13.1", + patch_strip = 2, + patches = ["specific-patch.txt"], + ), + ], + override = [ + _override( + base_url = "", + available_python_versions = ["3.12.4", "3.13.0", "3.13.1"], + minor_mapping = { + "3.13": "3.13.0", + }, + ), + ], + ), + ), + ) + + env.expect.that_str(py.default_python_version).equals("3.13") + env.expect.that_collection(py.config.default["tool_versions"].keys()).contains_exactly([ + "3.12.4", + "3.13.0", + "3.13.1", + ]) + env.expect.that_dict(py.config.default["tool_versions"]["3.13.0"]).contains_exactly({ + "sha256": {"aarch64-unknown-linux-gnu": "deadbeef"}, + "strip_prefix": {"aarch64-unknown-linux-gnu": "prefix"}, + "url": {"aarch64-unknown-linux-gnu": ["example.org"]}, + }) + env.expect.that_dict(py.config.default["tool_versions"]["3.13.1"]).contains_exactly({ + "coverage_tool": {"aarch64-unknown-linux-gnu": "specific_cov_tool"}, + "patch_strip": {"aarch64-unknown-linux-gnu": 2}, + "patches": {"aarch64-unknown-linux-gnu": ["specific-patch.txt"]}, + "sha256": {"aarch64-unknown-linux-gnu": "deadb00f"}, + "strip_prefix": {"aarch64-unknown-linux-gnu": "python"}, + "url": {"aarch64-unknown-linux-gnu": ["something.org", "else.org"]}, + }) + env.expect.that_dict(py.config.minor_mapping).contains_exactly({ + "3.13": "3.13.0", + }) + env.expect.that_collection(py.toolchains).contains_exactly([ + struct( + name = "python_3_13", + python_version = "3.13", + register_coverage_tool = False, + ), + ]) + +_tests.append(_test_add_new_version) + +def _test_register_all_versions(env): + py = parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + single_version_override = [ + _single_version_override( + python_version = "3.13.0", + sha256 = { + "aarch64-unknown-linux-gnu": "deadbeef", + }, + urls = ["example.org"], + ), + ], + single_version_platform_override = [ + _single_version_platform_override( + sha256 = "deadb00f", + urls = ["something.org"], + platform = "aarch64-unknown-linux-gnu", + python_version = "3.13.1", + ), + ], + override = [ + _override( + base_url = "", + available_python_versions = ["3.12.4", "3.13.0", "3.13.1"], + register_all_versions = True, + ), + ], + ), + ), + ) + + env.expect.that_str(py.default_python_version).equals("3.13") + env.expect.that_collection(py.config.default["tool_versions"].keys()).contains_exactly([ + "3.12.4", + "3.13.0", + "3.13.1", + ]) + env.expect.that_dict(py.config.minor_mapping).contains_exactly({ + # The mapping is calculated automatically + "3.12": "3.12.4", + "3.13": "3.13.1", + }) + env.expect.that_collection(py.toolchains).contains_exactly([ + struct( + name = name, + python_version = version, + register_coverage_tool = False, + ) + for name, version in { + "python_3_12": "3.12", + "python_3_12_4": "3.12.4", + "python_3_13": "3.13", + "python_3_13_0": "3.13.0", + "python_3_13_1": "3.13.1", + }.items() + ]) + +_tests.append(_test_register_all_versions) + +def _test_add_patches(env): + py = parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + single_version_override = [ + _single_version_override( + python_version = "3.13.0", + sha256 = { + "aarch64-apple-darwin": "deadbeef", + "aarch64-unknown-linux-gnu": "deadbeef", + }, + urls = ["example.org"], + patch_strip = 1, + patches = ["common.txt"], + strip_prefix = "prefix", + distutils_content = "", + distutils = None, + ), + ], + single_version_platform_override = [ + _single_version_platform_override( + sha256 = "deadb00f", + urls = ["something.org", "else.org"], + strip_prefix = "python", + platform = "aarch64-unknown-linux-gnu", + coverage_tool = "specific_cov_tool", + python_version = "3.13.0", + patch_strip = 2, + patches = ["specific-patch.txt"], + ), + ], + override = [ + _override( + base_url = "", + available_python_versions = ["3.13.0"], + minor_mapping = { + "3.13": "3.13.0", + }, + ), + ], + ), + ), + ) + + env.expect.that_str(py.default_python_version).equals("3.13") + env.expect.that_dict(py.config.default["tool_versions"]).contains_exactly({ + "3.13.0": { + "coverage_tool": {"aarch64-unknown-linux-gnu": "specific_cov_tool"}, + "patch_strip": {"aarch64-apple-darwin": 1, "aarch64-unknown-linux-gnu": 2}, + "patches": { + "aarch64-apple-darwin": ["common.txt"], + "aarch64-unknown-linux-gnu": ["specific-patch.txt"], + }, + "sha256": {"aarch64-apple-darwin": "deadbeef", "aarch64-unknown-linux-gnu": "deadb00f"}, + "strip_prefix": {"aarch64-apple-darwin": "prefix", "aarch64-unknown-linux-gnu": "python"}, + "url": { + "aarch64-apple-darwin": ["example.org"], + "aarch64-unknown-linux-gnu": ["something.org", "else.org"], + }, + }, + }) + env.expect.that_dict(py.config.minor_mapping).contains_exactly({ + "3.13": "3.13.0", + }) + env.expect.that_collection(py.toolchains).contains_exactly([ + struct( + name = "python_3_13", + python_version = "3.13", + register_coverage_tool = False, + ), + ]) + +_tests.append(_test_add_patches) + +def _test_fail_two_overrides(env): + errors = [] + parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + override = [ + _override(base_url = "foo"), + _override(base_url = "bar"), + ], + ), + ), + _fail = errors.append, + ) + env.expect.that_collection(errors).contains_exactly([ + "Only a single 'python.override' can be present", + ]) + +_tests.append(_test_fail_two_overrides) + +def _test_single_version_override_errors(env): + for test in [ + struct( + overrides = [ + _single_version_override(python_version = "3.12.4", distutils_content = "foo"), + _single_version_override(python_version = "3.12.4", distutils_content = "foo"), + ], + want_error = "Only a single 'python.single_version_override' can be present for '3.12.4'", + ), + struct( + overrides = [ + _single_version_override(python_version = "3.12.4+3", distutils_content = "foo"), + ], + want_error = "The 'python_version' attribute needs to specify an 'X.Y.Z' semver-compatible version, got: '3.12.4+3'", + ), + ]: + errors = [] + parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + single_version_override = test.overrides, + ), + ), + _fail = errors.append, + ) + env.expect.that_collection(errors).contains_exactly([test.want_error]) + +_tests.append(_test_single_version_override_errors) + +def _test_single_version_platform_override_errors(env): + for test in [ + struct( + overrides = [ + _single_version_platform_override(python_version = "3.12.4", platform = "foo", coverage_tool = "foo"), + _single_version_platform_override(python_version = "3.12.4", platform = "foo", coverage_tool = "foo"), + ], + want_error = "Only a single 'python.single_version_platform_override' can be present for '(\"3.12.4\", \"foo\")'", + ), + struct( + overrides = [ + _single_version_platform_override(python_version = "3.12", platform = "foo"), + ], + want_error = "The 'python_version' attribute needs to specify an 'X.Y.Z' semver-compatible version, got: '3.12'", + ), + struct( + overrides = [ + _single_version_platform_override(python_version = "3.12.1+my_build", platform = "foo"), + ], + want_error = "The 'python_version' attribute needs to specify an 'X.Y.Z' semver-compatible version, got: '3.12.1+my_build'", + ), + ]: + errors = [] + parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + single_version_platform_override = test.overrides, + ), + ), + _fail = errors.append, + ) + env.expect.that_collection(errors).contains_exactly([test.want_error]) + +_tests.append(_test_single_version_platform_override_errors) + +# TODO @aignas 2024-09-03: add failure tests: +# * incorrect platform failure +# * missing python_version failure + def python_test_suite(name): """Create the test suite. diff --git a/tests/semver/semver_test.bzl b/tests/semver/semver_test.bzl index 6395639810..9d13402c92 100644 --- a/tests/semver/semver_test.bzl +++ b/tests/semver/semver_test.bzl @@ -22,8 +22,8 @@ _tests = [] def _test_semver_from_major(env): actual = semver("3") env.expect.that_int(actual.major).equals(3) - env.expect.that_int(actual.minor).equals(0) - env.expect.that_int(actual.patch).equals(0) + env.expect.that_int(actual.minor).equals(None) + env.expect.that_int(actual.patch).equals(None) env.expect.that_str(actual.build).equals("") _tests.append(_test_semver_from_major) @@ -32,7 +32,7 @@ def _test_semver_from_major_minor_version(env): actual = semver("4.9") env.expect.that_int(actual.major).equals(4) env.expect.that_int(actual.minor).equals(9) - env.expect.that_int(actual.patch).equals(0) + env.expect.that_int(actual.patch).equals(None) env.expect.that_str(actual.build).equals("") _tests.append(_test_semver_from_major_minor_version)