diff --git a/CHANGELOG.md b/CHANGELOG.md index 0720a36a59..81340160a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,10 +36,20 @@ A brief description of the categories of changes: * (bzlmod): The `python` and internal `rules_python` extensions have been marked as `reproducible` and will not include any lock file entries from now on. - * (gazelle): Remove gazelle plugin's python deps and make it hermetic. Introduced a new Go-based helper leveraging tree-sitter for syntax analysis. Implemented the use of `pypi/stdlib-list` for standard library module verification. +* (pip.parse): Do not ignore yanked packages when using `experimental_index_url`. + This is to mimic what `uv` is doing. We will print a warning instead. +* (pip.parse): Add references to all supported wheels when using `experimental_index_url` + to allowing to correctly fetch the wheels for the right platform. See the + updated docs on how to use the feature. This is work towards addressing + [#735](https://github.com/bazelbuild/rules_python/issues/735) and + [#260](https://github.com/bazelbuild/rules_python/issues/260). The spoke + repository names when using this flag will have a structure of + `{pip_hub_prefix}_{wheel_name}_{py_tag}_{abi_tag}_{platform_tag}_{sha256}`, + which is an implementation detail which should not be relied on and is there + purely for better debugging experience. ### Fixed * (gazelle) Remove `visibility` from `NonEmptyAttr`. @@ -63,6 +73,13 @@ A brief description of the categories of changes: the `experimental_index_url` feature which will fetch metadata from PyPI or a different private index and write the contents to the lock file. Fixes [#1643](https://github.com/bazelbuild/rules_python/issues/1643). +* (pip.parse): Install `yanked` packages and print a warning instead of + ignoring them. This better matches the behaviour of `uv pip install`. +* (toolchains): Now matching of the default hermetic toolchain is more robust + and explicit and should fix rare edge-cases where the host toolchain + autodetection would match a different toolchain than expected. This may yield + to toolchain selection failures when the python toolchain is not registered, + but is requested via `//python/config_settings:python_version` flag setting. ### Added * (rules) Precompiling Python source at build time is available. but is @@ -104,6 +121,14 @@ A brief description of the categories of changes: {obj}`PyRuntimeInfo.zip_main_template`. * (toolchains) A replacement for the Bazel-builtn autodetecting toolchain is available. The `//python:autodetecting_toolchain` alias now uses it. +* (pip): Support fetching and using the wheels for other platforms. This + supports customizing whether the linux wheels are pulled for `musl` or + `glibc`, whether `universal2` or arch-specific MacOS wheels are preferred and + it also allows to select a particular `libc` version. All of this is done via + the `string_flags` in `@rules_python//python/config_settings`. If there are + no wheels that are supported for the target platform, `rules_python` will + fallback onto building the `sdist` from source. This behaviour can be + disabled if desired using one of the available string flags as well. [precompile-docs]: /precompiling diff --git a/MODULE.bazel b/MODULE.bazel index 38ee678015..7e86bda9b8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -96,7 +96,9 @@ dev_pip.parse( }, hub_name = "dev_pip", python_version = "3.11", - requirements_lock = "//docs/sphinx:requirements.txt", + requirements_by_platform = { + "//docs/sphinx:requirements.txt": "linux_*,osx_*", + }, ) dev_pip.parse( hub_name = "pypiserver", diff --git a/docs/sphinx/api/python/config_settings/index.md b/docs/sphinx/api/python/config_settings/index.md index 29779fd813..8c23bf6855 100644 --- a/docs/sphinx/api/python/config_settings/index.md +++ b/docs/sphinx/api/python/config_settings/index.md @@ -5,11 +5,18 @@ # //python/config_settings -:::{bzl:flag} precompile +:::{bzl:flag} python_version +Determines the default hermetic Python toolchain version. This can be set to +one of the values that `rules_python` maintains. +::: + +::::{bzl:flag} precompile Determines if Python source files should be compiled at build time. -NOTE: The flag value is overridden by the target level `precompile` attribute, +:::{note} +The flag value is overridden by the target level `precompile` attribute, except for the case of `force_enabled` and `forced_disabled`. +::: Values: @@ -27,14 +34,16 @@ Values: * `force_disabled`: Like `disabled`, except overrides target-level setting. This is useful useful for development, testing enabling precompilation more broadly, or as an escape hatch if build-time compiling is not available. -::: +:::: -:::{bzl:flag} precompile_source_retention +::::{bzl:flag} precompile_source_retention Determines, when a source file is compiled, if the source file is kept in the resulting output or not. -NOTE: This flag is overridden by the target level `precompile_source_retention` +:::{note} +This flag is overridden by the target level `precompile_source_retention` attribute. +::: Values: @@ -42,7 +51,7 @@ Values: * `omit_source`: Don't include the orignal py source. * `omit_if_generated_source`: Keep the original source if it's a regular source file, but omit it if it's a generated file. -::: +:::: :::{bzl:flag} precompile_add_to_runfiles Determines if a target adds its compiled files to its runfiles. @@ -59,15 +68,80 @@ Values: incrementally enabling precompilation on a per-binary basis. ::: -:::{bzl:flag} pyc_collection +::::{bzl:flag} pyc_collection Determine if `py_binary` collects transitive pyc files. -NOTE: This flag is overridden by the target level `pyc_collection` attribute. +:::{note} +This flag is overridden by the target level `pyc_collection` attribute. +::: Values: * `include_pyc`: Include `PyInfo.transitive_pyc_files` as part of the binary. * `disabled`: Don't include `PyInfo.transitive_pyc_files` as part of the binary. +:::: + +::::{bzl:flag} py_linux_libc +Set what libc is used for the target platform. This will affect which whl binaries will be pulled and what toolchain will be auto-detected. Currently `rules_python` only supplies toolchains compatible with `glibc`. + +Values: +* `glibc`: Use `glibc`, default. +* `muslc`: Use `muslc`. +:::{versionadded} 0.33.0 +::: +:::: + +::::{bzl:flag} pip_whl +Set what distributions are used in the `pip` integration. + +Values: +* `auto`: Prefer `whl` distributions if they are compatible with a target + platform, but fallback to `sdist`. This is the default. +* `only`: Only use `whl` distributions and error out if it is not available. +* `no`: Only use `sdist` distributions. The wheels will be built non-hermetically in the `whl_library` repository rule. +:::{versionadded} 0.33.0 ::: +:::: + +::::{bzl:flag} pip_whl_osx_arch +Set what wheel types we should prefer when building on the OSX platform. + +Values: +* `arch`: Prefer architecture specific wheels. +* `universal`: Prefer universal wheels that usually are bigger and contain binaries for both, Intel and ARM architectures in the same wheel. +:::{versionadded} 0.33.0 +::: +:::: + +::::{bzl:flag} pip_whl_glibc_version +Set the minimum `glibc` version that the `py_binary` using `whl` distributions from a PyPI index should support. + +Values: +* `""`: Select the lowest available version of each wheel giving you the maximum compatibility. This is the default. +* `X.Y`: The string representation of a `glibc` version. The allowed values depend on the `requirements.txt` lock file contents. +:::{versionadded} 0.33.0 +::: +:::: + +::::{bzl:flag} pip_whl_muslc_version +Set the minimum `muslc` version that the `py_binary` using `whl` distributions from a PyPI index should support. + +Values: +* `""`: Select the lowest available version of each wheel giving you the maximum compatibility. This is the default. +* `X.Y`: The string representation of a `muslc` version. The allowed values depend on the `requirements.txt` lock file contents. +:::{versionadded} 0.33.0 +::: +:::: + +::::{bzl:flag} pip_whl_osx_version +Set the minimum `osx` version that the `py_binary` using `whl` distributions from a PyPI index should support. + +Values: +* `""`: Select the lowest available version of each wheel giving you the maximum compatibility. This is the default. +* `X.Y`: The string representation of the MacOS version. The allowed values depend on the `requirements.txt` lock file contents. + +:::{versionadded} 0.33.0 +::: +:::: ::::{bzl:flag} bootstrap_impl Determine how programs implement their startup process. diff --git a/docs/sphinx/pip.md b/docs/sphinx/pip.md index fc29e41b5e..43d8fc4978 100644 --- a/docs/sphinx/pip.md +++ b/docs/sphinx/pip.md @@ -1,168 +1,4 @@ (pip-integration)= # Pip Integration -To pull in dependencies from PyPI, the `pip_parse` function is used, which -invokes `pip` to download and install dependencies from PyPI. - -In your WORKSPACE file: - -```starlark -load("@rules_python//python:pip.bzl", "pip_parse") - -pip_parse( - name = "pip_deps", - requirements_lock = ":requirements.txt", -) - -load("@pip_deps//:requirements.bzl", "install_deps") - -install_deps() -``` - -For `bzlmod` an equivalent `MODULE.bazel` would look like: -```starlark -pip = use_extension("//python/extensions:pip.bzl", "pip") -pip.parse( - hub_name = "pip_deps", - requirements_lock = ":requirements.txt", -) -use_repo(pip, "pip_deps") -``` - -You can then reference installed dependencies from a `BUILD` file with: - -```starlark -load("@pip_deps//:requirements.bzl", "requirement") - -py_library( - name = "bar", - ... - deps = [ - "//my/other:dep", - "@pip_deps//requests", - "@pip_deps//numpy", - ], -) -``` - -The rules also provide a convenience macro for translating the entries in the -`requirements.txt` file (e.g. `opencv-python`) to the right bazel label (e.g. -`@pip_deps//opencv_python`). The convention of bazel labels is lowercase -`snake_case`, but you can use the helper to avoid depending on this convention -as follows: - -```starlark -load("@pip_deps//:requirements.bzl", "requirement") - -py_library( - name = "bar", - ... - deps = [ - "//my/other:dep", - requirement("requests"), - requirement("numpy"), - ], -) -``` - -If you would like to access [entry points][whl_ep], see the `py_console_script_binary` rule documentation. - -[whl_ep]: https://packaging.python.org/specifications/entry-points/ - -(per-os-arch-requirements)= -## Requirements for a specific OS/Architecture - -In some cases you may need to use different requirements files for different OS, Arch combinations. This is enabled via the `requirements_by_platform` attribute in `pip.parse` extension and the `pip_parse` repository rule. The keys of the dictionary are labels to the file and the values are a list of comma separated target (os, arch) tuples. - -For example: -```starlark - # ... - requirements_by_platform = { - "requirements_linux_x86_64.txt": "linux_x86_64", - "requirements_osx.txt": "osx_*", - "requirements_linux_exotic.txt": "linux_exotic", - "requirements_some_platforms.txt": "linux_aarch64,windows_*", - }, - # For the list of standard platforms that the rules_python has toolchains for, default to - # the following requirements file. - requirements_lock = "requirements_lock.txt", -``` - -In case of duplicate platforms, `rules_python` will raise an error as there has -to be unambiguous mapping of the requirement files to the (os, arch) tuples. - -An alternative way is to use per-OS requirement attributes. -```starlark - # ... - requirements_windows = "requirements_windows.txt", - requirements_darwin = "requirements_darwin.txt", - # For the remaining platforms (which is basically only linux OS), use this file. - requirements_lock = "requirements_lock.txt", -) -``` - -(vendoring-requirements)= -## Vendoring the requirements.bzl file - -In some cases you may not want to generate the requirements.bzl file as a repository rule -while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module -such as a ruleset, you may want to include the requirements.bzl file rather than make your users -install the WORKSPACE setup to generate it. -See https://github.com/bazelbuild/rules_python/issues/608 - -This is the same workflow as Gazelle, which creates `go_repository` rules with -[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) - -To do this, use the "write to source file" pattern documented in -https://blog.aspect.dev/bazel-can-write-to-the-source-folder -to put a copy of the generated requirements.bzl into your project. -Then load the requirements.bzl file directly rather than from the generated repository. -See the example in rules_python/examples/pip_parse_vendored. - - -(credential-helper)= -## Credential Helper - -The "use Bazel downloader for python wheels" experimental feature includes support for the Bazel -[Credential Helper][cred-helper-design]. - -Your python artifact registry may provide a credential helper for you. Refer to your index's docs -to see if one is provided. - -See the [Credential Helper Spec][cred-helper-spec] for details. - -[cred-helper-design]: https://github.com/bazelbuild/proposals/blob/main/designs/2022-06-07-bazel-credential-helpers.md -[cred-helper-spec]: https://github.com/EngFlow/credential-helper-spec/blob/main/spec.md - - -### Basic Example: - -The simplest form of a credential helper is a bash script that accepts an arg and spits out JSON to -stdout. For a service like Google Artifact Registry that uses ['Basic' HTTP Auth][rfc7617] and does -not provide a credential helper that conforms to the [spec][cred-helper-spec], the script might -look like: - -```bash -#!/bin/bash -# cred_helper.sh -ARG=$1 # but we don't do anything with it as it's always "get" - -# formatting is optional -echo '{' -echo ' "headers": {' -echo ' "Authorization": ["Basic dGVzdDoxMjPCow=="]' -echo ' }' -echo '}' -``` - -Configure Bazel to use this credential helper for your python index `example.com`: - -``` -# .bazelrc -build --credential_helper=example.com=/full/path/to/cred_helper.sh -``` - -Bazel will call this file like `cred_helper.sh get` and use the returned JSON to inject headers -into whatever HTTP(S) request it performs against `example.com`. - -[rfc7617]: https://datatracker.ietf.org/doc/html/rfc7617 +See [PyPI dependencies](./pypi-dependencies). diff --git a/docs/sphinx/pypi-dependencies.md b/docs/sphinx/pypi-dependencies.md index f08f7fb7a7..db017d249f 100644 --- a/docs/sphinx/pypi-dependencies.md +++ b/docs/sphinx/pypi-dependencies.md @@ -1,3 +1,6 @@ +:::{default-domain} bzl +::: + # Using dependencies from PyPI Using PyPI packages (aka "pip install") involves two main steps. @@ -25,19 +28,21 @@ pip.parse( use_repo(pip, "my_deps") ``` For more documentation, including how the rules can update/create a requirements -file, see the bzlmod examples under the {gh-path}`examples` folder. +file, see the bzlmod examples under the {gh-path}`examples` folder or the documentation +for the {obj}`@rules_python//python/extensions:pip.bzl` extension. +```{note} We are using a host-platform compatible toolchain by default to setup pip dependencies. During the setup phase, we create some symlinks, which may be inefficient on Windows by default. In that case use the following `.bazelrc` options to improve performance if you have admin privileges: -``` -startup --windows_enable_symlinks -``` + + startup --windows_enable_symlinks This will enable symlinks on Windows and help with bootstrap performance of setting up the hermetic host python interpreter on this platform. Linux and OSX users should see no difference. +``` ### Using a WORKSPACE file @@ -59,16 +64,67 @@ load("@my_deps//:requirements.bzl", "install_deps") install_deps() ``` +(vendoring-requirements)= +#### Vendoring the requirements.bzl file + +In some cases you may not want to generate the requirements.bzl file as a repository rule +while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module +such as a ruleset, you may want to include the requirements.bzl file rather than make your users +install the WORKSPACE setup to generate it. +See https://github.com/bazelbuild/rules_python/issues/608 + +This is the same workflow as Gazelle, which creates `go_repository` rules with +[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) + +To do this, use the "write to source file" pattern documented in +https://blog.aspect.dev/bazel-can-write-to-the-source-folder +to put a copy of the generated requirements.bzl into your project. +Then load the requirements.bzl file directly rather than from the generated repository. +See the example in rules_python/examples/pip_parse_vendored. + +(per-os-arch-requirements)= +### Requirements for a specific OS/Architecture + +In some cases you may need to use different requirements files for different OS, Arch combinations. This is enabled via the `requirements_by_platform` attribute in `pip.parse` extension and the `pip_parse` repository rule. The keys of the dictionary are labels to the file and the values are a list of comma separated target (os, arch) tuples. + +For example: +```starlark + # ... + requirements_by_platform = { + "requirements_linux_x86_64.txt": "linux_x86_64", + "requirements_osx.txt": "osx_*", + "requirements_linux_exotic.txt": "linux_exotic", + "requirements_some_platforms.txt": "linux_aarch64,windows_*", + }, + # For the list of standard platforms that the rules_python has toolchains for, default to + # the following requirements file. + requirements_lock = "requirements_lock.txt", +``` + +In case of duplicate platforms, `rules_python` will raise an error as there has +to be unambiguous mapping of the requirement files to the (os, arch) tuples. + +An alternative way is to use per-OS requirement attributes. +```starlark + # ... + requirements_windows = "requirements_windows.txt", + requirements_darwin = "requirements_darwin.txt", + # For the remaining platforms (which is basically only linux OS), use this file. + requirements_lock = "requirements_lock.txt", +) +``` + ### pip rules -Note that since `pip_parse` is a repository rule and therefore executes pip at -WORKSPACE-evaluation time, Bazel has no information about the Python toolchain -and cannot enforce that the interpreter used to invoke pip matches the -interpreter used to run `py_binary` targets. By default, `pip_parse` uses the -system command `"python3"`. To override this, pass in the `python_interpreter` -attribute or `python_interpreter_target` attribute to `pip_parse`. +Note that since `pip_parse` and `pip.parse` are executed at evaluation time, +Bazel has no information about the Python toolchain and cannot enforce that the +interpreter used to invoke `pip` matches the interpreter used to run +`py_binary` targets. By default, `pip_parse` uses the system command +`"python3"`. To override this, pass in the `python_interpreter` attribute or +`python_interpreter_target` attribute to `pip_parse`. The `pip.parse` `bzlmod` extension +by default uses the hermetic python toolchain for the host platform. -You can have multiple `pip_parse`s in the same workspace. Or use the pip +You can have multiple `pip_parse`s in the same workspace, or use the pip extension multiple times when using bzlmod. This configuration will create multiple external repos that have no relation to one another and may result in downloading the same wheels numerous times. @@ -111,7 +167,7 @@ want to use `requirement()`, you can use the library labels directly instead. For `pip_parse`, the labels are of the following form: ```starlark -@{name}_{package}//:pkg +@{name}//{package} ``` Here `name` is the `name` attribute that was passed to `pip_parse` and @@ -121,30 +177,67 @@ update `name` from "old" to "new", then you can run the following buildozer command: ```shell -buildozer 'substitute deps @old_([^/]+)//:pkg @new_${1}//:pkg' //...:* +buildozer 'substitute deps @old//([^/]+) @new//${1}' //...:* ``` [requirements-drawbacks]: https://github.com/bazelbuild/rules_python/issues/414 +### Entry points + +If you would like to access [entry points][whl_ep], see the `py_console_script_binary` rule documentation, +which can help you create a `py_binary` target for a particular console script exposed by a package. + +[whl_ep]: https://packaging.python.org/specifications/entry-points/ + ### 'Extras' dependencies Any 'extras' specified in the requirements lock file will be automatically added as transitive dependencies of the package. In the example above, you'd just put -`requirement("useful_dep")`. +`requirement("useful_dep")` or `@pypi//useful_dep`. -### Packaging cycles +### Consuming Wheel Dists Directly -Sometimes PyPi packages contain dependency cycles -- for instance `sphinx` -depends on `sphinxcontrib-serializinghtml`. When using them as `requirement()`s, -ala +If you need to depend on the wheel dists themselves, for instance, to pass them +to some other packaging tool, you can get a handle to them with the +`whl_requirement` macro. For example: + +```starlark +load("@pypi//:requirements.bzl", "whl_requirement") + +filegroup( + name = "whl_files", + data = [ + # This is equivalent to "@pypi//boto3:whl" + whl_requirement("boto3"), + ] +) +``` + +### Creating a filegroup of files within a whl + +The rule {obj}`whl_filegroup` exists as an easy way to extract the necessary files +from a whl file without the need to modify the `BUILD.bazel` contents of the +whl repositories generated via `pip_repository`. Use it similarly to the `filegroup` +above. See the API docs for more information. + +(advance-topics)= +## Advanced topics + +(circular-deps)= +### Circular dependencies + +Sometimes PyPi packages contain dependency cycles -- for instance a particular +version `sphinx` (this is no longer the case in the latest version as of +2024-06-02) depends on `sphinxcontrib-serializinghtml`. When using them as +`requirement()`s, ala ``` py_binary( - name = "doctool", - ... - deps = [ - requirement("sphinx"), - ] + name = "doctool", + ... + deps = [ + requirement("sphinx"), + ], ) ``` @@ -166,15 +259,15 @@ issues by specifying groups of packages which form cycles. `pip_parse` will transparently fix the cycles for you and provide the cyclic dependencies simultaneously. -``` +```starlark pip_parse( - ... - experimental_requirement_cycles = { - "sphinx": [ - "sphinx", - "sphinxcontrib-serializinghtml", - ] - }, + ... + experimental_requirement_cycles = { + "sphinx": [ + "sphinx", + "sphinxcontrib-serializinghtml", + ] + }, ) ``` @@ -183,17 +276,17 @@ be distinct. `apache-airflow` for instance has dependency cycles with a number of its optional dependencies, which means those optional dependencies must all be a part of the `airflow` cycle. For instance -- -``` +```starlark pip_parse( - ... - experimental_requirement_cycles = { - "airflow": [ - "apache-airflow", - "apache-airflow-providers-common-sql", - "apache-airflow-providers-postgres", - "apache-airflow-providers-sqlite", - ] - } + ... + experimental_requirement_cycles = { + "airflow": [ + "apache-airflow", + "apache-airflow-providers-common-sql", + "apache-airflow-providers-postgres", + "apache-airflow-providers-sqlite", + ] + } ) ``` @@ -213,17 +306,98 @@ leg of the dependency manually. For instance by making `apache-airflow-providers-postgres` not explicitly depend on `apache-airflow` or perhaps `apache-airflow-providers-common-sql`. -## Consuming Wheel Dists Directly -If you need to depend on the wheel dists themselves, for instance, to pass them -to some other packaging tool, you can get a handle to them with the -`whl_requirement` macro. For example: +(bazel-downloader)= +### Bazel downloader and multi-platform wheel hub repository. -```starlark -filegroup( - name = "whl_files", - data = [ - whl_requirement("boto3"), - ] -) +The `bzlmod` `pip.parse` call supports pulling information from `PyPI` (or a +compatible mirror) and it will ensure that the [bazel +downloader][bazel_downloader] is used for downloading the wheels. This allows +the users to use the [credential helper](#credential-helper) to authenticate +with the mirror and it also ensures that the distribution downloads are cached. +It also avoids using `pip` altogether and results in much faster dependency +fetching. + +This can be enabled by `experimental_index_url` and related flags as shown in +the {gh-path}`examples/bzlmod/MODULE.bazel` example. + +When using this feature during the `pip` extension evaluation you will see the accessed indexes similar to below: +```console +Loading: 0 packages loaded + currently loading: docs/sphinx + Fetching module extension pip in @@//python/extensions:pip.bzl; starting + Fetching https://pypi.org/simple/twine/ ``` + +This does not mean that `rules_python` is fetching the wheels eagerly, but it +rather means that it is calling the PyPI server to get the Simple API response +to get the list of all available source and wheel distributions. Once it has +got all of the available distributions, it will select the right ones depending +on the `sha256` values in your `requirements_lock.txt` file. The compatible +distribution URLs will be then written to the `MODULE.bazel.lock` file. Currently +users wishing to use the lock file with `rules_python` with this feature have +to set an environment variable `RULES_PYTHON_OS_ARCH_LOCK_FILE=0` which will +become default in the next release. + +Fetching the distribution information from the PyPI allows `rules_python` to +know which `whl` should be used on which target platform and it will determine +that by parsing the `whl` filename based on [PEP600], [PEP656] standards. This +allows the user to configure the behaviour by using the following publicly +available flags: +* {obj}`--@rules_python//python/config_settings:py_linux_libc` for selecting the Linux libc variant. +* {obj}`--@rules_python//python/config_settings:pip_whl` for selecting `whl` distribution preference. +* {obj}`--@rules_python//python/config_settings:pip_whl_osx_arch` for selecting MacOS wheel preference. +* {obj}`--@rules_python//python/config_settings:pip_whl_glibc_version` for selecting the GLIBC version compatibility. +* {obj}`--@rules_python//python/config_settings:pip_whl_muslc_version` for selecting the musl version compatibility. +* {obj}`--@rules_python//python/config_settings:pip_whl_osx_version` for selecting MacOS version compatibility. + +[bazel_downloader]: https://bazel.build/rules/lib/builtins/repository_ctx#download +[pep600]: https://peps.python.org/pep-0600/ +[pep656]: https://peps.python.org/pep-0656/ + +(credential-helper)= +### Credential Helper + +The "use Bazel downloader for python wheels" experimental feature includes support for the Bazel +[Credential Helper][cred-helper-design]. + +Your python artifact registry may provide a credential helper for you. Refer to your index's docs +to see if one is provided. + +See the [Credential Helper Spec][cred-helper-spec] for details. + +[cred-helper-design]: https://github.com/bazelbuild/proposals/blob/main/designs/2022-06-07-bazel-credential-helpers.md +[cred-helper-spec]: https://github.com/EngFlow/credential-helper-spec/blob/main/spec.md + + +#### Basic Example: + +The simplest form of a credential helper is a bash script that accepts an arg and spits out JSON to +stdout. For a service like Google Artifact Registry that uses ['Basic' HTTP Auth][rfc7617] and does +not provide a credential helper that conforms to the [spec][cred-helper-spec], the script might +look like: + +```bash +#!/bin/bash +# cred_helper.sh +ARG=$1 # but we don't do anything with it as it's always "get" + +# formatting is optional +echo '{' +echo ' "headers": {' +echo ' "Authorization": ["Basic dGVzdDoxMjPCow=="]' +echo ' }' +echo '}' +``` + +Configure Bazel to use this credential helper for your python index `example.com`: + +``` +# .bazelrc +build --credential_helper=example.com=/full/path/to/cred_helper.sh +``` + +Bazel will call this file like `cred_helper.sh get` and use the returned JSON to inject headers +into whatever HTTP(S) request it performs against `example.com`. + +[rfc7617]: https://datatracker.ietf.org/doc/html/rfc7617 diff --git a/examples/bzlmod/libs/my_lib/__init__.py b/examples/bzlmod/libs/my_lib/__init__.py index 8ce96ea207..271e933417 100644 --- a/examples/bzlmod/libs/my_lib/__init__.py +++ b/examples/bzlmod/libs/my_lib/__init__.py @@ -19,4 +19,10 @@ def websockets_is_for_python_version(sanitized_version_check): # We are checking that the name of the repository folders # match the expected generated names. If we update the folder # structure or naming we will need to modify this test. - return f"_{sanitized_version_check}_websockets" in websockets.__file__ + want = f"_{sanitized_version_check}_websockets" + got_full = websockets.__file__ + if want not in got_full: + print(f"Failed, expected '{want}' to be a substring of '{got_full}'.") + return False + + return True diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index 9dab53c039..e2d2608c7c 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -7,6 +7,13 @@ load( "PrecompileSourceRetentionFlag", "PycCollectionFlag", ) +load( + "//python/private:pip_flags.bzl", + "INTERNAL_FLAGS", + "UniversalWhlFlag", + "UseWhlFlag", + "WhlLibcFlag", +) load(":config_settings.bzl", "construct_config_settings") filegroup( @@ -61,3 +68,63 @@ string_flag( # NOTE: Only public because its an implicit dependency visibility = ["//visibility:public"], ) + +# This is used for pip and hermetic toolchain resolution. +string_flag( + name = "py_linux_libc", + build_setting_default = WhlLibcFlag.GLIBC, + values = sorted(WhlLibcFlag.__members__.values()), + # NOTE: Only public because it is used in pip hub and toolchain repos. + visibility = ["//visibility:public"], +) + +# pip.parse related flags + +string_flag( + name = "pip_whl", + build_setting_default = UseWhlFlag.AUTO, + values = sorted(UseWhlFlag.__members__.values()), + # NOTE: Only public because it is used in pip hub repos. + visibility = ["//visibility:public"], +) + +string_flag( + name = "pip_whl_osx_arch", + build_setting_default = UniversalWhlFlag.ARCH, + values = sorted(UniversalWhlFlag.__members__.values()), + # NOTE: Only public because it is used in pip hub repos. + visibility = ["//visibility:public"], +) + +string_flag( + name = "pip_whl_glibc_version", + build_setting_default = "", + # NOTE: Only public because it is used in pip hub repos. + visibility = ["//visibility:public"], +) + +string_flag( + name = "pip_whl_muslc_version", + build_setting_default = "", + # NOTE: Only public because it is used in pip hub repos. + visibility = ["//visibility:public"], +) + +string_flag( + name = "pip_whl_osx_version", + build_setting_default = "", + # NOTE: Only public because it is used in pip hub repos. + visibility = ["//visibility:public"], +) + +# private pip whl related flags. Their values cannot be changed and they +# are an implementation detail of how `pip_config_settings` work. +[ + string_flag( + name = "_internal_pip_" + flag, + build_setting_default = "", + values = [""], + visibility = ["//visibility:public"], + ) + for flag in INTERNAL_FLAGS +] diff --git a/python/config_settings/transition.bzl b/python/config_settings/transition.bzl index cd54b21956..48b0447ede 100644 --- a/python/config_settings/transition.bzl +++ b/python/config_settings/transition.bzl @@ -53,12 +53,14 @@ def _transition_py_impl(ctx): for file in target[DefaultInfo].default_runfiles.files.to_list(): if file.short_path == expected_target_path: zipfile = file - zipfile_symlink = ctx.actions.declare_file(ctx.attr.name + ".zip") - ctx.actions.symlink( - is_executable = True, - output = zipfile_symlink, - target_file = zipfile, - ) + + if zipfile: + zipfile_symlink = ctx.actions.declare_file(ctx.attr.name + ".zip") + ctx.actions.symlink( + is_executable = True, + output = zipfile_symlink, + target_file = zipfile, + ) env = {} for k, v in ctx.attr.env.items(): env[k] = ctx.expand_location(v) @@ -75,7 +77,10 @@ def _transition_py_impl(ctx): elif BuiltinPyRuntimeInfo in target: py_runtime_info = target[BuiltinPyRuntimeInfo] else: - fail("target {} does not have rules_python PyRuntimeInfo or builtin PyRuntimeInfo".format(target)) + fail( + "target {} does not have rules_python PyRuntimeInfo or builtin PyRuntimeInfo. ".format(target) + + "There is likely no toolchain being matched to your configuration, use --toolchain_resolution_debug parameter to get more information", + ) providers = [ DefaultInfo( diff --git a/python/pip_install/tools/wheel_installer/wheel.py b/python/pip_install/tools/wheel_installer/wheel.py index b84c214018..3d6780de9a 100644 --- a/python/pip_install/tools/wheel_installer/wheel.py +++ b/python/pip_install/tools/wheel_installer/wheel.py @@ -51,6 +51,7 @@ class Arch(Enum): aarch64 = 3 ppc = 4 s390x = 5 + arm = 6 amd64 = x86_64 arm64 = aarch64 i386 = x86_32 diff --git a/python/pip_install/tools/wheel_installer/wheel_test.py b/python/pip_install/tools/wheel_installer/wheel_test.py index acf2315ee9..3ddfaf7f2e 100644 --- a/python/pip_install/tools/wheel_installer/wheel_test.py +++ b/python/pip_install/tools/wheel_installer/wheel_test.py @@ -371,17 +371,17 @@ def test_can_get_specific_from_string(self): def test_can_get_all_for_py_version(self): cp39 = wheel.Platform.all(minor_version=9) - self.assertEqual(15, len(cp39), f"Got {cp39}") + self.assertEqual(18, len(cp39), f"Got {cp39}") self.assertEqual(cp39, wheel.Platform.from_string("cp39_*")) def test_can_get_all_for_os(self): linuxes = wheel.Platform.all(wheel.OS.linux, minor_version=9) - self.assertEqual(5, len(linuxes)) + self.assertEqual(6, len(linuxes)) self.assertEqual(linuxes, wheel.Platform.from_string("cp39_linux_*")) def test_can_get_all_for_os_for_host_python(self): linuxes = wheel.Platform.all(wheel.OS.linux) - self.assertEqual(5, len(linuxes)) + self.assertEqual(6, len(linuxes)) self.assertEqual(linuxes, wheel.Platform.from_string("linux_*")) def test_specific_version_specializations(self): @@ -425,6 +425,7 @@ def test_linux_specializations(self): wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.aarch64), wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.ppc), wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.s390x), + wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.arm), ] self.assertEqual(want, all_specializations) @@ -441,6 +442,7 @@ def test_osx_specializations(self): wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.aarch64), wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.ppc), wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.s390x), + wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.arm), ] self.assertEqual(want, all_specializations) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index ab077c6f59..d73cee8ae4 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -144,6 +144,23 @@ bzl_library( srcs = ["parse_whl_name.bzl"], ) +bzl_library( + name = "pip_flags_bzl", + srcs = ["pip_flags.bzl"], + deps = [ + ":enum_bzl", + ], +) + +bzl_library( + name = "pip_repo_name_bzl", + srcs = ["pip_repo_name.bzl"], + deps = [ + ":normalize_name_bzl", + ":parse_whl_name_bzl", + ], +) + bzl_library( name = "pypi_index_bzl", srcs = ["pypi_index.bzl"], @@ -234,6 +251,8 @@ bzl_library( name = "py_toolchain_suite_bzl", srcs = ["py_toolchain_suite.bzl"], deps = [ + ":config_settings_bzl", + ":text_util_bzl", ":toolchain_types_bzl", "@bazel_skylib//lib:selects", ], diff --git a/python/private/bzlmod/BUILD.bazel b/python/private/bzlmod/BUILD.bazel index 2eab575726..3362f34ffd 100644 --- a/python/private/bzlmod/BUILD.bazel +++ b/python/private/bzlmod/BUILD.bazel @@ -36,6 +36,7 @@ bzl_library( "//python/private:normalize_name_bzl", "//python/private:parse_requirements_bzl", "//python/private:parse_whl_name_bzl", + "//python/private:pip_repo_name_bzl", "//python/private:version_label_bzl", ":bazel_features_bzl", ] + [ diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl index 8702f1fbe7..122debd2a2 100644 --- a/python/private/bzlmod/pip.bzl +++ b/python/private/bzlmod/pip.bzl @@ -26,11 +26,11 @@ load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") load("//python/private:parse_whl_name.bzl", "parse_whl_name") +load("//python/private:pip_repo_name.bzl", "pip_repo_name") load("//python/private:pypi_index.bzl", "simpleapi_download") load("//python/private:render_pkg_aliases.bzl", "whl_alias") load("//python/private:repo_utils.bzl", "repo_utils") load("//python/private:version_label.bzl", "version_label") -load("//python/private:whl_target_platforms.bzl", "select_whl") load(":pip_repository.bzl", "pip_repository") def _parse_version(version): @@ -199,19 +199,6 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s repository_platform = host_platform(module_ctx.os) for whl_name, requirements in requirements_by_platform.items(): - requirement = select_requirement( - requirements, - platform = repository_platform, - ) - if not requirement: - # Sometimes the package is not present for host platform if there - # are whls specified only in particular requirements files, in that - # case just continue, however, if the download_only flag is set up, - # then the user can also specify the target platform of the wheel - # packages they want to download, in that case there will be always - # a requirement here, so we will not be in this code branch. - continue - # We are not using the "sanitized name" because the user # would need to guess what name we modified the whl name # to. @@ -223,11 +210,9 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s # Construct args separately so that the lock file can be smaller and does not include unused # attrs. - repo_name = "{}_{}".format(pip_name, whl_name) whl_library_args = dict( repo = pip_name, dep_template = "@{}//{{name}}:{{target}}".format(hub_name), - requirement = requirement.requirement_line, ) maybe_args = dict( # The following values are safe to omit if they have false like values @@ -237,7 +222,6 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s environment = pip_attr.environment, envsubst = pip_attr.envsubst, experimental_target_platforms = pip_attr.experimental_target_platforms, - extra_pip_args = requirement.extra_pip_args, group_deps = group_deps, group_name = group_name, pip_data_exclude = pip_attr.pip_data_exclude, @@ -257,51 +241,83 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s ) whl_library_args.update({k: v for k, (v, default) in maybe_args_with_default.items() if v == default}) - if requirement.whls or requirement.sdist: - logger.debug(lambda: "Selecting a compatible dist for {} from dists:\n{}".format( - repository_platform, - json.encode( - struct( - whls = requirement.whls, - sdist = requirement.sdist, - ), - ), - )) - distribution = select_whl( - whls = requirement.whls, - want_platform = repository_platform, - ) or requirement.sdist - - logger.debug(lambda: "Selected: {}".format(distribution)) - - if distribution: - is_hub_reproducible = False - whl_library_args["requirement"] = requirement.srcs.requirement - whl_library_args["urls"] = [distribution.url] - whl_library_args["sha256"] = distribution.sha256 - whl_library_args["filename"] = distribution.filename - if pip_attr.netrc: - whl_library_args["netrc"] = pip_attr.netrc - if pip_attr.auth_patterns: - whl_library_args["auth_patterns"] = pip_attr.auth_patterns - - # pip is not used to download wheels and the python `whl_library` helpers are only extracting things - whl_library_args.pop("extra_pip_args", None) - - # This is no-op because pip is not used to download the wheel. - whl_library_args.pop("download_only", None) - else: - logger.warn("falling back to pip for installing the right file for {}".format(requirement.requirement_line)) + if get_index_urls: + # TODO @aignas 2024-05-26: move to a separate function + found_something = False + for requirement in requirements: + for distribution in requirement.whls + [requirement.sdist]: + if not distribution: + # sdist may be None + continue + + found_something = True + is_hub_reproducible = False + + if pip_attr.netrc: + whl_library_args["netrc"] = pip_attr.netrc + if pip_attr.auth_patterns: + whl_library_args["auth_patterns"] = pip_attr.auth_patterns + + # pip is not used to download wheels and the python `whl_library` helpers are only extracting things + whl_library_args.pop("extra_pip_args", None) + + # This is no-op because pip is not used to download the wheel. + whl_library_args.pop("download_only", None) + + repo_name = pip_repo_name(pip_name, distribution.filename, distribution.sha256) + whl_library_args["requirement"] = requirement.srcs.requirement + whl_library_args["urls"] = [distribution.url] + whl_library_args["sha256"] = distribution.sha256 + whl_library_args["filename"] = distribution.filename + whl_library_args["experimental_target_platforms"] = requirement.target_platforms + + # Pure python wheels or sdists may need to have a platform here + target_platforms = None + if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"): + if len(requirements) > 1: + target_platforms = requirement.target_platforms + + whl_library(name = repo_name, **dict(sorted(whl_library_args.items()))) + + whl_map[hub_name].setdefault(whl_name, []).append( + whl_alias( + repo = repo_name, + version = major_minor, + filename = distribution.filename, + target_platforms = target_platforms, + ), + ) + + if found_something: + continue + + requirement = select_requirement( + requirements, + platform = repository_platform, + ) + if not requirement: + # Sometimes the package is not present for host platform if there + # are whls specified only in particular requirements files, in that + # case just continue, however, if the download_only flag is set up, + # then the user can also specify the target platform of the wheel + # packages they want to download, in that case there will be always + # a requirement here, so we will not be in this code branch. + continue + elif get_index_urls: + logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line)) + + whl_library_args["requirement"] = requirement.requirement_line + if requirement.extra_pip_args: + whl_library_args["extra_pip_args"] = requirement.extra_pip_args # We sort so that the lock-file remains the same no matter the order of how the # args are manipulated in the code going before. + repo_name = "{}_{}".format(pip_name, whl_name) whl_library(name = repo_name, **dict(sorted(whl_library_args.items()))) whl_map[hub_name].setdefault(whl_name, []).append( whl_alias( repo = repo_name, version = major_minor, - # Call Label() to canonicalize because its used in a different context - config_setting = Label("//python/config_settings:is_python_" + major_minor), ), ) diff --git a/python/private/bzlmod/pip_repository.bzl b/python/private/bzlmod/pip_repository.bzl index 3a09766f65..0f962031d6 100644 --- a/python/private/bzlmod/pip_repository.bzl +++ b/python/private/bzlmod/pip_repository.bzl @@ -14,7 +14,11 @@ "" -load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias") +load( + "//python/private:render_pkg_aliases.bzl", + "render_multiplatform_pkg_aliases", + "whl_alias", +) load("//python/private:text_util.bzl", "render") _BUILD_FILE_CONTENTS = """\ @@ -26,12 +30,13 @@ exports_files(["requirements.bzl"]) def _pip_repository_impl(rctx): bzl_packages = rctx.attr.whl_map.keys() - aliases = render_pkg_aliases( + aliases = render_multiplatform_pkg_aliases( aliases = { key: [whl_alias(**v) for v in json.decode(values)] for key, values in rctx.attr.whl_map.items() }, default_version = rctx.attr.default_version, + default_config_setting = "//_config:is_python_" + rctx.attr.default_version, requirement_cycles = rctx.attr.groups, ) for path, contents in aliases.items(): diff --git a/python/private/config_settings.bzl b/python/private/config_settings.bzl index 75f88de4ac..92b96b3264 100644 --- a/python/private/config_settings.bzl +++ b/python/private/config_settings.bzl @@ -103,12 +103,16 @@ def is_python_config_setting(name, *, python_version, reuse_conditions = None, * fail("The 'python_version' must be known to 'rules_python', choose from the values: {}".format(VERSION_FLAG_VALUES.keys())) python_versions = VERSION_FLAG_VALUES[python_version] + extra_flag_values = kwargs.pop("flag_values", {}) + if _PYTHON_VERSION_FLAG in extra_flag_values: + fail("Cannot set '{}' in the flag values".format(_PYTHON_VERSION_FLAG)) + if len(python_versions) == 1: native.config_setting( name = name, flag_values = { _PYTHON_VERSION_FLAG: python_version, - }, + } | extra_flag_values, **kwargs ) return @@ -138,7 +142,7 @@ def is_python_config_setting(name, *, python_version, reuse_conditions = None, * for name_, flag_values_ in create_config_settings.items(): native.config_setting( name = name_, - flag_values = flag_values_, + flag_values = flag_values_ | extra_flag_values, **kwargs ) diff --git a/python/private/parse_requirements.bzl b/python/private/parse_requirements.bzl index c6a498539f..cb5024c841 100644 --- a/python/private/parse_requirements.bzl +++ b/python/private/parse_requirements.bzl @@ -451,6 +451,18 @@ def _add_dists(requirement, index_urls, python_version, logger = None): if logger: logger.warn("Could not find a whl or an sdist with sha256={}".format(sha256)) + yanked = {} + for dist in whls + [sdist]: + if dist and dist.yanked: + yanked.setdefault(dist.yanked, []).append(dist.filename) + if yanked: + logger.warn(lambda: "\n".join([ + "the following distributions got yanked:", + ] + [ + "reason: {}\n {}".format(reason, "\n".join(sorted(dists))) + for reason, dists in yanked.items() + ])) + # Filter out the wheels that are incompatible with the target_platforms. whls = select_whls( whls = whls, diff --git a/python/private/pip_config_settings.bzl b/python/private/pip_config_settings.bzl new file mode 100644 index 0000000000..2fe3c87a10 --- /dev/null +++ b/python/private/pip_config_settings.bzl @@ -0,0 +1,366 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module is used to construct the config settings for selecting which distribution is used in the pip hub repository. + +Bazel's selects work by selecting the most-specialized configuration setting +that matches the target platform. We can leverage this fact to ensure that the +most specialized wheels are used by default with the users being able to +configure string_flag values to select the less specialized ones. + +The list of specialization of the dists goes like follows: +* sdist +* py*-none-any.whl +* py*-abi3-any.whl +* py*-cpxy-any.whl +* cp*-none-any.whl +* cp*-abi3-any.whl +* cp*-cpxy-plat.whl +* py*-none-plat.whl +* py*-abi3-plat.whl +* py*-cpxy-plat.whl +* cp*-none-plat.whl +* cp*-abi3-plat.whl +* cp*-cpxy-plat.whl + +Note, that here the specialization of musl vs manylinux wheels is the same in +order to ensure that the matching fails if the user requests for `musl` and we don't have it or vice versa. +""" + +load(":config_settings.bzl", "is_python_config_setting") +load( + ":pip_flags.bzl", + "INTERNAL_FLAGS", + "UniversalWhlFlag", + "UseWhlFlag", + "WhlLibcFlag", +) + +FLAGS = struct( + **{ + f: str(Label("//python/config_settings:" + f)) + for f in [ + "python_version", + "pip_whl", + "pip_whl_glibc_version", + "pip_whl_muslc_version", + "pip_whl_osx_arch", + "pip_whl_osx_version", + "py_linux_libc", + ] + } +) + +# Here we create extra string flags that are just to work with the select +# selecting the most specialized match. We don't allow the user to change +# them. +_flags = struct( + **{ + f: str(Label("//python/config_settings:_internal_pip_" + f)) + for f in INTERNAL_FLAGS + } +) + +def pip_config_settings( + *, + python_versions = [], + glibc_versions = [], + muslc_versions = [], + osx_versions = [], + target_platforms = [], + name = None, + visibility = None, + alias_rule = None, + config_setting_rule = None): + """Generate all of the pip config settings. + + Args: + name (str): Currently unused. + python_versions (list[str]): The list of python versions to configure + config settings for. + glibc_versions (list[str]): The list of glibc version of the wheels to + configure config settings for. + muslc_versions (list[str]): The list of musl version of the wheels to + configure config settings for. + osx_versions (list[str]): The list of OSX OS versions to configure + config settings for. + target_platforms (list[str]): The list of "{os}_{cpu}" for deriving + constraint values for each condition. + visibility (list[str], optional): The visibility to be passed to the + exposed labels. All other labels will be private. + alias_rule (rule): The alias rule to use for creating the + objects. Can be overridden for unit tests reasons. + config_setting_rule (rule): The config setting rule to use for creating the + objects. Can be overridden for unit tests reasons. + """ + + glibc_versions = [""] + glibc_versions + muslc_versions = [""] + muslc_versions + osx_versions = [""] + osx_versions + target_platforms = [("", "")] + [ + t.split("_", 1) + for t in target_platforms + ] + + alias_rule = alias_rule or native.alias + + for version in python_versions: + is_python = "is_python_{}".format(version) + alias_rule( + name = is_python, + actual = Label("//python/config_settings:" + is_python), + visibility = visibility, + ) + + for os, cpu in target_platforms: + constraint_values = [] + suffix = "" + if os: + constraint_values.append("@platforms//os:" + os) + suffix += "_" + os + if cpu: + constraint_values.append("@platforms//cpu:" + cpu) + suffix += "_" + cpu + + _sdist_config_setting( + name = "sdist" + suffix, + constraint_values = constraint_values, + visibility = visibility, + config_setting_rule = config_setting_rule, + ) + for python_version in python_versions: + _sdist_config_setting( + name = "cp{}_sdist{}".format(python_version, suffix), + python_version = python_version, + constraint_values = constraint_values, + visibility = visibility, + config_setting_rule = config_setting_rule, + ) + + for python_version in [""] + python_versions: + _whl_config_settings( + suffix = suffix, + plat_flag_values = _plat_flag_values( + os = os, + cpu = cpu, + osx_versions = osx_versions, + glibc_versions = glibc_versions, + muslc_versions = muslc_versions, + ), + constraint_values = constraint_values, + python_version = python_version, + visibility = visibility, + config_setting_rule = config_setting_rule, + ) + +def _whl_config_settings(*, suffix, plat_flag_values, **kwargs): + # With the following three we cover different per-version wheels + python_version = kwargs.get("python_version") + py = "cp{}_py".format(python_version) if python_version else "py" + pycp = "cp{}_cp3x".format(python_version) if python_version else "cp3x" + + flag_values = {} + + for n, f in { + "{}_none_any{}".format(py, suffix): None, + "{}3_none_any{}".format(py, suffix): _flags.whl_py3, + "{}3_abi3_any{}".format(py, suffix): _flags.whl_py3_abi3, + "{}_none_any{}".format(pycp, suffix): _flags.whl_pycp3x, + "{}_abi3_any{}".format(pycp, suffix): _flags.whl_pycp3x_abi3, + "{}_cp_any{}".format(pycp, suffix): _flags.whl_pycp3x_abicp, + }.items(): + if f and f in flag_values: + fail("BUG") + elif f: + flag_values[f] = "" + + _whl_config_setting( + name = n, + flag_values = flag_values, + **kwargs + ) + + generic_flag_values = flag_values + + for (suffix, flag_values) in plat_flag_values: + flag_values = flag_values | generic_flag_values + + for n, f in { + "{}_none_{}".format(py, suffix): _flags.whl_plat, + "{}3_none_{}".format(py, suffix): _flags.whl_plat_py3, + "{}3_abi3_{}".format(py, suffix): _flags.whl_plat_py3_abi3, + "{}_none_{}".format(pycp, suffix): _flags.whl_plat_pycp3x, + "{}_abi3_{}".format(pycp, suffix): _flags.whl_plat_pycp3x_abi3, + "{}_cp_{}".format(pycp, suffix): _flags.whl_plat_pycp3x_abicp, + }.items(): + if f and f in flag_values: + fail("BUG") + elif f: + flag_values[f] = "" + + _whl_config_setting( + name = n, + flag_values = flag_values, + **kwargs + ) + +def _to_version_string(version, sep = "."): + if not version: + return "" + + return "{}{}{}".format(version[0], sep, version[1]) + +def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions): + ret = [] + if os == "": + return [] + elif os == "windows": + ret.append(("{}_{}".format(os, cpu), {})) + elif os == "osx": + for cpu_, arch in { + cpu: UniversalWhlFlag.ARCH, + cpu + "_universal2": UniversalWhlFlag.UNIVERSAL, + }.items(): + for osx_version in osx_versions: + flags = { + FLAGS.pip_whl_osx_version: _to_version_string(osx_version), + } + if arch == UniversalWhlFlag.ARCH: + flags[FLAGS.pip_whl_osx_arch] = arch + + if not osx_version: + suffix = "{}_{}".format(os, cpu_) + else: + suffix = "{}_{}_{}".format(os, _to_version_string(osx_version, "_"), cpu_) + + ret.append((suffix, flags)) + + elif os == "linux": + for os_prefix, linux_libc in { + os: WhlLibcFlag.GLIBC, + "many" + os: WhlLibcFlag.GLIBC, + "musl" + os: WhlLibcFlag.MUSL, + }.items(): + if linux_libc == WhlLibcFlag.GLIBC: + libc_versions = glibc_versions + libc_flag = FLAGS.pip_whl_glibc_version + elif linux_libc == WhlLibcFlag.MUSL: + libc_versions = muslc_versions + libc_flag = FLAGS.pip_whl_muslc_version + else: + fail("Unsupported libc type: {}".format(linux_libc)) + + for libc_version in libc_versions: + if libc_version and os_prefix == os: + continue + elif libc_version: + suffix = "{}_{}_{}".format(os_prefix, _to_version_string(libc_version, "_"), cpu) + else: + suffix = "{}_{}".format(os_prefix, cpu) + + ret.append(( + suffix, + { + FLAGS.py_linux_libc: linux_libc, + libc_flag: _to_version_string(libc_version), + }, + )) + else: + fail("Unsupported os: {}".format(os)) + + return ret + +def _whl_config_setting(*, name, flag_values, visibility, config_setting_rule = None, **kwargs): + config_setting_rule = config_setting_rule or _config_setting_or + config_setting_rule( + name = "is_" + name, + flag_values = flag_values | { + FLAGS.pip_whl: UseWhlFlag.ONLY, + }, + default = flag_values | { + _flags.whl_py2_py3: "", + FLAGS.pip_whl: UseWhlFlag.AUTO, + }, + visibility = visibility, + **kwargs + ) + +def _sdist_config_setting(*, name, visibility, config_setting_rule = None, **kwargs): + config_setting_rule = config_setting_rule or _config_setting_or + config_setting_rule( + name = "is_" + name, + flag_values = {FLAGS.pip_whl: UseWhlFlag.NO}, + default = {FLAGS.pip_whl: UseWhlFlag.AUTO}, + visibility = visibility, + **kwargs + ) + +def _config_setting_or(*, name, flag_values, default, visibility, **kwargs): + match_name = "_{}".format(name) + default_name = "_{}_default".format(name) + + native.alias( + name = name, + actual = select({ + "//conditions:default": default_name, + match_name: match_name, + }), + visibility = visibility, + ) + + _config_setting( + name = match_name, + flag_values = flag_values, + visibility = visibility, + **kwargs + ) + _config_setting( + name = default_name, + flag_values = default, + visibility = visibility, + **kwargs + ) + +def _config_setting(python_version = "", **kwargs): + if python_version: + # NOTE @aignas 2024-05-26: with this we are getting about 24k internal + # config_setting targets in our unit tests. Whilst the number of the + # external dependencies does not dictate this number, it does mean that + # bazel will take longer to parse stuff. This would be especially + # noticeable in repos, which use multiple hub repos within a single + # workspace. + # + # A way to reduce the number of targets would be: + # * put them to a central location and teach this code to just alias them, + # maybe we should create a pip_config_settings repo within the pip + # extension, which would collect config settings for all hub_repos. + # * put them in rules_python - this has the drawback of exposing things like + # is_cp3.10_linux and users may start depending upon the naming + # convention and this API is very unstable. + is_python_config_setting( + python_version = python_version, + **kwargs + ) + else: + # We need this to ensure that there are no ambiguous matches when python_version + # is unset, which usually happens when we are not using the python version aware + # rules. + flag_values = kwargs.pop("flag_values", {}) | { + FLAGS.python_version: "", + } + native.config_setting( + flag_values = flag_values, + **kwargs + ) diff --git a/python/private/pip_flags.bzl b/python/private/pip_flags.bzl new file mode 100644 index 0000000000..c8154ff383 --- /dev/null +++ b/python/private/pip_flags.bzl @@ -0,0 +1,69 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Values and helpers for pip_repository related flags. + +NOTE: The transitive loads of this should be kept minimal. This avoids loading +unnecessary files when all that are needed are flag definitions. +""" + +load(":enum.bzl", "enum") + +# Determines if we should use whls for third party +# +# buildifier: disable=name-conventions +UseWhlFlag = enum( + # Automatically decide the effective value based on environment, target + # platform and the presence of distributions for a particular package. + AUTO = "auto", + # Do not use `sdist` and fail if there are no available whls suitable for the target platform. + ONLY = "only", + # Do not use whl distributions and instead build the whls from `sdist`. + NO = "no", +) + +# Determines whether universal wheels should be preferred over arch platform specific ones. +# +# buildifier: disable=name-conventions +UniversalWhlFlag = enum( + # Prefer platform-specific wheels over universal wheels. + ARCH = "arch", + # Prefer universal wheels over platform-specific wheels. + UNIVERSAL = "universal", +) + +# Determines which libc flavor is preferred when selecting the linux whl distributions. +# +# buildifier: disable=name-conventions +WhlLibcFlag = enum( + # Prefer glibc wheels (e.g. manylinux_2_17_x86_64 or linux_x86_64) + GLIBC = "glibc", + # Prefer musl wheels (e.g. musllinux_2_17_x86_64) + MUSL = "musl", +) + +INTERNAL_FLAGS = [ + "whl_plat", + "whl_plat_py3", + "whl_plat_py3_abi3", + "whl_plat_pycp3x", + "whl_plat_pycp3x_abi3", + "whl_plat_pycp3x_abicp", + "whl_py2_py3", + "whl_py3", + "whl_py3_abi3", + "whl_pycp3x", + "whl_pycp3x_abi3", + "whl_pycp3x_abicp", +] diff --git a/python/private/pip_repo_name.bzl b/python/private/pip_repo_name.bzl new file mode 100644 index 0000000000..bef4304e15 --- /dev/null +++ b/python/private/pip_repo_name.bzl @@ -0,0 +1,52 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A function to convert a dist name to a valid bazel repo name. +""" + +load(":normalize_name.bzl", "normalize_name") +load(":parse_whl_name.bzl", "parse_whl_name") + +def pip_repo_name(prefix, filename, sha256): + """Return a valid whl_library repo name given a distribution filename. + + Args: + prefix: str, the prefix of the whl_library. + filename: str, the filename of the distribution. + sha256: str, the sha256 of the distribution. + + Returns: + a string that can be used in `whl_library`. + """ + parts = [prefix] + + if not filename.endswith(".whl"): + # Then the filename is basically foo-3.2.1. + parts.append(normalize_name(filename.rpartition("-")[0])) + parts.append("sdist") + else: + parsed = parse_whl_name(filename) + name = normalize_name(parsed.distribution) + python_tag, _, _ = parsed.python_tag.partition(".") + abi_tag, _, _ = parsed.abi_tag.partition(".") + platform_tag, _, _ = parsed.platform_tag.partition(".") + + parts.append(name) + parts.append(python_tag) + parts.append(abi_tag) + parts.append(platform_tag) + + parts.append(sha256[:8]) + + return "_".join(parts) diff --git a/python/private/py_toolchain_suite.bzl b/python/private/py_toolchain_suite.bzl index 9971a8a4c3..174c36f782 100644 --- a/python/private/py_toolchain_suite.bzl +++ b/python/private/py_toolchain_suite.bzl @@ -22,7 +22,7 @@ load( "TARGET_TOOLCHAIN_TYPE", ) -def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_python_version_constraint, **kwargs): +def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_python_version_constraint, flag_values, **kwargs): """For internal use only. Args: @@ -30,6 +30,7 @@ def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_pyth user_repository_name: The name of the user repository. python_version: The full (X.Y.Z) version of the interpreter. set_python_version_constraint: True or False as a string. + flag_values: Extra flag values to match for this toolchain. **kwargs: extra args passed to the `toolchain` calls. """ @@ -38,23 +39,39 @@ def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_pyth # string as we cannot have list of bools in build rule attribues. # This if statement does not appear to work unless it is in the # toolchain file. - if set_python_version_constraint == "True": + if set_python_version_constraint in ["True", "False"]: major_minor, _, _ = python_version.rpartition(".") + python_versions = [major_minor, python_version] + if set_python_version_constraint == "False": + python_versions.append("") + match_any = [] + for i, v in enumerate(python_versions): + name = "{prefix}_{python_version}_{i}".format( + prefix = prefix, + python_version = python_version, + i = i, + ) + match_any.append(name) + native.config_setting( + name = name, + flag_values = flag_values | { + Label("@rules_python//python/config_settings:python_version"): v, + }, + visibility = ["//visibility:private"], + ) + + name = "{prefix}_version_setting_{python_version}".format( + prefix = prefix, + python_version = python_version, + visibility = ["//visibility:private"], + ) selects.config_setting_group( - name = prefix + "_version_setting", - match_any = [ - Label("//python/config_settings:is_python_%s" % v) - for v in [ - major_minor, - python_version, - ] - ], + name = name, + match_any = match_any, visibility = ["//visibility:private"], ) - target_settings = [prefix + "_version_setting"] - elif set_python_version_constraint == "False": - target_settings = [] + target_settings = [name] else: fail(("Invalid set_python_version_constraint value: got {} {}, wanted " + "either the string 'True' or the string 'False'; " + diff --git a/python/private/render_pkg_aliases.bzl b/python/private/render_pkg_aliases.bzl index bc1bab2049..a37c32882e 100644 --- a/python/private/render_pkg_aliases.bzl +++ b/python/private/render_pkg_aliases.bzl @@ -30,7 +30,9 @@ load( "WHEEL_FILE_PUBLIC_LABEL", ) load(":normalize_name.bzl", "normalize_name") +load(":parse_whl_name.bzl", "parse_whl_name") load(":text_util.bzl", "render") +load(":whl_target_platforms.bzl", "whl_target_platforms") NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -51,10 +53,27 @@ If the value is missing, then the "default" Python version is being used, which has a "null" version value and will not match version constraints. """ +NO_MATCH_ERROR_MESSAGE_TEMPLATE_V2 = """\ +No matching wheel for current configuration's Python version. + +The current build configuration's Python version doesn't match any of the Python +wheels available for this wheel. This wheel supports the following Python +configuration settings: + {config_settings} + +To determine the current configuration's Python version, run: + `bazel config ` (shown further below) +and look for + {rules_python}//python/config_settings:python_version + +If the value is missing, then the "default" Python version is being used, +which has a "null" version value and will not match version constraints. +""" + def _render_whl_library_alias( *, name, - default_version, + default_config_setting, aliases, target_name, **kwargs): @@ -78,7 +97,7 @@ def _render_whl_library_alias( for alias in sorted(aliases, key = lambda x: x.version): actual = "@{repo}//:{name}".format(repo = alias.repo, name = target_name) selects.setdefault(actual, []).append(alias.config_setting) - if alias.version == default_version: + if alias.config_setting == default_config_setting: selects[actual].append("//conditions:default") no_match_error = None @@ -102,21 +121,23 @@ def _render_whl_library_alias( **kwargs ) -def _render_common_aliases(*, name, aliases, default_version = None, group_name = None): +def _render_common_aliases(*, name, aliases, default_config_setting = None, group_name = None): lines = [ """load("@bazel_skylib//lib:selects.bzl", "selects")""", """package(default_visibility = ["//visibility:public"])""", ] - versions = None + config_settings = None if aliases: - versions = sorted([v.version for v in aliases if v.version]) + config_settings = sorted([v.config_setting for v in aliases if v.config_setting]) - if not versions or default_version in versions: + if not config_settings or default_config_setting in config_settings: pass else: - error_msg = NO_MATCH_ERROR_MESSAGE_TEMPLATE.format( - supported_versions = ", ".join(versions), + error_msg = NO_MATCH_ERROR_MESSAGE_TEMPLATE_V2.format( + config_settings = render.indent( + "\n".join(config_settings), + ).lstrip(), rules_python = "rules_python", ) @@ -126,7 +147,7 @@ def _render_common_aliases(*, name, aliases, default_version = None, group_name # This is to simplify the code in _render_whl_library_alias and to ensure # that we don't pass a 'default_version' that is not in 'versions'. - default_version = None + default_config_setting = None lines.append( render.alias( @@ -138,7 +159,7 @@ def _render_common_aliases(*, name, aliases, default_version = None, group_name [ _render_whl_library_alias( name = name, - default_version = default_version, + default_config_setting = default_config_setting, aliases = aliases, target_name = target_name, visibility = ["//_groups:__subpackages__"] if name.startswith("_") else None, @@ -167,7 +188,7 @@ def _render_common_aliases(*, name, aliases, default_version = None, group_name return "\n\n".join(lines) -def render_pkg_aliases(*, aliases, default_version = None, requirement_cycles = None): +def render_pkg_aliases(*, aliases, default_config_setting = None, requirement_cycles = None): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -177,7 +198,7 @@ def render_pkg_aliases(*, aliases, default_version = None, requirement_cycles = Args: aliases: dict, the keys are normalized distribution names and values are the whl_alias instances. - default_version: the default version to be used for the aliases. + default_config_setting: the default to be used for the aliases. requirement_cycles: any package groups to also add. Returns: @@ -206,16 +227,17 @@ def render_pkg_aliases(*, aliases, default_version = None, requirement_cycles = "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( name = normalize_name(name), aliases = pkg_aliases, - default_version = default_version, + default_config_setting = default_config_setting, group_name = whl_group_mapping.get(normalize_name(name)), ).strip() for name, pkg_aliases in aliases.items() } + if requirement_cycles: files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) return files -def whl_alias(*, repo, version = None, config_setting = None, extra_targets = None): +def whl_alias(*, repo, version = None, config_setting = None, filename = None, target_platforms = None): """The bzl_packages value used by by the render_pkg_aliases function. This contains the minimum amount of information required to generate correct @@ -228,9 +250,10 @@ def whl_alias(*, repo, version = None, config_setting = None, extra_targets = No constructed. This is mainly used for better error messages when there is no match found during a select. config_setting: optional(Label or str), the config setting that we should use. Defaults - to "@rules_python//python/config_settings:is_python_{version}". - extra_targets: optional(list[str]), the extra targets that we need to create - aliases for. + to "//_config:is_python_{version}". + filename: optional(str), the distribution filename to derive the config_setting. + target_platforms: optional(list[str]), the list of target_platforms for this + distribution. Returns: a struct with the validated and parsed values. @@ -239,12 +262,361 @@ def whl_alias(*, repo, version = None, config_setting = None, extra_targets = No fail("'repo' must be specified") if version: - config_setting = config_setting or Label("//python/config_settings:is_python_" + version) + config_setting = config_setting or ("//_config:is_python_" + version) config_setting = str(config_setting) return struct( repo = repo, version = version, config_setting = config_setting, - extra_targets = extra_targets or [], + filename = filename, + target_platforms = target_platforms, + ) + +def render_multiplatform_pkg_aliases(*, aliases, default_version = None, **kwargs): + """Render the multi-platform pkg aliases. + + Args: + aliases: dict[str, list(whl_alias)] A list of aliases that will be + transformed from ones having `filename` to ones having `config_setting`. + default_version: str, the default python version. Defaults to None. + **kwargs: extra arguments passed to render_pkg_aliases. + + Returns: + A dict of file paths and their contents. + """ + + flag_versions = get_whl_flag_versions( + aliases = [ + a + for bunch in aliases.values() + for a in bunch + ], + ) + + config_setting_aliases = { + pkg: multiplatform_whl_aliases( + aliases = pkg_aliases, + default_version = default_version, + glibc_versions = flag_versions.get("glibc_versions", []), + muslc_versions = flag_versions.get("muslc_versions", []), + osx_versions = flag_versions.get("osx_versions", []), + ) + for pkg, pkg_aliases in aliases.items() + } + + contents = render_pkg_aliases( + aliases = config_setting_aliases, + **kwargs ) + contents["_config/BUILD.bazel"] = _render_pip_config_settings(**flag_versions) + return contents + +def multiplatform_whl_aliases(*, aliases, default_version = None, **kwargs): + """convert a list of aliases from filename to config_setting ones. + + Args: + aliases: list(whl_alias): The aliases to process. Any aliases that have + the filename set will be converted to a list of aliases, each with + an appropriate config_setting value. + default_version: string | None, the default python version to use. + **kwargs: Extra parameters passed to get_filename_config_settings. + + Returns: + A dict with aliases to be used in the hub repo. + """ + + ret = [] + versioned_additions = {} + for alias in aliases: + if not alias.filename: + ret.append(alias) + continue + + config_settings, all_versioned_settings = get_filename_config_settings( + # TODO @aignas 2024-05-27: pass the parsed whl to reduce the + # number of duplicate operations. + filename = alias.filename, + target_platforms = alias.target_platforms, + python_version = alias.version, + python_default = default_version == alias.version, + **kwargs + ) + + for setting in config_settings: + ret.append(whl_alias( + repo = alias.repo, + version = alias.version, + config_setting = "//_config" + setting, + )) + + # Now for the versioned platform config settings, we need to select one + # that best fits the bill and if there are multiple wheels, e.g. + # manylinux_2_17_x86_64 and manylinux_2_28_x86_64, then we need to select + # the former when the glibc is in the range of [2.17, 2.28) and then chose + # the later if it is [2.28, ...). If the 2.28 wheel was not present in + # the hub, then we would need to use 2.17 for all the glibc version + # configurations. + # + # Here we add the version settings to a dict where we key the range of + # versions that the whl spans. If the wheel supports musl and glibc at + # the same time, we do this for each supported platform, hence the + # double dict. + for default_setting, versioned in all_versioned_settings.items(): + versions = sorted(versioned) + min_version = versions[0] + max_version = versions[-1] + + versioned_additions.setdefault(default_setting, {})[(min_version, max_version)] = struct( + repo = alias.repo, + python_version = alias.version, + settings = versioned, + ) + + versioned = {} + for default_setting, candidates in versioned_additions.items(): + # Sort the candidates by the range of versions the span, so that we + # start with the lowest version. + for _, candidate in sorted(candidates.items()): + # Set the default with the first candidate, which gives us the highest + # compatibility. If the users want to use a higher-version than the default + # they can configure the glibc_version flag. + versioned.setdefault(default_setting, whl_alias( + version = candidate.python_version, + config_setting = "//_config" + default_setting, + repo = candidate.repo, + )) + + # We will be overwriting previously added entries, but that is intended. + for _, setting in sorted(candidate.settings.items()): + versioned[setting] = whl_alias( + version = candidate.python_version, + config_setting = "//_config" + setting, + repo = candidate.repo, + ) + + ret.extend(versioned.values()) + return ret + +def _render_pip_config_settings(python_versions = [], target_platforms = [], osx_versions = [], glibc_versions = [], muslc_versions = []): + return """\ +load("@rules_python//python/private:pip_config_settings.bzl", "pip_config_settings") + +pip_config_settings( + name = "pip_config_settings", + glibc_versions = {glibc_versions}, + muslc_versions = {muslc_versions}, + osx_versions = {osx_versions}, + python_versions = {python_versions}, + target_platforms = {target_platforms}, + visibility = ["//:__subpackages__"], +)""".format( + glibc_versions = render.indent(render.list(glibc_versions)).lstrip(), + muslc_versions = render.indent(render.list(muslc_versions)).lstrip(), + osx_versions = render.indent(render.list(osx_versions)).lstrip(), + python_versions = render.indent(render.list(python_versions)).lstrip(), + target_platforms = render.indent(render.list(target_platforms)).lstrip(), + ) + +def get_whl_flag_versions(aliases): + """Return all of the flag versions that is used by the aliases + + Args: + aliases: list[whl_alias] + + Returns: + dict, which may have keys: + * python_versions + """ + python_versions = {} + glibc_versions = {} + target_platforms = {} + muslc_versions = {} + osx_versions = {} + + for a in aliases: + if not a.version and not a.filename: + continue + + if a.version: + python_versions[a.version] = None + + if not a.filename: + continue + + if a.filename.endswith(".whl") and not a.filename.endswith("-any.whl"): + parsed = parse_whl_name(a.filename) + else: + for plat in a.target_platforms or []: + target_platforms[plat] = None + continue + + for platform_tag in parsed.platform_tag.split("."): + parsed = whl_target_platforms(platform_tag) + + for p in parsed: + target_platforms[p.target_platform] = None + + if platform_tag.startswith("win") or platform_tag.startswith("linux"): + continue + + head, _, tail = platform_tag.partition("_") + major, _, tail = tail.partition("_") + minor, _, tail = tail.partition("_") + if tail: + version = (int(major), int(minor)) + if "many" in head: + glibc_versions[version] = None + elif "musl" in head: + muslc_versions[version] = None + elif "mac" in head: + osx_versions[version] = None + else: + fail(platform_tag) + + return { + k: sorted(v) + for k, v in { + "glibc_versions": glibc_versions, + "muslc_versions": muslc_versions, + "osx_versions": osx_versions, + "python_versions": python_versions, + "target_platforms": target_platforms, + }.items() + if v + } + +def get_filename_config_settings( + *, + filename, + target_platforms, + glibc_versions, + muslc_versions, + osx_versions, + python_version = "", + python_default = True): + """Get the filename config settings. + + Args: + filename: the distribution filename (can be a whl or an sdist). + target_platforms: list[str], target platforms in "{os}_{cpu}" format. + glibc_versions: list[tuple[int, int]], list of versions. + muslc_versions: list[tuple[int, int]], list of versions. + osx_versions: list[tuple[int, int]], list of versions. + python_version: the python version to generate the config_settings for. + python_default: if we should include the setting when python_version is not set. + + Returns: + A tuple: + * A list of config settings that are generated by ./pip_config_settings.bzl + * The list of default version settings. + """ + prefixes = [] + suffixes = [] + if (0, 0) in glibc_versions: + fail("Invalid version in 'glibc_versions': cannot specify (0, 0) as a value") + if (0, 0) in muslc_versions: + fail("Invalid version in 'muslc_versions': cannot specify (0, 0) as a value") + if (0, 0) in osx_versions: + fail("Invalid version in 'osx_versions': cannot specify (0, 0) as a value") + + glibc_versions = sorted(glibc_versions) + muslc_versions = sorted(muslc_versions) + osx_versions = sorted(osx_versions) + setting_supported_versions = {} + + if filename.endswith(".whl"): + parsed = parse_whl_name(filename) + if parsed.python_tag == "py2.py3": + py = "py" + elif parsed.python_tag.startswith("cp"): + py = "cp3x" + else: + py = "py3" + + if parsed.abi_tag.startswith("cp"): + abi = "cp" + else: + abi = parsed.abi_tag + + if parsed.platform_tag == "any": + prefixes = ["{}_{}_any".format(py, abi)] + suffixes = target_platforms + else: + prefixes = ["{}_{}".format(py, abi)] + suffixes = _whl_config_setting_sufixes( + platform_tag = parsed.platform_tag, + glibc_versions = glibc_versions, + muslc_versions = muslc_versions, + osx_versions = osx_versions, + setting_supported_versions = setting_supported_versions, + ) + else: + prefixes = ["sdist"] + suffixes = target_platforms + + if python_default and python_version: + prefixes += ["cp{}_{}".format(python_version, p) for p in prefixes] + elif python_version: + prefixes = ["cp{}_{}".format(python_version, p) for p in prefixes] + elif python_default: + pass + else: + fail("BUG: got no python_version and it is not default") + + versioned = { + ":is_{}_{}".format(p, suffix): { + version: ":is_{}_{}".format(p, setting) + for version, setting in versions.items() + } + for p in prefixes + for suffix, versions in setting_supported_versions.items() + } + + if suffixes or versioned: + return [":is_{}_{}".format(p, s) for p in prefixes for s in suffixes], versioned + else: + return [":is_{}".format(p) for p in prefixes], setting_supported_versions + +def _whl_config_setting_sufixes( + platform_tag, + glibc_versions, + muslc_versions, + osx_versions, + setting_supported_versions): + suffixes = [] + for platform_tag in platform_tag.split("."): + for p in whl_target_platforms(platform_tag): + prefix = p.os + suffix = p.cpu + if "manylinux" in platform_tag: + prefix = "manylinux" + versions = glibc_versions + elif "musllinux" in platform_tag: + prefix = "musllinux" + versions = muslc_versions + elif p.os in ["linux", "windows"]: + versions = [(0, 0)] + elif p.os == "osx": + versions = osx_versions + if "universal2" in platform_tag: + suffix += "_universal2" + else: + fail("Unsupported whl os: {}".format(p.os)) + + default_version_setting = "{}_{}".format(prefix, suffix) + supported_versions = {} + for v in versions: + if v == (0, 0): + suffixes.append(default_version_setting) + elif v >= p.version: + supported_versions[v] = "{}_{}_{}_{}".format( + prefix, + v[0], + v[1], + suffix, + ) + if supported_versions: + setting_supported_versions[default_version_setting] = supported_versions + + return suffixes diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index 018c4433ed..dd59e477d8 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -31,6 +31,7 @@ load( "WINDOWS_NAME", ) load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") +load("//python/private:text_util.bzl", "render") def get_repository_name(repository_workspace): dummy_label = "//:_" @@ -64,10 +65,15 @@ py_toolchain_suite( user_repository_name = "{user_repository_name}_{platform}", prefix = "{prefix}{platform}", target_compatible_with = {compatible_with}, + flag_values = {flag_values}, python_version = "{python_version}", set_python_version_constraint = "{set_python_version_constraint}", )""".format( - compatible_with = meta.compatible_with, + compatible_with = render.indent(render.list(meta.compatible_with)).lstrip(), + flag_values = render.indent(render.dict( + meta.flag_values, + key_repr = lambda x: repr(str(x)), # this is to correctly display labels + )).lstrip(), platform = platform, set_python_version_constraint = set_python_version_constraint, user_repository_name = user_repository_name, diff --git a/python/private/whl_target_platforms.bzl b/python/private/whl_target_platforms.bzl index 678c841feb..08177ca1b3 100644 --- a/python/private/whl_target_platforms.bzl +++ b/python/private/whl_target_platforms.bzl @@ -265,6 +265,16 @@ def whl_target_platforms(platform_tag, abi_tag = ""): if abi_tag not in ["", "none", "abi3"]: abi = abi_tag + # TODO @aignas 2024-05-29: this code is present in many places, I think + _, _, tail = platform_tag.partition("_") + maybe_arch = tail + major, _, tail = tail.partition("_") + minor, _, tail = tail.partition("_") + if not tail or not major.isdigit() or not minor.isdigit(): + tail = maybe_arch + major = 0 + minor = 0 + for prefix, os in _OS_PREFIXES.items(): if platform_tag.startswith(prefix): return [ @@ -272,6 +282,7 @@ def whl_target_platforms(platform_tag, abi_tag = ""): os = os, cpu = cpu, abi = abi, + version = (int(major), int(minor)), target_platform = "_".join([abi, os, cpu] if abi else [os, cpu]), ) for cpu in cpus diff --git a/python/versions.bzl b/python/versions.bzl index 08882d3ade..26b975d068 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -501,6 +501,7 @@ PLATFORMS = { "@platforms//os:macos", "@platforms//cpu:aarch64", ], + flag_values = {}, os_name = MACOS_NAME, # Matches the value returned from: # repository_ctx.execute(["uname", "-m"]).stdout.strip() @@ -511,6 +512,9 @@ PLATFORMS = { "@platforms//os:linux", "@platforms//cpu:aarch64", ], + flag_values = { + Label("//python/config_settings:py_linux_libc"): "glibc", + }, os_name = LINUX_NAME, # Note: this string differs between OSX and Linux # Matches the value returned from: @@ -522,6 +526,9 @@ PLATFORMS = { "@platforms//os:linux", "@platforms//cpu:armv7", ], + flag_values = { + Label("//python/config_settings:py_linux_libc"): "glibc", + }, os_name = LINUX_NAME, arch = "armv7", ), @@ -530,6 +537,9 @@ PLATFORMS = { "@platforms//os:linux", "@platforms//cpu:ppc", ], + flag_values = { + Label("//python/config_settings:py_linux_libc"): "glibc", + }, os_name = LINUX_NAME, # Note: this string differs between OSX and Linux # Matches the value returned from: @@ -541,6 +551,9 @@ PLATFORMS = { "@platforms//os:linux", "@platforms//cpu:riscv64", ], + flag_values = { + Label("//python/config_settings:py_linux_libc"): "glibc", + }, os_name = LINUX_NAME, arch = "riscv64", ), @@ -549,6 +562,9 @@ PLATFORMS = { "@platforms//os:linux", "@platforms//cpu:s390x", ], + flag_values = { + Label("//python/config_settings:py_linux_libc"): "glibc", + }, os_name = LINUX_NAME, # Note: this string differs between OSX and Linux # Matches the value returned from: @@ -560,6 +576,7 @@ PLATFORMS = { "@platforms//os:macos", "@platforms//cpu:x86_64", ], + flag_values = {}, os_name = MACOS_NAME, arch = "x86_64", ), @@ -568,6 +585,7 @@ PLATFORMS = { "@platforms//os:windows", "@platforms//cpu:x86_64", ], + flag_values = {}, os_name = WINDOWS_NAME, arch = "x86_64", ), @@ -576,6 +594,9 @@ PLATFORMS = { "@platforms//os:linux", "@platforms//cpu:x86_64", ], + flag_values = { + Label("//python/config_settings:py_linux_libc"): "glibc", + }, os_name = LINUX_NAME, arch = "x86_64", ), diff --git a/tests/config_settings/transition/multi_version_tests.bzl b/tests/config_settings/transition/multi_version_tests.bzl index 32f7209c9f..f3707dba20 100644 --- a/tests/config_settings/transition/multi_version_tests.bzl +++ b/tests/config_settings/transition/multi_version_tests.bzl @@ -16,9 +16,16 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:util.bzl", rt_util = "util") -load("//python:versions.bzl", "TOOL_VERSIONS") load("//python/config_settings:transition.bzl", py_binary_transitioned = "py_binary", py_test_transitioned = "py_test") +# NOTE @aignas 2024-06-04: we are using here something that is registered in the MODULE.Bazel +# and if you find tests failing, it could be because of the toolchain resolution issues here. +# +# If the toolchain is not resolved then you will have a weird message telling +# you that your transition target does not have a PyRuntime provider, which is +# caused by there not being a toolchain detected for the target. +_PYTHON_VERSION = "3.11" + _tests = [] def _test_py_test_with_transition(name): @@ -26,7 +33,7 @@ def _test_py_test_with_transition(name): py_test_transitioned, name = name + "_subject", srcs = [name + "_subject.py"], - python_version = TOOL_VERSIONS.keys()[0], + python_version = _PYTHON_VERSION, ) analysis_test( @@ -46,7 +53,7 @@ def _test_py_binary_with_transition(name): py_binary_transitioned, name = name + "_subject", srcs = [name + "_subject.py"], - python_version = TOOL_VERSIONS.keys()[0], + python_version = _PYTHON_VERSION, ) analysis_test( diff --git a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl index a38d657962..a0689f70b9 100644 --- a/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pip_hub_repository/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -16,7 +16,19 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility -load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias") # buildifier: disable=bzl-visibility +load( + "//python/private:pip_config_settings.bzl", + "pip_config_settings", +) # buildifier: disable=bzl-visibility +load( + "//python/private:render_pkg_aliases.bzl", + "get_filename_config_settings", + "get_whl_flag_versions", + "multiplatform_whl_aliases", + "render_multiplatform_pkg_aliases", + "render_pkg_aliases", + "whl_alias", +) # buildifier: disable=bzl-visibility def _normalize_label_strings(want): """normalize expected strings. @@ -99,8 +111,9 @@ alias( _tests.append(_test_legacy_aliases) def _test_bzlmod_aliases(env): - actual = render_pkg_aliases( - default_version = "3.2", + # Use this function as it is used in pip_repository + actual = render_multiplatform_pkg_aliases( + default_config_setting = "//:my_config_setting", aliases = { "bar-baz": [ whl_alias(version = "3.2", repo = "pypi_32_bar_baz", config_setting = "//:my_config_setting"), @@ -167,14 +180,28 @@ alias( ), )""" + env.expect.that_str(actual.pop("_config/BUILD.bazel")).equals( + """\ +load("@rules_python//python/private:pip_config_settings.bzl", "pip_config_settings") + +pip_config_settings( + name = "pip_config_settings", + glibc_versions = [], + muslc_versions = [], + osx_versions = [], + python_versions = ["3.2"], + target_platforms = [], + visibility = ["//:__subpackages__"], +)""", + ) env.expect.that_collection(actual.keys()).contains_exactly([want_key]) env.expect.that_str(actual[want_key]).equals(want_content) _tests.append(_test_bzlmod_aliases) def _test_bzlmod_aliases_with_no_default_version(env): - actual = render_pkg_aliases( - default_version = None, + actual = render_multiplatform_pkg_aliases( + default_config_setting = None, aliases = { "bar-baz": [ whl_alias( @@ -198,11 +225,10 @@ _NO_MATCH_ERROR = \"\"\"\\ No matching wheel for current configuration's Python version. The current build configuration's Python version doesn't match any of the Python -versions available for this wheel. This wheel supports the following Python versions: - 3.1, 3.2 - -As matched by the `@rules_python//python/config_settings:is_python_` -configuration settings. +wheels available for this wheel. This wheel supports the following Python +configuration settings: + //_config:is_python_3.1 + @@//python/config_settings:is_python_3.2 To determine the current configuration's Python version, run: `bazel config ` (shown further below) @@ -222,7 +248,7 @@ alias( name = "pkg", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:pkg", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:pkg", "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:pkg", }, no_match_error = _NO_MATCH_ERROR, @@ -233,7 +259,7 @@ alias( name = "whl", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:whl", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:whl", "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:whl", }, no_match_error = _NO_MATCH_ERROR, @@ -244,7 +270,7 @@ alias( name = "data", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:data", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:data", "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:data", }, no_match_error = _NO_MATCH_ERROR, @@ -255,13 +281,14 @@ alias( name = "dist_info", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:dist_info", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:dist_info", "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:dist_info", }, no_match_error = _NO_MATCH_ERROR, ), )""" + actual.pop("_config/BUILD.bazel") env.expect.that_collection(actual.keys()).contains_exactly([want_key]) env.expect.that_str(actual[want_key]).equals(_normalize_label_strings(want_content)) @@ -274,9 +301,10 @@ def _test_bzlmod_aliases_for_non_root_modules(env): # as _test_bzlmod_aliases. # # However, if the root module uses a different default version than the - # non-root module, then we will have a no-match-error because the default_version - # is not in the list of the versions in the whl_map. - default_version = "3.3", + # non-root module, then we will have a no-match-error because the + # default_config_setting is not in the list of the versions in the + # whl_map. + default_config_setting = "//_config:is_python_3.3", aliases = { "bar-baz": [ whl_alias(version = "3.2", repo = "pypi_32_bar_baz"), @@ -295,11 +323,10 @@ _NO_MATCH_ERROR = \"\"\"\\ No matching wheel for current configuration's Python version. The current build configuration's Python version doesn't match any of the Python -versions available for this wheel. This wheel supports the following Python versions: - 3.1, 3.2 - -As matched by the `@rules_python//python/config_settings:is_python_` -configuration settings. +wheels available for this wheel. This wheel supports the following Python +configuration settings: + //_config:is_python_3.1 + //_config:is_python_3.2 To determine the current configuration's Python version, run: `bazel config ` (shown further below) @@ -319,8 +346,8 @@ alias( name = "pkg", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:pkg", - "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:pkg", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:pkg", + "//_config:is_python_3.2": "@pypi_32_bar_baz//:pkg", }, no_match_error = _NO_MATCH_ERROR, ), @@ -330,8 +357,8 @@ alias( name = "whl", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:whl", - "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:whl", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:whl", + "//_config:is_python_3.2": "@pypi_32_bar_baz//:whl", }, no_match_error = _NO_MATCH_ERROR, ), @@ -341,8 +368,8 @@ alias( name = "data", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:data", - "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:data", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:data", + "//_config:is_python_3.2": "@pypi_32_bar_baz//:data", }, no_match_error = _NO_MATCH_ERROR, ), @@ -352,21 +379,21 @@ alias( name = "dist_info", actual = selects.with_or( { - "@@//python/config_settings:is_python_3.1": "@pypi_31_bar_baz//:dist_info", - "@@//python/config_settings:is_python_3.2": "@pypi_32_bar_baz//:dist_info", + "//_config:is_python_3.1": "@pypi_31_bar_baz//:dist_info", + "//_config:is_python_3.2": "@pypi_32_bar_baz//:dist_info", }, no_match_error = _NO_MATCH_ERROR, ), )""" env.expect.that_collection(actual.keys()).contains_exactly([want_key]) - env.expect.that_str(actual[want_key]).equals(_normalize_label_strings(want_content)) + env.expect.that_str(actual[want_key]).equals(want_content) _tests.append(_test_bzlmod_aliases_for_non_root_modules) def _test_aliases_are_created_for_all_wheels(env): actual = render_pkg_aliases( - default_version = "3.2", + default_config_setting = "//_config:is_python_3.2", aliases = { "bar": [ whl_alias(version = "3.1", repo = "pypi_31_bar"), @@ -390,7 +417,7 @@ _tests.append(_test_aliases_are_created_for_all_wheels) def _test_aliases_with_groups(env): actual = render_pkg_aliases( - default_version = "3.2", + default_config_setting = "//_config:is_python_3.2", aliases = { "bar": [ whl_alias(version = "3.1", repo = "pypi_31_bar"), @@ -432,6 +459,491 @@ def _test_aliases_with_groups(env): _tests.append(_test_aliases_with_groups) +def _test_empty_flag_versions(env): + got = get_whl_flag_versions( + aliases = [], + ) + want = {} + env.expect.that_dict(got).contains_exactly(want) + +_tests.append(_test_empty_flag_versions) + +def _test_get_python_versions(env): + got = get_whl_flag_versions( + aliases = [ + whl_alias(repo = "foo", version = "3.3"), + whl_alias(repo = "foo", version = "3.2"), + ], + ) + want = { + "python_versions": ["3.2", "3.3"], + } + env.expect.that_dict(got).contains_exactly(want) + +_tests.append(_test_get_python_versions) + +def _test_get_python_versions_from_filenames(env): + got = get_whl_flag_versions( + aliases = [ + whl_alias( + repo = "foo", + version = "3.3", + filename = "foo-0.0.0-py3-none-" + plat + ".whl", + ) + for plat in [ + "linux_x86_64", + "manylinux_2_17_x86_64", + "manylinux_2_14_aarch64.musllinux_1_1_aarch64", + "musllinux_1_0_x86_64", + "manylinux2014_x86_64.manylinux_2_17_x86_64", + "macosx_11_0_arm64", + "macosx_10_9_x86_64", + "macosx_10_9_universal2", + "windows_x86_64", + ] + ], + ) + want = { + "glibc_versions": [(2, 14), (2, 17)], + "muslc_versions": [(1, 0), (1, 1)], + "osx_versions": [(10, 9), (11, 0)], + "python_versions": ["3.3"], + "target_platforms": [ + "linux_aarch64", + "linux_x86_64", + "osx_aarch64", + "osx_x86_64", + "windows_x86_64", + ], + } + env.expect.that_dict(got).contains_exactly(want) + +_tests.append(_test_get_python_versions_from_filenames) + +def _test_target_platforms_from_alias_target_platforms(env): + got = get_whl_flag_versions( + aliases = [ + whl_alias( + repo = "foo", + version = "3.3", + filename = "foo-0.0.0-py3-none-" + plat + ".whl", + ) + for plat in [ + "windows_x86_64", + ] + ] + [ + whl_alias( + repo = "foo", + version = "3.3", + filename = "foo-0.0.0-py3-none-any.whl", + target_platforms = [ + "linux_x86_64", + ], + ), + ], + ) + want = { + "python_versions": ["3.3"], + "target_platforms": [ + "linux_x86_64", + "windows_x86_64", + ], + } + env.expect.that_dict(got).contains_exactly(want) + +_tests.append(_test_target_platforms_from_alias_target_platforms) + +def _test_config_settings( + env, + *, + filename, + want, + want_versions = {}, + target_platforms = [], + glibc_versions = [], + muslc_versions = [], + osx_versions = [], + python_version = "", + python_default = True): + got, got_default_version_settings = get_filename_config_settings( + filename = filename, + target_platforms = target_platforms, + glibc_versions = glibc_versions, + muslc_versions = muslc_versions, + osx_versions = osx_versions, + python_version = python_version, + python_default = python_default, + ) + env.expect.that_collection(got).contains_exactly(want) + env.expect.that_dict(got_default_version_settings).contains_exactly(want_versions) + +def _test_sdist(env): + # Do the first test for multiple extensions + for ext in [".tar.gz", ".zip"]: + _test_config_settings( + env, + filename = "foo-0.0.1" + ext, + want = [":is_sdist"], + ) + + ext = ".zip" + _test_config_settings( + env, + filename = "foo-0.0.1" + ext, + target_platforms = [ + "linux_aarch64", + ], + want = [":is_sdist_linux_aarch64"], + ) + + _test_config_settings( + env, + filename = "foo-0.0.1" + ext, + python_version = "3.2", + want = [ + ":is_sdist", + ":is_cp3.2_sdist", + ], + ) + + _test_config_settings( + env, + filename = "foo-0.0.1" + ext, + python_version = "3.2", + python_default = True, + target_platforms = [ + "linux_aarch64", + "linux_x86_64", + ], + want = [ + ":is_sdist_linux_aarch64", + ":is_cp3.2_sdist_linux_aarch64", + ":is_sdist_linux_x86_64", + ":is_cp3.2_sdist_linux_x86_64", + ], + ) + +_tests.append(_test_sdist) + +def _test_py2_py3_none_any(env): + _test_config_settings( + env, + filename = "foo-0.0.1-py2.py3-none-any.whl", + want = [":is_py_none_any"], + ) + + _test_config_settings( + env, + filename = "foo-0.0.1-py2.py3-none-any.whl", + target_platforms = [ + "linux_aarch64", + ], + want = [":is_py_none_any_linux_aarch64"], + ) + + _test_config_settings( + env, + filename = "foo-0.0.1-py2.py3-none-any.whl", + python_version = "3.2", + python_default = True, + want = [ + ":is_py_none_any", + ":is_cp3.2_py_none_any", + ], + ) + + _test_config_settings( + env, + filename = "foo-0.0.1-py2.py3-none-any.whl", + python_version = "3.2", + python_default = False, + target_platforms = [ + "osx_x86_64", + ], + want = [ + ":is_cp3.2_py_none_any_osx_x86_64", + ], + ) + +_tests.append(_test_py2_py3_none_any) + +def _test_py3_none_any(env): + _test_config_settings( + env, + filename = "foo-0.0.1-py3-none-any.whl", + want = [":is_py3_none_any"], + ) + + _test_config_settings( + env, + filename = "foo-0.0.1-py3-none-any.whl", + target_platforms = ["linux_x86_64"], + want = [":is_py3_none_any_linux_x86_64"], + ) + +_tests.append(_test_py3_none_any) + +def _test_py3_none_macosx_10_9_universal2(env): + _test_config_settings( + env, + filename = "foo-0.0.1-py3-none-macosx_10_9_universal2.whl", + osx_versions = [ + (10, 9), + (11, 0), + ], + want = [], + want_versions = { + ":is_py3_none_osx_aarch64_universal2": { + (10, 9): ":is_py3_none_osx_10_9_aarch64_universal2", + (11, 0): ":is_py3_none_osx_11_0_aarch64_universal2", + }, + ":is_py3_none_osx_x86_64_universal2": { + (10, 9): ":is_py3_none_osx_10_9_x86_64_universal2", + (11, 0): ":is_py3_none_osx_11_0_x86_64_universal2", + }, + }, + ) + +_tests.append(_test_py3_none_macosx_10_9_universal2) + +def _test_cp37_abi3_linux_x86_64(env): + _test_config_settings( + env, + filename = "foo-0.0.1-cp37-abi3-linux_x86_64.whl", + want = [ + ":is_cp3x_abi3_linux_x86_64", + ], + ) + + _test_config_settings( + env, + filename = "foo-0.0.1-cp37-abi3-linux_x86_64.whl", + python_version = "3.2", + python_default = True, + want = [ + ":is_cp3x_abi3_linux_x86_64", + # TODO @aignas 2024-05-29: update the pip_config_settings to generate this + ":is_cp3.2_cp3x_abi3_linux_x86_64", + ], + ) + +_tests.append(_test_cp37_abi3_linux_x86_64) + +def _test_cp37_abi3_windows_x86_64(env): + _test_config_settings( + env, + filename = "foo-0.0.1-cp37-abi3-windows_x86_64.whl", + want = [ + ":is_cp3x_abi3_windows_x86_64", + ], + ) + +_tests.append(_test_cp37_abi3_windows_x86_64) + +def _test_cp37_abi3_manylinux_2_17_x86_64(env): + _test_config_settings( + env, + filename = "foo-0.0.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + glibc_versions = [ + (2, 16), + (2, 17), + (2, 18), + ], + want = [], + want_versions = { + ":is_cp3x_abi3_manylinux_x86_64": { + (2, 17): ":is_cp3x_abi3_manylinux_2_17_x86_64", + (2, 18): ":is_cp3x_abi3_manylinux_2_18_x86_64", + }, + }, + ) + +_tests.append(_test_cp37_abi3_manylinux_2_17_x86_64) + +def _test_cp37_abi3_manylinux_2_17_musllinux_1_1_aarch64(env): + # I've seen such a wheel being built for `uv` + _test_config_settings( + env, + filename = "foo-0.0.1-cp37-cp37-manylinux_2_17_arm64.musllinux_1_1_arm64.whl", + glibc_versions = [ + (2, 16), + (2, 17), + (2, 18), + ], + muslc_versions = [ + (1, 1), + ], + want = [], + want_versions = { + ":is_cp3x_cp_manylinux_aarch64": { + (2, 17): ":is_cp3x_cp_manylinux_2_17_aarch64", + (2, 18): ":is_cp3x_cp_manylinux_2_18_aarch64", + }, + ":is_cp3x_cp_musllinux_aarch64": { + (1, 1): ":is_cp3x_cp_musllinux_1_1_aarch64", + }, + }, + ) + +_tests.append(_test_cp37_abi3_manylinux_2_17_musllinux_1_1_aarch64) + +def _test_multiplatform_whl_aliases_empty(env): + # Check that we still work with an empty requirements.txt + got = multiplatform_whl_aliases(aliases = [], default_version = None) + env.expect.that_collection(got).contains_exactly([]) + +_tests.append(_test_multiplatform_whl_aliases_empty) + +def _test_multiplatform_whl_aliases_nofilename(env): + aliases = [ + whl_alias( + repo = "foo", + config_setting = "//:label", + version = "3.1", + ), + ] + got = multiplatform_whl_aliases(aliases = aliases, default_version = None) + env.expect.that_collection(got).contains_exactly(aliases) + +_tests.append(_test_multiplatform_whl_aliases_nofilename) + +def _test_multiplatform_whl_aliases_filename(env): + aliases = [ + whl_alias( + repo = "foo-py3-0.0.3", + filename = "foo-0.0.3-py3-none-any.whl", + version = "3.2", + ), + whl_alias( + repo = "foo-py3-0.0.1", + filename = "foo-0.0.1-py3-none-any.whl", + version = "3.1", + ), + whl_alias( + repo = "foo-0.0.2", + filename = "foo-0.0.2-py3-none-any.whl", + version = "3.1", + target_platforms = [ + "linux_x86_64", + "linux_aarch64", + ], + ), + ] + got = multiplatform_whl_aliases( + aliases = aliases, + default_version = "3.1", + glibc_versions = [], + muslc_versions = [], + osx_versions = [], + ) + want = [ + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_any", repo = "foo-py3-0.0.1", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_any_linux_aarch64", repo = "foo-0.0.2", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_any_linux_x86_64", repo = "foo-0.0.2", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.2_py3_none_any", repo = "foo-py3-0.0.3", version = "3.2"), + whl_alias(config_setting = "//_config:is_py3_none_any", repo = "foo-py3-0.0.1", version = "3.1"), + whl_alias(config_setting = "//_config:is_py3_none_any_linux_aarch64", repo = "foo-0.0.2", version = "3.1"), + whl_alias(config_setting = "//_config:is_py3_none_any_linux_x86_64", repo = "foo-0.0.2", version = "3.1"), + ] + env.expect.that_collection(got).contains_exactly(want) + +_tests.append(_test_multiplatform_whl_aliases_filename) + +def _test_multiplatform_whl_aliases_filename_versioned(env): + aliases = [ + whl_alias( + repo = "glibc-2.17", + filename = "foo-0.0.1-py3-none-manylinux_2_17_x86_64.whl", + version = "3.1", + ), + whl_alias( + repo = "glibc-2.18", + filename = "foo-0.0.1-py3-none-manylinux_2_18_x86_64.whl", + version = "3.1", + ), + whl_alias( + repo = "musl", + filename = "foo-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + version = "3.1", + ), + ] + got = multiplatform_whl_aliases( + aliases = aliases, + default_version = None, + glibc_versions = [(2, 17), (2, 18)], + muslc_versions = [(1, 1), (1, 2)], + osx_versions = [], + ) + want = [ + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_manylinux_2_17_x86_64", repo = "glibc-2.17", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_manylinux_2_18_x86_64", repo = "glibc-2.18", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_manylinux_x86_64", repo = "glibc-2.17", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_musllinux_1_1_x86_64", repo = "musl", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_musllinux_1_2_x86_64", repo = "musl", version = "3.1"), + whl_alias(config_setting = "//_config:is_cp3.1_py3_none_musllinux_x86_64", repo = "musl", version = "3.1"), + ] + env.expect.that_collection(got).contains_exactly(want) + +_tests.append(_test_multiplatform_whl_aliases_filename_versioned) + +def _test_config_settings_exist(env): + for py_tag in ["py2.py3", "py3", "py311", "cp311"]: + if py_tag == "py2.py3": + abis = ["none"] + elif py_tag.startswith("py"): + abis = ["none", "abi3"] + else: + abis = ["none", "abi3", "cp311"] + + for abi_tag in abis: + for platform_tag, kwargs in { + "any": {}, + "macosx_11_0_arm64": { + "osx_versions": [(11, 0)], + "target_platforms": ["osx_aarch64"], + }, + "manylinux_2_17_x86_64": { + "glibc_versions": [(2, 17), (2, 18)], + "target_platforms": ["linux_x86_64"], + }, + "manylinux_2_18_x86_64": { + "glibc_versions": [(2, 17), (2, 18)], + "target_platforms": ["linux_x86_64"], + }, + "musllinux_1_1_aarch64": { + "muslc_versions": [(1, 2), (1, 1), (1, 0)], + "target_platforms": ["linux_aarch64"], + }, + }.items(): + aliases = [ + whl_alias( + repo = "repo", + filename = "foo-0.0.1-{}-{}-{}.whl".format(py_tag, abi_tag, platform_tag), + version = "3.11", + ), + ] + available_config_settings = [] + mock_rule = lambda name, **kwargs: available_config_settings.append(name) + pip_config_settings( + python_versions = ["3.11"], + alias_rule = mock_rule, + config_setting_rule = mock_rule, + **kwargs + ) + + got_aliases = multiplatform_whl_aliases( + aliases = aliases, + default_version = None, + glibc_versions = kwargs.get("glibc_versions", []), + muslc_versions = kwargs.get("muslc_versions", []), + osx_versions = kwargs.get("osx_versions", []), + ) + got = [a.config_setting.partition(":")[-1] for a in got_aliases] + + env.expect.that_collection(available_config_settings).contains_at_least(got) + +_tests.append(_test_config_settings_exist) + def render_pkg_aliases_test_suite(name): """Create the test suite. diff --git a/tests/private/pip_config_settings/BUILD.bazel b/tests/private/pip_config_settings/BUILD.bazel new file mode 100644 index 0000000000..c3752e0ac3 --- /dev/null +++ b/tests/private/pip_config_settings/BUILD.bazel @@ -0,0 +1,5 @@ +load(":pip_config_settings_tests.bzl", "pip_config_settings_test_suite") + +pip_config_settings_test_suite( + name = "pip_config_settings", +) diff --git a/tests/private/pip_config_settings/pip_config_settings_tests.bzl b/tests/private/pip_config_settings/pip_config_settings_tests.bzl new file mode 100644 index 0000000000..a66e7f47d5 --- /dev/null +++ b/tests/private/pip_config_settings/pip_config_settings_tests.bzl @@ -0,0 +1,544 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for construction of Python version matching config settings.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:truth.bzl", "subjects") +load("@rules_testing//lib:util.bzl", test_util = "util") +load("//python/private:pip_config_settings.bzl", "pip_config_settings") # buildifier: disable=bzl-visibility + +def _subject_impl(ctx): + _ = ctx # @unused + return [DefaultInfo()] + +_subject = rule( + implementation = _subject_impl, + attrs = { + "dist": attr.string(), + }, +) + +_flag = struct( + platform = lambda x: ("//command_line_option:platforms", str(Label("//tests/support:" + x))), + pip_whl = lambda x: (str(Label("//python/config_settings:pip_whl")), str(x)), + pip_whl_glibc_version = lambda x: (str(Label("//python/config_settings:pip_whl_glibc_version")), str(x)), + pip_whl_muslc_version = lambda x: (str(Label("//python/config_settings:pip_whl_muslc_version")), str(x)), + pip_whl_osx_version = lambda x: (str(Label("//python/config_settings:pip_whl_osx_version")), str(x)), + pip_whl_osx_arch = lambda x: (str(Label("//python/config_settings:pip_whl_osx_arch")), str(x)), + py_linux_libc = lambda x: (str(Label("//python/config_settings:py_linux_libc")), str(x)), + python_version = lambda x: (str(Label("//python/config_settings:python_version")), str(x)), +) + +def _analysis_test(*, name, dist, want, config_settings = [_flag.platform("linux_aarch64")]): + subject_name = name + "_subject" + test_util.helper_target( + _subject, + name = subject_name, + dist = select( + dist | { + "//conditions:default": "no_match", + }, + ), + ) + config_settings = dict(config_settings) + if not config_settings: + fail("For reproducibility on different platforms, the config setting must be specified") + + analysis_test( + name = name, + target = subject_name, + impl = lambda env, target: _match(env, target, want), + config_settings = config_settings, + ) + +def _match(env, target, want): + target = env.expect.that_target(target) + target.attr("dist", factory = subjects.str).equals(want) + +_tests = [] + +# Tests when we only have an `sdist` present. + +def _test_sdist_default(name): + _analysis_test( + name = name, + dist = { + "is_sdist": "sdist", + }, + want = "sdist", + ) + +_tests.append(_test_sdist_default) + +def _test_sdist_no_whl(name): + _analysis_test( + name = name, + dist = { + "is_sdist": "sdist", + }, + config_settings = [ + _flag.platform("linux_aarch64"), + _flag.pip_whl("no"), + ], + want = "sdist", + ) + +_tests.append(_test_sdist_no_whl) + +def _test_sdist_no_sdist(name): + _analysis_test( + name = name, + dist = { + "is_sdist": "sdist", + }, + config_settings = [ + _flag.platform("linux_aarch64"), + _flag.pip_whl("only"), + ], + # We will use `no_match_error` in the real case to indicate that `sdist` is not + # allowed to be used. + want = "no_match", + ) + +_tests.append(_test_sdist_no_sdist) + +def _test_basic_whl_default(name): + _analysis_test( + name = name, + dist = { + "is_py_none_any": "whl", + "is_sdist": "sdist", + }, + want = "whl", + ) + +_tests.append(_test_basic_whl_default) + +def _test_basic_whl_nowhl(name): + _analysis_test( + name = name, + dist = { + "is_py_none_any": "whl", + "is_sdist": "sdist", + }, + config_settings = [ + _flag.platform("linux_aarch64"), + _flag.pip_whl("no"), + ], + want = "sdist", + ) + +_tests.append(_test_basic_whl_nowhl) + +def _test_basic_whl_nosdist(name): + _analysis_test( + name = name, + dist = { + "is_py_none_any": "whl", + "is_sdist": "sdist", + }, + config_settings = [ + _flag.platform("linux_aarch64"), + _flag.pip_whl("only"), + ], + want = "whl", + ) + +_tests.append(_test_basic_whl_nosdist) + +def _test_whl_default(name): + _analysis_test( + name = name, + dist = { + "is_py3_none_any": "whl", + "is_py_none_any": "basic_whl", + }, + want = "whl", + ) + +_tests.append(_test_whl_default) + +def _test_whl_nowhl(name): + _analysis_test( + name = name, + dist = { + "is_py3_none_any": "whl", + "is_py_none_any": "basic_whl", + }, + config_settings = [ + _flag.platform("linux_aarch64"), + _flag.pip_whl("no"), + ], + want = "no_match", + ) + +_tests.append(_test_whl_nowhl) + +def _test_whl_nosdist(name): + _analysis_test( + name = name, + dist = { + "is_py3_none_any": "whl", + }, + config_settings = [ + _flag.platform("linux_aarch64"), + _flag.pip_whl("only"), + ], + want = "whl", + ) + +_tests.append(_test_whl_nosdist) + +def _test_abi_whl_is_prefered(name): + _analysis_test( + name = name, + dist = { + "is_py3_abi3_any": "abi_whl", + "is_py3_none_any": "whl", + }, + want = "abi_whl", + ) + +_tests.append(_test_abi_whl_is_prefered) + +def _test_whl_with_constraints_is_prefered(name): + _analysis_test( + name = name, + dist = { + "is_py3_none_any": "default_whl", + "is_py3_none_any_linux_aarch64": "whl", + "is_py3_none_any_linux_x86_64": "amd64_whl", + }, + want = "whl", + ) + +_tests.append(_test_whl_with_constraints_is_prefered) + +def _test_cp_whl_is_prefered_over_py3(name): + _analysis_test( + name = name, + dist = { + "is_cp3x_none_any": "cp", + "is_py3_abi3_any": "py3_abi3", + "is_py3_none_any": "py3", + }, + want = "cp", + ) + +_tests.append(_test_cp_whl_is_prefered_over_py3) + +def _test_cp_abi_whl_is_prefered_over_py3(name): + _analysis_test( + name = name, + dist = { + "is_cp3x_abi3_any": "cp", + "is_py3_abi3_any": "py3", + }, + want = "cp", + ) + +_tests.append(_test_cp_abi_whl_is_prefered_over_py3) + +def _test_cp_version_is_selected_when_python_version_is_specified(name): + _analysis_test( + name = name, + dist = { + "is_cp3.10_cp3x_none_any": "cp310", + "is_cp3.8_cp3x_none_any": "cp38", + "is_cp3.9_cp3x_none_any": "cp39", + "is_cp3x_none_any": "cp_default", + }, + want = "cp310", + config_settings = [ + _flag.python_version("3.10.9"), + _flag.platform("linux_aarch64"), + ], + ) + +_tests.append(_test_cp_version_is_selected_when_python_version_is_specified) + +def _test_py_none_any_versioned(name): + _analysis_test( + name = name, + dist = { + "is_cp3.10_py_none_any": "whl", + "is_cp3.9_py_none_any": "too-low", + }, + want = "whl", + config_settings = [ + _flag.python_version("3.10.9"), + _flag.platform("linux_aarch64"), + ], + ) + +_tests.append(_test_py_none_any_versioned) + +def _test_cp_cp_whl(name): + _analysis_test( + name = name, + dist = { + "is_cp3.10_cp3x_cp_linux_aarch64": "whl", + }, + want = "whl", + config_settings = [ + _flag.python_version("3.10.9"), + _flag.platform("linux_aarch64"), + ], + ) + +_tests.append(_test_cp_cp_whl) + +def _test_cp_version_sdist_is_selected(name): + _analysis_test( + name = name, + dist = { + "is_cp3.10_sdist": "sdist", + }, + want = "sdist", + config_settings = [ + _flag.python_version("3.10.9"), + _flag.platform("linux_aarch64"), + ], + ) + +_tests.append(_test_cp_version_sdist_is_selected) + +def _test_platform_whl_is_prefered_over_any_whl_with_constraints(name): + _analysis_test( + name = name, + dist = { + "is_py3_abi3_any": "better_default_whl", + "is_py3_abi3_any_linux_aarch64": "better_default_any_whl", + "is_py3_none_any": "default_whl", + "is_py3_none_any_linux_aarch64": "whl", + "is_py3_none_linux_aarch64": "platform_whl", + }, + want = "platform_whl", + ) + +_tests.append(_test_platform_whl_is_prefered_over_any_whl_with_constraints) + +def _test_abi3_platform_whl_preference(name): + _analysis_test( + name = name, + dist = { + "is_py3_abi3_linux_aarch64": "abi3_platform", + "is_py3_none_linux_aarch64": "platform", + }, + want = "abi3_platform", + ) + +_tests.append(_test_abi3_platform_whl_preference) + +def _test_glibc(name): + _analysis_test( + name = name, + dist = { + "is_cp3x_cp_manylinux_aarch64": "glibc", + "is_py3_abi3_linux_aarch64": "abi3_platform", + }, + want = "glibc", + ) + +_tests.append(_test_glibc) + +def _test_glibc_versioned(name): + _analysis_test( + name = name, + dist = { + "is_cp3x_cp_manylinux_2_14_aarch64": "glibc", + "is_cp3x_cp_manylinux_2_17_aarch64": "glibc", + "is_py3_abi3_linux_aarch64": "abi3_platform", + }, + want = "glibc", + config_settings = [ + _flag.py_linux_libc("glibc"), + _flag.pip_whl_glibc_version("2.17"), + _flag.platform("linux_aarch64"), + ], + ) + +_tests.append(_test_glibc_versioned) + +def _test_glibc_compatible_exists(name): + _analysis_test( + name = name, + dist = { + # Code using the conditions will need to construct selects, which + # do the version matching correctly. + "is_cp3x_cp_manylinux_2_14_aarch64": "2_14_whl_via_2_14_branch", + "is_cp3x_cp_manylinux_2_17_aarch64": "2_14_whl_via_2_17_branch", + }, + want = "2_14_whl_via_2_17_branch", + config_settings = [ + _flag.py_linux_libc("glibc"), + _flag.pip_whl_glibc_version("2.17"), + _flag.platform("linux_aarch64"), + ], + ) + +_tests.append(_test_glibc_compatible_exists) + +def _test_musl(name): + _analysis_test( + name = name, + dist = { + "is_cp3x_cp_musllinux_aarch64": "musl", + }, + want = "musl", + config_settings = [ + _flag.py_linux_libc("musl"), + _flag.platform("linux_aarch64"), + ], + ) + +_tests.append(_test_musl) + +def _test_windows(name): + _analysis_test( + name = name, + dist = { + "is_cp3x_cp_windows_x86_64": "whl", + }, + want = "whl", + config_settings = [ + _flag.platform("windows_x86_64"), + ], + ) + +_tests.append(_test_windows) + +def _test_osx(name): + _analysis_test( + name = name, + dist = { + # We prefer arch specific whls over universal + "is_cp3x_cp_osx_x86_64": "whl", + "is_cp3x_cp_osx_x86_64_universal2": "universal_whl", + }, + want = "whl", + config_settings = [ + _flag.platform("mac_x86_64"), + ], + ) + +_tests.append(_test_osx) + +def _test_osx_universal_default(name): + _analysis_test( + name = name, + dist = { + # We default to universal if only that exists + "is_cp3x_cp_osx_x86_64_universal2": "whl", + }, + want = "whl", + config_settings = [ + _flag.platform("mac_x86_64"), + ], + ) + +_tests.append(_test_osx_universal_default) + +def _test_osx_universal_only(name): + _analysis_test( + name = name, + dist = { + # If we prefer universal, then we use that + "is_cp3x_cp_osx_x86_64": "whl", + "is_cp3x_cp_osx_x86_64_universal2": "universal", + }, + want = "universal", + config_settings = [ + _flag.pip_whl_osx_arch("universal"), + _flag.platform("mac_x86_64"), + ], + ) + +_tests.append(_test_osx_universal_only) + +def _test_osx_os_version(name): + _analysis_test( + name = name, + dist = { + # Similarly to the libc version, the user of the config settings will have to + # construct the select so that the version selection is correct. + "is_cp3x_cp_osx_10_9_x86_64": "whl", + }, + want = "whl", + config_settings = [ + _flag.pip_whl_osx_version("10.9"), + _flag.platform("mac_x86_64"), + ], + ) + +_tests.append(_test_osx_os_version) + +def _test_all(name): + _analysis_test( + name = name, + dist = { + "is_" + f: f + for f in [ + "{py}_{abi}_{plat}".format(py = valid_py, abi = valid_abi, plat = valid_plat) + # we have py2.py3, py3, cp3x + for valid_py in ["py", "py3", "cp3x"] + # cp abi usually comes with a version and we only need one + # config setting variant for all of them because the python + # version will discriminate between different versions. + for valid_abi in ["none", "abi3", "cp"] + for valid_plat in [ + "any", + "manylinux_2_17_x86_64", + "manylinux_2_17_aarch64", + "osx_x86_64", + "windows_x86_64", + ] + if not ( + valid_abi == "abi3" and valid_py == "py" or + valid_abi == "cp" and valid_py != "cp3x" + ) + ] + }, + want = "cp3x_cp_manylinux_2_17_x86_64", + config_settings = [ + _flag.pip_whl_glibc_version("2.17"), + _flag.platform("linux_x86_64"), + ], + ) + +_tests.append(_test_all) + +def pip_config_settings_test_suite(name): # buildifier: disable=function-docstring + test_suite( + name = name, + tests = _tests, + ) + + pip_config_settings( + name = "dummy", + python_versions = ["3.8", "3.9", "3.10"], + glibc_versions = [(2, 14), (2, 17)], + muslc_versions = [(1, 1)], + osx_versions = [(10, 9), (11, 0)], + target_platforms = [ + "windows_x86_64", + "windows_aarch64", + "linux_x86_64", + "linux_ppc", + "linux_aarch64", + "osx_x86_64", + "osx_aarch64", + ], + ) diff --git a/tests/private/pip_repo_name/BUILD.bazel b/tests/private/pip_repo_name/BUILD.bazel new file mode 100644 index 0000000000..7c6782daaf --- /dev/null +++ b/tests/private/pip_repo_name/BUILD.bazel @@ -0,0 +1,3 @@ +load(":pip_repo_name_tests.bzl", "pip_repo_name_test_suite") + +pip_repo_name_test_suite(name = "pip_repo_name_tests") diff --git a/tests/private/pip_repo_name/pip_repo_name_tests.bzl b/tests/private/pip_repo_name/pip_repo_name_tests.bzl new file mode 100644 index 0000000000..574d558277 --- /dev/null +++ b/tests/private/pip_repo_name/pip_repo_name_tests.bzl @@ -0,0 +1,52 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:pip_repo_name.bzl", "pip_repo_name") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_simple(env): + got = pip_repo_name("prefix", "foo-1.2.3-py3-none-any.whl", "deadbeef") + env.expect.that_str(got).equals("prefix_foo_py3_none_any_deadbeef") + +_tests.append(_test_simple) + +def _test_sdist(env): + got = pip_repo_name("prefix", "foo-1.2.3.tar.gz", "deadbeef000deadbeef") + env.expect.that_str(got).equals("prefix_foo_sdist_deadbeef") + +_tests.append(_test_sdist) + +def _test_platform_whl(env): + got = pip_repo_name( + "prefix", + "foo-1.2.3-cp39.cp310-abi3-manylinux1_x86_64.manylinux_2_17_x86_64.whl", + "deadbeef000deadbeef", + ) + + # We only need the first segment of each + env.expect.that_str(got).equals("prefix_foo_cp39_abi3_manylinux_2_5_x86_64_deadbeef") + +_tests.append(_test_platform_whl) + +def pip_repo_name_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/private/whl_target_platforms/whl_target_platforms_tests.bzl b/tests/private/whl_target_platforms/whl_target_platforms_tests.bzl index a06147b946..07f3158b31 100644 --- a/tests/private/whl_target_platforms/whl_target_platforms_tests.bzl +++ b/tests/private/whl_target_platforms/whl_target_platforms_tests.bzl @@ -22,20 +22,20 @@ _tests = [] def _test_simple(env): tests = { "macosx_10_9_arm64": [ - struct(os = "osx", cpu = "aarch64", abi = None, target_platform = "osx_aarch64"), + struct(os = "osx", cpu = "aarch64", abi = None, target_platform = "osx_aarch64", version = (10, 9)), ], "macosx_10_9_universal2": [ - struct(os = "osx", cpu = "x86_64", abi = None, target_platform = "osx_x86_64"), - struct(os = "osx", cpu = "aarch64", abi = None, target_platform = "osx_aarch64"), + struct(os = "osx", cpu = "x86_64", abi = None, target_platform = "osx_x86_64", version = (10, 9)), + struct(os = "osx", cpu = "aarch64", abi = None, target_platform = "osx_aarch64", version = (10, 9)), ], - "manylinux1_i686.manylinux_2_17_i686": [ - struct(os = "linux", cpu = "x86_32", abi = None, target_platform = "linux_x86_32"), + "manylinux_2_17_i686": [ + struct(os = "linux", cpu = "x86_32", abi = None, target_platform = "linux_x86_32", version = (2, 17)), ], "musllinux_1_1_ppc64le": [ - struct(os = "linux", cpu = "ppc", abi = None, target_platform = "linux_ppc"), + struct(os = "linux", cpu = "ppc", abi = None, target_platform = "linux_ppc", version = (1, 1)), ], "win_amd64": [ - struct(os = "windows", cpu = "x86_64", abi = None, target_platform = "windows_x86_64"), + struct(os = "windows", cpu = "x86_64", abi = None, target_platform = "windows_x86_64", version = (0, 0)), ], } @@ -49,20 +49,22 @@ _tests.append(_test_simple) def _test_with_abi(env): tests = { "macosx_10_9_arm64": [ - struct(os = "osx", cpu = "aarch64", abi = "cp39", target_platform = "cp39_osx_aarch64"), + struct(os = "osx", cpu = "aarch64", abi = "cp39", target_platform = "cp39_osx_aarch64", version = (10, 9)), ], "macosx_10_9_universal2": [ - struct(os = "osx", cpu = "x86_64", abi = "cp310", target_platform = "cp310_osx_x86_64"), - struct(os = "osx", cpu = "aarch64", abi = "cp310", target_platform = "cp310_osx_aarch64"), + struct(os = "osx", cpu = "x86_64", abi = "cp310", target_platform = "cp310_osx_x86_64", version = (10, 9)), + struct(os = "osx", cpu = "aarch64", abi = "cp310", target_platform = "cp310_osx_aarch64", version = (10, 9)), ], + # This should use version 0 because there are two platform_tags. This is + # just to ensure that the code is robust "manylinux1_i686.manylinux_2_17_i686": [ - struct(os = "linux", cpu = "x86_32", abi = "cp38", target_platform = "cp38_linux_x86_32"), + struct(os = "linux", cpu = "x86_32", abi = "cp38", target_platform = "cp38_linux_x86_32", version = (0, 0)), ], "musllinux_1_1_ppc64le": [ - struct(os = "linux", cpu = "ppc", abi = "cp311", target_platform = "cp311_linux_ppc"), + struct(os = "linux", cpu = "ppc", abi = "cp311", target_platform = "cp311_linux_ppc", version = (1, 1)), ], "win_amd64": [ - struct(os = "windows", cpu = "x86_64", abi = "cp311", target_platform = "cp311_windows_x86_64"), + struct(os = "windows", cpu = "x86_64", abi = "cp311", target_platform = "cp311_windows_x86_64", version = (0, 0)), ], } diff --git a/tests/support/BUILD.bazel b/tests/support/BUILD.bazel index 3b77cde0c5..e5d5189a3b 100644 --- a/tests/support/BUILD.bazel +++ b/tests/support/BUILD.bazel @@ -52,6 +52,14 @@ platform( ], ) +platform( + name = "linux_aarch64", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], +) + platform( name = "mac_x86_64", constraint_values = [ @@ -68,6 +76,14 @@ platform( ], ) +platform( + name = "win_aarch64", + constraint_values = [ + "@platforms//os:windows", + "@platforms//cpu:aarch64", + ], +) + py_runtime( name = "platform_runtime", implementation_name = "fakepy",