diff --git a/CHANGELOG.md b/CHANGELOG.md index 821053be93..1e67ca8f7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,15 @@ A brief description of the categories of changes: * (py_wheel) Added `requires_file` and `extra_requires_files` attributes. +* (whl_library) *experimental_target_platforms* now supports specifying the + Python version explicitly and the output `BUILD.bazel` file will be correct + irrespective of the python interpreter that is generating the file and + extracting the `whl` distribution. Multiple python target version can be + specified and the code generation will generate version specific dependency + closures but that is not yet ready to be used and may break the build if + the default python version is not selected using + `common --@rules_python//python/config_settings:python_version=X.Y.Z`. + ## 0.29.0 - 2024-01-22 [0.29.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.29.0 @@ -59,6 +68,7 @@ A brief description of the categories of changes: platform-specific content in `MODULE.bazel.lock` files; Follow [#1643](https://github.com/bazelbuild/rules_python/issues/1643) for removing platform-specific content in `MODULE.bazel.lock` files. + * (wheel) The stamp variables inside the distribution name are no longer lower-cased when normalizing under PEP440 conventions. diff --git a/examples/bzlmod/.bazelrc b/examples/bzlmod/.bazelrc index 6f557e67b9..e9a73c58aa 100644 --- a/examples/bzlmod/.bazelrc +++ b/examples/bzlmod/.bazelrc @@ -1,4 +1,4 @@ -common --experimental_enable_bzlmod +common --enable_bzlmod coverage --java_runtime_version=remotejdk_11 diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index e49b586fed..ceb0010bd4 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -107,9 +107,10 @@ pip.parse( # You can use one of the values below to specify the target platform # to generate the dependency graph for. experimental_target_platforms = [ - "all", - "linux_*", - "host", + # Specifying the target platforms explicitly + "cp39_linux_x86_64", + "cp39_linux_*", + "cp39_*", ], hub_name = "pip", python_version = "3.9", @@ -137,8 +138,13 @@ pip.parse( # You can use one of the values below to specify the target platform # to generate the dependency graph for. experimental_target_platforms = [ - "all", + # Using host python version "linux_*", + "osx_*", + "windows_*", + # Or specifying an exact platform + "linux_x86_64", + # Or the following to get the `host` platform only "host", ], hub_name = "pip", diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index fe58472f53..542e312120 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -493,13 +493,19 @@ WARNING: It may not work as expected in cases where the python interpreter implementation that is being used at runtime is different between different platforms. This has been tested for CPython only. -Special values: `all` (for generating deps for all platforms), `host` (for -generating deps for the host platform only). `linux_*` and other `_*` values. -In the future we plan to set `all` as the default to this attribute. - For specific target platforms use values of the form `_` where `` is one of `linux`, `osx`, `windows` and arch is one of `x86_64`, `x86_32`, `aarch64`, `s390x` and `ppc64le`. + +You can also target a specific Python version by using `cp3__`. +If multiple python versions are specified as target platforms, then select statements +of the `lib` and `whl` targets will include usage of version aware toolchain config +settings like `@rules_python//python/config_settings:is_python_3.y`. + +Special values: `host` (for generating deps for the host platform only) and +`_*` values. For example, `cp39_*`, `linux_*`, `cp39_linux_*`. + +NOTE: this is not for cross-compiling Python wheels but rather for parsing the `whl` METADATA correctly. """, ), "extra_pip_args": attr.string_list( @@ -749,7 +755,7 @@ def _whl_library_impl(rctx): # NOTE @aignas 2023-12-04: if the wheel is a platform specific # wheel, we only include deps for that target platform target_platforms = [ - "{}_{}".format(p.os, p.cpu) + "{}_{}_{}".format(parsed_whl.abi_tag, p.os, p.cpu) for p in whl_target_platforms(parsed_whl.platform_tag) ] diff --git a/python/pip_install/private/generate_whl_library_build_bazel.bzl b/python/pip_install/private/generate_whl_library_build_bazel.bzl index 568b00e4df..19650d16d7 100644 --- a/python/pip_install/private/generate_whl_library_build_bazel.bzl +++ b/python/pip_install/private/generate_whl_library_build_bazel.bzl @@ -48,8 +48,7 @@ py_binary( """ _BUILD_TEMPLATE = """\ -load("@rules_python//python:defs.bzl", "py_library", "py_binary") -load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +{loads} package(default_visibility = ["//visibility:public"]) @@ -102,22 +101,37 @@ alias( ) """ +def _plat_label(plat): + if plat.startswith("@//"): + return "@@" + str(Label("//:BUILD.bazel")).partition("//")[0].strip("@") + plat.strip("@") + elif plat.startswith("@"): + return str(Label(plat)) + else: + return ":is_" + plat + def _render_list_and_select(deps, deps_by_platform, tmpl): - deps = render.list([tmpl.format(d) for d in deps]) + deps = render.list([tmpl.format(d) for d in sorted(deps)]) if not deps_by_platform: return deps deps_by_platform = { - p if p.startswith("@") else ":is_" + p: [ + _plat_label(p): [ tmpl.format(d) - for d in deps + for d in sorted(deps) ] - for p, deps in deps_by_platform.items() + for p, deps in sorted(deps_by_platform.items()) } # Add the default, which means that we will be just using the dependencies in # `deps` for platforms that are not handled in a special way by the packages + # + # FIXME @aignas 2024-01-24: This currently works as expected only if the default + # value of the @rules_python//python/config_settings:python_version is set in + # the `.bazelrc`. If it is unset, then the we don't get the expected behaviour + # in cases where we are using a simple `py_binary` using the default toolchain + # without forcing any transitions. If the `python_version` config setting is set + # via .bazelrc, then everything works correctly. deps_by_platform["//conditions:default"] = [] deps_by_platform = render.select(deps_by_platform, value_repr = render.list) @@ -126,6 +140,87 @@ def _render_list_and_select(deps, deps_by_platform, tmpl): else: return "{} + {}".format(deps, deps_by_platform) +def _render_config_settings(dependencies_by_platform): + py_version_by_os_arch = {} + for p in dependencies_by_platform: + # p can be one of the following formats: + # * @platforms//os:{value} + # * @platforms//cpu:{value} + # * @//python/config_settings:is_python_3.{minor_version} + # * {os}_{cpu} + # * cp3{minor_version}_{os}_{cpu} + if p.startswith("@"): + continue + + abi, _, tail = p.partition("_") + if not abi.startswith("cp"): + tail = p + abi = "" + os, _, arch = tail.partition("_") + os = "" if os == "anyos" else os + arch = "" if arch == "anyarch" else arch + + py_version_by_os_arch.setdefault((os, arch), []).append(abi) + + if not py_version_by_os_arch: + return None, None + + loads = [] + additional_content = [] + for (os, arch), abis in py_version_by_os_arch.items(): + constraint_values = [] + if os: + constraint_values.append("@platforms//os:{}".format(os)) + if arch: + constraint_values.append("@platforms//cpu:{}".format(arch)) + + os_arch = (os or "anyos") + "_" + (arch or "anyarch") + additional_content.append( + """\ +config_setting( + name = "is_{name}", + constraint_values = {values}, + visibility = ["//visibility:private"], +)""".format( + name = os_arch, + values = render.indent(render.list(sorted([str(Label(c)) for c in constraint_values]))).strip(), + ), + ) + + if abis == [""]: + if not os or not arch: + fail("BUG: both os and arch should be set in this case") + continue + + for abi in abis: + if not loads: + loads.append("""load("@bazel_skylib//lib:selects.bzl", "selects")""") + minor_version = int(abi[len("cp3"):]) + setting = "@@{rules_python}//python/config_settings:is_python_3.{version}".format( + rules_python = str(Label("//:BUILD.bazel")).partition("//")[0].strip("@"), + version = minor_version, + ) + settings = [ + ":is_" + os_arch, + setting, + ] + + plat = "{}_{}".format(abi, os_arch) + + additional_content.append( + """\ +selects.config_setting_group( + name = "{name}", + match_all = {values}, + visibility = ["//visibility:private"], +)""".format( + name = _plat_label(plat).lstrip(":"), + values = render.indent(render.list(sorted(settings))).strip(), + ), + ) + + return loads, "\n\n".join(additional_content) + def generate_whl_library_build_bazel( *, repo_prefix, @@ -228,24 +323,17 @@ def generate_whl_library_build_bazel( if deps } - for p in dependencies_by_platform: - if p.startswith("@"): - continue - - os, _, cpu = p.partition("_") + loads = [ + """load("@rules_python//python:defs.bzl", "py_library", "py_binary")""", + """load("@bazel_skylib//rules:copy_file.bzl", "copy_file")""", + ] - additional_content.append( - """\ -config_setting( - name = "is_{os}_{cpu}", - constraint_values = [ - "@platforms//cpu:{cpu}", - "@platforms//os:{os}", - ], - visibility = ["//visibility:private"], -) -""".format(os = os, cpu = cpu), - ) + loads_, config_settings_content = _render_config_settings(dependencies_by_platform) + if config_settings_content: + for line in loads_: + if line not in loads: + loads.append(line) + additional_content.append(config_settings_content) lib_dependencies = _render_list_and_select( deps = dependencies, @@ -277,6 +365,7 @@ config_setting( contents = "\n".join( [ _BUILD_TEMPLATE.format( + loads = "\n".join(loads), py_library_public_label = PY_LIBRARY_PUBLIC_LABEL, py_library_impl_label = PY_LIBRARY_IMPL_LABEL, py_library_actual_label = library_impl_label, diff --git a/python/pip_install/tools/wheel_installer/arguments_test.py b/python/pip_install/tools/wheel_installer/arguments_test.py index 840c2fa6cc..cafb85f8ed 100644 --- a/python/pip_install/tools/wheel_installer/arguments_test.py +++ b/python/pip_install/tools/wheel_installer/arguments_test.py @@ -58,7 +58,8 @@ def test_platform_aggregation(self) -> None: args=[ "--platform=host", "--platform=linux_*", - "--platform=all", + "--platform=osx_*", + "--platform=windows_*", "--requirement=foo", ] ) diff --git a/python/pip_install/tools/wheel_installer/wheel.py b/python/pip_install/tools/wheel_installer/wheel.py index 2275f770a5..750ebfcf7a 100644 --- a/python/pip_install/tools/wheel_installer/wheel.py +++ b/python/pip_install/tools/wheel_installer/wheel.py @@ -84,16 +84,31 @@ def _as_int(value: Optional[Union[OS, Arch]]) -> int: return int(value.value) +def host_interpreter_minor_version() -> int: + return sys.version_info.minor + + @dataclass(frozen=True) class Platform: - os: OS + os: Optional[OS] = None arch: Optional[Arch] = None + minor_version: Optional[int] = None + + def __post_init__(self): + if not self.os and not self.arch and not self.minor_version: + raise ValueError( + "At least one of os, arch, minor_version must be specified" + ) @classmethod - def all(cls, want_os: Optional[OS] = None) -> List["Platform"]: + def all( + cls, + want_os: Optional[OS] = None, + minor_version: Optional[int] = None, + ) -> List["Platform"]: return sorted( [ - cls(os=os, arch=arch) + cls(os=os, arch=arch, minor_version=minor_version) for os in OS for arch in Arch if not want_os or want_os == os @@ -121,7 +136,14 @@ def all_specializations(self) -> Iterator["Platform"]: yield self if self.arch is None: for arch in Arch: - yield Platform(os=self.os, arch=arch) + yield Platform(os=self.os, arch=arch, minor_version=self.minor_version) + if self.os is None: + for os in OS: + yield Platform(os=os, arch=self.arch, minor_version=self.minor_version) + if self.arch is None and self.os is None: + for os in OS: + for arch in Arch: + yield Platform(os=os, arch=arch, minor_version=self.minor_version) def __lt__(self, other: Any) -> bool: """Add a comparison method, so that `sorted` returns the most specialized platforms first.""" @@ -137,10 +159,25 @@ def __lt__(self, other: Any) -> bool: return self_os < other_os def __str__(self) -> str: + if self.minor_version is None: + assert ( + self.os is not None + ), f"if minor_version is None, OS must be specified, got {repr(self)}" + if self.arch is None: + return f"@platforms//os:{self.os}" + else: + return f"{self.os}_{self.arch}" + + if self.arch is None and self.os is None: + return f"@//python/config_settings:is_python_3.{self.minor_version}" + if self.arch is None: - return f"@platforms//os:{self.os}" + return f"cp3{self.minor_version}_{self.os}_anyarch" + + if self.os is None: + return f"cp3{self.minor_version}_anyos_{self.arch}" - return f"{self.os}_{self.arch}" + return f"cp3{self.minor_version}_{self.os}_{self.arch}" @classmethod def from_string(cls, platform: Union[str, List[str]]) -> List["Platform"]: @@ -150,14 +187,33 @@ def from_string(cls, platform: Union[str, List[str]]) -> List["Platform"]: for p in platform: if p == "host": ret.update(cls.host()) - elif p == "all": - ret.update(cls.all()) - elif p.endswith("*"): - os, _, _ = p.partition("_") - ret.update(cls.all(OS[os])) + continue + + abi, _, tail = p.partition("_") + if not abi.startswith("cp"): + # The first item is not an abi + tail = p + abi = "" + os, _, arch = tail.partition("_") + arch = arch or "*" + + minor_version = int(abi[len("cp3") :]) if abi else None + + if arch != "*": + ret.add( + cls( + os=OS[os] if os != "*" else None, + arch=Arch[arch], + minor_version=minor_version, + ) + ) else: - os, _, arch = p.partition("_") - ret.add(cls(os=OS[os], arch=Arch[arch])) + ret.update( + cls.all( + want_os=OS[os] if os != "*" else None, + minor_version=minor_version, + ) + ) return sorted(ret) @@ -227,6 +283,9 @@ def platform_machine(self) -> str: return "" def env_markers(self, extra: str) -> Dict[str, str]: + # If it is None, use the host version + minor_version = self.minor_version or host_interpreter_minor_version() + return { "extra": extra, "os_name": self.os_name, @@ -235,11 +294,14 @@ def env_markers(self, extra: str) -> Dict[str, str]: "platform_system": self.platform_system, "platform_release": "", # unset "platform_version": "", # unset + "python_version": f"3.{minor_version}", + # FIXME @aignas 2024-01-14: is putting zero last a good idea? Maybe we should + # use `20` or something else to avoid having weird issues where the full version is used for + # matching and the author decides to only support 3.y.5 upwards. + "implementation_version": f"3.{minor_version}.0", + "python_full_version": f"3.{minor_version}.0", # we assume that the following are the same as the interpreter used to setup the deps: - # "implementation_version": "X.Y.Z", # "implementation_name": "cpython" - # "python_version": "X.Y", - # "python_full_version": "X.Y.Z", # "platform_python_implementation: "CPython", } @@ -251,16 +313,36 @@ class FrozenDeps: class Deps: + """Deps is a dependency builder that has a build() method to return FrozenDeps.""" + def __init__( self, name: str, + requires_dist: List[str], *, - requires_dist: Optional[List[str]], extras: Optional[Set[str]] = None, platforms: Optional[Set[Platform]] = None, ): + """Create a new instance and parse the requires_dist + + Args: + name (str): The name of the whl distribution + requires_dist (list[Str]): The Requires-Dist from the METADATA of the whl + distribution. + extras (set[str], optional): The list of requested extras, defaults to None. + platforms (set[Platform], optional): The list of target platforms, defaults to + None. If the list of platforms has multiple `minor_version` values, it + will change the code to generate the select statements using + `@rules_python//python/config_settings:is_python_3.y` conditions. + """ self.name: str = Deps._normalize(name) self._platforms: Set[Platform] = platforms or set() + self._target_versions = {p.minor_version for p in platforms or {}} + self._add_version_select = platforms and len(self._target_versions) > 2 + if None in self._target_versions and len(self._target_versions) > 2: + raise ValueError( + f"all python versions need to be specified explicitly, got: {platforms}" + ) # Sort so that the dictionary order in the FrozenDeps is deterministic # without the final sort because Python retains insertion order. That way @@ -301,18 +383,39 @@ def _add(self, dep: str, platform: Optional[Platform]): self._select[p].add(dep) - if len(self._select[platform]) != 1: + if len(self._select[platform]) == 1: + # We are adding a new item to the select and we need to ensure that + # existing dependencies from less specialized platforms are propagated + # to the newly added dependency set. + for p, deps in self._select.items(): + # Check if the existing platform overlaps with the given platform + if p == platform or platform not in p.all_specializations(): + continue + + self._select[platform].update(self._select[p]) + + def _maybe_add_common_dep(self, dep): + if len(self._target_versions) < 2: return - # We are adding a new item to the select and we need to ensure that - # existing dependencies from less specialized platforms are propagated - # to the newly added dependency set. - for p, deps in self._select.items(): - # Check if the existing platform overlaps with the given platform - if p == platform or platform not in p.all_specializations(): - continue + platforms = [Platform(minor_version=v) for v in self._target_versions] - self._select[platform].update(self._select[p]) + # If the dep is targeting all target python versions, lets add it to + # the common dependency list to simplify the select statements. + for p in platforms: + if p not in self._select: + return + + if dep not in self._select[p]: + return + + # All of the python version-specific branches have the dep, so lets add + # it to the common deps. + self._deps.add(dep) + for p in platforms: + self._select[p].remove(dep) + if not self._select[p]: + self._select.pop(p) @staticmethod def _normalize(name: str) -> str: @@ -400,8 +503,9 @@ def _add_req(self, req: Requirement, extras: Set[str]) -> None: ] ) match_arch = "platform_machine" in marker_str + match_version = "version" in marker_str - if not (match_os or match_arch): + if not (match_os or match_arch or match_version): if any(req.marker.evaluate({"extra": extra}) for extra in extras): self._add(req.name, None) return @@ -414,8 +518,17 @@ def _add_req(self, req: Requirement, extras: Set[str]) -> None: if match_arch: self._add(req.name, plat) - else: + elif match_os and self._add_version_select: + self._add(req.name, Platform(plat.os, minor_version=plat.minor_version)) + elif match_os: self._add(req.name, Platform(plat.os)) + elif match_version and self._add_version_select: + self._add(req.name, Platform(minor_version=plat.minor_version)) + elif match_version: + self._add(req.name, None) + + # Merge to common if possible after processing all platforms + self._maybe_add_common_dep(req.name) def build(self) -> FrozenDeps: return FrozenDeps( diff --git a/python/pip_install/tools/wheel_installer/wheel_test.py b/python/pip_install/tools/wheel_installer/wheel_test.py index 5e95ee37dd..f7c847fc9e 100644 --- a/python/pip_install/tools/wheel_installer/wheel_test.py +++ b/python/pip_install/tools/wheel_installer/wheel_test.py @@ -15,12 +15,6 @@ def test_simple(self): self.assertEqual({}, got.deps_select) def test_can_add_os_specific_deps(self): - platforms = { - "linux_x86_64", - "osx_x86_64", - "osx_aarch64", - "windows_x86_64", - } deps = wheel.Deps( "foo", requires_dist=[ @@ -29,7 +23,49 @@ def test_can_add_os_specific_deps(self): "posix_dep; os_name=='posix'", "win_dep; os_name=='nt'", ], - platforms=set(wheel.Platform.from_string(platforms)), + platforms={ + wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.x86_64), + wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.x86_64), + wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.aarch64), + wheel.Platform(os=wheel.OS.windows, arch=wheel.Arch.x86_64), + }, + ) + + got = deps.build() + + self.assertEqual(["bar"], got.deps) + self.assertEqual( + { + "@platforms//os:linux": ["posix_dep"], + "@platforms//os:osx": ["an_osx_dep", "posix_dep"], + "@platforms//os:windows": ["win_dep"], + }, + got.deps_select, + ) + + def test_can_add_os_specific_deps_with_specific_python_version(self): + deps = wheel.Deps( + "foo", + requires_dist=[ + "bar", + "an_osx_dep; sys_platform=='darwin'", + "posix_dep; os_name=='posix'", + "win_dep; os_name=='nt'", + ], + platforms={ + wheel.Platform( + os=wheel.OS.linux, arch=wheel.Arch.x86_64, minor_version=8 + ), + wheel.Platform( + os=wheel.OS.osx, arch=wheel.Arch.x86_64, minor_version=8 + ), + wheel.Platform( + os=wheel.OS.osx, arch=wheel.Arch.aarch64, minor_version=8 + ), + wheel.Platform( + os=wheel.OS.windows, arch=wheel.Arch.x86_64, minor_version=8 + ), + }, ) got = deps.build() @@ -45,17 +81,16 @@ def test_can_add_os_specific_deps(self): ) def test_deps_are_added_to_more_specialized_platforms(self): - platforms = { - "osx_x86_64", - "osx_aarch64", - } got = wheel.Deps( "foo", requires_dist=[ "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'", "mac_dep; sys_platform=='darwin'", ], - platforms=set(wheel.Platform.from_string(platforms)), + platforms={ + wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.x86_64), + wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.aarch64), + }, ).build() self.assertEqual( @@ -70,17 +105,16 @@ def test_deps_are_added_to_more_specialized_platforms(self): ) def test_deps_from_more_specialized_platforms_are_propagated(self): - platforms = { - "osx_x86_64", - "osx_aarch64", - } got = wheel.Deps( "foo", requires_dist=[ "a_mac_dep; sys_platform=='darwin'", "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'", ], - platforms=set(wheel.Platform.from_string(platforms)), + platforms={ + wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.x86_64), + wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.aarch64), + }, ).build() self.assertEqual([], got.deps) @@ -93,12 +127,6 @@ def test_deps_from_more_specialized_platforms_are_propagated(self): ) def test_non_platform_markers_are_added_to_common_deps(self): - platforms = { - "linux_x86_64", - "osx_x86_64", - "osx_aarch64", - "windows_x86_64", - } got = wheel.Deps( "foo", requires_dist=[ @@ -106,7 +134,12 @@ def test_non_platform_markers_are_added_to_common_deps(self): "baz; implementation_name=='cpython'", "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'", ], - platforms=set(wheel.Platform.from_string(platforms)), + platforms={ + wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.x86_64), + wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.x86_64), + wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.aarch64), + wheel.Platform(os=wheel.OS.windows, arch=wheel.Arch.x86_64), + }, ).build() self.assertEqual(["bar", "baz"], got.deps) @@ -152,6 +185,91 @@ def test_self_dependencies_can_come_in_any_order(self): self.assertEqual(["bar", "baz", "zdep"], got.deps) self.assertEqual({}, got.deps_select) + def test_can_get_deps_based_on_specific_python_version(self): + requires_dist = [ + "bar", + "baz; python_version < '3.8'", + "posix_dep; os_name=='posix' and python_version >= '3.8'", + ] + + py38_deps = wheel.Deps( + "foo", + requires_dist=requires_dist, + platforms=[ + wheel.Platform( + os=wheel.OS.linux, arch=wheel.Arch.x86_64, minor_version=8 + ), + ], + ).build() + py37_deps = wheel.Deps( + "foo", + requires_dist=requires_dist, + platforms=[ + wheel.Platform( + os=wheel.OS.linux, arch=wheel.Arch.x86_64, minor_version=7 + ), + ], + ).build() + + self.assertEqual(["bar", "baz"], py37_deps.deps) + self.assertEqual({}, py37_deps.deps_select) + self.assertEqual(["bar"], py38_deps.deps) + self.assertEqual({"@platforms//os:linux": ["posix_dep"]}, py38_deps.deps_select) + + def test_can_get_version_select(self): + requires_dist = [ + "bar", + "baz; python_version < '3.8'", + "posix_dep; os_name=='posix'", + "posix_dep_with_version; os_name=='posix' and python_version >= '3.8'", + ] + + deps = wheel.Deps( + "foo", + requires_dist=requires_dist, + platforms=[ + wheel.Platform( + os=wheel.OS.linux, arch=wheel.Arch.x86_64, minor_version=minor + ) + for minor in [7, 8, 9] + ], + ) + got = deps.build() + + self.assertEqual(["bar"], got.deps) + self.assertEqual( + { + "@//python/config_settings:is_python_3.7": ["baz"], + "cp37_linux_anyarch": ["baz", "posix_dep"], + "cp38_linux_anyarch": ["posix_dep", "posix_dep_with_version"], + "cp39_linux_anyarch": ["posix_dep", "posix_dep_with_version"], + }, + got.deps_select, + ) + + def test_deps_spanning_all_target_py_versions_are_added_to_common(self): + requires_dist = [ + "bar", + "baz (<2,>=1.11) ; python_version < '3.8'", + "baz (<2,>=1.14) ; python_version >= '3.8'", + ] + + deps = wheel.Deps( + "foo", + requires_dist=requires_dist, + platforms=wheel.Platform.from_string(["cp37_*", "cp38_*", "cp39_*"]), + ) + got = deps.build() + + self.assertEqual(["bar", "baz"], got.deps) + self.assertEqual({}, got.deps_select) + + +class MinorVersionTest(unittest.TestCase): + def test_host(self): + host = wheel.host_interpreter_minor_version() + self.assertIsNotNone(host) + class PlatformTest(unittest.TestCase): def test_can_get_host(self): @@ -160,16 +278,64 @@ def test_can_get_host(self): self.assertEqual(1, len(wheel.Platform.from_string("host"))) self.assertEqual(host, wheel.Platform.from_string("host")) - def test_can_get_all(self): - all_platforms = wheel.Platform.all() - self.assertEqual(15, len(all_platforms)) - self.assertEqual(all_platforms, wheel.Platform.from_string("all")) + def test_can_get_linux_x86_64_without_py_version(self): + got = wheel.Platform.from_string("linux_x86_64") + want = wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.x86_64) + self.assertEqual(want, got[0]) + + def test_can_get_specific_from_string(self): + got = wheel.Platform.from_string("cp33_linux_x86_64") + want = wheel.Platform( + os=wheel.OS.linux, arch=wheel.Arch.x86_64, minor_version=3 + ) + self.assertEqual(want, got[0]) + + def test_can_get_all_for_py_version(self): + cp39 = wheel.Platform.all(minor_version=9) + self.assertEqual(15, len(cp39), f"Got {cp39}") + self.assertEqual(cp39, wheel.Platform.from_string("cp39_*")) def test_can_get_all_for_os(self): + linuxes = wheel.Platform.all(wheel.OS.linux, minor_version=9) + self.assertEqual(5, len(linuxes)) + self.assertEqual(linuxes, wheel.Platform.from_string("cp39_linux_*")) + + def test_can_get_all_for_os_for_host_python(self): linuxes = wheel.Platform.all(wheel.OS.linux) self.assertEqual(5, len(linuxes)) self.assertEqual(linuxes, wheel.Platform.from_string("linux_*")) + def test_specific_version_specializations(self): + any_py33 = wheel.Platform(minor_version=3) + + # When + all_specializations = list(any_py33.all_specializations()) + + want = ( + [any_py33] + + [ + wheel.Platform(arch=arch, minor_version=any_py33.minor_version) + for arch in wheel.Arch + ] + + [ + wheel.Platform(os=os, minor_version=any_py33.minor_version) + for os in wheel.OS + ] + + wheel.Platform.all(minor_version=any_py33.minor_version) + ) + self.assertEqual(want, all_specializations) + + def test_aarch64_specializations(self): + any_aarch64 = wheel.Platform(arch=wheel.Arch.aarch64) + all_specializations = list(any_aarch64.all_specializations()) + want = [ + wheel.Platform(os=None, arch=wheel.Arch.aarch64), + wheel.Platform(os=wheel.OS.linux, arch=wheel.Arch.aarch64), + wheel.Platform(os=wheel.OS.osx, arch=wheel.Arch.aarch64), + wheel.Platform(os=wheel.OS.windows, arch=wheel.Arch.aarch64), + ] + self.assertEqual(want, all_specializations) + def test_linux_specializations(self): any_linux = wheel.Platform(os=wheel.OS.linux) all_specializations = list(any_linux.all_specializations()) diff --git a/tests/pip_install/whl_library/generate_build_bazel_tests.bzl b/tests/pip_install/whl_library/generate_build_bazel_tests.bzl index b89477fd4c..72423aaec4 100644 --- a/tests/pip_install/whl_library/generate_build_bazel_tests.bzl +++ b/tests/pip_install/whl_library/generate_build_bazel_tests.bzl @@ -36,6 +36,82 @@ filegroup( srcs = glob(["data/**"], allow_empty = True), ) +filegroup( + name = "_whl", + srcs = ["foo.whl"], + data = [ + "@pypi_bar_baz//:whl", + "@pypi_foo//:whl", + ], + visibility = ["//visibility:private"], +) + +py_library( + name = "_pkg", + srcs = glob( + ["site-packages/**/*.py"], + exclude=[], + # Empty sources are allowed to support wheels that don't have any + # pure-Python code, e.g. pymssql, which is written in Cython. + allow_empty = True, + ), + data = [] + glob( + ["site-packages/**/*"], + exclude=["**/* *", "**/*.py", "**/*.pyc", "**/*.pyc.*", "**/*.dist-info/RECORD"], + ), + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["site-packages"], + deps = [ + "@pypi_bar_baz//:pkg", + "@pypi_foo//:pkg", + ], + tags = ["tag1", "tag2"], + visibility = ["//visibility:private"], +) + +alias( + name = "pkg", + actual = "_pkg", +) + +alias( + name = "whl", + actual = "_whl", +) +""" + actual = generate_whl_library_build_bazel( + repo_prefix = "pypi_", + whl_name = "foo.whl", + dependencies = ["foo", "bar-baz"], + dependencies_by_platform = {}, + data_exclude = [], + tags = ["tag1", "tag2"], + entry_points = {}, + annotation = None, + ) + env.expect.that_str(actual).equals(want) + +_tests.append(_test_simple) + +def _test_dep_selects(env): + want = """\ +load("@rules_python//python:defs.bzl", "py_library", "py_binary") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "dist_info", + srcs = glob(["site-packages/*.dist-info/**"], allow_empty = True), +) + +filegroup( + name = "data", + srcs = glob(["data/**"], allow_empty = True), +) + filegroup( name = "_whl", srcs = ["foo.whl"], @@ -44,7 +120,13 @@ filegroup( "@pypi_foo//:whl", ] + select( { - "@platforms//os:windows": ["@pypi_colorama//:whl"], + "@//python/config_settings:is_python_3.9": ["@pypi_py39_dep//:whl"], + "@platforms//cpu:aarch64": ["@pypi_arm_dep//:whl"], + "@platforms//os:windows": ["@pypi_win_dep//:whl"], + ":is_cp310_linux_ppc": ["@pypi_py310_linux_ppc_dep//:whl"], + ":is_cp39_anyos_aarch64": ["@pypi_py39_arm_dep//:whl"], + ":is_cp39_linux_anyarch": ["@pypi_py39_linux_dep//:whl"], + ":is_linux_x86_64": ["@pypi_linux_intel_dep//:whl"], "//conditions:default": [], }, ), @@ -72,7 +154,13 @@ py_library( "@pypi_foo//:pkg", ] + select( { - "@platforms//os:windows": ["@pypi_colorama//:pkg"], + "@//python/config_settings:is_python_3.9": ["@pypi_py39_dep//:pkg"], + "@platforms//cpu:aarch64": ["@pypi_arm_dep//:pkg"], + "@platforms//os:windows": ["@pypi_win_dep//:pkg"], + ":is_cp310_linux_ppc": ["@pypi_py310_linux_ppc_dep//:pkg"], + ":is_cp39_anyos_aarch64": ["@pypi_py39_arm_dep//:pkg"], + ":is_cp39_linux_anyarch": ["@pypi_py39_linux_dep//:pkg"], + ":is_linux_x86_64": ["@pypi_linux_intel_dep//:pkg"], "//conditions:default": [], }, ), @@ -89,20 +177,85 @@ alias( name = "whl", actual = "_whl", ) + +config_setting( + name = "is_linux_ppc", + constraint_values = [ + "@platforms//cpu:ppc", + "@platforms//os:linux", + ], + visibility = ["//visibility:private"], +) + +selects.config_setting_group( + name = "is_cp310_linux_ppc", + match_all = [ + ":is_linux_ppc", + "@//python/config_settings:is_python_3.10", + ], + visibility = ["//visibility:private"], +) + +config_setting( + name = "is_anyos_aarch64", + constraint_values = ["@platforms//cpu:aarch64"], + visibility = ["//visibility:private"], +) + +selects.config_setting_group( + name = "is_cp39_anyos_aarch64", + match_all = [ + ":is_anyos_aarch64", + "@//python/config_settings:is_python_3.9", + ], + visibility = ["//visibility:private"], +) + +config_setting( + name = "is_linux_anyarch", + constraint_values = ["@platforms//os:linux"], + visibility = ["//visibility:private"], +) + +selects.config_setting_group( + name = "is_cp39_linux_anyarch", + match_all = [ + ":is_linux_anyarch", + "@//python/config_settings:is_python_3.9", + ], + visibility = ["//visibility:private"], +) + +config_setting( + name = "is_linux_x86_64", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:linux", + ], + visibility = ["//visibility:private"], +) """ actual = generate_whl_library_build_bazel( repo_prefix = "pypi_", whl_name = "foo.whl", dependencies = ["foo", "bar-baz"], - dependencies_by_platform = {"@platforms//os:windows": ["colorama"]}, + dependencies_by_platform = { + "@//python/config_settings:is_python_3.9": ["py39_dep"], + "@platforms//cpu:aarch64": ["arm_dep"], + "@platforms//os:windows": ["win_dep"], + "cp310_linux_ppc": ["py310_linux_ppc_dep"], + "cp39_anyos_aarch64": ["py39_arm_dep"], + "cp39_linux_anyarch": ["py39_linux_dep"], + "linux_x86_64": ["linux_intel_dep"], + }, data_exclude = [], tags = ["tag1", "tag2"], entry_points = {}, annotation = None, ) - env.expect.that_str(actual).equals(want) + env.expect.that_str(actual.replace("@@", "@")).equals(want) -_tests.append(_test_simple) +_tests.append(_test_dep_selects) def _test_with_annotation(env): want = """\ @@ -308,11 +461,11 @@ filegroup( srcs = ["foo.whl"], data = ["@pypi_bar_baz//:whl"] + select( { + "@platforms//os:linux": ["@pypi_box//:whl"], ":is_linux_x86_64": [ "@pypi_box//:whl", "@pypi_box_amd64//:whl", ], - "@platforms//os:linux": ["@pypi_box//:whl"], "//conditions:default": [], }, ), @@ -337,11 +490,11 @@ py_library( imports = ["site-packages"], deps = ["@pypi_bar_baz//:pkg"] + select( { + "@platforms//os:linux": ["@pypi_box//:pkg"], ":is_linux_x86_64": [ "@pypi_box//:pkg", "@pypi_box_amd64//:pkg", ], - "@platforms//os:linux": ["@pypi_box//:pkg"], "//conditions:default": [], }, ), @@ -375,7 +528,7 @@ config_setting( dependencies_by_platform = { "linux_x86_64": ["box", "box-amd64"], "windows_x86_64": ["fox"], - "@platforms//os:linux": ["box"], # buildifier: disable=unsorted-dict-items + "@platforms//os:linux": ["box"], # buildifier: disable=unsorted-dict-items to check that we sort inside the test }, tags = [], entry_points = {}, @@ -384,7 +537,7 @@ config_setting( group_name = "qux", group_deps = ["foo", "fox", "qux"], ) - env.expect.that_str(actual).equals(want) + env.expect.that_str(actual.replace("@@", "@")).equals(want) _tests.append(_test_group_member)