Handle compatible libc-based manylinux/musllinux platform tags

news/10760.feature.rst

@@ -0,0 +1 @@
+Support glibc-based manylinux and musllinux platform tags.

src/pip/_internal/utils/compatibility_tags.py

@@ -1,6 +1,5 @@
 """Generate and work with PEP 425 Compatibility Tags.
 """
-
 import re
 from typing import List, Optional, Tuple
 
@@ -15,6 +14,8 @@
     mac_platforms,
 )
 
+from pip._internal.utils.packaging import filter_manylinux_tags, filter_musllinux_tags
+
 _osx_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)")
 
 
@@ -43,10 +44,16 @@ def _mac_platforms(arch: str) -> List[str]:
     return arches
 
 
-def _custom_manylinux_platforms(arch: str) -> List[str]:
+def _manylinux_platforms(arch: str) -> List[str]:
     arches = [arch]
     arch_prefix, arch_sep, arch_suffix = arch.partition("_")
-    if arch_prefix == "manylinux2014":
+
+    if arch_prefix == "manylinux":
+        curr_glibc_major, curr_glibc_minor, curr_arch = arch_suffix.split("_", 2)
+        curr_glibc = (int(curr_glibc_major), int(curr_glibc_minor))
+        arches = list(filter_manylinux_tags(curr_glibc, curr_arch)) or arches
+
+    elif arch_prefix == "manylinux2014":
         # manylinux1/manylinux2010 wheels run on most manylinux2014 systems
         # with the exception of wheels depending on ncurses. PEP 599 states
         # manylinux1/manylinux2010 wheels should be considered
@@ -64,12 +71,24 @@ def _custom_manylinux_platforms(arch: str) -> List[str]:
     return arches
 
 
+def _musllinux_platforms(arch: str) -> List[str]:
+    arches = [arch]
+
+    *_, arch_suffix = arch.partition("_")
+    curr_musl_major, curr_musl_minor, curr_arch = arch_suffix.split("_", 2)
+    curr_musl = (int(curr_musl_major), int(curr_musl_minor))
+
+    arches = list(filter_musllinux_tags(curr_musl, curr_arch)) or arches
+    return arches
+
+
 def _get_custom_platforms(arch: str) -> List[str]:
-    arch_prefix, arch_sep, arch_suffix = arch.partition("_")
     if arch.startswith("macosx"):
         arches = _mac_platforms(arch)
-    elif arch_prefix in ["manylinux2014", "manylinux2010"]:
-        arches = _custom_manylinux_platforms(arch)
+    elif arch.startswith("manylinux"):
+        arches = _manylinux_platforms(arch)
+    elif arch.startswith("musllinux"):
+        arches = _musllinux_platforms(arch)
     else:
         arches = [arch]
     return arches

src/pip/_internal/utils/packaging.py

@@ -1,16 +1,23 @@
 import functools
 import logging
 import re
-from typing import NewType, Optional, Tuple, cast
+from typing import Generator, NewType, Optional, Tuple, cast
 
 from pip._vendor.packaging import specifiers, version
 from pip._vendor.packaging.requirements import Requirement
 from pip._vendor.packaging.specifiers import SpecifierSet
+from pip._vendor.packaging.tags import platform_tags
 
 NormalizedExtra = NewType("NormalizedExtra", str)
 
 logger = logging.getLogger(__name__)
 
+_LEGACY_MANYLINUX_MAP = {
+    "manylinux2014": (2, 17),
+    "manylinux2010": (2, 12),
+    "manylinux1": (2, 5),
+}
+
 
 def check_requires_python(
     requires_python: Optional[str], version_info: Tuple[int, ...]
@@ -68,3 +75,30 @@ def is_pinned(specifier: SpecifierSet) -> bool:
             continue
         return True
     return False
+
+
+def filter_manylinux_tags(
+    glibc: Tuple[int, int], arch: str
+) -> Generator[str, None, None]:
+    for tag in filter(lambda t: t.startswith("manylinux"), platform_tags()):
+        tag_prefix, _, tag_suffix = tag.partition("_")
+        if tag_prefix in _LEGACY_MANYLINUX_MAP:
+            tag_glibc = _LEGACY_MANYLINUX_MAP[tag_prefix]
+            tag_arch = tag_suffix
+        else:
+            tag_glibc_major, tag_glibc_minor, tag_arch = tag_suffix.split("_", 2)
+            tag_glibc = (int(tag_glibc_major), int(tag_glibc_minor))
+
+        if arch == tag_arch and tag_glibc <= glibc:
+            yield tag
+
+
+def filter_musllinux_tags(
+    musl: Tuple[int, int], arch: str
+) -> Generator[str, None, None]:
+    for tag in filter(lambda t: t.startswith("musllinux"), platform_tags()):
+        *_, tag_suffix = tag.partition("_")
+        tag_musl_major, tag_musl_minor, tag_arch = tag_suffix.split("_", 2)
+        tag_musl = (int(tag_musl_major), int(tag_musl_minor))
+        if tag_arch == arch and tag_musl <= musl:
+            yield tag

tests/unit/test_utils_compatibility_tags.py

@@ -1,11 +1,27 @@
+import os
+import platform
+import sys
 import sysconfig
-from typing import Any, Callable, Dict, List, Tuple
+import types
+from typing import Any, Callable, Dict, List, Optional, Tuple
 from unittest.mock import patch
 
 import pytest
+from pip._vendor.packaging.tags import _manylinux, _musllinux
 
 from pip._internal.utils import compatibility_tags
 
+ManylinuxModule = Callable[[pytest.MonkeyPatch], types.ModuleType]
+
+
+@pytest.fixture
+def manylinux_module(monkeypatch: pytest.MonkeyPatch) -> types.ModuleType:
+    monkeypatch.setattr(_manylinux, "_get_glibc_version", lambda *args: (2, 20))
+    module_name = "_manylinux"
+    module = types.ModuleType(module_name)
+    monkeypatch.setitem(sys.modules, module_name, module)
+    return module
+
 
 @pytest.mark.parametrize(
     "version_info, expected",
@@ -56,6 +72,196 @@ def test_no_hyphen_tag(self) -> None:
             assert "-" not in tag.platform
 
 
+class TestManylinuxTags:
+    def teardown_method(self) -> None:
+        _manylinux._get_glibc_version.cache_clear()
+
+    @pytest.mark.parametrize(
+        "manylinux,expected,glibc_ver",
+        [
+            (
+                "manylinux_2_12_x86_64",
+                [
+                    "manylinux_2_12_x86_64",
+                    "manylinux2010_x86_64",
+                    "manylinux_2_11_x86_64",
+                    "manylinux_2_10_x86_64",
+                    "manylinux_2_9_x86_64",
+                    "manylinux_2_8_x86_64",
+                    "manylinux_2_7_x86_64",
+                    "manylinux_2_6_x86_64",
+                    "manylinux_2_5_x86_64",
+                    "manylinux1_x86_64",
+                ],
+                "2.12",
+            ),
+            (
+                "manylinux_2_17_x86_64",
+                [
+                    "manylinux_2_17_x86_64",
+                    "manylinux2014_x86_64",
+                    "manylinux_2_16_x86_64",
+                    "manylinux_2_15_x86_64",
+                    "manylinux_2_14_x86_64",
+                    "manylinux_2_13_x86_64",
+                    "manylinux_2_12_x86_64",
+                    "manylinux2010_x86_64",
+                    "manylinux_2_11_x86_64",
+                    "manylinux_2_10_x86_64",
+                    "manylinux_2_9_x86_64",
+                    "manylinux_2_8_x86_64",
+                    "manylinux_2_7_x86_64",
+                    "manylinux_2_6_x86_64",
+                    "manylinux_2_5_x86_64",
+                    "manylinux1_x86_64",
+                ],
+                "2.17",
+            ),
+            (
+                "manylinux_2_5_x86_64",
+                [
+                    "manylinux_2_5_x86_64",
+                    "manylinux1_x86_64",
+                ],
+                "2.5",
+            ),
+            (
+                "manylinux_2_17_s390x",
+                [
+                    "manylinux_2_17_s390x",
+                    "manylinux2014_s390x",
+                ],
+                "2.17",
+            ),
+            (
+                "manylinux_2_17_x86_64",
+                [
+                    "manylinux_2_12_x86_64",
+                    "manylinux2010_x86_64",
+                    "manylinux_2_11_x86_64",
+                    "manylinux_2_10_x86_64",
+                    "manylinux_2_9_x86_64",
+                    "manylinux_2_8_x86_64",
+                    "manylinux_2_7_x86_64",
+                    "manylinux_2_6_x86_64",
+                    "manylinux_2_5_x86_64",
+                    "manylinux1_x86_64",
+                ],
+                "2.12",
+            ),
+        ],
+    )
+    def test_manylinux(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+        manylinux: str,
+        expected: List[str],
+        glibc_ver: str,
+    ) -> None:
+        *_, arch = manylinux.split("_", 3)
+        monkeypatch.setattr(platform, "system", lambda: "Linux")
+        monkeypatch.setattr(sysconfig, "get_platform", lambda: f"linux_{arch}")
+        monkeypatch.setattr(platform, "machine", lambda: arch)
+        monkeypatch.setattr(
+            os, "confstr", lambda x: f"glibc {glibc_ver}", raising=False
+        )
+        groups: Dict[Tuple[str, str], List[str]] = {}
+        supported = compatibility_tags.get_supported(platforms=[manylinux])
+        for tag in supported:
+            groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform)
+
+        for arches in groups.values():
+            if "any" in arches:
+                continue
+            assert arches == expected
+
+
+class TestManylinuxCompatibleTags:
+    @pytest.mark.parametrize(
+        "machine, major, minor, tf", [("x86_64", 2, 20, False), ("s390x", 2, 22, True)]
+    )
+    def test_use_manylinux_compatible(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+        manylinux_module: ManylinuxModule,
+        machine: str,
+        major: int,
+        minor: int,
+        tf: bool,
+    ) -> None:
+        def manylinux_compatible(tag_major: int, tag_minor: int, tag_arch: str) -> bool:
+            if tag_major == 2 and tag_minor == 22:
+                return tag_arch == "s390x"
+            return False
+
+        monkeypatch.setattr(platform, "system", lambda: "Linux")
+        monkeypatch.setattr(
+            _manylinux,
+            "_get_glibc_version",
+            lambda: (major, minor),
+        )
+        monkeypatch.setattr(
+            manylinux_module,
+            "manylinux_compatible",
+            manylinux_compatible,
+            raising=False,
+        )
+        groups: Dict[Tuple[str, str], List[str]] = {}
+        manylinux = f"manylinux_{major}_{minor}_{machine}"
+        supported = compatibility_tags.get_supported(platforms=[manylinux])
+        for tag in supported:
+            groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform)
+
+        if tf:
+            expected = [f"manylinux_2_22_{machine}"]
+        else:
+            expected = [f"manylinux_{major}_{minor}_{machine}"]
+        for arches in groups.values():
+            if "any" in arches:
+                continue
+            assert arches == expected
+
+    def test_linux_use_manylinux_compatible_none(
+        self, monkeypatch: pytest.MonkeyPatch, manylinux_module: ManylinuxModule
+    ) -> None:
+        def manylinux_compatible(
+            tag_major: int, tag_minor: int, tag_arch: str
+        ) -> Optional[bool]:
+            if tag_major == 2 and tag_minor < 25:
+                return False
+            return None
+
+        monkeypatch.setattr(platform, "system", lambda: "Linux")
+        monkeypatch.setattr(_manylinux, "_get_glibc_version", lambda: (2, 30))
+        monkeypatch.setattr(sysconfig, "get_platform", lambda: "linux_x86_64")
+        monkeypatch.setattr(
+            manylinux_module,
+            "manylinux_compatible",
+            manylinux_compatible,
+            raising=False,
+        )
+
+        groups: Dict[Tuple[str, str], List[str]] = {}
+        supported = compatibility_tags.get_supported(
+            platforms=["manylinux_2_30_x86_64"]
+        )
+        for tag in supported:
+            groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform)
+
+        expected = [
+            "manylinux_2_30_x86_64",
+            "manylinux_2_29_x86_64",
+            "manylinux_2_28_x86_64",
+            "manylinux_2_27_x86_64",
+            "manylinux_2_26_x86_64",
+            "manylinux_2_25_x86_64",
+        ]
+        for arches in groups.values():
+            if "any" in arches:
+                continue
+            assert arches == expected
+
+
 class TestManylinux2010Tags:
     @pytest.mark.parametrize(
         "manylinux2010,manylinux1",
@@ -106,3 +312,45 @@ def test_manylinuxA_implies_manylinuxB(
             if arches == ["any"]:
                 continue
             assert arches[:3] == expected_arches
+
+
+class TestMusllinuxTags:
+    @pytest.mark.parametrize(
+        "musllinux,arch,musl_ver",
+        [
+            ("musllinux_1_4", "x86_64", (1, 4)),
+            ("musllinux_1_4", "i686", (1, 2)),
+        ],
+    )
+    def test_musllinux(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+        musllinux: str,
+        arch: str,
+        musl_ver: Tuple[int, int],
+    ) -> None:
+        monkeypatch.setattr(
+            _manylinux,
+            "_get_glibc_version",
+            lambda: (-1, -1),
+        )
+        monkeypatch.setattr(
+            _musllinux,
+            "_get_musl_version",
+            lambda _: _musllinux._MuslVersion(*musl_ver),
+        )
+        monkeypatch.setattr(platform, "system", lambda: "Linux")
+        monkeypatch.setattr(sysconfig, "get_platform", lambda: f"linux_{arch}")
+        groups: Dict[Tuple[str, str], List[str]] = {}
+        supported = compatibility_tags.get_supported(platforms=[f"{musllinux}_{arch}"])
+        for tag in supported:
+            groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform)
+
+        expected = [
+            f"musllinux_{musl_ver[0]}_{minor}_{arch}"
+            for minor in range(musl_ver[1], -1, -1)
+        ]
+        for arches in groups.values():
+            if "any" in arches:
+                continue
+            assert arches == expected