diff --git a/src/auditwheel/main_repair.py b/src/auditwheel/main_repair.py index b1d9f878..a89de776 100644 --- a/src/auditwheel/main_repair.py +++ b/src/auditwheel/main_repair.py @@ -6,20 +6,15 @@ from auditwheel.patcher import Patchelf -from .policy import ( - POLICY_PRIORITY_HIGHEST, - get_policy_by_name, - get_policy_name, - get_priority_by_name, - load_policies, -) +from .policy import WheelPolicies from .tools import EnvironmentDefault logger = logging.getLogger(__name__) def configure_parser(sub_parsers): - policies = load_policies() + wheel_policy = WheelPolicies() + policies = wheel_policy.policies policy_names = [p["name"] for p in policies] policy_names += [alias for p in policies for alias in p["aliases"]] epilog = """PLATFORMS: @@ -32,7 +27,7 @@ def configure_parser(sub_parsers): if len(p["aliases"]) > 0: epilog += f" (aliased by {', '.join(p['aliases'])})" epilog += "\n" - highest_policy = get_policy_name(POLICY_PRIORITY_HIGHEST) + highest_policy = wheel_policy.get_policy_name(wheel_policy.priority_highest) help = """Vendor in external shared library dependencies of a wheel. If multiple wheels are specified, an error processing one wheel will abort processing of subsequent wheels. @@ -114,6 +109,8 @@ def execute(args, p): from .repair import repair_wheel from .wheel_abi import NonPlatformWheel, analyze_wheel_abi + wheel_policy = WheelPolicies() + for wheel_file in args.WHEEL_FILE: if not isfile(wheel_file): p.error("cannot access %s. No such file" % wheel_file) @@ -124,15 +121,15 @@ def execute(args, p): os.makedirs(args.WHEEL_DIR) try: - wheel_abi = analyze_wheel_abi(wheel_file) + wheel_abi = analyze_wheel_abi(wheel_policy, wheel_file) except NonPlatformWheel: logger.info(NonPlatformWheel.LOG_MESSAGE) return 1 - policy = get_policy_by_name(args.PLAT) + policy = wheel_policy.get_policy_by_name(args.PLAT) reqd_tag = policy["priority"] - if reqd_tag > get_priority_by_name(wheel_abi.sym_tag): + if reqd_tag > wheel_policy.get_priority_by_name(wheel_abi.sym_tag): msg = ( 'cannot repair "%s" to "%s" ABI because of the presence ' "of too-recent versioned symbols. You'll need to compile " @@ -140,7 +137,7 @@ def execute(args, p): ) p.error(msg) - if reqd_tag > get_priority_by_name(wheel_abi.ucs_tag): + if reqd_tag > wheel_policy.get_priority_by_name(wheel_abi.ucs_tag): msg = ( 'cannot repair "%s" to "%s" ABI because it was compiled ' "against a UCS2 build of Python. You'll need to compile " @@ -149,7 +146,7 @@ def execute(args, p): ) p.error(msg) - if reqd_tag > get_priority_by_name(wheel_abi.blacklist_tag): + if reqd_tag > wheel_policy.get_priority_by_name(wheel_abi.blacklist_tag): msg = ( 'cannot repair "%s" to "%s" ABI because it depends on ' "black-listed symbols." % (wheel_file, args.PLAT) @@ -158,7 +155,7 @@ def execute(args, p): abis = [policy["name"]] + policy["aliases"] if not args.ONLY_PLAT: - if reqd_tag < get_priority_by_name(wheel_abi.overall_tag): + if reqd_tag < wheel_policy.get_priority_by_name(wheel_abi.overall_tag): logger.info( ( "Wheel is eligible for a higher priority tag. " @@ -168,11 +165,12 @@ def execute(args, p): args.PLAT, wheel_abi.overall_tag, ) - higher_policy = get_policy_by_name(wheel_abi.overall_tag) + higher_policy = wheel_policy.get_policy_by_name(wheel_abi.overall_tag) abis = [higher_policy["name"]] + higher_policy["aliases"] + abis patcher = Patchelf() out_wheel = repair_wheel( + wheel_policy, wheel_file, abis=abis, lib_sdir=args.LIB_SDIR, diff --git a/src/auditwheel/main_show.py b/src/auditwheel/main_show.py index 28355738..576c0eb6 100644 --- a/src/auditwheel/main_show.py +++ b/src/auditwheel/main_show.py @@ -2,6 +2,8 @@ import logging +from auditwheel.policy import WheelPolicies + logger = logging.getLogger(__name__) @@ -23,22 +25,17 @@ def execute(args, p): import json from os.path import basename, isfile - from .policy import ( - POLICY_PRIORITY_HIGHEST, - POLICY_PRIORITY_LOWEST, - get_policy_name, - get_priority_by_name, - load_policies, - ) from .wheel_abi import NonPlatformWheel, analyze_wheel_abi + wheel_policy = WheelPolicies() + fn = basename(args.WHEEL_FILE) if not isfile(args.WHEEL_FILE): p.error("cannot access %s. No such file" % args.WHEEL_FILE) try: - winfo = analyze_wheel_abi(args.WHEEL_FILE) + winfo = analyze_wheel_abi(wheel_policy, args.WHEEL_FILE) except NonPlatformWheel: logger.info(NonPlatformWheel.LOG_MESSAGE) return 1 @@ -52,7 +49,10 @@ def execute(args, p): % (fn, winfo.overall_tag) ) - if get_priority_by_name(winfo.pyfpe_tag) < POLICY_PRIORITY_HIGHEST: + if ( + wheel_policy.get_priority_by_name(winfo.pyfpe_tag) + < wheel_policy.priority_highest + ): printp( "This wheel uses the PyFPE_jbuf function, which is not compatible with the" " manylinux1 tag. (see https://www.python.org/dev/peps/pep-0513/" @@ -61,7 +61,7 @@ def execute(args, p): if args.verbose < 1: return - if get_priority_by_name(winfo.ucs_tag) < POLICY_PRIORITY_HIGHEST: + if wheel_policy.get_priority_by_name(winfo.ucs_tag) < wheel_policy.priority_highest: printp( "This wheel is compiled against a narrow unicode (UCS2) " "version of Python, which is not compatible with the " @@ -81,7 +81,7 @@ def execute(args, p): "system-provided shared libraries: %s" % ", ".join(libs_with_versions) ) - if get_priority_by_name(winfo.sym_tag) < POLICY_PRIORITY_HIGHEST: + if wheel_policy.get_priority_by_name(winfo.sym_tag) < wheel_policy.priority_highest: printp( ( 'This constrains the platform tag to "%s". ' @@ -95,15 +95,17 @@ def execute(args, p): if args.verbose < 1: return - libs = winfo.external_refs[get_policy_name(POLICY_PRIORITY_LOWEST)]["libs"] + libs = winfo.external_refs[ + wheel_policy.get_policy_name(wheel_policy.priority_lowest) + ]["libs"] if len(libs) == 0: printp("The wheel requires no external shared libraries! :)") else: printp("The following external shared libraries are required " "by the wheel:") print(json.dumps(dict(sorted(libs.items())), indent=4)) - for p in sorted(load_policies(), key=lambda p: p["priority"]): - if p["priority"] > get_priority_by_name(winfo.overall_tag): + for p in sorted(wheel_policy.policies, key=lambda p: p["priority"]): + if p["priority"] > wheel_policy.get_priority_by_name(winfo.overall_tag): libs = winfo.external_refs[p["name"]]["libs"] if len(libs): printp( diff --git a/src/auditwheel/policy/__init__.py b/src/auditwheel/policy/__init__.py index 97fd3888..93cd7399 100644 --- a/src/auditwheel/policy/__init__.py +++ b/src/auditwheel/policy/__init__.py @@ -3,21 +3,209 @@ import json import logging import platform as _platform_module +import re import sys from collections import defaultdict from os.path import abspath, dirname, join from pathlib import Path +from typing import Any, Generator + +from auditwheel.elfutils import filter_undefined_symbols, is_subdir from ..libc import Libc, get_libc from ..musllinux import find_musl_libc, get_musl_version _HERE = Path(__file__).parent +LIBPYTHON_RE = re.compile(r"^libpython\d+\.\d+m?.so(.\d)*$") logger = logging.getLogger(__name__) # https://docs.python.org/3/library/platform.html#platform.architecture bits = 8 * (8 if sys.maxsize > 2**32 else 4) +_POLICY_JSON_MAP = { + Libc.GLIBC: _HERE / "manylinux-policy.json", + Libc.MUSL: _HERE / "musllinux-policy.json", +} + + +class WheelPolicies: + def __init__(self) -> None: + libc_variant = get_libc() + policies_path = _POLICY_JSON_MAP[libc_variant] + policies = json.loads(policies_path.read_text()) + self._policies = [] + self._musl_policy = _get_musl_policy() + self._arch_name = get_arch_name() + self._libc_variant = get_libc() + + _validate_pep600_compliance(policies) + for policy in policies: + if self._musl_policy is not None and policy["name"] not in { + "linux", + self._musl_policy, + }: + continue + if ( + self._arch_name in policy["symbol_versions"].keys() + or policy["name"] == "linux" + ): + if policy["name"] != "linux": + policy["symbol_versions"] = policy["symbol_versions"][ + self._arch_name + ] + policy["name"] = policy["name"] + "_" + self._arch_name + policy["aliases"] = [ + alias + "_" + self._arch_name for alias in policy["aliases"] + ] + policy["lib_whitelist"] = _fixup_musl_libc_soname( + policy["lib_whitelist"] + ) + self._policies.append(policy) + + if self._libc_variant == Libc.MUSL: + assert len(self._policies) == 2, self._policies + + @property + def policies(self): + return self._policies + + @property + def priority_highest(self): + return max(p["priority"] for p in self._policies) + + @property + def priority_lowest(self): + return min(p["priority"] for p in self._policies) + + def get_policy_by_name(self, name: str) -> dict | None: + matches = [ + p for p in self._policies if p["name"] == name or name in p["aliases"] + ] + if len(matches) == 0: + return None + if len(matches) > 1: + raise RuntimeError("Internal error. Policies should be unique") + return matches[0] + + def get_policy_name(self, priority: int) -> str | None: + matches = [p["name"] for p in self._policies if p["priority"] == priority] + if len(matches) == 0: + return None + if len(matches) > 1: + raise RuntimeError("Internal error. priorities should be unique") + return matches[0] + + def get_priority_by_name(self, name: str) -> int | None: + policy = self.get_policy_by_name(name) + return None if policy is None else policy["priority"] + + def versioned_symbols_policy(self, versioned_symbols: dict[str, set[str]]) -> int: + def policy_is_satisfied( + policy_name: str, policy_sym_vers: dict[str, set[str]] + ) -> bool: + policy_satisfied = True + for name in set(required_vers) & set(policy_sym_vers): + if not required_vers[name].issubset(policy_sym_vers[name]): + for symbol in required_vers[name] - policy_sym_vers[name]: + logger.debug( + "Package requires %s, incompatible with " + "policy %s which requires %s", + symbol, + policy_name, + policy_sym_vers[name], + ) + policy_satisfied = False + return policy_satisfied + + required_vers: dict[str, set[str]] = {} + for symbols in versioned_symbols.values(): + for symbol in symbols: + sym_name, _, _ = symbol.partition("_") + required_vers.setdefault(sym_name, set()).add(symbol) + matching_policies: list[int] = [] + for p in self.policies: + policy_sym_vers = { + sym_name: {sym_name + "_" + version for version in versions} + for sym_name, versions in p["symbol_versions"].items() + } + if policy_is_satisfied(p["name"], policy_sym_vers): + matching_policies.append(p["priority"]) + + if len(matching_policies) == 0: + # the base policy (generic linux) should always match + raise RuntimeError("Internal error") + + return max(matching_policies) + + def lddtree_external_references(self, lddtree: dict, wheel_path: str) -> dict: + # XXX: Document the lddtree structure, or put it in something + # more stable than a big nested dict + def filter_libs( + libs: set[str], whitelist: set[str] + ) -> Generator[str, None, None]: + for lib in libs: + if "ld-linux" in lib or lib in ["ld64.so.2", "ld64.so.1"]: + # always exclude ELF dynamic linker/loader + # 'ld64.so.2' on s390x + # 'ld64.so.1' on ppc64le + # 'ld-linux*' on other platforms + continue + if LIBPYTHON_RE.match(lib): + # always exclude libpythonXY + continue + if lib in whitelist: + # exclude any libs in the whitelist + continue + yield lib + + def get_req_external(libs: set[str], whitelist: set[str]) -> set[str]: + # get all the required external libraries + libs = libs.copy() + reqs = set() + while libs: + lib = libs.pop() + reqs.add(lib) + for dep in filter_libs(lddtree["libs"][lib]["needed"], whitelist): + if dep not in reqs: + libs.add(dep) + return reqs + + ret: dict[str, dict[str, Any]] = {} + for p in self.policies: + needed_external_libs: set[str] = set() + blacklist = {} + + if not (p["name"] == "linux" and p["priority"] == 0): + # special-case the generic linux platform here, because it + # doesn't have a whitelist. or, you could say its + # whitelist is the complete set of all libraries. so nothing + # is considered "external" that needs to be copied in. + whitelist = set(p["lib_whitelist"]) + blacklist_libs = set(p["blacklist"].keys()) & set(lddtree["needed"]) + blacklist = {k: p["blacklist"][k] for k in blacklist_libs} + blacklist = filter_undefined_symbols(lddtree["realpath"], blacklist) + needed_external_libs = get_req_external( + set(filter_libs(lddtree["needed"], whitelist)), whitelist + ) + + pol_ext_deps = {} + for lib in needed_external_libs: + if is_subdir(lddtree["libs"][lib]["realpath"], wheel_path): + # we didn't filter libs that resolved via RPATH out + # earlier because we wanted to make sure to pick up + # our elf's indirect dependencies. But now we want to + # filter these ones out, since they're not "external". + logger.debug("RPATH FTW: %s", lib) + continue + pol_ext_deps[lib] = lddtree["libs"][lib]["realpath"] + ret[p["name"]] = { + "libs": pol_ext_deps, + "priority": p["priority"], + "blacklist": blacklist, + } + return ret + def get_arch_name() -> str: machine = _platform_module.machine() @@ -65,12 +253,6 @@ def _validate_pep600_compliance(policies) -> None: symbol_versions[arch] = symbol_versions_arch -_POLICY_JSON_MAP = { - Libc.GLIBC: _HERE / "manylinux-policy.json", - Libc.MUSL: _HERE / "musllinux-policy.json", -} - - def _get_musl_policy(): if _LIBC != Libc.MUSL: return None @@ -78,9 +260,6 @@ def _get_musl_policy(): return f"musllinux_{musl_version.major}_{musl_version.minor}" -_MUSL_POLICY = _get_musl_policy() - - def _fixup_musl_libc_soname(whitelist): if _LIBC != Libc.MUSL: return whitelist @@ -105,60 +284,6 @@ def _fixup_musl_libc_soname(whitelist): return new_whitelist -with _POLICY_JSON_MAP[_LIBC].open() as f: - _POLICIES = [] - _policies_temp = json.load(f) - _validate_pep600_compliance(_policies_temp) - for _p in _policies_temp: - if _MUSL_POLICY is not None and _p["name"] not in {"linux", _MUSL_POLICY}: - continue - if _ARCH_NAME in _p["symbol_versions"].keys() or _p["name"] == "linux": - if _p["name"] != "linux": - _p["symbol_versions"] = _p["symbol_versions"][_ARCH_NAME] - _p["name"] = _p["name"] + "_" + _ARCH_NAME - _p["aliases"] = [alias + "_" + _ARCH_NAME for alias in _p["aliases"]] - _p["lib_whitelist"] = _fixup_musl_libc_soname(_p["lib_whitelist"]) - _POLICIES.append(_p) - if _LIBC == Libc.MUSL: - assert len(_POLICIES) == 2, _POLICIES - -POLICY_PRIORITY_HIGHEST = max(p["priority"] for p in _POLICIES) -POLICY_PRIORITY_LOWEST = min(p["priority"] for p in _POLICIES) - - -def load_policies(): - return _POLICIES - - -def _load_policy_schema(): - with open(join(dirname(abspath(__file__)), "policy-schema.json")) as f_: - schema = json.load(f_) - return schema - - -def get_policy_by_name(name: str) -> dict | None: - matches = [p for p in _POLICIES if p["name"] == name or name in p["aliases"]] - if len(matches) == 0: - return None - if len(matches) > 1: - raise RuntimeError("Internal error. Policies should be unique") - return matches[0] - - -def get_policy_name(priority: int) -> str | None: - matches = [p["name"] for p in _POLICIES if p["priority"] == priority] - if len(matches) == 0: - return None - if len(matches) > 1: - raise RuntimeError("Internal error. priorities should be unique") - return matches[0] - - -def get_priority_by_name(name: str) -> int | None: - policy = get_policy_by_name(name) - return None if policy is None else policy["priority"] - - def get_replace_platforms(name: str) -> list[str]: """Extract platform tag replacement rules from policy @@ -181,14 +306,12 @@ def get_replace_platforms(name: str) -> list[str]: return ["linux_" + "_".join(name.split("_")[1:])] -# These have to be imported here to avoid a circular import. -from .external_references import lddtree_external_references # noqa -from .versioned_symbols import versioned_symbols_policy # noqa +def _load_policy_schema(): + with open(join(dirname(abspath(__file__)), "policy-schema.json")) as f_: + schema = json.load(f_) + return schema + __all__ = [ - "lddtree_external_references", - "versioned_symbols_policy", - "load_policies", - "POLICY_PRIORITY_HIGHEST", - "POLICY_PRIORITY_LOWEST", + "WheelPolicies", ] diff --git a/src/auditwheel/policy/external_references.py b/src/auditwheel/policy/external_references.py deleted file mode 100644 index 23afde85..00000000 --- a/src/auditwheel/policy/external_references.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import annotations - -import logging -import re -from typing import Any, Generator - -from ..elfutils import filter_undefined_symbols, is_subdir -from . import load_policies - -log = logging.getLogger(__name__) -LIBPYTHON_RE = re.compile(r"^libpython\d+\.\d+m?.so(.\d)*$") - - -def lddtree_external_references(lddtree: dict, wheel_path: str) -> dict: - # XXX: Document the lddtree structure, or put it in something - # more stable than a big nested dict - policies = load_policies() - - def filter_libs(libs: set[str], whitelist: set[str]) -> Generator[str, None, None]: - for lib in libs: - if "ld-linux" in lib or lib in ["ld64.so.2", "ld64.so.1"]: - # always exclude ELF dynamic linker/loader - # 'ld64.so.2' on s390x - # 'ld64.so.1' on ppc64le - # 'ld-linux*' on other platforms - continue - if LIBPYTHON_RE.match(lib): - # always exclude libpythonXY - continue - if lib in whitelist: - # exclude any libs in the whitelist - continue - yield lib - - def get_req_external(libs: set[str], whitelist: set[str]) -> set[str]: - # get all the required external libraries - libs = libs.copy() - reqs = set() - while libs: - lib = libs.pop() - reqs.add(lib) - for dep in filter_libs(lddtree["libs"][lib]["needed"], whitelist): - if dep not in reqs: - libs.add(dep) - return reqs - - ret: dict[str, dict[str, Any]] = {} - for p in policies: - needed_external_libs: set[str] = set() - blacklist = {} - - if not (p["name"] == "linux" and p["priority"] == 0): - # special-case the generic linux platform here, because it - # doesn't have a whitelist. or, you could say its - # whitelist is the complete set of all libraries. so nothing - # is considered "external" that needs to be copied in. - whitelist = set(p["lib_whitelist"]) - blacklist_libs = set(p["blacklist"].keys()) & set(lddtree["needed"]) - blacklist = {k: p["blacklist"][k] for k in blacklist_libs} - blacklist = filter_undefined_symbols(lddtree["realpath"], blacklist) - needed_external_libs = get_req_external( - set(filter_libs(lddtree["needed"], whitelist)), whitelist - ) - - pol_ext_deps = {} - for lib in needed_external_libs: - if is_subdir(lddtree["libs"][lib]["realpath"], wheel_path): - # we didn't filter libs that resolved via RPATH out - # earlier because we wanted to make sure to pick up - # our elf's indirect dependencies. But now we want to - # filter these ones out, since they're not "external". - log.debug("RPATH FTW: %s", lib) - continue - pol_ext_deps[lib] = lddtree["libs"][lib]["realpath"] - ret[p["name"]] = { - "libs": pol_ext_deps, - "priority": p["priority"], - "blacklist": blacklist, - } - return ret diff --git a/src/auditwheel/policy/versioned_symbols.py b/src/auditwheel/policy/versioned_symbols.py deleted file mode 100644 index 7d4e0a18..00000000 --- a/src/auditwheel/policy/versioned_symbols.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import annotations - -import logging - -from . import load_policies - -log = logging.getLogger(__name__) - - -def versioned_symbols_policy(versioned_symbols: dict[str, set[str]]) -> int: - def policy_is_satisfied( - policy_name: str, policy_sym_vers: dict[str, set[str]] - ) -> bool: - policy_satisfied = True - for name in set(required_vers) & set(policy_sym_vers): - if not required_vers[name].issubset(policy_sym_vers[name]): - for symbol in required_vers[name] - policy_sym_vers[name]: - log.debug( - "Package requires %s, incompatible with " - "policy %s which requires %s", - symbol, - policy_name, - policy_sym_vers[name], - ) - policy_satisfied = False - return policy_satisfied - - required_vers: dict[str, set[str]] = {} - for symbols in versioned_symbols.values(): - for symbol in symbols: - sym_name, _, _ = symbol.partition("_") - required_vers.setdefault(sym_name, set()).add(symbol) - matching_policies: list[int] = [] - for p in load_policies(): - policy_sym_vers = { - sym_name: {sym_name + "_" + version for version in versions} - for sym_name, versions in p["symbol_versions"].items() - } - if policy_is_satisfied(p["name"], policy_sym_vers): - matching_policies.append(p["priority"]) - - if len(matching_policies) == 0: - # the base policy (generic linux) should always match - raise RuntimeError("Internal error") - - return max(matching_policies) diff --git a/src/auditwheel/repair.py b/src/auditwheel/repair.py index 69e5df27..c172b471 100644 --- a/src/auditwheel/repair.py +++ b/src/auditwheel/repair.py @@ -16,7 +16,7 @@ from .elfutils import elf_read_dt_needed, elf_read_rpaths, is_subdir from .hashfile import hashfile -from .policy import get_replace_platforms +from .policy import WheelPolicies, get_replace_platforms from .wheel_abi import get_wheel_elfdata from .wheeltools import InWheelCtx, add_platforms @@ -32,6 +32,7 @@ def repair_wheel( + wheel_policy: WheelPolicies, wheel_path: str, abis: list[str], lib_sdir: str, @@ -41,7 +42,7 @@ def repair_wheel( exclude: list[str], strip: bool = False, ) -> str | None: - external_refs_by_fn = get_wheel_elfdata(wheel_path)[1] + external_refs_by_fn = get_wheel_elfdata(wheel_policy, wheel_path)[1] # Do not repair a pure wheel, i.e. has no external refs if not external_refs_by_fn: diff --git a/src/auditwheel/wheel_abi.py b/src/auditwheel/wheel_abi.py index ca72d04f..e4a9446b 100644 --- a/src/auditwheel/wheel_abi.py +++ b/src/auditwheel/wheel_abi.py @@ -19,14 +19,7 @@ ) from .genericpkgctx import InGenericPkgCtx from .lddtree import lddtree -from .policy import ( - POLICY_PRIORITY_HIGHEST, - POLICY_PRIORITY_LOWEST, - get_policy_name, - lddtree_external_references, - load_policies, - versioned_symbols_policy, -) +from .policy import WheelPolicies log = logging.getLogger(__name__) WheelAbIInfo = namedtuple( @@ -59,7 +52,7 @@ class NonPlatformWheel(WheelAbiError): @functools.lru_cache -def get_wheel_elfdata(wheel_fn: str): +def get_wheel_elfdata(wheel_policy: WheelPolicies, wheel_fn: str): full_elftree = {} nonpy_elftree = {} full_external_refs = {} @@ -104,7 +97,7 @@ def get_wheel_elfdata(wheel_fn: str): uses_ucs2_symbols |= any( True for _ in elf_find_ucs2_symbols(elf) ) - full_external_refs[fn] = lddtree_external_references( + full_external_refs[fn] = wheel_policy.lddtree_external_references( elftree, ctx.path ) else: @@ -147,7 +140,7 @@ def get_wheel_elfdata(wheel_fn: str): # Even if a non-pyextension ELF file is not needed, we # should include it as an external reference, because # it might require additional external libraries. - full_external_refs[fn] = lddtree_external_references( + full_external_refs[fn] = wheel_policy.lddtree_external_references( nonpy_elftree[fn], ctx.path ) @@ -204,7 +197,9 @@ def get_versioned_symbols(libs): return result -def get_symbol_policies(versioned_symbols, external_versioned_symbols, external_refs): +def get_symbol_policies( + wheel_policy, versioned_symbols, external_versioned_symbols, external_refs +): """Get symbol policies Since white-list is different per policy, this function inspects versioned_symbol per policy when including external refs @@ -226,14 +221,16 @@ def get_symbol_policies(versioned_symbols, external_versioned_symbols, external_ ext_symbols = external_versioned_symbols[soname] for k in iter(ext_symbols): policy_symbols[k].update(ext_symbols[k]) - result.append((versioned_symbols_policy(policy_symbols), policy_symbols)) + result.append( + (wheel_policy.versioned_symbols_policy(policy_symbols), policy_symbols) + ) return result -def analyze_wheel_abi(wheel_fn: str) -> WheelAbIInfo: +def analyze_wheel_abi(wheel_policy: WheelPolicies, wheel_fn: str) -> WheelAbIInfo: external_refs = { p["name"]: {"libs": {}, "blacklist": {}, "priority": p["priority"]} - for p in load_policies() + for p in wheel_policy.policies } ( @@ -242,7 +239,7 @@ def analyze_wheel_abi(wheel_fn: str) -> WheelAbIInfo: versioned_symbols, has_ucs2, uses_PyFPE_jbuf, - ) = get_wheel_elfdata(wheel_fn) + ) = get_wheel_elfdata(wheel_policy, wheel_fn) for fn in elftree_by_fn.keys(): update(external_refs, external_refs_by_fn[fn]) @@ -253,9 +250,9 @@ def analyze_wheel_abi(wheel_fn: str) -> WheelAbIInfo: external_libs = get_external_libs(external_refs) external_versioned_symbols = get_versioned_symbols(external_libs) symbol_policies = get_symbol_policies( - versioned_symbols, external_versioned_symbols, external_refs + wheel_policy, versioned_symbols, external_versioned_symbols, external_refs ) - symbol_policy = versioned_symbols_policy(versioned_symbols) + symbol_policy = wheel_policy.versioned_symbols_policy(versioned_symbols) # let's keep the highest priority policy and # corresponding versioned_symbols @@ -265,30 +262,30 @@ def analyze_wheel_abi(wheel_fn: str) -> WheelAbIInfo: ref_policy = max( (e["priority"] for e in external_refs.values() if len(e["libs"]) == 0), - default=POLICY_PRIORITY_LOWEST, + default=wheel_policy.priority_lowest, ) blacklist_policy = max( (e["priority"] for e in external_refs.values() if len(e["blacklist"]) == 0), - default=POLICY_PRIORITY_LOWEST, + default=wheel_policy.priority_lowest, ) if has_ucs2: - ucs_policy = POLICY_PRIORITY_LOWEST + ucs_policy = wheel_policy.priority_lowest else: - ucs_policy = POLICY_PRIORITY_HIGHEST + ucs_policy = wheel_policy.priority_highest if uses_PyFPE_jbuf: - pyfpe_policy = POLICY_PRIORITY_LOWEST + pyfpe_policy = wheel_policy.priority_lowest else: - pyfpe_policy = POLICY_PRIORITY_HIGHEST - - ref_tag = get_policy_name(ref_policy) - sym_tag = get_policy_name(symbol_policy) - ucs_tag = get_policy_name(ucs_policy) - pyfpe_tag = get_policy_name(pyfpe_policy) - blacklist_tag = get_policy_name(blacklist_policy) - overall_tag = get_policy_name( + pyfpe_policy = wheel_policy.priority_highest + + ref_tag = wheel_policy.get_policy_name(ref_policy) + sym_tag = wheel_policy.get_policy_name(symbol_policy) + ucs_tag = wheel_policy.get_policy_name(ucs_policy) + pyfpe_tag = wheel_policy.get_policy_name(pyfpe_policy) + blacklist_tag = wheel_policy.get_policy_name(blacklist_policy) + overall_tag = wheel_policy.get_policy_name( min(symbol_policy, ref_policy, ucs_policy, pyfpe_policy, blacklist_policy) ) diff --git a/tests/integration/test_bundled_wheels.py b/tests/integration/test_bundled_wheels.py index fe51f5e6..b43973b4 100644 --- a/tests/integration/test_bundled_wheels.py +++ b/tests/integration/test_bundled_wheels.py @@ -13,6 +13,7 @@ import pytest from auditwheel import main_repair +from auditwheel.policy import WheelPolicies from auditwheel.wheel_abi import analyze_wheel_abi HERE = Path(__file__).parent.resolve() @@ -27,13 +28,17 @@ ], ) def test_analyze_wheel_abi(file, external_libs): - winfo = analyze_wheel_abi(str(HERE / file)) + wheel_policies = WheelPolicies() + winfo = analyze_wheel_abi(wheel_policies, str(HERE / file)) assert set(winfo.external_refs["manylinux_2_5_x86_64"]["libs"]) == external_libs @pytest.mark.skipif(platform.machine() != "x86_64", reason="only supported on x86_64") def test_analyze_wheel_abi_pyfpe(): - winfo = analyze_wheel_abi(str(HERE / "fpewheel-0.0.0-cp35-cp35m-linux_x86_64.whl")) + wheel_policies = WheelPolicies() + winfo = analyze_wheel_abi( + wheel_policies, str(HERE / "fpewheel-0.0.0-cp35-cp35m-linux_x86_64.whl") + ) assert ( winfo.sym_tag == "manylinux_2_5_x86_64" ) # for external symbols, it could get manylinux1 diff --git a/tests/integration/test_manylinux.py b/tests/integration/test_manylinux.py index b34ba532..39b33dc4 100644 --- a/tests/integration/test_manylinux.py +++ b/tests/integration/test_manylinux.py @@ -16,7 +16,7 @@ import pytest from elftools.elf.elffile import ELFFile -from auditwheel.policy import get_arch_name, get_priority_by_name +from auditwheel.policy import WheelPolicies, get_arch_name logger = logging.getLogger(__name__) @@ -859,11 +859,12 @@ def test_build_wheel_with_image_dependencies( "auditwheel -v repair --plat {policy} -w /io /io/{orig_wheel}" ) - policy_priority = get_priority_by_name(policy) + wheel_policy = WheelPolicies() + policy_priority = wheel_policy.get_priority_by_name(policy) older_policies = [ f"{p}_{PLATFORM}" for p in MANYLINUX_IMAGES.keys() - if policy_priority < get_priority_by_name(f"{p}_{PLATFORM}") + if policy_priority < wheel_policy.get_priority_by_name(f"{p}_{PLATFORM}") ] for target_policy in older_policies: # we shall fail to repair the wheel when targeting an older policy than diff --git a/tests/integration/test_policy_files.py b/tests/integration/test_policy_files.py index e92f7742..6d83757f 100644 --- a/tests/integration/test_policy_files.py +++ b/tests/integration/test_policy_files.py @@ -2,27 +2,25 @@ from jsonschema import validate -from auditwheel.policy import ( - POLICY_PRIORITY_HIGHEST, - POLICY_PRIORITY_LOWEST, - _load_policy_schema, - load_policies, - versioned_symbols_policy, -) +from auditwheel.policy import WheelPolicies, _load_policy_schema def test_policy(): - policy = load_policies() + wheel_policy = WheelPolicies() policy_schema = _load_policy_schema() - validate(policy, policy_schema) + validate(wheel_policy.policies, policy_schema) def test_policy_checks_glibc(): - policy = versioned_symbols_policy({"some_library.so": {"GLIBC_2.17"}}) - assert policy > POLICY_PRIORITY_LOWEST - policy = versioned_symbols_policy({"some_library.so": {"GLIBC_999"}}) - assert policy == POLICY_PRIORITY_LOWEST - policy = versioned_symbols_policy({"some_library.so": {"OPENSSL_1_1_0"}}) - assert policy == POLICY_PRIORITY_HIGHEST - policy = versioned_symbols_policy({"some_library.so": {"IAMALIBRARY"}}) - assert policy == POLICY_PRIORITY_HIGHEST + wheel_policy = WheelPolicies() + + policy = wheel_policy.versioned_symbols_policy({"some_library.so": {"GLIBC_2.17"}}) + assert policy > wheel_policy.priority_lowest + policy = wheel_policy.versioned_symbols_policy({"some_library.so": {"GLIBC_999"}}) + assert policy == wheel_policy.priority_lowest + policy = wheel_policy.versioned_symbols_policy( + {"some_library.so": {"OPENSSL_1_1_0"}} + ) + assert policy == wheel_policy.priority_highest + policy = wheel_policy.versioned_symbols_policy({"some_library.so": {"IAMALIBRARY"}}) + assert policy == wheel_policy.priority_highest diff --git a/tests/unit/test_policy.py b/tests/unit/test_policy.py index adbdad8c..181c6b29 100644 --- a/tests/unit/test_policy.py +++ b/tests/unit/test_policy.py @@ -5,12 +5,10 @@ import pytest from auditwheel.policy import ( + WheelPolicies, _validate_pep600_compliance, get_arch_name, - get_policy_name, - get_priority_by_name, get_replace_platforms, - lddtree_external_references, ) @@ -156,53 +154,53 @@ def test_pep600_compliance(): class TestPolicyAccess: def test_get_by_priority(self): _arch = get_arch_name() - assert get_policy_name(65) == f"manylinux_2_27_{_arch}" - assert get_policy_name(70) == f"manylinux_2_24_{_arch}" - assert get_policy_name(80) == f"manylinux_2_17_{_arch}" + wheel_policy = WheelPolicies() + assert wheel_policy.get_policy_name(65) == f"manylinux_2_27_{_arch}" + assert wheel_policy.get_policy_name(70) == f"manylinux_2_24_{_arch}" + assert wheel_policy.get_policy_name(80) == f"manylinux_2_17_{_arch}" if _arch in {"x86_64", "i686"}: - assert get_policy_name(90) == f"manylinux_2_12_{_arch}" - assert get_policy_name(100) == f"manylinux_2_5_{_arch}" - assert get_policy_name(0) == f"linux_{_arch}" + assert wheel_policy.get_policy_name(90) == f"manylinux_2_12_{_arch}" + assert wheel_policy.get_policy_name(100) == f"manylinux_2_5_{_arch}" + assert wheel_policy.get_policy_name(0) == f"linux_{_arch}" def test_get_by_priority_missing(self): - assert get_policy_name(101) is None + wheel_policy = WheelPolicies() + assert wheel_policy.get_policy_name(101) is None - @patch( - "auditwheel.policy._POLICIES", - [ + def test_get_by_priority_duplicate(self): + wheel_policy = WheelPolicies() + wheel_policy._policies = [ {"name": "duplicate", "priority": 0}, {"name": "duplicate", "priority": 0}, - ], - ) - def test_get_by_priority_duplicate(self): + ] with pytest.raises(RuntimeError): - get_policy_name(0) + wheel_policy.get_policy_name(0) def test_get_by_name(self): _arch = get_arch_name() - assert get_priority_by_name(f"manylinux_2_27_{_arch}") == 65 - assert get_priority_by_name(f"manylinux_2_24_{_arch}") == 70 - assert get_priority_by_name(f"manylinux2014_{_arch}") == 80 - assert get_priority_by_name(f"manylinux_2_17_{_arch}") == 80 + wheel_policy = WheelPolicies() + assert wheel_policy.get_priority_by_name(f"manylinux_2_27_{_arch}") == 65 + assert wheel_policy.get_priority_by_name(f"manylinux_2_24_{_arch}") == 70 + assert wheel_policy.get_priority_by_name(f"manylinux2014_{_arch}") == 80 + assert wheel_policy.get_priority_by_name(f"manylinux_2_17_{_arch}") == 80 if _arch in {"x86_64", "i686"}: - assert get_priority_by_name(f"manylinux2010_{_arch}") == 90 - assert get_priority_by_name(f"manylinux_2_12_{_arch}") == 90 - assert get_priority_by_name(f"manylinux1_{_arch}") == 100 - assert get_priority_by_name(f"manylinux_2_5_{_arch}") == 100 + assert wheel_policy.get_priority_by_name(f"manylinux2010_{_arch}") == 90 + assert wheel_policy.get_priority_by_name(f"manylinux_2_12_{_arch}") == 90 + assert wheel_policy.get_priority_by_name(f"manylinux1_{_arch}") == 100 + assert wheel_policy.get_priority_by_name(f"manylinux_2_5_{_arch}") == 100 def test_get_by_name_missing(self): - assert get_priority_by_name("nosuchpolicy") is None + wheel_policy = WheelPolicies() + assert wheel_policy.get_priority_by_name("nosuchpolicy") is None - @patch( - "auditwheel.policy._POLICIES", - [ + def test_get_by_name_duplicate(self): + wheel_policy = WheelPolicies() + wheel_policy._policies = [ {"name": "duplicate", "priority": 0}, {"name": "duplicate", "priority": 0}, - ], - ) - def test_get_by_name_duplicate(self): + ] with pytest.raises(RuntimeError): - get_priority_by_name("duplicate") + wheel_policy.get_priority_by_name("duplicate") class TestLddTreeExternalReferences: @@ -227,7 +225,10 @@ def test_filter_libs(self): "needed": libs, "libs": {lib: {"needed": [], "realpath": "/path/to/lib"} for lib in libs}, } - full_external_refs = lddtree_external_references(lddtree, "/path/to/wheel") + wheel_policy = WheelPolicies() + full_external_refs = wheel_policy.lddtree_external_references( + lddtree, "/path/to/wheel" + ) # Assert that each policy only has the unfiltered libs. for policy in full_external_refs: diff --git a/tests/unit/test_wheel_abi.py b/tests/unit/test_wheel_abi.py index 882d3e15..d0def8e8 100644 --- a/tests/unit/test_wheel_abi.py +++ b/tests/unit/test_wheel_abi.py @@ -6,6 +6,7 @@ import pytest from auditwheel import wheel_abi +from auditwheel.policy import WheelPolicies class TestGetWheelElfdata: @@ -44,8 +45,9 @@ def test_finds_shared_library_in_purelib(self, filenames, message, monkeypatch): "elf_file_filter", lambda fns: [(fn, pretend.stub()) for fn in fns], ) + wheel_policy = WheelPolicies() with pytest.raises(RuntimeError) as exec_info: - wheel_abi.get_wheel_elfdata("/fakepath") + wheel_abi.get_wheel_elfdata(wheel_policy, "/fakepath") assert exec_info.value.args == (message,)