From e34bb79aceb1e2ada772a7d8054148e81c8d7184 Mon Sep 17 00:00:00 2001 From: Laszlo Kiss-Kollar Date: Mon, 5 Jul 2021 21:05:09 +0100 Subject: [PATCH] feature: add support for musllinux Detect the running libc at startup & select the correct policy file based on this. For musllinux, we select only one musllinux policy depending on the running version of musl. --- auditwheel/error.py | 6 +++ auditwheel/lddtree.py | 34 ++++++++++++-- auditwheel/libc.py | 23 ++++++++++ auditwheel/musllinux.py | 60 ++++++++++++++++++++++++ auditwheel/policy/__init__.py | 61 +++++++++++++++++++++++-- auditwheel/policy/musllinux-policy.json | 48 +++++++++++++++++++ tests/unit/test_musllinux.py | 49 ++++++++++++++++++++ 7 files changed, 273 insertions(+), 8 deletions(-) create mode 100644 auditwheel/error.py create mode 100644 auditwheel/libc.py create mode 100644 auditwheel/musllinux.py create mode 100644 auditwheel/policy/musllinux-policy.json create mode 100644 tests/unit/test_musllinux.py diff --git a/auditwheel/error.py b/auditwheel/error.py new file mode 100644 index 00000000..c95348c8 --- /dev/null +++ b/auditwheel/error.py @@ -0,0 +1,6 @@ +class AuditwheelException(Exception): + pass + + +class InvalidLibc(AuditwheelException): + pass diff --git a/auditwheel/lddtree.py b/auditwheel/lddtree.py index 195bab30..53f7c7ae 100644 --- a/auditwheel/lddtree.py +++ b/auditwheel/lddtree.py @@ -17,9 +17,12 @@ import errno import logging import functools +from pathlib import Path from typing import List, Dict, Optional, Any, Tuple from elftools.elf.elffile import ELFFile +from .libc import get_libc, Libc + log = logging.getLogger(__name__) __all__ = ['lddtree'] @@ -195,11 +198,32 @@ def load_ld_paths(root: str = '/', prefix: str = '') -> Dict[str, List[str]]: # on a per-ELF basis so it can get turned into the right thing. ldpaths['env'] = parse_ld_paths(env_ldpath, path='') - # Load up /etc/ld.so.conf. - ldpaths['conf'] = parse_ld_so_conf(root + prefix + '/etc/ld.so.conf', - root=root) - # the trusted directories are not necessarily in ld.so.conf - ldpaths['conf'].extend(['/lib', '/lib64/', '/usr/lib', '/usr/lib64']) + libc = get_libc() + if libc == Libc.MUSL: + # from https://git.musl-libc.org/cgit/musl/tree/ldso + # /dynlink.c?id=3f701faace7addc75d16dea8a6cd769fa5b3f260#n1063 + root_prefix = Path(root) / prefix + ld_musl = list((root_prefix / 'etc').glob("ld-musl-*.path")) + assert len(ld_musl) <= 1 + if len(ld_musl) == 0: + ldpaths['conf'] = [ + root + '/lib', + root + '/usr/local/lib', + root + '/usr/lib' + ] + else: + ldpaths['conf'] = [] + for ldpath in ld_musl[0].read_text().split(':'): + ldpath_stripped = ldpath.strip() + if ldpath_stripped == "": + continue + ldpaths['conf'].append(root + ldpath_stripped) + else: + # Load up /etc/ld.so.conf. + ldpaths['conf'] = parse_ld_so_conf(root + prefix + '/etc/ld.so.conf', + root=root) + # the trusted directories are not necessarily in ld.so.conf + ldpaths['conf'].extend(['/lib', '/lib64/', '/usr/lib', '/usr/lib64']) log.debug('linker ldpaths: %s', ldpaths) return ldpaths diff --git a/auditwheel/libc.py b/auditwheel/libc.py new file mode 100644 index 00000000..5ff9ca48 --- /dev/null +++ b/auditwheel/libc.py @@ -0,0 +1,23 @@ +import logging +from enum import IntEnum + +from .error import InvalidLibc +from .musllinux import find_musl_libc + + +logger = logging.getLogger(__name__) + + +class Libc(IntEnum): + GLIBC = 1, + MUSL = 2, + + +def get_libc() -> Libc: + try: + find_musl_libc() + logger.debug("Detected musl libc") + return Libc.MUSL + except InvalidLibc: + logger.debug("Falling back to GNU libc") + return Libc.GLIBC diff --git a/auditwheel/musllinux.py b/auditwheel/musllinux.py new file mode 100644 index 00000000..32f6ef27 --- /dev/null +++ b/auditwheel/musllinux.py @@ -0,0 +1,60 @@ +import logging +import pathlib +import subprocess +import re +from typing import NamedTuple + +from auditwheel.error import InvalidLibc + +LOG = logging.getLogger(__name__) + + +class MuslVersion(NamedTuple): + major: int + minor: int + patch: int + + +def find_musl_libc() -> pathlib.Path: + try: + ldd = subprocess.check_output(["ldd", "/bin/ls"], errors='strict') + except (subprocess.CalledProcessError, FileNotFoundError): + LOG.error("Failed to determine libc version", exc_info=True) + raise InvalidLibc + + match = re.search( + r"libc\.musl-(?P\w+)\.so.1 " # TODO drop the platform + r"=> (?P[/\-\w.]+)", + ldd) + + if not match: + raise InvalidLibc + + return pathlib.Path(match.group("path")) + + +def get_musl_version(ld_path: pathlib.Path) -> MuslVersion: + try: + ld = subprocess.run( + [ld_path], + check=False, + errors='strict', + stderr=subprocess.PIPE + ).stderr + except FileNotFoundError: + LOG.error("Failed to determine musl version", exc_info=True) + raise InvalidLibc + + match = re.search( + r"Version " + r"(?P\d+)." + r"(?P\d+)." + r"(?P\d+)", + ld) + if not match: + raise InvalidLibc + + return MuslVersion( + int(match.group("major")), + int(match.group("minor")), + int(match.group("patch"))) diff --git a/auditwheel/policy/__init__.py b/auditwheel/policy/__init__.py index 96898bac..99556818 100644 --- a/auditwheel/policy/__init__.py +++ b/auditwheel/policy/__init__.py @@ -2,10 +2,16 @@ import json import platform as _platform_module from collections import defaultdict +from pathlib import Path from typing import Dict, List, Optional, Set from os.path import join, dirname, abspath import logging +from ..libc import get_libc, Libc +from ..musllinux import find_musl_libc, get_musl_version + + +_HERE = Path(__file__).parent logger = logging.getLogger(__name__) @@ -22,6 +28,7 @@ def get_arch_name() -> str: _ARCH_NAME = get_arch_name() +_LIBC = get_libc() def _validate_pep600_compliance(policies) -> None: @@ -56,18 +63,64 @@ def _validate_pep600_compliance(policies) -> None: symbol_versions[arch] = symbol_versions_arch -with open(join(dirname(abspath(__file__)), 'manylinux-policy.json')) as f: +_POLICY_JSON_MAP = { + Libc.GLIBC: _HERE / 'manylinux-policy.json', + Libc.MUSL: _HERE / 'musllinux-policy.json', +} + + +def _get_musl_policy(): + if _LIBC != Libc.MUSL: + return None + musl_version = get_musl_version(find_musl_libc()) + return f'musllinux_{musl_version.major}_{musl_version.minor}' + + +_MUSL_POLICY = _get_musl_policy() + + +def _fixup_musl_libc_soname(whitelist): + if _LIBC != Libc.MUSL: + return whitelist + soname_map = { + "libc.so": { + "x86_64": "libc.musl-x86_64.so.1", + "i686": "libc.musl-x86.so.1", + "aarch64": "libc.musl-aarch64.so.1", + "s390x": "libc.musl-s390x.so.1", + "ppc64le": "libc.musl-ppc64le.so.1", + "armv7l": "libc.musl-armv7.so.1", + } + } + new_whitelist = [] + for soname in whitelist: + if soname in soname_map: + new_soname = soname_map[soname][_ARCH_NAME] + logger.debug(f"Replacing whitelisted '{soname}' by '{new_soname}'") + new_whitelist.append(new_soname) + else: + new_whitelist.append(soname) + return new_whitelist + + +with _POLICY_JSON_MAP[_LIBC].open() as f: _POLICIES = [] _policies_temp = json.load(f) _validate_pep600_compliance(_policies_temp) for _p in _policies_temp: + if _MUSL_POLICY is not None and \ + _p['name'] not in {'linux', _MUSL_POLICY}: + continue if _ARCH_NAME in _p['symbol_versions'].keys() or _p['name'] == 'linux': if _p['name'] != 'linux': _p['symbol_versions'] = _p['symbol_versions'][_ARCH_NAME] _p['name'] = _p['name'] + '_' + _ARCH_NAME _p['aliases'] = [alias + '_' + _ARCH_NAME for alias in _p['aliases']] + _p['lib_whitelist'] = _fixup_musl_libc_soname(_p['lib_whitelist']) _POLICIES.append(_p) + if _LIBC == Libc.MUSL: + assert len(_POLICIES) == 2, _POLICIES POLICY_PRIORITY_HIGHEST = max(p['priority'] for p in _POLICIES) POLICY_PRIORITY_LOWEST = min(p['priority'] for p in _POLICIES) @@ -78,8 +131,8 @@ def load_policies(): def _load_policy_schema(): - with open(join(dirname(abspath(__file__)), 'policy-schema.json')) as f: - schema = json.load(f) + with open(join(dirname(abspath(__file__)), 'policy-schema.json')) as f_: + schema = json.load(f_) return schema @@ -124,6 +177,8 @@ def get_replace_platforms(name: str) -> List[str]: return [] if name.startswith('manylinux_'): return ['linux_' + '_'.join(name.split('_')[3:])] + if name.startswith('musllinux_'): + return ['linux_' + '_'.join(name.split('_')[3:])] return ['linux_' + '_'.join(name.split('_')[1:])] diff --git a/auditwheel/policy/musllinux-policy.json b/auditwheel/policy/musllinux-policy.json new file mode 100644 index 00000000..6cf8c318 --- /dev/null +++ b/auditwheel/policy/musllinux-policy.json @@ -0,0 +1,48 @@ +[ + {"name": "linux", + "aliases": [], + "priority": 0, + "symbol_versions": {}, + "lib_whitelist": [] + }, + {"name": "musllinux_1_1", + "aliases": [], + "priority": 100, + "symbol_versions": { + "i686": { + }, + "x86_64": { + }, + "aarch64": { + }, + "ppc64le": { + }, + "s390x": { + }, + "armv7l": { + } + }, + "lib_whitelist": [ + "libc.so" + ]}, + {"name": "musllinux_1_2", + "aliases": [], + "priority": 90, + "symbol_versions": { + "i686": { + }, + "x86_64": { + }, + "aarch64": { + }, + "ppc64le": { + }, + "s390x": { + }, + "armv7l": { + } + }, + "lib_whitelist": [ + "libc.so" + ]} +] diff --git a/tests/unit/test_musllinux.py b/tests/unit/test_musllinux.py new file mode 100644 index 00000000..4e56180c --- /dev/null +++ b/tests/unit/test_musllinux.py @@ -0,0 +1,49 @@ +import subprocess +from unittest.mock import patch + +import pytest + +from auditwheel.musllinux import find_musl_libc, get_musl_version +from auditwheel.error import InvalidLibc + + +@patch("auditwheel.musllinux.subprocess.check_output") +def test_find_musllinux_no_ldd(check_output_mock): + check_output_mock.side_effect = FileNotFoundError() + with pytest.raises(InvalidLibc): + find_musl_libc() + + +@patch("auditwheel.musllinux.subprocess.check_output") +def test_find_musllinux_ldd_error(check_output_mock): + check_output_mock.side_effect = subprocess.CalledProcessError(1, "ldd") + with pytest.raises(InvalidLibc): + find_musl_libc() + + +@patch("auditwheel.musllinux.subprocess.check_output") +def test_find_musllinux_not_found(check_output_mock): + check_output_mock.return_value = "" + with pytest.raises(InvalidLibc): + find_musl_libc() + + +def test_get_musl_version_invalid_path(): + with pytest.raises(InvalidLibc): + get_musl_version("/tmp/no/executable/here") + + +@patch("auditwheel.musllinux.subprocess.run") +def test_get_musl_version_invalid_version(run_mock): + run_mock.return_value = subprocess.CompletedProcess([], 1, None, "Version 1.1") + with pytest.raises(InvalidLibc): + get_musl_version("anything") + + +@patch("auditwheel.musllinux.subprocess.run") +def test_get_musl_version_valid_version(run_mock): + run_mock.return_value = subprocess.CompletedProcess([], 1, None, "Version 5.6.7") + version = get_musl_version("anything") + assert version.major == 5 + assert version.minor == 6 + assert version.patch == 7