Skip to content

Commit

Permalink
feat: add PyPy macOS arm64 (#1372)
Browse files Browse the repository at this point in the history
* feat: add PyPy macOS arm64
* fix: use PyPy 7.3.11
* fix: filter-out unsupported PyPy cross-compilation configs
* Add quiet notice about the auto-skipping of arm64 pp builds on x86_64

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>
Co-authored-by: Henry Schreiner <henryschreineriii@gmail.com>
Co-authored-by: Joe Rickerby <joerick@mac.com>
  • Loading branch information
3 people authored Jan 14, 2023
1 parent 06c4927 commit 92cb1d8
Showing 8 changed files with 70 additions and 16 deletions.
5 changes: 3 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -32,12 +32,13 @@ What does it do?
| CPython 3.10 ||||| ✅² ||||||
| CPython 3.11 ||||| ✅² ||||||
| PyPy 3.7 v7.3 || N/A || N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A |
| PyPy 3.8 v7.3 || N/A || N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A |
| PyPy 3.9 v7.3 || N/A || N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A |
| PyPy 3.8 v7.3 || ✅⁴ || N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A |
| PyPy 3.9 v7.3 || ✅⁴ || N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A |

<sup>¹ PyPy is only supported for manylinux wheels.</sup><br>
<sup>² Windows arm64 support is experimental.</sup><br>
<sup>³ Alpine 3.14 and very briefly 3.15's default python3 [was not able to load](https://github.com/pypa/cibuildwheel/issues/934) musllinux wheels. This has been fixed; please upgrade the python package if using Alpine from before the fix.</sup><br>
<sup>⁴ Cross-compilation not supported with PyPy - to build these wheels you need to run cibuildwheel on an Apple Silicon machine.</sup><br>

- Builds manylinux, musllinux, macOS 10.9+, and Windows wheels for CPython and PyPy
- Works on GitHub Actions, Azure Pipelines, Travis CI, AppVeyor, CircleCI, GitLab CI, and Cirrus CI
18 changes: 12 additions & 6 deletions bin/update_pythons.py
Original file line number Diff line number Diff line change
@@ -159,8 +159,8 @@ def update_version_windows(self, spec: Specifier) -> ConfigWinCP:
)

def update_version_macos(self, spec: Specifier) -> ConfigMacOS:
if self.arch != "64":
msg = "Other archs not supported yet on macOS"
if self.arch not in {"64", "ARM64"}:
msg = f"'{self.arch}' arch not supported yet on macOS"
raise RuntimeError(msg)

releases = [r for r in self.releases if spec.contains(r["python_version"])]
@@ -172,12 +172,14 @@ def update_version_macos(self, spec: Specifier) -> ConfigMacOS:

release = releases[-1]
version = release["python_version"]
identifier = f"pp{version.major}{version.minor}-macosx_x86_64"
arch = "x86_64" if self.arch == "64" else self.arch.lower()
identifier = f"pp{version.major}{version.minor}-macosx_{arch}"

arch = "x64" if self.arch == "64" else self.arch.lower()
(url,) = (
rf["download_url"]
for rf in release["files"]
if "" in rf["platform"] == "darwin" and rf["arch"] == "x64"
if "" in rf["platform"] == "darwin" and rf["arch"] == arch
)

return ConfigMacOS(
@@ -251,6 +253,7 @@ def __init__(self) -> None:

self.macos_cpython = CPythonVersions()
self.macos_pypy = PyPyVersions("64")
self.macos_pypy_arm64 = PyPyVersions("ARM64")

def update_config(self, config: dict[str, str]) -> None:
identifier = config["identifier"]
@@ -261,11 +264,14 @@ def update_config(self, config: dict[str, str]) -> None:
config_update: AnyConfig | None = None

# We need to use ** in update due to MyPy (probably a bug)
if "macos" in identifier:
if "macosx" in identifier:
if identifier.startswith("cp"):
config_update = self.macos_cpython.update_version_macos(identifier, version, spec)
elif identifier.startswith("pp"):
config_update = self.macos_pypy.update_version_macos(spec)
if "macosx_x86_64" in identifier:
config_update = self.macos_pypy.update_version_macos(spec)
elif "macosx_arm64" in identifier:
config_update = self.macos_pypy_arm64.update_version_macos(spec)
elif "win32" in identifier:
if identifier.startswith("cp"):
config_update = self.windows_32.update_version_windows(spec)
4 changes: 4 additions & 0 deletions cibuildwheel/logger.py
Original file line number Diff line number Diff line change
@@ -126,6 +126,10 @@ def step_end_with_error(self, error: BaseException | str) -> None:
self.step_end(success=False)
self.error(error)

def quiet(self, message: str) -> None:
c = self.colors
print(f"{c.gray}{message}{c.end}", file=sys.stderr)

def notice(self, message: str) -> None:
if self.fold_mode == "github":
print(f"::notice::{message}\n", file=sys.stderr)
26 changes: 25 additions & 1 deletion cibuildwheel/macos.py
Original file line number Diff line number Diff line change
@@ -84,7 +84,31 @@ def get_python_configurations(
]

# skip builds as required by BUILD/SKIP
return [c for c in python_configurations if build_selector(c.identifier)]
python_configurations = [c for c in python_configurations if build_selector(c.identifier)]

# filter-out some cross-compilation configs with PyPy:
# can't build arm64 on x86_64
# rosetta allows to build x86_64 on arm64
if platform.machine() == "x86_64":
python_configurations_before = set(python_configurations)
python_configurations = [
c
for c in python_configurations
if not (c.identifier.startswith("pp") and c.identifier.endswith("arm64"))
]
removed_elements = python_configurations_before - set(python_configurations)
if removed_elements:
ids = ", ".join(c.identifier for c in removed_elements)
log.quiet(
unwrap(
f"""
Note: {ids} {'was' if len(removed_elements) == 1 else 'were'}
selected, but can't be built on x86_64 so will be skipped automatically.
"""
)
)

return python_configurations


def install_cpython(tmp: Path, version: str, url: str) -> Path:
2 changes: 2 additions & 0 deletions cibuildwheel/resources/build-platforms.toml
Original file line number Diff line number Diff line change
@@ -89,7 +89,9 @@ python_configurations = [
{ identifier = "cp311-macosx_universal2", version = "3.11", url = "https://www.python.org/ftp/python/3.11.1/python-3.11.1-macos11.pkg" },
{ identifier = "pp37-macosx_x86_64", version = "3.7", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.9-osx64.tar.bz2" },
{ identifier = "pp38-macosx_x86_64", version = "3.8", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-macos_x86_64.tar.bz2" },
{ identifier = "pp38-macosx_arm64", version = "3.8", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-macos_arm64.tar.bz2" },
{ identifier = "pp39-macosx_x86_64", version = "3.9", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.11-macos_x86_64.tar.bz2" },
{ identifier = "pp39-macosx_arm64", version = "3.9", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.11-macos_arm64.tar.bz2" },
]

[windows]
4 changes: 2 additions & 2 deletions docs/options.md
Original file line number Diff line number Diff line change
@@ -227,8 +227,8 @@ When setting the options, you can use shell-style globbing syntax, as per [fnmat
| Python 3.10 | cp310-macosx_x86_64<br/>cp310-macosx_universal2<br/>cp310-macosx_arm64 | cp310-win_amd64<br/>cp310-win32<br/>cp310-win_arm64 | cp310-manylinux_x86_64<br/>cp310-manylinux_i686<br/>cp310-musllinux_x86_64<br/>cp310-musllinux_i686 | cp310-manylinux_aarch64<br/>cp310-manylinux_ppc64le<br/>cp310-manylinux_s390x<br/>cp310-musllinux_aarch64<br/>cp310-musllinux_ppc64le<br/>cp310-musllinux_s390x |
| Python 3.11 | cp311-macosx_x86_64<br/>cp311-macosx_universal2<br/>cp311-macosx_arm64 | cp311-win_amd64<br/>cp311-win32<br/>cp311-win_arm64 | cp311-manylinux_x86_64<br/>cp311-manylinux_i686<br/>cp311-musllinux_x86_64<br/>cp311-musllinux_i686 | cp311-manylinux_aarch64<br/>cp311-manylinux_ppc64le<br/>cp311-manylinux_s390x<br/>cp311-musllinux_aarch64<br/>cp311-musllinux_ppc64le<br/>cp311-musllinux_s390x |
| PyPy3.7 v7.3 | pp37-macosx_x86_64 | pp37-win_amd64 | pp37-manylinux_x86_64<br/>pp37-manylinux_i686 | pp37-manylinux_aarch64 |
| PyPy3.8 v7.3 | pp38-macosx_x86_64 | pp38-win_amd64 | pp38-manylinux_x86_64<br/>pp38-manylinux_i686 | pp38-manylinux_aarch64 |
| PyPy3.9 v7.3 | pp39-macosx_x86_64 | pp39-win_amd64 | pp39-manylinux_x86_64<br/>pp39-manylinux_i686 | pp39-manylinux_aarch64 |
| PyPy3.8 v7.3 | pp38-macosx_x86_64<br/>pp38-macosx_arm64 | pp38-win_amd64 | pp38-manylinux_x86_64<br/>pp38-manylinux_i686 | pp38-manylinux_aarch64 |
| PyPy3.9 v7.3 | pp39-macosx_x86_64<br/>pp39-macosx_arm64 | pp39-win_amd64 | pp39-manylinux_x86_64<br/>pp39-manylinux_i686 | pp39-manylinux_aarch64 |

The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to cibuildwheel.
The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details).
18 changes: 14 additions & 4 deletions test/test_macos_archs.py
Original file line number Diff line number Diff line change
@@ -65,7 +65,7 @@ def test_cross_compiled_test(tmp_path, capfd, build_universal2):
actual_wheels = utils.cibuildwheel_run(
project_dir,
add_env={
"CIBW_BUILD": "cp39-*",
"CIBW_BUILD": "cp39-*" if build_universal2 else "*p39-*",
"CIBW_TEST_COMMAND": '''python -c "import platform; print('running tests on ' + platform.machine())"''',
"CIBW_ARCHS": "universal2" if build_universal2 else "x86_64 arm64",
"CIBW_BUILD_VERBOSITY": "3",
@@ -76,7 +76,8 @@ def test_cross_compiled_test(tmp_path, capfd, build_universal2):

assert DEPLOYMENT_TARGET_TOO_LOW_WARNING not in captured.err

if platform.machine() == "x86_64":
platform_machine = platform.machine()
if platform_machine == "x86_64":
# ensure that tests were run on only x86_64
assert "running tests on x86_64" in captured.out
assert "running tests on arm64" not in captured.out
@@ -89,15 +90,24 @@ def test_cross_compiled_test(tmp_path, capfd, build_universal2):
assert (
"While arm64 wheels can be built on x86_64, they cannot be tested" in captured.err
)
elif platform.machine() == "arm64":
elif platform_machine == "arm64":
# ensure that tests were run on both x86_64 and arm64
assert "running tests on x86_64" in captured.out
assert "running tests on arm64" in captured.out
assert (
"While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested"
not in captured.err
)
assert (
"While arm64 wheels can be built on x86_64, they cannot be tested" not in captured.err
)

if build_universal2:
expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp39" in w and "universal2" in w]
else:
expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp39" in w and "universal2" not in w]
expected_wheels = [w for w in ALL_MACOS_WHEELS if "p39-" in w and "universal2" not in w]
if platform_machine == "x86_64":
expected_wheels = [w for w in expected_wheels if not ("pp39" in w and "arm64" in w)]

assert set(actual_wheels) == set(expected_wheels)

9 changes: 8 additions & 1 deletion test/utils.py
Original file line number Diff line number Diff line change
@@ -177,7 +177,14 @@ def expected_wheels(

if platform == "macos" and machine_arch == "arm64":
# arm64 macs are only supported by cp38+
python_abi_tags = ["cp38-cp38", "cp39-cp39", "cp310-cp310", "cp311-cp311"]
python_abi_tags = [
"cp38-cp38",
"cp39-cp39",
"cp310-cp310",
"cp311-cp311",
"pp38-pypy38_pp73",
"pp39-pypy39_pp73",
]

wheels = []

0 comments on commit 92cb1d8

Please sign in to comment.