From 345467c6cfff27a5576ac80755eecaa9316b196d Mon Sep 17 00:00:00 2001 From: mayeut Date: Fri, 17 May 2024 21:44:08 +0200 Subject: [PATCH] feat: add support for free-threaded (no-gil) Python 3.13 --- bin/update_pythons.py | 29 ++++++++++++++----- cibuildwheel/linux.py | 4 +++ cibuildwheel/macos.py | 4 +++ cibuildwheel/resources/build-platforms.toml | 13 +++++++++ .../resources/constraints-python313.txt | 2 +- cibuildwheel/resources/constraints.in | 3 +- cibuildwheel/util.py | 9 ++++-- cibuildwheel/windows.py | 26 ++++++++++++----- test/test_abi_variants.py | 16 +++++++--- test/utils.py | 2 ++ 10 files changed, 85 insertions(+), 23 deletions(-) diff --git a/bin/update_pythons.py b/bin/update_pythons.py index 05a68b0d4..ecf8e3ef2 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -58,7 +58,7 @@ class ConfigMacOS(TypedDict): class WindowsVersions: - def __init__(self, arch_str: ArchStr) -> None: + def __init__(self, arch_str: ArchStr, free_threaded: bool) -> None: response = requests.get("https://api.nuget.org/v3/index.json") response.raise_for_status() api_info = response.json() @@ -72,7 +72,11 @@ def __init__(self, arch_str: ArchStr) -> None: self.arch_str = arch_str self.arch = ARCH_DICT[arch_str] + self.free_threaded = free_threaded + package = PACKAGE_DICT[arch_str] + if free_threaded: + package = f"{package}-freethreaded" response = requests.get(f"{endpoint}{package}/index.json") response.raise_for_status() @@ -92,8 +96,9 @@ def update_version_windows(self, spec: Specifier) -> ConfigWinCP | None: if not versions: return None + flags = "t" if self.free_threaded else "" version = versions[0] - identifier = f"cp{version.major}{version.minor}-{self.arch}" + identifier = f"cp{version.major}{version.minor}{flags}-{self.arch}" return ConfigWinCP( identifier=identifier, version=self.version_dict[version], @@ -233,9 +238,12 @@ def update_version_macos( class AllVersions: def __init__(self) -> None: - self.windows_32 = WindowsVersions("32") - self.windows_64 = WindowsVersions("64") - self.windows_arm64 = WindowsVersions("ARM64") + self.windows_32 = WindowsVersions("32", False) + self.windows_t_32 = WindowsVersions("32", True) + self.windows_64 = WindowsVersions("64", False) + self.windows_t_64 = WindowsVersions("64", True) + self.windows_arm64 = WindowsVersions("ARM64", False) + self.windows_t_arm64 = WindowsVersions("ARM64", True) self.windows_pypy_64 = PyPyVersions("64") self.macos_cpython = CPythonVersions() @@ -259,14 +267,19 @@ def update_config(self, config: MutableMapping[str, str]) -> None: config_update = self.macos_pypy.update_version_macos(spec) elif "macosx_arm64" in identifier: config_update = self.macos_pypy_arm64.update_version_macos(spec) - elif "win32" in identifier: - if identifier.startswith("cp"): - config_update = self.windows_32.update_version_windows(spec) + elif "t-win32" in identifier and identifier.startswith("cp"): + config_update = self.windows_t_32.update_version_windows(spec) + elif "win32" in identifier and identifier.startswith("cp"): + config_update = self.windows_32.update_version_windows(spec) + elif "t-win_amd64" in identifier and identifier.startswith("cp"): + config_update = self.windows_t_64.update_version_windows(spec) elif "win_amd64" in identifier: if identifier.startswith("cp"): config_update = self.windows_64.update_version_windows(spec) elif identifier.startswith("pp"): config_update = self.windows_pypy_64.update_version_windows(spec) + elif "t-win_arm64" in identifier and identifier.startswith("cp"): + config_update = self.windows_t_arm64.update_version_windows(spec) elif "win_arm64" in identifier and identifier.startswith("cp"): config_update = self.windows_arm64.update_version_windows(spec) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 55e61f2d3..1fc686126 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -337,6 +337,10 @@ def build_in_container( virtualenv_env = env.copy() virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" + # TODO remove me once virtualenv provides pip>=24.1b1 + if config.version == "3.13": + container.call(["pip", "install", "pip>=24.1b1"], env=virtualenv_env) + if build_options.before_test: before_test_prepared = prepare_command( build_options.before_test, diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 6b219a102..3df239618 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -575,6 +575,10 @@ def build(options: Options, tmp_path: Path) -> None: # check that we are using the Python from the virtual environment call_with_arch("which", "python", env=virtualenv_env) + # TODO remove me once virtualenv provides pip>=24.1b1 + if config.version == "3.13": + call("python", "-m", "pip", "install", "pip>=24.1b1", env=virtualenv_env) + if build_options.before_test: before_test_prepared = prepare_command( build_options.before_test, diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index f95d329cc..2f0393e11 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -8,6 +8,7 @@ python_configurations = [ { identifier = "cp311-manylinux_x86_64", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-manylinux_x86_64", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-manylinux_x86_64", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_x86_64", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "cp36-manylinux_i686", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-manylinux_i686", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-manylinux_i686", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -16,6 +17,7 @@ python_configurations = [ { identifier = "cp311-manylinux_i686", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-manylinux_i686", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-manylinux_i686", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_i686", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "pp37-manylinux_x86_64", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" }, { identifier = "pp38-manylinux_x86_64", version = "3.8", path_str = "/opt/python/pp38-pypy38_pp73" }, { identifier = "pp39-manylinux_x86_64", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" }, @@ -28,6 +30,7 @@ python_configurations = [ { identifier = "cp311-manylinux_aarch64", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-manylinux_aarch64", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-manylinux_aarch64", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_aarch64", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "cp36-manylinux_ppc64le", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-manylinux_ppc64le", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-manylinux_ppc64le", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -36,6 +39,7 @@ python_configurations = [ { identifier = "cp311-manylinux_ppc64le", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-manylinux_ppc64le", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-manylinux_ppc64le", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_ppc64le", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "cp36-manylinux_s390x", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-manylinux_s390x", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-manylinux_s390x", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -44,6 +48,7 @@ python_configurations = [ { identifier = "cp311-manylinux_s390x", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-manylinux_s390x", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-manylinux_s390x", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_s390x", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "pp37-manylinux_aarch64", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" }, { identifier = "pp38-manylinux_aarch64", version = "3.8", path_str = "/opt/python/pp38-pypy38_pp73" }, { identifier = "pp39-manylinux_aarch64", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" }, @@ -60,6 +65,7 @@ python_configurations = [ { identifier = "cp311-musllinux_x86_64", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-musllinux_x86_64", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-musllinux_x86_64", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_x86_64", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "cp36-musllinux_i686", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-musllinux_i686", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-musllinux_i686", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -68,6 +74,7 @@ python_configurations = [ { identifier = "cp311-musllinux_i686", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-musllinux_i686", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-musllinux_i686", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_i686", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "cp36-musllinux_aarch64", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-musllinux_aarch64", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-musllinux_aarch64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -76,6 +83,7 @@ python_configurations = [ { identifier = "cp311-musllinux_aarch64", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-musllinux_aarch64", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-musllinux_aarch64", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_aarch64", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "cp36-musllinux_ppc64le", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-musllinux_ppc64le", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-musllinux_ppc64le", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -84,6 +92,7 @@ python_configurations = [ { identifier = "cp311-musllinux_ppc64le", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-musllinux_ppc64le", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-musllinux_ppc64le", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_ppc64le", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "cp36-musllinux_s390x", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-musllinux_s390x", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-musllinux_s390x", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -92,6 +101,7 @@ python_configurations = [ { identifier = "cp311-musllinux_s390x", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-musllinux_s390x", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-musllinux_s390x", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_s390x", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, ] [macos] @@ -142,12 +152,15 @@ python_configurations = [ { identifier = "cp312-win32", version = "3.12.3", arch = "32" }, { identifier = "cp312-win_amd64", version = "3.12.3", arch = "64" }, { identifier = "cp313-win32", version = "3.13.0-b1", arch = "32" }, + { identifier = "cp313t-win32", version = "3.13.0-b1", arch = "32" }, { identifier = "cp313-win_amd64", version = "3.13.0-b1", arch = "64" }, + { identifier = "cp313t-win_amd64", version = "3.13.0-b1", arch = "64" }, { identifier = "cp39-win_arm64", version = "3.9.10", arch = "ARM64" }, { identifier = "cp310-win_arm64", version = "3.10.11", arch = "ARM64" }, { identifier = "cp311-win_arm64", version = "3.11.9", arch = "ARM64" }, { identifier = "cp312-win_arm64", version = "3.12.3", arch = "ARM64" }, { identifier = "cp313-win_arm64", version = "3.13.0-b1", arch = "ARM64" }, + { identifier = "cp313t-win_arm64", version = "3.13.0-b1", arch = "ARM64" }, { identifier = "pp37-win_amd64", version = "3.7", arch = "64", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.9-win64.zip" }, { identifier = "pp38-win_amd64", version = "3.8", arch = "64", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-win64.zip" }, { identifier = "pp39-win_amd64", version = "3.9", arch = "64", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.16-win64.zip" }, diff --git a/cibuildwheel/resources/constraints-python313.txt b/cibuildwheel/resources/constraints-python313.txt index 474fa6de0..5f4f1bf97 100644 --- a/cibuildwheel/resources/constraints-python313.txt +++ b/cibuildwheel/resources/constraints-python313.txt @@ -16,7 +16,7 @@ packaging==24.0 # via # build # delocate -pip==24.0 +pip==24.1b1 # via -r cibuildwheel/resources/constraints.in platformdirs==4.2.2 # via virtualenv diff --git a/cibuildwheel/resources/constraints.in b/cibuildwheel/resources/constraints.in index 50bfabb6e..4f01a4b8c 100644 --- a/cibuildwheel/resources/constraints.in +++ b/cibuildwheel/resources/constraints.in @@ -1,4 +1,5 @@ -pip +pip>=24.1b1 ; python_version >= '3.13' +pip ; python_version < '3.13' build delocate virtualenv diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 66be99ca7..e704911cb 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -238,13 +238,15 @@ class BuildSelector: requires_python: SpecifierSet | None = None # a pattern that skips prerelease versions, when include_prereleases is False. - PRERELEASE_SKIP: ClassVar[str] = "cp313-*" + PRERELEASE_SKIP: ClassVar[str] = "cp313-* cp313t-*" prerelease_pythons: bool = False def __call__(self, build_id: str) -> bool: # Filter build selectors by python_requires if set if self.requires_python is not None: py_ver_str = build_id.split("-")[0] + if py_ver_str.endswith("t"): + py_ver_str = py_ver_str[:-1] major = int(py_ver_str[2]) minor = int(py_ver_str[3:]) version = Version(f"{major}.{minor}.99") @@ -645,10 +647,13 @@ def find_compatible_wheel(wheels: Sequence[T], identifier: str) -> T | None: """ interpreter, platform = identifier.split("-") + free_threaded = interpreter.endswith("t") + if free_threaded: + interpreter = interpreter[:-1] for wheel in wheels: _, _, _, tags = parse_wheel_filename(wheel.name) for tag in tags: - if tag.abi == "abi3": + if tag.abi == "abi3" and not free_threaded: # ABI3 wheels must start with cp3 for impl and tag if not (interpreter.startswith("cp3") and tag.interpreter.startswith("cp3")): continue diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index e661716d4..a8e82b65e 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -44,7 +44,9 @@ ) -def get_nuget_args(version: str, arch: str, output_directory: Path) -> list[str]: +def get_nuget_args( + version: str, arch: str, free_threaded: bool, output_directory: Path +) -> list[str]: package_name = { "32": "pythonx86", "64": "python", @@ -53,6 +55,8 @@ def get_nuget_args(version: str, arch: str, output_directory: Path) -> list[str] "x86": "pythonx86", "AMD64": "python", }[arch] + if free_threaded: + package_name = f"{package_name}-freethreaded" return [ package_name, "-Version", @@ -106,11 +110,12 @@ def _ensure_nuget() -> Path: return nuget -def install_cpython(version: str, arch: str) -> Path: +def install_cpython(version: str, arch: str, free_threaded: bool) -> Path: base_output_dir = CIBW_CACHE_PATH / "nuget-cpython" - nuget_args = get_nuget_args(version, arch, base_output_dir) + nuget_args = get_nuget_args(version, arch, free_threaded, base_output_dir) installation_path = base_output_dir / (nuget_args[0] + "." + version) / "tools" - with FileLock(str(base_output_dir) + f"-{version}-{arch}.lock"): + free_threaded_str = "-freethreaded" if free_threaded else "" + with FileLock(str(base_output_dir) + f"-{version}{free_threaded_str}-{arch}.lock"): if not installation_path.exists(): nuget = _ensure_nuget() call(nuget, "install", *nuget_args) @@ -224,18 +229,21 @@ def setup_python( log.step(f"Installing Python {implementation_id}...") if implementation_id.startswith("cp"): native_arch = platform_module.machine() + free_threaded = implementation_id.endswith("t") if python_configuration.arch == "ARM64" != native_arch: # To cross-compile for ARM64, we need a native CPython to run the # build, and a copy of the ARM64 import libraries ('.\libs\*.lib') # for any extension modules. python_libs_base = install_cpython( - python_configuration.version, python_configuration.arch + python_configuration.version, python_configuration.arch, free_threaded ) python_libs_base = python_libs_base.parent / "libs" log.step(f"Installing native Python {native_arch} for cross-compilation...") - base_python = install_cpython(python_configuration.version, native_arch) + base_python = install_cpython(python_configuration.version, native_arch, free_threaded) else: - base_python = install_cpython(python_configuration.version, python_configuration.arch) + base_python = install_cpython( + python_configuration.version, python_configuration.arch, free_threaded + ) elif implementation_id.startswith("pp"): assert python_configuration.url is not None base_python = install_pypy(tmp, python_configuration.arch, python_configuration.url) @@ -521,6 +529,10 @@ def build(options: Options, tmp_path: Path) -> None: # check that we are using the Python from the virtual environment call("where", "python", env=virtualenv_env) + # TODO remove me once virtualenv provides pip>=24.1b1 + if config.version.startswith("3.13."): + call("python", "-m", "pip", "install", "--pre", "-U", "pip", env=virtualenv_env) + if build_options.before_test: before_test_prepared = prepare_command( build_options.before_test, diff --git a/test/test_abi_variants.py b/test/test_abi_variants.py index d203c06fe..4446ed56c 100644 --- a/test/test_abi_variants.py +++ b/test/test_abi_variants.py @@ -13,9 +13,14 @@ limited_api_project = test_projects.new_c_project( setup_py_add=textwrap.dedent( r""" + import sysconfig + + IS_CPYTHON = sys.implementation.name == "cpython" + Py_GIL_DISABLED = sysconfig.get_config_var("Py_GIL_DISABLED") + CAN_USE_ABI3 = IS_CPYTHON and not Py_GIL_DISABLED cmdclass = {} extension_kwargs = {} - if sys.version_info[:2] >= (3, 8): + if CAN_USE_ABI3 and sys.version_info[:2] >= (3, 8): from wheel.bdist_wheel import bdist_wheel as _bdist_wheel class bdist_wheel_abi3(_bdist_wheel): @@ -47,7 +52,8 @@ def test_abi3(tmp_path): actual_wheels = utils.cibuildwheel_run( project_dir, add_env={ - "CIBW_SKIP": "pp* ", # PyPy does not have a Py_LIMITED_API equivalent + # free_threaded and PyPy do not have a Py_LIMITED_API equivalent, just build one of those + "CIBW_BUILD": "cp3?-* cp31?-* cp313t-* pp310-*" }, ) @@ -55,7 +61,9 @@ def test_abi3(tmp_path): expected_wheels = [ w.replace("cp38-cp38", "cp38-abi3") for w in utils.expected_wheels("spam", "0.1.0") - if "-pp" not in w and "-cp39" not in w and "-cp31" not in w + if ("-pp310" in w or "-pp" not in w) + and "-cp39" not in w + and ("-cp313t" in w or "-cp31" not in w) ] assert set(actual_wheels) == set(expected_wheels) @@ -177,7 +185,7 @@ def test_abi_none(tmp_path, capfd): "CIBW_TEST_REQUIRES": "pytest", "CIBW_TEST_COMMAND": "pytest {project}/test", # limit the number of builds for test performance reasons - "CIBW_BUILD": "cp38-* cp310-* pp39-*", + "CIBW_BUILD": "cp38-* cp310-* cp313t-* pp310-*", }, ) diff --git a/test/utils.py b/test/utils.py index f6f610c41..4bc02f47d 100644 --- a/test/utils.py +++ b/test/utils.py @@ -176,6 +176,8 @@ def expected_wheels( "cp312-cp312", "cp313-cp313", ] + if platform != "macos": + python_abi_tags.append("cp313-cp313t") if machine_arch in ["x86_64", "AMD64", "x86", "aarch64"]: python_abi_tags += [