From f674657c6c5f6cfa9c7d407b12c65d96e09994b5 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 16 Nov 2024 15:38:30 -0800 Subject: [PATCH 1/4] pythonbuild: make download_to_path() concurrently safe I'm about to enable support for parallel CPython builds. This function was currently not safe if called in parallel. This was causing setuptools / pip downloads to race writing to the same file and failing builds. --- pythonbuild/utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index 1c1ebef0..3108670f 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -12,7 +12,9 @@ import os import pathlib import platform +import random import stat +import string import subprocess import sys import tarfile @@ -269,7 +271,15 @@ def download_to_path(url: str, path: pathlib.Path, size: int, sha256: str): path.unlink() - tmp = path.with_name("%s.tmp" % path.name) + # Need to write to random path to avoid race conditions. If there is a + # race, worst case we'll download the same file N>1 times. Meh. + tmp = path.with_name( + "%s.tmp%s" + % ( + path.name, + "".join(random.choices(string.ascii_uppercase + string.digits, k=8)), + ) + ) for attempt in range(5): try: From 2f481159eacca1cdb6f492bb358cbf2edfcaa262 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 16 Nov 2024 12:09:25 -0800 Subject: [PATCH 2/4] unix: move Python major minor version parsing to Makefile This is less ergonomic (oh Make). But it will make it easier to build multiple Python versions at the same time in a future commit. --- cpython-unix/Makefile | 9 ++++----- cpython-unix/build-main.py | 8 +++----- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index 2827fc88..d1c9a86b 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -4,6 +4,7 @@ OUTDIR := $(ROOT)/build BUILD := $(HERE)/build.py NULL := +SPACE := $(subst ,, ) ifndef PYBUILD_TARGET_TRIPLE $(error PYBUILD_TARGET_TRIPLE not defined) @@ -25,9 +26,7 @@ ifndef PYBUILD_PYTHON_VERSION $(error PYBUILD_PYTHON_VERSION not defined) endif -ifndef PYBUILD_PYTHON_MAJOR_VERSION - $(error PYBUILD_PYTHON_MAJOR_VERSION not defined) -endif +PYTHON_MAJOR_VERSION := $(subst $(SPACE),.,$(wordlist 1,2,$(subst .,$(SPACE),$(PYBUILD_PYTHON_VERSION)))) TARGET_TRIPLE := $(PYBUILD_TARGET_TRIPLE) HOST_PLATFORM := $(PYBUILD_HOST_PLATFORM) @@ -47,7 +46,7 @@ endif # Always write out settings files. $(shell $(RUN_BUILD) placeholder_archive makefiles) -include $(OUTDIR)/Makefile.$(HOST_PLATFORM).$(TARGET_TRIPLE).$(PYBUILD_PYTHON_MAJOR_VERSION) +include $(OUTDIR)/Makefile.$(HOST_PLATFORM).$(TARGET_TRIPLE).$(PYTHON_MAJOR_VERSION) include $(OUTDIR)/versions/VERSION.* # Always write out expanded Dockerfiles. @@ -274,7 +273,7 @@ PYTHON_DEPENDS := \ $(PYTHON_SUPPORT_FILES) \ $(OUTDIR)/versions/VERSION.pip \ $(OUTDIR)/versions/VERSION.setuptools \ - $(OUTDIR)/cpython-$(PYBUILD_PYTHON_MAJOR_VERSION)-$(PYBUILD_PYTHON_VERSION)-$(HOST_PLATFORM).tar \ + $(OUTDIR)/cpython-$(PYTHON_MAJOR_VERSION)-$(PYBUILD_PYTHON_VERSION)-$(HOST_PLATFORM).tar \ $(if $(NEED_AUTOCONF),$(OUTDIR)/autoconf-$(AUTOCONF_VERSION)-$(PACKAGE_SUFFIX).tar) \ $(if $(NEED_BDB),$(OUTDIR)/bdb-$(BDB_VERSION)-$(PACKAGE_SUFFIX).tar) \ $(if $(NEED_BZIP2),$(OUTDIR)/bzip2-$(BZIP2_VERSION)-$(PACKAGE_SUFFIX).tar) \ diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index 1e310b22..e837dd5d 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -156,7 +156,7 @@ def main(): return 1 cpython_version = env["PYBUILD_PYTHON_VERSION"] - env["PYBUILD_PYTHON_MAJOR_VERSION"] = ".".join(cpython_version.split(".")[0:2]) + python_majmin = ".".join(cpython_version.split(".")[0:2]) if "PYBUILD_RELEASE_TAG" in os.environ: release_tag = os.environ["PYBUILD_RELEASE_TAG"] @@ -164,12 +164,10 @@ def main(): release_tag = release_tag_from_git() # Guard against accidental misuse of the free-threaded flag with older versions - if "freethreaded" in args.options and env["PYBUILD_PYTHON_MAJOR_VERSION"] not in ( - "3.13" - ): + if "freethreaded" in args.options and python_majmin not in ("3.13",): print( "Invalid build option: 'freethreaded' is only compatible with CPython 3.13+ (got %s)" - % env["PYBUILD_PYTHON_MAJOR_VERSION"] + % cpython_version ) return 1 From 1d66a39ee560747ad8a6c9c8c1e679743884b238 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 16 Nov 2024 12:58:45 -0800 Subject: [PATCH 3/4] unix: only generate 1 Makefile per build configuration Previously we generated the same Makefile for every Python version. We can simplify things by only writing 1 Makefile per target configuration. --- cpython-unix/Makefile | 2 +- pythonbuild/utils.py | 42 +++++++++++++++++++----------------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index d1c9a86b..42dd31ca 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -46,7 +46,7 @@ endif # Always write out settings files. $(shell $(RUN_BUILD) placeholder_archive makefiles) -include $(OUTDIR)/Makefile.$(HOST_PLATFORM).$(TARGET_TRIPLE).$(PYTHON_MAJOR_VERSION) +include $(OUTDIR)/Makefile.$(HOST_PLATFORM).$(TARGET_TRIPLE) include $(OUTDIR)/versions/VERSION.* # Always write out expanded Dockerfiles. diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index 3108670f..d8c61abc 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -143,35 +143,31 @@ def write_triples_makefiles( for triple, settings in targets.items(): for host_platform in settings["host_platforms"]: - for python in settings["pythons_supported"]: - makefile_path = dest_dir / ( - "Makefile.%s.%s.%s" % (host_platform, triple, python) - ) + makefile_path = dest_dir / ("Makefile.%s.%s" % (host_platform, triple)) - lines = [] - for need in settings.get("needs", []): - lines.append( - "NEED_%s := 1\n" - % need.upper().replace("-", "_").replace(".", "_") - ) + lines = [] + for need in settings.get("needs", []): + lines.append( + "NEED_%s := 1\n" % need.upper().replace("-", "_").replace(".", "_") + ) - image_suffix = settings.get("docker_image_suffix", "") + image_suffix = settings.get("docker_image_suffix", "") - lines.append("DOCKER_IMAGE_BUILD := build%s\n" % image_suffix) - lines.append("DOCKER_IMAGE_XCB := xcb%s\n" % image_suffix) + lines.append("DOCKER_IMAGE_BUILD := build%s\n" % image_suffix) + lines.append("DOCKER_IMAGE_XCB := xcb%s\n" % image_suffix) - entry = clang_toolchain(host_platform, triple) - lines.append( - "CLANG_FILENAME := %s-%s-%s.tar\n" - % (entry, DOWNLOADS[entry]["version"], host_platform) - ) + entry = clang_toolchain(host_platform, triple) + lines.append( + "CLANG_FILENAME := %s-%s-%s.tar\n" + % (entry, DOWNLOADS[entry]["version"], host_platform) + ) - lines.append( - "PYTHON_SUPPORT_FILES := $(PYTHON_SUPPORT_FILES) %s\n" - % (support_search_dir / "extension-modules.yml") - ) + lines.append( + "PYTHON_SUPPORT_FILES := $(PYTHON_SUPPORT_FILES) %s\n" + % (support_search_dir / "extension-modules.yml") + ) - write_if_different(makefile_path, "".join(lines).encode("ascii")) + write_if_different(makefile_path, "".join(lines).encode("ascii")) def write_package_versions(dest_path: pathlib.Path): From 46d61c79afd04328330689d818bfe8a3a62ee8ea Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 16 Nov 2024 12:42:16 -0800 Subject: [PATCH 4/4] unix: templatize per version Makefile variables and targets This refactor allows targets for each major Python version to be defined simultaneously. This opens the door to a single `make` invocation building multiple versions in parallel. --- cpython-unix/Makefile | 111 ++++++++++++++++++------------------------ pythonbuild/utils.py | 6 +++ 2 files changed, 54 insertions(+), 63 deletions(-) diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index 42dd31ca..779bc12e 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -6,6 +6,8 @@ BUILD := $(HERE)/build.py NULL := SPACE := $(subst ,, ) +ALL_PYTHON_VERSIONS := 3.9 3.10 3.11 3.12 3.13 + ifndef PYBUILD_TARGET_TRIPLE $(error PYBUILD_TARGET_TRIPLE not defined) endif @@ -68,7 +70,7 @@ PYTHON_DEP_DEPENDS := \ $(TOOLCHAIN_DEPENDS) \ $(NULL) -default: $(OUTDIR)/cpython-$(PYBUILD_PYTHON_VERSION)-$(PACKAGE_SUFFIX).tar +default: $(OUTDIR)/cpython-$(CPYTHON_$(PYTHON_MAJOR_VERSION)_VERSION)-$(PACKAGE_SUFFIX).tar ifndef PYBUILD_NO_DOCKER $(OUTDIR)/image-%.tar: $(OUTDIR)/%.Dockerfile @@ -254,65 +256,48 @@ PYTHON_HOST_DEPENDS := \ $(OUTDIR)/m4-$(M4_VERSION)-$(PACKAGE_SUFFIX).tar \ $(NULL) -$(OUTDIR)/cpython-3.9-$(CPYTHON_3.9_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.9-host - -$(OUTDIR)/cpython-3.10-$(CPYTHON_3.10_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.10-host - -$(OUTDIR)/cpython-3.11-$(CPYTHON_3.11_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.11-host - -$(OUTDIR)/cpython-3.12-$(CPYTHON_3.12_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.12-host - -$(OUTDIR)/cpython-3.13-$(CPYTHON_3.13_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.13-host - -PYTHON_DEPENDS := \ - $(PYTHON_SUPPORT_FILES) \ - $(OUTDIR)/versions/VERSION.pip \ - $(OUTDIR)/versions/VERSION.setuptools \ - $(OUTDIR)/cpython-$(PYTHON_MAJOR_VERSION)-$(PYBUILD_PYTHON_VERSION)-$(HOST_PLATFORM).tar \ - $(if $(NEED_AUTOCONF),$(OUTDIR)/autoconf-$(AUTOCONF_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_BDB),$(OUTDIR)/bdb-$(BDB_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_BZIP2),$(OUTDIR)/bzip2-$(BZIP2_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_EXPAT),$(OUTDIR)/expat-$(EXPAT_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_LIBEDIT),$(OUTDIR)/libedit-$(LIBEDIT_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_LIBFFI_3_3),$(OUTDIR)/libffi-3.3-$(LIBFFI_3.3_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_LIBFFI),$(OUTDIR)/libffi-$(LIBFFI_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_m4),$(OUTDIR)/m4-$(M4_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_MPDECIMAL),$(OUTDIR)/mpdecimal-$(MPDECIMAL_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_NCURSES),$(OUTDIR)/ncurses-$(NCURSES_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_OPENSSL_1_1),$(OUTDIR)/openssl-1.1-$(OPENSSL_1.1_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_OPENSSL_3_0),$(OUTDIR)/openssl-3.0-$(OPENSSL_3.0_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_PATCHELF),$(OUTDIR)/patchelf-$(PATCHELF_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_SQLITE),$(OUTDIR)/sqlite-$(SQLITE_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_TCL),$(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_TK),$(OUTDIR)/tk-$(TK_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_TIX),$(OUTDIR)/tix-$(TIX_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_UUID),$(OUTDIR)/uuid-$(UUID_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_XZ),$(OUTDIR)/xz-$(XZ_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(if $(NEED_ZLIB),$(OUTDIR)/zlib-$(ZLIB_VERSION)-$(PACKAGE_SUFFIX).tar) \ - $(NULL) - -ALL_PYTHON_DEPENDS = \ - $(PYTHON_DEP_DEPENDS) \ - $(HERE)/build-cpython.sh \ - $(PYTHON_DEPENDS) \ - $(NULL) - -$(OUTDIR)/cpython-$(CPYTHON_3.9_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.9 - -$(OUTDIR)/cpython-$(CPYTHON_3.10_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.10 - -$(OUTDIR)/cpython-$(CPYTHON_3.11_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.11 - -$(OUTDIR)/cpython-$(CPYTHON_3.12_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.12 - -$(OUTDIR)/cpython-$(CPYTHON_3.13_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS) - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.13 +# Each X.Y Python version has its own set of variables and targets. This independent +# definition allows multiple Python versions to be built using the same Makefile +# invocation. +define python_version_template +PYTHON_DEPENDS_$(1) := \ + $$(PYTHON_SUPPORT_FILES) \ + $$(OUTDIR)/versions/VERSION.pip \ + $$(OUTDIR)/versions/VERSION.setuptools \ + $$(OUTDIR)/cpython-$(1)-$$(CPYTHON_$(1)_VERSION)-$$(HOST_PLATFORM).tar \ + $$(if$$(NEED_AUTOCONF),$$(OUTDIR)/autoconf-$$(AUTOCONF_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_BDB),$$(OUTDIR)/bdb-$$(BDB_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_BZIP2),$$(OUTDIR)/bzip2-$$(BZIP2_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_EXPAT),$$(OUTDIR)/expat-$$(EXPAT_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_LIBEDIT),$$(OUTDIR)/libedit-$$(LIBEDIT_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_LIBFFI_3_3),$$(OUTDIR)/libffi-3.3-$$(LIBFFI_3.3_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_LIBFFI),$$(OUTDIR)/libffi-$$(LIBFFI_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_m4),$$(OUTDIR)/m4-$$(M4_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_MPDECIMAL),$$(OUTDIR)/mpdecimal-$$(MPDECIMAL_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_NCURSES),$$(OUTDIR)/ncurses-$$(NCURSES_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_OPENSSL_1_1),$$(OUTDIR)/openssl-1.1-$$(OPENSSL_1.1_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_OPENSSL_3_0),$$(OUTDIR)/openssl-3.0-$$(OPENSSL_3.0_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_PATCHELF),$$(OUTDIR)/patchelf-$$(PATCHELF_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_SQLITE),$$(OUTDIR)/sqlite-$$(SQLITE_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_TCL),$$(OUTDIR)/tcl-$$(TCL_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_TK),$$(OUTDIR)/tk-$$(TK_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_TIX),$$(OUTDIR)/tix-$$(TIX_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_UUID),$$(OUTDIR)/uuid-$$(UUID_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_XZ),$$(OUTDIR)/xz-$$(XZ_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(if $$(NEED_ZLIB),$$(OUTDIR)/zlib-$$(ZLIB_VERSION)-$$(PACKAGE_SUFFIX).tar) \ + $$(NULL) + +ALL_PYTHON_DEPENDS_$(1) = \ + $$(PYTHON_DEP_DEPENDS) \ + $$(HERE)/build-cpython.sh \ + $$(PYTHON_DEPENDS_$(1)) \ + $$(NULL) + +$$(OUTDIR)/cpython-$(1)-$$(CPYTHON_$(1)_VERSION)-$$(HOST_PLATFORM).tar: $$(PYTHON_HOST_DEPENDS) + $$(RUN_BUILD) --docker-image $$(DOCKER_IMAGE_BUILD) cpython-$(1)-host + +$$(OUTDIR)/cpython-$$(CPYTHON_$(1)_VERSION)-$$(PACKAGE_SUFFIX).tar: $$(ALL_PYTHON_DEPENDS_$(1)) + $$(RUN_BUILD) --docker-image $$(DOCKER_IMAGE_BUILD) cpython-$(1) +endef + +$(foreach local_version,$(ALL_PYTHON_VERSIONS),$(eval $(call python_version_template,$(local_version)))) diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index d8c61abc..49113a7e 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -143,6 +143,12 @@ def write_triples_makefiles( for triple, settings in targets.items(): for host_platform in settings["host_platforms"]: + # IMPORTANT: if we ever vary the content of these Makefiles by + # Python versions, the variable names will need add the Python + # version and the Makefile references updated to point to specific + # versions. If we don't do that, multi-version builds will fail + # to work correctly. + makefile_path = dest_dir / ("Makefile.%s.%s" % (host_platform, triple)) lines = []