Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Build system refactor to prepare for parallel Python version building #419

Merged
merged 4 commits into from
Dec 16, 2024
Merged

Build system refactor to prepare for parallel Python version building #419

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f674657
pythonbuild: make download_to_path() concurrently safe
indygreg Nov 16, 2024
2f48115
unix: move Python major minor version parsing to Makefile
indygreg Nov 16, 2024
1d66a39
unix: only generate 1 Makefile per build configuration
indygreg Nov 16, 2024
46d61c7
unix: templatize per version Makefile variables and targets
indygreg Nov 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 51 additions & 67 deletions cpython-unix/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ OUTDIR := $(ROOT)/build

BUILD := $(HERE)/build.py
NULL :=
SPACE := $(subst ,, )

ALL_PYTHON_VERSIONS := 3.9 3.10 3.11 3.12 3.13

ifndef PYBUILD_TARGET_TRIPLE
$(error PYBUILD_TARGET_TRIPLE not defined)
Expand All @@ -25,9 +28,7 @@ ifndef PYBUILD_PYTHON_VERSION
$(error PYBUILD_PYTHON_VERSION not defined)
endif

ifndef PYBUILD_PYTHON_MAJOR_VERSION
$(error PYBUILD_PYTHON_MAJOR_VERSION not defined)
endif
PYTHON_MAJOR_VERSION := $(subst $(SPACE),.,$(wordlist 1,2,$(subst .,$(SPACE),$(PYBUILD_PYTHON_VERSION))))

TARGET_TRIPLE := $(PYBUILD_TARGET_TRIPLE)
HOST_PLATFORM := $(PYBUILD_HOST_PLATFORM)
Expand All @@ -47,7 +48,7 @@ endif

# Always write out settings files.
$(shell $(RUN_BUILD) placeholder_archive makefiles)
include $(OUTDIR)/Makefile.$(HOST_PLATFORM).$(TARGET_TRIPLE).$(PYBUILD_PYTHON_MAJOR_VERSION)
include $(OUTDIR)/Makefile.$(HOST_PLATFORM).$(TARGET_TRIPLE)
include $(OUTDIR)/versions/VERSION.*

# Always write out expanded Dockerfiles.
Expand All @@ -69,7 +70,7 @@ PYTHON_DEP_DEPENDS := \
$(TOOLCHAIN_DEPENDS) \
$(NULL)

default: $(OUTDIR)/cpython-$(PYBUILD_PYTHON_VERSION)-$(PACKAGE_SUFFIX).tar
default: $(OUTDIR)/cpython-$(CPYTHON_$(PYTHON_MAJOR_VERSION)_VERSION)-$(PACKAGE_SUFFIX).tar

ifndef PYBUILD_NO_DOCKER
$(OUTDIR)/image-%.tar: $(OUTDIR)/%.Dockerfile
Expand Down Expand Up @@ -255,65 +256,48 @@ PYTHON_HOST_DEPENDS := \
$(OUTDIR)/m4-$(M4_VERSION)-$(PACKAGE_SUFFIX).tar \
$(NULL)

$(OUTDIR)/cpython-3.9-$(CPYTHON_3.9_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS)
$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.9-host

$(OUTDIR)/cpython-3.10-$(CPYTHON_3.10_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS)
$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.10-host

$(OUTDIR)/cpython-3.11-$(CPYTHON_3.11_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS)
$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.11-host

$(OUTDIR)/cpython-3.12-$(CPYTHON_3.12_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS)
$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.12-host

$(OUTDIR)/cpython-3.13-$(CPYTHON_3.13_VERSION)-$(HOST_PLATFORM).tar: $(PYTHON_HOST_DEPENDS)
$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.13-host

PYTHON_DEPENDS := \
$(PYTHON_SUPPORT_FILES) \
$(OUTDIR)/versions/VERSION.pip \
$(OUTDIR)/versions/VERSION.setuptools \
$(OUTDIR)/cpython-$(PYBUILD_PYTHON_MAJOR_VERSION)-$(PYBUILD_PYTHON_VERSION)-$(HOST_PLATFORM).tar \
$(if $(NEED_AUTOCONF),$(OUTDIR)/autoconf-$(AUTOCONF_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_BDB),$(OUTDIR)/bdb-$(BDB_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_BZIP2),$(OUTDIR)/bzip2-$(BZIP2_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_EXPAT),$(OUTDIR)/expat-$(EXPAT_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_LIBEDIT),$(OUTDIR)/libedit-$(LIBEDIT_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_LIBFFI_3_3),$(OUTDIR)/libffi-3.3-$(LIBFFI_3.3_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_LIBFFI),$(OUTDIR)/libffi-$(LIBFFI_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_m4),$(OUTDIR)/m4-$(M4_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_MPDECIMAL),$(OUTDIR)/mpdecimal-$(MPDECIMAL_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_NCURSES),$(OUTDIR)/ncurses-$(NCURSES_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_OPENSSL_1_1),$(OUTDIR)/openssl-1.1-$(OPENSSL_1.1_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_OPENSSL_3_0),$(OUTDIR)/openssl-3.0-$(OPENSSL_3.0_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_PATCHELF),$(OUTDIR)/patchelf-$(PATCHELF_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_SQLITE),$(OUTDIR)/sqlite-$(SQLITE_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_TCL),$(OUTDIR)/tcl-$(TCL_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_TK),$(OUTDIR)/tk-$(TK_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_TIX),$(OUTDIR)/tix-$(TIX_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_UUID),$(OUTDIR)/uuid-$(UUID_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_XZ),$(OUTDIR)/xz-$(XZ_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(if $(NEED_ZLIB),$(OUTDIR)/zlib-$(ZLIB_VERSION)-$(PACKAGE_SUFFIX).tar) \
$(NULL)

ALL_PYTHON_DEPENDS = \
$(PYTHON_DEP_DEPENDS) \
$(HERE)/build-cpython.sh \
$(PYTHON_DEPENDS) \
$(NULL)

$(OUTDIR)/cpython-$(CPYTHON_3.9_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS)
$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.9

$(OUTDIR)/cpython-$(CPYTHON_3.10_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS)
$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.10

$(OUTDIR)/cpython-$(CPYTHON_3.11_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS)
$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.11

$(OUTDIR)/cpython-$(CPYTHON_3.12_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS)
$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.12

$(OUTDIR)/cpython-$(CPYTHON_3.13_VERSION)-$(PACKAGE_SUFFIX).tar: $(ALL_PYTHON_DEPENDS)
$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) cpython-3.13
# Each X.Y Python version has its own set of variables and targets. This independent
# definition allows multiple Python versions to be built using the same Makefile
# invocation.
define python_version_template
PYTHON_DEPENDS_$(1) := \
$$(PYTHON_SUPPORT_FILES) \
$$(OUTDIR)/versions/VERSION.pip \
$$(OUTDIR)/versions/VERSION.setuptools \
$$(OUTDIR)/cpython-$(1)-$$(CPYTHON_$(1)_VERSION)-$$(HOST_PLATFORM).tar \
$$(if$$(NEED_AUTOCONF),$$(OUTDIR)/autoconf-$$(AUTOCONF_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_BDB),$$(OUTDIR)/bdb-$$(BDB_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_BZIP2),$$(OUTDIR)/bzip2-$$(BZIP2_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_EXPAT),$$(OUTDIR)/expat-$$(EXPAT_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_LIBEDIT),$$(OUTDIR)/libedit-$$(LIBEDIT_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_LIBFFI_3_3),$$(OUTDIR)/libffi-3.3-$$(LIBFFI_3.3_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_LIBFFI),$$(OUTDIR)/libffi-$$(LIBFFI_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_m4),$$(OUTDIR)/m4-$$(M4_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_MPDECIMAL),$$(OUTDIR)/mpdecimal-$$(MPDECIMAL_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_NCURSES),$$(OUTDIR)/ncurses-$$(NCURSES_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_OPENSSL_1_1),$$(OUTDIR)/openssl-1.1-$$(OPENSSL_1.1_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_OPENSSL_3_0),$$(OUTDIR)/openssl-3.0-$$(OPENSSL_3.0_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_PATCHELF),$$(OUTDIR)/patchelf-$$(PATCHELF_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_SQLITE),$$(OUTDIR)/sqlite-$$(SQLITE_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_TCL),$$(OUTDIR)/tcl-$$(TCL_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_TK),$$(OUTDIR)/tk-$$(TK_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_TIX),$$(OUTDIR)/tix-$$(TIX_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_UUID),$$(OUTDIR)/uuid-$$(UUID_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_XZ),$$(OUTDIR)/xz-$$(XZ_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(if $$(NEED_ZLIB),$$(OUTDIR)/zlib-$$(ZLIB_VERSION)-$$(PACKAGE_SUFFIX).tar) \
$$(NULL)

ALL_PYTHON_DEPENDS_$(1) = \
$$(PYTHON_DEP_DEPENDS) \
$$(HERE)/build-cpython.sh \
$$(PYTHON_DEPENDS_$(1)) \
$$(NULL)

$$(OUTDIR)/cpython-$(1)-$$(CPYTHON_$(1)_VERSION)-$$(HOST_PLATFORM).tar: $$(PYTHON_HOST_DEPENDS)
$$(RUN_BUILD) --docker-image $$(DOCKER_IMAGE_BUILD) cpython-$(1)-host

$$(OUTDIR)/cpython-$$(CPYTHON_$(1)_VERSION)-$$(PACKAGE_SUFFIX).tar: $$(ALL_PYTHON_DEPENDS_$(1))
$$(RUN_BUILD) --docker-image $$(DOCKER_IMAGE_BUILD) cpython-$(1)
endef

$(foreach local_version,$(ALL_PYTHON_VERSIONS),$(eval $(call python_version_template,$(local_version))))
8 changes: 3 additions & 5 deletions cpython-unix/build-main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,20 +156,18 @@ def main():
return 1
cpython_version = env["PYBUILD_PYTHON_VERSION"]

env["PYBUILD_PYTHON_MAJOR_VERSION"] = ".".join(cpython_version.split(".")[0:2])
python_majmin = ".".join(cpython_version.split(".")[0:2])

if "PYBUILD_RELEASE_TAG" in os.environ:
release_tag = os.environ["PYBUILD_RELEASE_TAG"]
else:
release_tag = release_tag_from_git()

# Guard against accidental misuse of the free-threaded flag with older versions
if "freethreaded" in args.options and env["PYBUILD_PYTHON_MAJOR_VERSION"] not in (
"3.13"
):
if "freethreaded" in args.options and python_majmin not in ("3.13",):
print(
"Invalid build option: 'freethreaded' is only compatible with CPython 3.13+ (got %s)"
% env["PYBUILD_PYTHON_MAJOR_VERSION"]
% cpython_version
)
return 1

Expand Down
60 changes: 36 additions & 24 deletions pythonbuild/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@
import os
import pathlib
import platform
import random
import stat
import string
import subprocess
import sys
import tarfile
Expand Down Expand Up @@ -141,35 +143,37 @@ def write_triples_makefiles(

for triple, settings in targets.items():
for host_platform in settings["host_platforms"]:
for python in settings["pythons_supported"]:
makefile_path = dest_dir / (
"Makefile.%s.%s.%s" % (host_platform, triple, python)
)

lines = []
for need in settings.get("needs", []):
lines.append(
"NEED_%s := 1\n"
% need.upper().replace("-", "_").replace(".", "_")
)

image_suffix = settings.get("docker_image_suffix", "")
# IMPORTANT: if we ever vary the content of these Makefiles by
# Python versions, the variable names will need add the Python
# version and the Makefile references updated to point to specific
# versions. If we don't do that, multi-version builds will fail
# to work correctly.

lines.append("DOCKER_IMAGE_BUILD := build%s\n" % image_suffix)
lines.append("DOCKER_IMAGE_XCB := xcb%s\n" % image_suffix)
makefile_path = dest_dir / ("Makefile.%s.%s" % (host_platform, triple))

entry = clang_toolchain(host_platform, triple)
lines = []
for need in settings.get("needs", []):
lines.append(
"CLANG_FILENAME := %s-%s-%s.tar\n"
% (entry, DOWNLOADS[entry]["version"], host_platform)
"NEED_%s := 1\n" % need.upper().replace("-", "_").replace(".", "_")
)

lines.append(
"PYTHON_SUPPORT_FILES := $(PYTHON_SUPPORT_FILES) %s\n"
% (support_search_dir / "extension-modules.yml")
)
image_suffix = settings.get("docker_image_suffix", "")

write_if_different(makefile_path, "".join(lines).encode("ascii"))
lines.append("DOCKER_IMAGE_BUILD := build%s\n" % image_suffix)
lines.append("DOCKER_IMAGE_XCB := xcb%s\n" % image_suffix)

entry = clang_toolchain(host_platform, triple)
lines.append(
"CLANG_FILENAME := %s-%s-%s.tar\n"
% (entry, DOWNLOADS[entry]["version"], host_platform)
)

lines.append(
"PYTHON_SUPPORT_FILES := $(PYTHON_SUPPORT_FILES) %s\n"
% (support_search_dir / "extension-modules.yml")
)

write_if_different(makefile_path, "".join(lines).encode("ascii"))


def write_package_versions(dest_path: pathlib.Path):
Expand Down Expand Up @@ -269,7 +273,15 @@ def download_to_path(url: str, path: pathlib.Path, size: int, sha256: str):

path.unlink()

tmp = path.with_name("%s.tmp" % path.name)
# Need to write to random path to avoid race conditions. If there is a
# race, worst case we'll download the same file N>1 times. Meh.
tmp = path.with_name(
"%s.tmp%s"
% (
path.name,
"".join(random.choices(string.ascii_uppercase + string.digits, k=8)),
)
)

for attempt in range(5):
try:
Expand Down
Loading