From abd022383b321e49203bb828144dd9e6d846cb25 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 8 Jun 2022 17:24:30 +0200 Subject: [PATCH] [3.11] gh-93584: Make all install+tests targets depends on all (GH-93589) (GH-93603) All install targets use the "all" target as synchronization point to prevent race conditions with PGO builds. PGO builds use recursive make, which can lead to two parallel `./python setup.py build` processes that step on each others toes. "test" targets now correctly compile PGO build in a clean repo. (cherry picked from commit 243ed5439c32e8517aa745bc2ca9774d99c99d0f) Co-authored-by: Christian Heimes --- Makefile.pre.in | 28 +++++++++++-------- ...2-06-08-14-28-03.gh-issue-93584.0xfHOK.rst | 2 ++ 2 files changed, 18 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2022-06-08-14-28-03.gh-issue-93584.0xfHOK.rst diff --git a/Makefile.pre.in b/Makefile.pre.in index f4b23f6e765700..ccd7f52a5601d0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -580,8 +580,8 @@ LIBEXPAT_HEADERS= \ # Default target all: @DEF_MAKE_ALL_RULE@ -build_all: check-clean-src $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks \ - Programs/_testembed python-config +build_all: check-clean-src $(BUILDPYTHON) platform oldsharedmods sharedmods \ + gdbhooks Programs/_testembed python-config build_wasm: check-clean-src $(BUILDPYTHON) platform oldsharedmods python-config # Check that the source is clean when building out of source. @@ -1671,7 +1671,7 @@ cleantest: all # Run a basic set of regression tests. # This excludes some tests that are particularly resource-intensive. -test: @DEF_MAKE_RULE@ platform +test: all $(TESTRUNNER) $(TESTOPTS) # Run the full test suite twice - once without .pyc files, and once with. @@ -1681,7 +1681,7 @@ test: @DEF_MAKE_RULE@ platform # the bytecode read from a .pyc file had the bug, sometimes the directly # generated bytecode. This is sometimes a very shy bug needing a lot of # sample data. -testall: @DEF_MAKE_RULE@ platform +testall: all -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f $(TESTPYTHON) -E $(srcdir)/Lib/compileall.py -find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f @@ -1690,7 +1690,7 @@ testall: @DEF_MAKE_RULE@ platform # Run the test suite for both architectures in a Universal build on OSX. # Must be run on an Intel box. -testuniversal: @DEF_MAKE_RULE@ platform +testuniversal: all @if [ `arch` != 'i386' ]; then \ echo "This can only be used on OSX/i386" ;\ exit 1 ;\ @@ -1701,7 +1701,7 @@ testuniversal: @DEF_MAKE_RULE@ platform # Like testall, but with only one pass and without multiple processes. # Run an optional script to include information about the build environment. -buildbottest: all platform +buildbottest: all -@if which pybuildbot.identify >/dev/null 2>&1; then \ pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \ fi @@ -1716,7 +1716,7 @@ QUICKTESTOPTS= $(TESTOPTS) -x test_subprocess test_io test_lib2to3 \ test_multiprocessing_forkserver \ test_mailbox test_nntplib test_socket test_poll \ test_select test_zipfile test_concurrent_futures -quicktest: @DEF_MAKE_RULE@ platform +quicktest: all $(TESTRUNNER) $(QUICKTESTOPTS) # SSL tests @@ -1727,6 +1727,10 @@ multisslcompile: all multissltest: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/ssl/multissltests.py +# All install targets use the "all" target as synchronization point to +# prevent race conditions with PGO builds. PGO builds use recursive make, +# which can lead to two parallel `./python setup.py build` processes that +# step on each others toes. install: @FRAMEWORKINSTALLFIRST@ commoninstall bininstall maninstall @FRAMEWORKINSTALLLAST@ if test "x$(ENSUREPIP)" != "xno" ; then \ case $(ENSUREPIP) in \ @@ -1755,7 +1759,7 @@ commoninstall: check-clean-src @FRAMEWORKALTINSTALLFIRST@ \ # Install shared libraries enabled by Setup DESTDIRS= $(exec_prefix) $(LIBDIR) $(BINLIBDEST) $(DESTSHARED) -oldsharedinstall: $(DESTSHARED) $(SHAREDMODS) +oldsharedinstall: $(DESTSHARED) all @for i in X $(SHAREDMODS); do \ if test $$i != X; then \ echo $(INSTALL_SHARED) $$i $(DESTSHARED)/`basename $$i`; \ @@ -2161,7 +2165,7 @@ LIBPL= @LIBPL@ # pkgconfig directory LIBPC= $(LIBDIR)/pkgconfig -libainstall: @DEF_MAKE_RULE@ python-config +libainstall: all python-config @for i in $(LIBDIR) $(LIBPL) $(LIBPC) $(BINDIR); \ do \ if test ! -d $(DESTDIR)$$i; then \ @@ -2215,7 +2219,7 @@ libainstall: @DEF_MAKE_RULE@ python-config # Install the dynamically loadable modules # This goes into $(exec_prefix) -sharedinstall: sharedmods +sharedinstall: all $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/setup.py install \ --prefix=$(prefix) \ --install-scripts=$(BINDIR) \ @@ -2445,7 +2449,7 @@ distclean: clobber -exec rm -f {} ';' # Check that all symbols exported by libpython start with "Py" or "_Py" -smelly: @DEF_MAKE_RULE@ +smelly: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/smelly.py # Find files with funny names @@ -2480,7 +2484,7 @@ funny: -o -print # Perform some verification checks on any modified files. -patchcheck: @DEF_MAKE_RULE@ +patchcheck: all $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py check-limited-abi: all diff --git a/Misc/NEWS.d/next/Build/2022-06-08-14-28-03.gh-issue-93584.0xfHOK.rst b/Misc/NEWS.d/next/Build/2022-06-08-14-28-03.gh-issue-93584.0xfHOK.rst new file mode 100644 index 00000000000000..07ca5fad592b2d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-06-08-14-28-03.gh-issue-93584.0xfHOK.rst @@ -0,0 +1,2 @@ +Address race condition in ``Makefile`` when installing a PGO build. All +``test`` and ``install`` targets now depend on ``all`` target.