diff --git a/Dockerfile b/Dockerfile index f84509ff55..a631238a88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,7 @@ RUN apt-get update && \ # matches our production except -march=haswell, we have to downgrade -march because of GHA ENV OPT="-pipe -fno-semantic-interposition -march=haswell -mabm -maes -mno-pku -mno-sgx --param l1-cache-line-size=64 --param l1-cache-size=32 --param l2-cache-size=33792" -# Bolt: -Wl,--emit-relocs -fno-reorder-blocks-and-partition +ADD patches/cpython_configure_ac.patch patches/cpython_makefile.patch patches/cpython_rules.patch / # runtime environment RUN echo 'deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted' >>/etc/apt/sources.list && \ @@ -61,18 +61,27 @@ RUN echo 'deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted' >>/et python3-distutils html2text libjs-sphinxdoc && \ echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ locale-gen && \ + echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy main" >>/etc/apt/sources.list.d/llvm.list && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + apt-get update && \ + apt-get install -y bolt-16 llvm-16 && \ + export LLVM_SYMBOLIZER_PATH=/usr/lib/llvm-16/bin/llvm-symbolizer && \ add-apt-repository -s ppa:deadsnakes/ppa && \ mkdir /cpython && \ cd /cpython && \ apt-get source python$PYTHON_VERSION && \ apt-get -s build-dep python$PYTHON_VERSION | grep "Inst " | cut -d" " -f2 | sort | tr '\n' ' ' >build_bloat && \ DEBIAN_FRONTEND="noninteractive" TZ="Europe/Madrid" apt-get build-dep -y python$PYTHON_VERSION && \ - rm /etc/apt/sources.list.d/deadsnakes* && \ + rm /etc/apt/sources.list.d/deadsnakes* /etc/apt/sources.list.d/llvm.list && \ cd python$PYTHON_VERSION* && \ sed -i 's/__main__/__skip__/g' Tools/scripts/run_tests.py && \ dch --bin-nmu -Dunstable "Optimized build" && \ echo 11 >debian/compat && \ sed -i 's/debhelper (>= 9)/debhelper (>= 11)/g' debian/control.in && \ + patch configure.ac >/et python$PYTHON_VERSION-venv* \ python$PYTHON_VERSION-full* && \ echo "========" && ls && \ - apt-get purge -y dpkg-dev devscripts software-properties-common html2text $(cat build_bloat) && \ + apt-get purge -y dpkg-dev devscripts software-properties-common html2text bolt-16 llvm-16 $(cat build_bloat) && \ apt-get autoremove -y && \ dpkg -i *python3.11*.deb && \ dpkg -i python3-minimal*.deb libpython3-stdlib*.deb && \ diff --git a/patches/cpython_configure_ac.patch b/patches/cpython_configure_ac.patch new file mode 100644 index 0000000000..9c602f968a --- /dev/null +++ b/patches/cpython_configure_ac.patch @@ -0,0 +1,80 @@ +--- configure.ac 2022-11-16 11:37:19.773231374 +0100 ++++ configure.ac.bolt 2022-11-16 11:37:11.161182410 +0100 +@@ -1751,7 +1751,16 @@ + # non-suffixed name in their versioned llvm directory. + + llvm_bin_dir='' +-llvm_path="${PATH}" ++ ++AC_ARG_WITH(llvm, ++ AS_HELP_STRING([--with-llvm=PATH], ++ [specify the directory where LLVM binaries are installed]), ++[ ++ llvm_path=${withval} ++],[ ++ llvm_path="${PATH}" ++]) ++ + if test "${CC}" = "clang" + then + clang_bin=`which clang` +@@ -1889,6 +1898,59 @@ + LDFLAGS_NODIST="$LDFLAGS_NODIST $LTOFLAGS" + fi + ++# Enable bolt flags ++Py_BOLT='false' ++AC_MSG_CHECKING(for --enable-bolt) ++AC_ARG_ENABLE(bolt, AS_HELP_STRING( ++ [--enable-bolt], ++ [enable usage of the llvm-bolt post-link optimizer (default is no)]), ++[ ++if test "$enableval" != no ++then ++ Py_BOLT='true' ++ AC_MSG_RESULT(yes); ++else ++ Py_BOLT='false' ++ AC_MSG_RESULT(no); ++fi], ++[AC_MSG_RESULT(no)]) ++ ++AC_SUBST(PREBOLT_RULE) ++if test "$Py_BOLT" = 'true' ; then ++ PREBOLT_RULE="${DEF_MAKE_ALL_RULE}" ++ DEF_MAKE_ALL_RULE="bolt-opt" ++ DEF_MAKE_RULE="build_all" ++ ++ # These flags are required for bolt to work: ++ CFLAGS_NODIST="$CFLAGS_NODIST -fno-reorder-blocks-and-partition" ++ LDFLAGS_NODIST="$LDFLAGS_NODIST -Wl,--emit-relocs" ++ ++ # These flags are required to get good performance from bolt: ++ CFLAGS_NODIST="$CFLAGS_NODIST -fno-pie" ++ # We want to add these no-pie flags to linking executables but not shared libraries: ++ LINKCC="$LINKCC -fno-pie -no-pie" ++ # Designate the DWARF version into 4 since the LLVM-BOLT does not support DWARF5 yet. ++ CFLAGS="$CFLAGS -gdwarf-4" ++ LDFLAGS="$LDFLAGS -gdwarf-4" ++ AC_SUBST(LLVM_BOLT) ++ AC_PATH_TOOL(LLVM_BOLT, llvm-bolt, '', ${llvm_path}) ++ if test -n "${LLVM_BOLT}" -a -x "${LLVM_BOLT}" ++ then ++ AC_MSG_RESULT("Found llvm-bolt") ++ else ++ AC_MSG_ERROR([llvm-bolt is required for a --enable-bolt build but could not be found.]) ++ fi ++ ++ AC_SUBST(MERGE_FDATA) ++ AC_PATH_TOOL(MERGE_FDATA, merge-fdata, '', ${llvm_path}) ++ if test -n "${MERGE_FDATA}" -a -x "${MERGE_FDATA}" ++ then ++ AC_MSG_RESULT("Found merge-fdata") ++ else ++ AC_MSG_ERROR([merge-fdata is required for a --enable-bolt build but could not be found.]) ++ fi ++fi ++ + # Enable PGO flags. + AC_SUBST(PGO_PROF_GEN_FLAG) + AC_SUBST(PGO_PROF_USE_FLAG) diff --git a/patches/cpython_makefile.patch b/patches/cpython_makefile.patch new file mode 100644 index 0000000000..41c7b6be4f --- /dev/null +++ b/patches/cpython_makefile.patch @@ -0,0 +1,19 @@ +--- Makefile.pre.in 2022-11-09 19:45:00.000000000 +0100 ++++ Makefile.pre.in.bolt 2022-11-16 12:06:59.565683052 +0100 +@@ -659,6 +659,16 @@ + -rm -f profile-clean-stamp + $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS_NODIST) $(PGO_PROF_USE_FLAG)" LDFLAGS_NODIST="$(LDFLAGS_NODIST)" + ++bolt-opt: @PREBOLT_RULE@ ++ rm -f *.fdata ++ @LLVM_BOLT@ ./$(BUILDPYTHON) -instrument -instrumentation-file-append-pid -instrumentation-file=$(abspath $(BUILDPYTHON).bolt) -o $(BUILDPYTHON).bolt_inst ++ ./$(BUILDPYTHON).bolt_inst $(PROFILE_TASK) || true ++ @MERGE_FDATA@ $(BUILDPYTHON).*.fdata > $(BUILDPYTHON).fdata ++ @LLVM_BOLT@ ./$(BUILDPYTHON) -o $(BUILDPYTHON).bolt -data=$(BUILDPYTHON).fdata -update-debug-sections -reorder-blocks=ext-tsp -reorder-functions=hfsort+ -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=all -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot ++ rm -f *.fdata ++ rm -f $(BUILDPYTHON).bolt_inst ++ mv $(BUILDPYTHON).bolt $(BUILDPYTHON) ++ + # Compile and run with gcov + .PHONY=coverage coverage-lcov coverage-report + coverage: diff --git a/patches/cpython_rules.patch b/patches/cpython_rules.patch new file mode 100644 index 0000000000..c757b62995 --- /dev/null +++ b/patches/cpython_rules.patch @@ -0,0 +1,12 @@ +--- rules 2022-11-16 12:34:58.128078268 +0100 ++++ rules.bolt 2022-11-16 12:37:40.129579671 +0100 +@@ -394,7 +394,8 @@ + CPPFLAGS="$(DPKG_CPPFLAGS)" LDFLAGS="$(DPKG_LDFLAGS) $(LTO_CFLAGS)" \ + $(config_site) \ + ../configure \ +- $(common_configure_args) ++ $(common_configure_args) \ ++ --with-llvm=/usr/lib/llvm-16/bin --enable-bolt + + $(call __post_configure,$(buildd_static)) + touch $@