Skip to content

Commit

Permalink
[DEV-5385] Build CPython using BOLT
Browse files Browse the repository at this point in the history
Using the patch adapted from python/cpython#95908
  • Loading branch information
vmarkovtsev committed Nov 16, 2022
1 parent be3db6c commit 25c9972
Show file tree
Hide file tree
Showing 2 changed files with 90 additions and 3 deletions.
13 changes: 10 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ RUN apt-get update && \

# matches our production except -march=haswell, we have to downgrade -march because of GHA
ENV OPT="-pipe -fno-semantic-interposition -march=haswell -mabm -maes -mno-pku -mno-sgx --param l1-cache-line-size=64 --param l1-cache-size=32 --param l2-cache-size=33792"
# Bolt: -Wl,--emit-relocs -fno-reorder-blocks-and-partition
ADD patches/cpython_configure_ac.patch /

# runtime environment
RUN echo 'deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted' >>/etc/apt/sources.list && \
Expand All @@ -61,18 +61,25 @@ RUN echo 'deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted' >>/et
python3-distutils html2text libjs-sphinxdoc && \
echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
locale-gen && \
echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" >>/etc/apt/sources.list.d/llvm.list && \
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
apt-get update && \
apt-get install -y bolt-15 && \
add-apt-repository -s ppa:deadsnakes/ppa && \
mkdir /cpython && \
cd /cpython && \
apt-get source python$PYTHON_VERSION && \
apt-get -s build-dep python$PYTHON_VERSION | grep "Inst " | cut -d" " -f2 | sort | tr '\n' ' ' >build_bloat && \
DEBIAN_FRONTEND="noninteractive" TZ="Europe/Madrid" apt-get build-dep -y python$PYTHON_VERSION && \
rm /etc/apt/sources.list.d/deadsnakes* && \
rm /etc/apt/sources.list.d/deadsnakes* /etc/apt/sources.list.d/llvm.list && \
cd python$PYTHON_VERSION* && \
sed -i 's/__main__/__skip__/g' Tools/scripts/run_tests.py && \
dch --bin-nmu -Dunstable "Optimized build" && \
echo 11 >debian/compat && \
sed -i 's/debhelper (>= 9)/debhelper (>= 11)/g' debian/control.in && \
patch configure.ac </cpython_configure_ac.patch && \
rm /cpython_configure_ac.patch && \
sed -i 's\with-computed-gotos\with-computed-gotos --with-llvm=/usr/lib/llvm-15/bin --enable-bolt\g' debian/rules && \
DEB_CFLAGS_SET="$OPT" DEB_LDFLAGS_SET="$OPT" dpkg-buildpackage -uc -b -j$(getconf _NPROCESSORS_ONLN) && \
cd .. && \
apt-get source python3 && \
Expand Down Expand Up @@ -101,7 +108,7 @@ RUN echo 'deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted' >>/et
python$PYTHON_VERSION-venv* \
python$PYTHON_VERSION-full* && \
echo "========" && ls && \
apt-get purge -y dpkg-dev devscripts software-properties-common html2text $(cat build_bloat) && \
apt-get purge -y dpkg-dev devscripts software-properties-common html2text bolt-15 $(cat build_bloat) && \
apt-get autoremove -y && \
dpkg -i *python3.11*.deb && \
dpkg -i python3-minimal*.deb libpython3-stdlib*.deb && \
Expand Down
80 changes: 80 additions & 0 deletions patches/cpython_configure_ac.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
--- configure.ac 2022-11-16 11:37:19.773231374 +0100
+++ configure.ac.bolt 2022-11-16 11:37:11.161182410 +0100
@@ -1751,7 +1751,16 @@
# non-suffixed name in their versioned llvm directory.

llvm_bin_dir=''
-llvm_path="${PATH}"
+
+AC_ARG_WITH(llvm,
+ AS_HELP_STRING([--with-llvm=PATH],
+ [specify the directory where LLVM binaries are installed]),
+[
+ llvm_path=${withval}
+],[
+ llvm_path="${PATH}"
+])
+
if test "${CC}" = "clang"
then
clang_bin=`which clang`
@@ -1889,6 +1898,59 @@
LDFLAGS_NODIST="$LDFLAGS_NODIST $LTOFLAGS"
fi

+# Enable bolt flags
+Py_BOLT='false'
+AC_MSG_CHECKING(for --enable-bolt)
+AC_ARG_ENABLE(bolt, AS_HELP_STRING(
+ [--enable-bolt],
+ [enable usage of the llvm-bolt post-link optimizer (default is no)]),
+[
+if test "$enableval" != no
+then
+ Py_BOLT='true'
+ AC_MSG_RESULT(yes);
+else
+ Py_BOLT='false'
+ AC_MSG_RESULT(no);
+fi],
+[AC_MSG_RESULT(no)])
+
+AC_SUBST(PREBOLT_RULE)
+if test "$Py_BOLT" = 'true' ; then
+ PREBOLT_RULE="${DEF_MAKE_ALL_RULE}"
+ DEF_MAKE_ALL_RULE="bolt-opt"
+ DEF_MAKE_RULE="build_all"
+
+ # These flags are required for bolt to work:
+ CFLAGS_NODIST="$CFLAGS_NODIST -fno-reorder-blocks-and-partition"
+ LDFLAGS_NODIST="$LDFLAGS_NODIST -Wl,--emit-relocs"
+
+ # These flags are required to get good performance from bolt:
+ CFLAGS_NODIST="$CFLAGS_NODIST -fno-pie"
+ # We want to add these no-pie flags to linking executables but not shared libraries:
+ LINKCC="$LINKCC -fno-pie -no-pie"
+ # Designate the DWARF version into 4 since the LLVM-BOLT does not support DWARF5 yet.
+ CFLAGS="$CFLAGS -gdwarf-4"
+ LDFLAGS="$LDFLAGS -gdwarf-4"
+ AC_SUBST(LLVM_BOLT)
+ AC_PATH_TOOL(LLVM_BOLT, llvm-bolt, '', ${llvm_path})
+ if test -n "${LLVM_BOLT}" -a -x "${LLVM_BOLT}"
+ then
+ AC_MSG_RESULT("Found llvm-bolt")
+ else
+ AC_MSG_ERROR([llvm-bolt is required for a --enable-bolt build but could not be found.])
+ fi
+
+ AC_SUBST(MERGE_FDATA)
+ AC_PATH_TOOL(MERGE_FDATA, merge-fdata, '', ${llvm_path})
+ if test -n "${MERGE_FDATA}" -a -x "${MERGE_FDATA}"
+ then
+ AC_MSG_RESULT("Found merge-fdata")
+ else
+ AC_MSG_ERROR([merge-fdata is required for a --enable-bolt build but could not be found.])
+ fi
+fi
+
# Enable PGO flags.
AC_SUBST(PGO_PROF_GEN_FLAG)
AC_SUBST(PGO_PROF_USE_FLAG)

0 comments on commit 25c9972

Please sign in to comment.