feat: support jemalloc

.github/workflows/ci-pull-request.yaml

@@ -62,3 +62,31 @@ jobs:
         run: |
           source ./scripts/config_hdfs.sh
           ./run.sh test --on_travis
+
+  test-with-jemalloc:
+    name: Test With Jemalloc
+    needs: lint
+    runs-on: ubuntu-latest
+    container:
+      image: registry.cn-beijing.aliyuncs.com/apachepegasus/thirdparties-bin:ubuntu1804
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - name: Unpack prebuilt third-parties
+        run: unzip /root/thirdparties-bin.zip -d ./rdsn/thirdparty
+      - name: Compilation
+        run: ./run.sh build -c --skip_thirdparty --use_jemalloc
+      -
+        name: Packaging Server
+        run: ./run.sh pack_server --use-jemalloc
+      -
+        name: Packaging Tools
+        run: ./run.sh pack_tools --use-jemalloc
+      - name: Unit Testing
+        run: |
+          source ./scripts/config_hdfs.sh
+          ./run.sh test --on_travis

run.sh

@@ -90,6 +90,7 @@ function usage_build()
                                    type: address|leak|thread|undefined"
     echo "   --skip_thirdparty     whether to skip building thirdparties, default no"
     echo "   --enable_rocksdb_portable      build a portable rocksdb binary"
+    echo "   --use_jemalloc        build with jemalloc"
 }
 function run_build()
 {
@@ -112,6 +113,7 @@ function run_build()
     SANITIZER=""
     TEST_MODULE=""
     ENABLE_ROCKSDB_PORTABLE=NO
+    USE_JEMALLOC=NO
     while [[ $# > 0 ]]; do
         key="$1"
         case $key in
@@ -175,6 +177,10 @@ function run_build()
             --enable_rocksdb_portable)
                 ENABLE_ROCKSDB_PORTABLE=YES
                 ;;
+            --use_jemalloc)
+                USE_JEMALLOC=YES
+                DISABLE_GPERF=YES
+                ;;
             *)
                 echo "ERROR: unknown option \"$key\""
                 echo
@@ -235,6 +241,9 @@ function run_build()
     if [ "$ENABLE_ROCKSDB_PORTABLE" == "YES" ]; then
         OPT="$OPT --enable_rocksdb_portable"
     fi
+    if [ "$USE_JEMALLOC" == "YES" ]; then
+        OPT="$OPT --use_jemalloc"
+    fi
     ./run.sh build $OPT --notest
     if [ $? -ne 0 ]; then
         echo "ERROR: build rdsn failed"
@@ -246,7 +255,8 @@ function run_build()
     C_COMPILER="$C_COMPILER" CXX_COMPILER="$CXX_COMPILER" BUILD_TYPE="$BUILD_TYPE" \
         CLEAR="$CLEAR" PART_CLEAR="$PART_CLEAR" JOB_NUM="$JOB_NUM" \
         WARNING_ALL="$WARNING_ALL" ENABLE_GCOV="$ENABLE_GCOV" SANITIZER="$SANITIZER"\
-        RUN_VERBOSE="$RUN_VERBOSE" TEST_MODULE="$TEST_MODULE" DISABLE_GPERF="$DISABLE_GPERF" ./build.sh
+        RUN_VERBOSE="$RUN_VERBOSE" TEST_MODULE="$TEST_MODULE" DISABLE_GPERF="$DISABLE_GPERF"\
+        USE_JEMALLOC="$USE_JEMALLOC" ./build.sh
     if [ $? -ne 0 ]; then
         echo "ERROR: build pegasus failed"
         exit 1

scripts/pack_server.sh

@@ -24,6 +24,7 @@ function usage() {
     echo "  -p|--update-package-template <minos-package-template-file-path>"
     echo "  -g|--custom-gcc"
     echo "  -k|--keytab-file"
+    echo "  -j|--use-jemalloc"
     exit 0
 }
 
@@ -74,6 +75,7 @@ fi
 
 custom_gcc="false"
 keytab_file=""
+use_jemalloc="no"
 
 while [[ $# > 0 ]]; do
     option_key="$1"
@@ -92,6 +94,9 @@ while [[ $# > 0 ]]; do
         keytab_file="$2"
         shift
         ;;
+    -j | --use-jemalloc)
+        use_jemalloc="yes"
+        ;;
     esac
     shift
 done
@@ -102,7 +107,14 @@ copy_file ./DSN_ROOT/lib/libdsn_meta_server.so ${pack}/bin
 copy_file ./DSN_ROOT/lib/libdsn_replica_server.so ${pack}/bin
 copy_file ./DSN_ROOT/lib/libdsn_utils.so ${pack}/bin
 copy_file ./rdsn/thirdparty/output/lib/libPoco*.so.* ${pack}/bin
-copy_file ./rdsn/thirdparty/output/lib/libtcmalloc_and_profiler.so.4 ${pack}/bin
+
+if [ "$use_jemalloc" == "yes" ]; then
+    copy_file ./rdsn/thirdparty/output/lib/libjemalloc.so.2 ${pack}/bin
+    copy_file ./rdsn/thirdparty/output/lib/libprofiler.so.0 ${pack}/bin
+else
+    copy_file ./rdsn/thirdparty/output/lib/libtcmalloc_and_profiler.so.4 ${pack}/bin
+fi
+
 copy_file ./rdsn/thirdparty/output/lib/libboost*.so.1.69.0 ${pack}/bin
 copy_file ./rdsn/thirdparty/output/lib/libhdfs* ${pack}/bin
 copy_file ./rdsn/thirdparty/output/lib/libsasl2.so.3 ${pack}/bin

scripts/pack_tools.sh

@@ -24,6 +24,7 @@ function usage()
     echo "  -h"
     echo "  -p|--update-package-template <minos-package-template-file-path>"
     echo "  -g|--custom-gcc"
+    echo "  -j|--use-jemalloc"
     exit 0
 }
 
@@ -79,6 +80,7 @@ if [ -n "$MINOS_CONFIG_FILE" ]; then
 fi
 
 custom_gcc="false"
+use_jemalloc="no"
 
 while [[ $# > 0 ]]; do
     option_key="$1"
@@ -93,6 +95,9 @@ while [[ $# > 0 ]]; do
         -h|--help)
             usage
             ;;
+        -j|--use-jemalloc)
+            use_jemalloc="yes"
+            ;;
     esac
     shift
 done
@@ -111,7 +116,14 @@ cp -v -r ./DSN_ROOT/bin/pegasus_pressureclient ${pack}/DSN_ROOT/bin/
 mkdir -p ${pack}/DSN_ROOT/lib
 copy_file ./DSN_ROOT/lib/*.so* ${pack}/DSN_ROOT/lib/
 copy_file ./rdsn/thirdparty/output/lib/libPoco*.so.* ${pack}/DSN_ROOT/lib/
-copy_file ./rdsn/thirdparty/output/lib/libtcmalloc_and_profiler.so.4 ${pack}/DSN_ROOT/lib/
+
+if [ "$use_jemalloc" == "yes" ]; then
+    copy_file ./rdsn/thirdparty/output/lib/libjemalloc.so.2 ${pack}/DSN_ROOT/lib/
+    copy_file ./rdsn/thirdparty/output/lib/libprofiler.so.0 ${pack}/DSN_ROOT/lib/
+else
+    copy_file ./rdsn/thirdparty/output/lib/libtcmalloc_and_profiler.so.4 ${pack}/DSN_ROOT/lib/
+fi
+
 copy_file ./rdsn/thirdparty/output/lib/libboost*.so.1.69.0 ${pack}/DSN_ROOT/lib/
 copy_file ./rdsn/thirdparty/output/lib/libhdfs* ${pack}/DSN_ROOT/lib
 copy_file ./rdsn/thirdparty/output/lib/libsasl2.so.3 ${pack}/DSN_ROOT/lib/

src/build.sh

@@ -93,6 +93,14 @@ else
     echo "ENABLE_GCOV=NO"
 fi
 
+if [ "$USE_JEMALLOC" == "YES" ]
+then
+    echo "USE_JEMALLOC=YES"
+    CMAKE_OPTIONS="$CMAKE_OPTIONS -DUSE_JEMALLOC=ON"
+else
+    echo "USE_JEMALLOC=NO"
+fi
+
 if [ ! -z "$SANITIZER" ]
 then
     echo "SANITIZER=$SANITIZER"
@@ -111,6 +119,14 @@ else
     echo "DISABLE_GPERF=NO"
 fi
 
+if [ "$USE_JEMALLOC" == "YES" ]
+then
+    echo "USE_JEMALLOC=YES"
+    CMAKE_OPTIONS="$CMAKE_OPTIONS -DUSE_JEMALLOC=ON"
+else
+    echo "USE_JEMALLOC=NO"
+fi
+
 CMAKE_OPTIONS="$CMAKE_OPTIONS -DBoost_NO_BOOST_CMAKE=ON -DBOOST_ROOT=${ROOT}/rdsn/thirdparty/output -DBoost_NO_SYSTEM_PATHS=ON"
 
 echo "#############################################################################"

src/server/CMakeLists.txt

@@ -50,6 +50,12 @@ set(MY_PROJ_LIBS
     PocoJSON
     )
 
+message(STATUS "USE_JEMALLOC = ${USE_JEMALLOC}")
+if(USE_JEMALLOC)
+    message(STATUS "jemalloc is enabled")
+    set(MY_PROJ_LIBS JeMalloc::JeMalloc ${MY_PROJ_LIBS})
+endif()
+
 set(MY_BOOST_LIBS Boost::system Boost::filesystem Boost::regex)
 
 set(MY_BINPLACES "${CMAKE_CURRENT_SOURCE_DIR}/config.ini")