Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore(pack): Support to pack server binaries separately #2034

Merged
merged 1 commit into from
May 31, 2024
Merged

chore(pack): Support to pack server binaries separately #2034

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/actions/build_pegasus/action.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,12 @@ runs:
shell: bash
- name: Pack Server
run: |
./run.sh pack_server -j
./run.sh pack_server -j ${PACK_OPTIONS}
rm -rf pegasus-server-*
shell: bash
- name: Pack Tools
run: |
./run.sh pack_tools -j
./run.sh pack_tools -j ${PACK_OPTIONS}
rm -rf pegasus-tools-*
shell: bash
- name: Clear Build Files
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/lint_and_test_cpp.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,8 @@ jobs:
runs-on: ubuntu-latest
env:
USE_JEMALLOC: OFF
BUILD_OPTIONS: -t debug --test
BUILD_OPTIONS: -t debug --test --separate_servers
PACK_OPTIONS: --separate_servers
container:
image: apache/pegasus:thirdparties-bin-centos7-${{ github.base_ref }}
steps:
Expand Down
6 changes: 0 additions & 6 deletions scripts/pack_client.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,6 @@ then
exit 1
fi

if [ ! -f ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server ]
then
echo "ERROR: ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server not found"
exit 1
fi

if [ ! -f ${BUILD_LATEST_DIR}/CMakeCache.txt ]
then
echo "ERROR: ${BUILD_LATEST_DIR}/CMakeCache.txt not found"
Expand Down
6 changes: 5 additions & 1 deletion scripts/pack_common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,11 @@ set -e

function get_stdcpp_lib()
{
libname=`ldd ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server 2>/dev/null | grep libstdc++`
if [[ $2 == "false" ]]; then
libname=`ldd ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server 2>/dev/null | grep libstdc++`
else
libname=`ldd ${BUILD_LATEST_DIR}/output/bin/pegasus_meta_server/pegasus_meta_server 2>/dev/null | grep libstdc++`
acelyc111 marked this conversation as resolved.
Show resolved Hide resolved
fi
libname=`echo $libname | cut -f1 -d" "`
if [ $1 = "true" ]; then
gcc_path=`which gcc`
Expand Down
35 changes: 22 additions & 13 deletions scripts/pack_server.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ function usage() {
echo " -g|--custom-gcc"
echo " -k|--keytab-file"
echo " -j|--use-jemalloc"
echo " -s|--separate_servers"
acelyc111 marked this conversation as resolved.
Show resolved Hide resolved
exit 0
}

Expand All @@ -39,11 +40,6 @@ if [ ! -f src/include/pegasus/git_commit.h ]; then
exit 1
fi

if [ ! -f ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server ]; then
echo "ERROR: ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server not found"
exit 1
fi

if [ ! -f ${BUILD_LATEST_DIR}/CMakeCache.txt ]; then
echo "ERROR: ${BUILD_LATEST_DIR}/CMakeCache.txt not found"
exit 1
Expand Down Expand Up @@ -77,7 +73,8 @@ fi

custom_gcc="false"
keytab_file=""
use_jemalloc="off"
use_jemalloc="false"
separate_servers="false"
acelyc111 marked this conversation as resolved.
Show resolved Hide resolved

while [[ $# > 0 ]]; do
option_key="$1"
Expand All @@ -97,7 +94,10 @@ while [[ $# > 0 ]]; do
shift
;;
-j | --use-jemalloc)
use_jemalloc="on"
use_jemalloc="true"
;;
-s | --separate_servers)
separate_servers="true"
;;
*)
echo "ERROR: unknown option \"$option_key\""
Expand All @@ -110,12 +110,17 @@ while [[ $# > 0 ]]; do
done

mkdir -p ${pack}/bin
copy_file ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server ${pack}/bin
if [[ $separate_servers == "false" ]]; then
copy_file ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server ${pack}/bin
else
copy_file ${BUILD_LATEST_DIR}/output/bin/pegasus_meta_server/pegasus_meta_server ${pack}/bin
copy_file ${BUILD_LATEST_DIR}/output/bin/pegasus_replica_server/pegasus_replica_server ${pack}/bin
fi
copy_file ${BUILD_LATEST_DIR}/output/lib/libdsn_meta_server.so ${pack}/bin
copy_file ${BUILD_LATEST_DIR}/output/lib/libdsn_replica_server.so ${pack}/bin
copy_file ${BUILD_LATEST_DIR}/output/lib/libdsn_utils.so ${pack}/bin

if [ "$use_jemalloc" == "on" ]; then
if [ "$use_jemalloc" == "true" ]; then
copy_file ${THIRDPARTY_ROOT}/output/lib/libjemalloc.so.2 ${pack}/bin
copy_file ${THIRDPARTY_ROOT}/output/lib/libprofiler.so.0 ${pack}/bin
else
Expand All @@ -130,14 +135,18 @@ copy_file ./src/server/config.ini ${pack}/bin
copy_file ./src/server/config.min.ini ${pack}/bin
copy_file ./scripts/config_hdfs.sh ${pack}/bin

copy_file "$(get_stdcpp_lib $custom_gcc)" "${pack}/bin"
copy_file "$(get_stdcpp_lib $custom_gcc $separate_servers)" "${pack}/bin"

pack_server_lib() {
pack_system_lib "${pack}/bin" server "$1"
if [[ $2 == "false" ]]; then
pack_system_lib "${pack}/bin" server "$1"
else
pack_system_lib "${pack}/bin" meta_server "$1"
fi
}

pack_server_lib crypto
pack_server_lib ssl
pack_server_lib crypto $separate_servers
pack_server_lib ssl $separate_servers

# Pack hadoop-related files.
# If you want to use hdfs service to backup/restore/bulkload pegasus tables,
Expand Down
24 changes: 17 additions & 7 deletions scripts/pack_tools.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ function usage()
echo " -p|--update-package-template <minos-package-template-file-path>"
echo " -g|--custom-gcc"
echo " -j|--use-jemalloc"
echo " -s|--separate_servers"
exit 0
}

Expand Down Expand Up @@ -82,7 +83,8 @@ if [ -n "$MINOS_CONFIG_FILE" ]; then
fi

custom_gcc="false"
use_jemalloc="off"
use_jemalloc="false"
separate_servers="false"

while [[ $# > 0 ]]; do
option_key="$1"
Expand All @@ -98,7 +100,10 @@ while [[ $# > 0 ]]; do
usage
;;
-j|--use-jemalloc)
use_jemalloc="on"
use_jemalloc="true"
;;
-s | --separate_servers)
separate_servers="true"
;;
*)
echo "ERROR: unknown option \"$option_key\""
Expand All @@ -114,7 +119,12 @@ mkdir -p ${pack}
copy_file ./run.sh ${pack}/

mkdir -p ${pack}/bin
cp -v -r ${BUILD_LATEST_DIR}/output/bin/pegasus_server ${pack}/bin/
if [[ $separate_servers == "false" ]]; then
copy_file ${BUILD_LATEST_DIR}/output/bin/pegasus_server/pegasus_server ${pack}/bin
else
copy_file ${BUILD_LATEST_DIR}/output/bin/pegasus_meta_server/pegasus_meta_server ${pack}/bin
copy_file ${BUILD_LATEST_DIR}/output/bin/pegasus_replica_server/pegasus_replica_server ${pack}/bin
fi
cp -v -r ${BUILD_LATEST_DIR}/output/bin/pegasus_shell ${pack}/bin/
cp -v -r ${BUILD_LATEST_DIR}/output/bin/pegasus_bench ${pack}/bin/
cp -v -r ${BUILD_LATEST_DIR}/output/bin/pegasus_kill_test ${pack}/bin/
Expand All @@ -124,7 +134,7 @@ cp -v -r ${BUILD_LATEST_DIR}/output/bin/pegasus_pressureclient ${pack}/bin/
mkdir -p ${pack}/lib
copy_file ${BUILD_LATEST_DIR}/output/lib/*.so* ${pack}/lib/

if [ "$use_jemalloc" == "on" ]; then
if [ "$use_jemalloc" == "true" ]; then
copy_file ${THIRDPARTY_ROOT}/output/lib/libjemalloc.so.2 ${pack}/lib/
copy_file ${THIRDPARTY_ROOT}/output/lib/libprofiler.so.0 ${pack}/lib/
else
Expand All @@ -134,14 +144,14 @@ fi
copy_file ${THIRDPARTY_ROOT}/output/lib/libboost*.so.1.69.0 ${pack}/lib/
copy_file ${THIRDPARTY_ROOT}/output/lib/libhdfs* ${pack}/lib/
copy_file ${THIRDPARTY_ROOT}/output/lib/librocksdb.so.8 ${pack}/lib/
copy_file `get_stdcpp_lib $custom_gcc` ${pack}/lib/
copy_file `get_stdcpp_lib $custom_gcc $separate_servers` ${pack}/lib/

pack_tools_lib() {
pack_system_lib "${pack}/lib" shell "$1"
}

pack_tools_lib crypto
pack_tools_lib ssl
pack_tools_lib crypto $separate_servers
pack_tools_lib ssl $separate_servers

chmod -x ${pack}/lib/*

Expand Down
Loading