diff --git a/.cicd/build.sh b/.cicd/build.sh index b15f6f2500e..2543a6256aa 100755 --- a/.cicd/build.sh +++ b/.cicd/build.sh @@ -1,40 +1,47 @@ #!/bin/bash set -eo pipefail +[[ "$ENABLE_INSTALL" == 'true' ]] || echo '--- :evergreen_tree: Configuring Environment' . ./.cicd/helpers/general.sh mkdir -p "$BUILD_DIR" -CMAKE_EXTRAS="-DCMAKE_BUILD_TYPE='Release' -DENABLE_MULTIVERSION_PROTOCOL_TEST=true -DBUILD_MONGO_DB_PLUGIN=true" +CMAKE_EXTRAS="-DCMAKE_BUILD_TYPE=\"Release\" -DENABLE_MULTIVERSION_PROTOCOL_TEST=\"true\" -DBUILD_MONGO_DB_PLUGIN=\"true\"" if [[ "$(uname)" == 'Darwin' && "$FORCE_LINUX" != 'true' ]]; then # You can't use chained commands in execute if [[ "$GITHUB_ACTIONS" == 'true' ]]; then - export PINNED=false + export PINNED='false' fi - [[ ! "$PINNED" == 'false' ]] && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=$HELPERS_DIR/clang.make" + [[ ! "$PINNED" == 'false' ]] && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=\"$HELPERS_DIR/clang.make\"" cd "$BUILD_DIR" + echo '+++ :hammer_and_wrench: Building EOSIO' CMAKE_COMMAND="cmake $CMAKE_EXTRAS .." echo "$ $CMAKE_COMMAND" eval $CMAKE_COMMAND MAKE_COMMAND="make -j '$JOBS'" echo "$ $MAKE_COMMAND" eval $MAKE_COMMAND + cd .. else # Linux - ARGS=${ARGS:-"--rm --init -v $(pwd):$MOUNTED_DIR"} - PRE_COMMANDS="cd '$MOUNTED_DIR/build'" + ARGS=${ARGS:-"--rm --init -v \"\$(pwd):$MOUNTED_DIR\""} + PRE_COMMANDS="cd \"$MOUNTED_DIR/build\"" # PRE_COMMANDS: Executed pre-cmake # CMAKE_EXTRAS: Executed within and right before the cmake path (cmake CMAKE_EXTRAS ..) - [[ ! "$IMAGE_TAG" =~ 'unpinned' ]] && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE='$MOUNTED_DIR/.cicd/helpers/clang.make'" + [[ ! "$IMAGE_TAG" =~ 'unpinned' ]] && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=\"$MOUNTED_DIR/.cicd/helpers/clang.make\"" if [[ "$IMAGE_TAG" == 'amazon_linux-2-unpinned' ]]; then - CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang'" + CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER=\"clang++\" -DCMAKE_C_COMPILER=\"clang\"" elif [[ "$IMAGE_TAG" == 'centos-7.7-unpinned' ]]; then - PRE_COMMANDS="$PRE_COMMANDS && source /opt/rh/devtoolset-8/enable && source /opt/rh/rh-python36/enable" + PRE_COMMANDS="$PRE_COMMANDS && source \"/opt/rh/devtoolset-8/enable\" && source \"/opt/rh/rh-python36/enable\"" elif [[ "$IMAGE_TAG" == 'ubuntu-18.04-unpinned' ]]; then - CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang'" + CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER=\"clang++\" -DCMAKE_C_COMPILER=\"clang\"" fi - BUILD_COMMANDS="cmake $CMAKE_EXTRAS .. && make -j '$JOBS'" + CMAKE_COMMAND="cmake \$CMAKE_EXTRAS .." + MAKE_COMMAND="make -j $JOBS" + BUILD_COMMANDS="echo \"+++ :hammer_and_wrench: Building EOSIO\" && echo \"$ $CMAKE_COMMAND\" && eval $CMAKE_COMMAND && echo \"$ $MAKE_COMMAND\" && eval $MAKE_COMMAND" # Docker Commands if [[ "$BUILDKITE" == 'true' ]]; then # Generate Base Images - "$CICD_DIR/generate-base-images.sh" - [[ "$ENABLE_INSTALL" == 'true' ]] && COMMANDS="cp -r $MOUNTED_DIR /root/eosio && cd /root/eosio/build &&" + BASE_IMAGE_COMMAND="\"$CICD_DIR/generate-base-images.sh\"" + echo "$ $BASE_IMAGE_COMMAND" + eval $BASE_IMAGE_COMMAND + [[ "$ENABLE_INSTALL" == 'true' ]] && COMMANDS="cp -r \"$MOUNTED_DIR\" \"/root/eosio\" && cd \"/root/eosio/build\" &&" COMMANDS="$COMMANDS $BUILD_COMMANDS" [[ "$ENABLE_INSTALL" == 'true' ]] && COMMANDS="$COMMANDS && make install" elif [[ "$GITHUB_ACTIONS" == 'true' ]]; then @@ -43,7 +50,16 @@ else # Linux fi . "$HELPERS_DIR/file-hash.sh" "$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile" COMMANDS="$PRE_COMMANDS && $COMMANDS" - DOCKER_RUN="docker run $ARGS $(buildkite-intrinsics) '$FULL_TAG' bash -c '$COMMANDS'" + DOCKER_RUN="docker run $ARGS $(buildkite-intrinsics) --env CMAKE_EXTRAS='$CMAKE_EXTRAS' '$FULL_TAG' bash -c '$COMMANDS'" echo "$ $DOCKER_RUN" eval $DOCKER_RUN fi +if [[ "$BUILDKITE" == 'true' && "$ENABLE_INSTALL" != 'true' ]]; then + echo '--- :arrow_up: Uploading Artifacts' + echo 'Compressing build directory.' + tar -pczf 'build.tar.gz' build + echo 'Uploading build directory.' + buildkite-agent artifact upload 'build.tar.gz' + echo 'Done uploading artifacts.' +fi +[[ "$ENABLE_INSTALL" == 'true' ]] || echo '--- :white_check_mark: Done!' diff --git a/.cicd/docker-tag.sh b/.cicd/docker-tag.sh index 01542a18724..2e922d1a9a5 100755 --- a/.cicd/docker-tag.sh +++ b/.cicd/docker-tag.sh @@ -1,6 +1,6 @@ #!/bin/bash set -eo pipefail -echo '+++ :evergreen_tree: Configuring Environment' +echo '--- :evergreen_tree: Configuring Environment' . ./.cicd/helpers/general.sh PREFIX='base-ubuntu-18.04' SANITIZED_BRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's.^/..' | sed 's/[:/]/_/g') @@ -52,7 +52,7 @@ for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do fi done # cleanup -echo '+++ :put_litter_in_its_place: Cleaning Up' +echo '--- :put_litter_in_its_place: Cleaning Up' for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do if [[ ! -z "$REGISTRY" ]]; then echo "Cleaning up from $REGISTRY." diff --git a/.cicd/generate-base-images.sh b/.cicd/generate-base-images.sh index 1661a2a79db..e2c1f0337d8 100755 --- a/.cicd/generate-base-images.sh +++ b/.cicd/generate-base-images.sh @@ -3,7 +3,8 @@ set -eo pipefail . ./.cicd/helpers/general.sh . "$HELPERS_DIR/file-hash.sh" "$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile" # search for base image in docker registries -echo "+++ :mag_right: Looking for $HASHED_IMAGE_TAG" +echo '--- :docker: Build or Pull Base Image :minidisc:' +echo "Looking for '$HASHED_IMAGE_TAG' container in our registries." EXISTS_ALL='true' EXISTS_DOCKER_HUB='false' EXISTS_ECR='false' diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index c6f29b42ede..f31daa44152 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -97,9 +97,7 @@ echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then cat </dev/null | wc -l`)) != 0 ]] && tar czf core.tar.gz core.* || : # collect core dumps echo 'Exporting xUnit XML' @@ -36,7 +39,8 @@ if [[ "$BUILDKITE" == 'true' ]]; then echo 'Done uploading artifacts.' fi # re-throw -if [[ "$EXIT_STATUS" != 0 ]]; then +if [[ "$EXIT_STATUS" != '0' ]]; then echo "Failing due to non-zero exit status from ctest: $EXIT_STATUS" exit $EXIT_STATUS fi +echo '--- :white_check_mark: Done!' diff --git a/scripts/long-running-test.sh b/scripts/long-running-test.sh index f10de222686..a3ab7bad43b 100755 --- a/scripts/long-running-test.sh +++ b/scripts/long-running-test.sh @@ -1,16 +1,16 @@ #!/bin/bash set -eo pipefail # variables -echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':evergreen_tree: ')Configuring Environment" +echo "--- $([[ "$BUILDKITE" == 'true' ]] && echo ':evergreen_tree: ')Configuring Environment" GIT_ROOT="$(dirname $BASH_SOURCE[0])/.." -[[ -z "$TEST" ]] && export TEST=$1 +[[ -z "$TEST" ]] && export TEST="$1" if [[ "$(uname)" == 'Linux' ]]; then . /etc/os-release if [[ "$ID" == 'centos' ]]; then [[ -f /opt/rh/rh-python36/enable ]] && source /opt/rh/rh-python36/enable fi fi -cd $GIT_ROOT/build +cd "$GIT_ROOT/build" # mongoDB if [[ ! -z "$(pgrep mongod)" ]]; then echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':leaves: ')Killing old MongoDB" @@ -26,13 +26,14 @@ fi if [[ -z "$TEST" ]]; then # run all serial tests # count tests echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':microscope: ')Running Long-Running Tests" - TEST_COUNT=$(ctest -N -L long_running_tests | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}') - if [[ $TEST_COUNT > 0 ]]; then + TEST_COUNT=$(ctest -N -L 'long_running_tests' | grep -i 'Total Tests: ' | cut -d ':' -f '2' | awk '{print $1}') + if [[ "$TEST_COUNT" > '0' ]]; then echo "$TEST_COUNT tests found." # run tests set +e # defer ctest error handling to end - echo '$ ctest -L long_running_tests --output-on-failure -T Test' - ctest -L long_running_tests --output-on-failure -T Test + CTEST_COMMAND="ctest -L 'long_running_tests' --output-on-failure -T 'Test'" + echo "$ $CTEST_COMMAND" + eval $CTEST_COMMAND EXIT_STATUS=$? echo 'Done running long-running tests.' else @@ -42,13 +43,14 @@ if [[ -z "$TEST" ]]; then # run all serial tests else # run specific serial test # ensure test exists echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':microscope: ')Running $TEST" - TEST_COUNT=$(ctest -N -R ^$TEST$ | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}') - if [[ $TEST_COUNT > 0 ]]; then + TEST_COUNT=$(ctest -N -R "^$TEST$" | grep -i 'Total Tests: ' | cut -d ':' -f '2' | awk '{print $1}') + if [[ "$TEST_COUNT" > '0' ]]; then echo "$TEST found." # run tests set +e # defer ctest error handling to end - echo "$ ctest -R ^$TEST$ --output-on-failure -T Test" - ctest -R ^$TEST$ --output-on-failure -T Test + CTEST_COMMAND="ctest -R '^$TEST$' --output-on-failure -T 'Test'" + echo "$ $CTEST_COMMAND" + eval $CTEST_COMMAND EXIT_STATUS=$? echo "Done running $TEST." else @@ -60,4 +62,4 @@ if [[ ! -z "$(pgrep mongod)" ]]; then echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':leaves: ')Killing MongoDB" $(pgrep mongod | xargs kill -9) || : fi -exit $EXIT_STATUS \ No newline at end of file +exit $EXIT_STATUS diff --git a/scripts/parallel-test.sh b/scripts/parallel-test.sh index bd3f9150ab9..91106c36281 100755 --- a/scripts/parallel-test.sh +++ b/scripts/parallel-test.sh @@ -1,8 +1,8 @@ #!/bin/bash set -eo pipefail # variables -echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':evergreen_tree: ')Configuring Environment" -[[ -z "$JOBS" ]] && export JOBS=$(getconf _NPROCESSORS_ONLN) +echo "--- $([[ "$BUILDKITE" == 'true' ]] && echo ':evergreen_tree: ')Configuring Environment" +[[ -z "$JOBS" ]] && export JOBS="$(getconf _NPROCESSORS_ONLN)" GIT_ROOT="$(dirname $BASH_SOURCE[0])/.." if [[ "$(uname)" == 'Linux' ]]; then . /etc/os-release @@ -10,11 +10,11 @@ if [[ "$(uname)" == 'Linux' ]]; then [[ -f /opt/rh/rh-python36/enable ]] && source /opt/rh/rh-python36/enable fi fi -cd $GIT_ROOT/build +cd "$GIT_ROOT/build" # count tests echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':microscope: ')Running Parallelizable Tests" -TEST_COUNT=$(ctest -N -LE _tests | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}') -if [[ $TEST_COUNT > 0 ]]; then +TEST_COUNT=$(ctest -N -LE '_tests' | grep -i 'Total Tests: ' | cut -d ':' -f '2' | awk '{print $1}') +if [[ "$TEST_COUNT" > '0' ]]; then echo "$TEST_COUNT tests found." else echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':no_entry: ')ERROR: No tests registered with ctest! Exiting..." @@ -22,8 +22,9 @@ else fi # run tests set +e # defer ctest error handling to end -echo "$ ctest -j $JOBS -LE _tests --output-on-failure -T Test" -ctest -j $JOBS -LE _tests --output-on-failure -T Test +CTEST_COMMAND="ctest -j '$JOBS' -LE '_tests' --output-on-failure -T 'Test'" +echo "$ $CTEST_COMMAND" +eval $CTEST_COMMAND EXIT_STATUS=$? echo 'Done running parallelizable tests.' -exit $EXIT_STATUS \ No newline at end of file +exit $EXIT_STATUS diff --git a/scripts/serial-test.sh b/scripts/serial-test.sh index f5b7a24e812..8c46d2a7506 100755 --- a/scripts/serial-test.sh +++ b/scripts/serial-test.sh @@ -1,16 +1,16 @@ #!/bin/bash set -eo pipefail # variables -echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':evergreen_tree: ')Configuring Environment" +echo "--- $([[ "$BUILDKITE" == 'true' ]] && echo ':evergreen_tree: ')Configuring Environment" GIT_ROOT="$(dirname $BASH_SOURCE[0])/.." -[[ -z "$TEST" ]] && export TEST=$1 +[[ -z "$TEST" ]] && export TEST="$1" if [[ "$(uname)" == 'Linux' ]]; then . /etc/os-release if [[ "$ID" == 'centos' ]]; then [[ -f /opt/rh/rh-python36/enable ]] && source /opt/rh/rh-python36/enable fi fi -cd $GIT_ROOT/build +cd "$GIT_ROOT/build" # mongoDB if [[ ! -z "$(pgrep mongod)" ]]; then echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':leaves: ')Killing old MongoDB" @@ -26,13 +26,14 @@ fi if [[ -z "$TEST" ]]; then # run all serial tests # count tests echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':microscope: ')Running Non-Parallelizable Tests" - TEST_COUNT=$(ctest -N -L nonparallelizable_tests | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}') - if [[ $TEST_COUNT > 0 ]]; then + TEST_COUNT=$(ctest -N -L 'nonparallelizable_tests' | grep -i 'Total Tests: ' | cut -d ':' -f '2' | awk '{print $1}') + if [[ "$TEST_COUNT" > '0' ]]; then echo "$TEST_COUNT tests found." # run tests set +e # defer ctest error handling to end - echo '$ ctest -L nonparallelizable_tests --output-on-failure -T Test' - ctest -L nonparallelizable_tests --output-on-failure -T Test + CTEST_COMMAND="ctest -L 'nonparallelizable_tests' --output-on-failure -T 'Test'" + echo "$ $CTEST_COMMAND" + eval $CTEST_COMMAND EXIT_STATUS=$? echo 'Done running non-parallelizable tests.' else @@ -43,12 +44,13 @@ else # run specific serial test # ensure test exists echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':microscope: ')Running $TEST" TEST_COUNT=$(ctest -N -R ^$TEST$ | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}') - if [[ $TEST_COUNT > 0 ]]; then + if [[ "$TEST_COUNT" > '0' ]]; then echo "$TEST found." # run tests set +e # defer ctest error handling to end - echo "$ ctest -R ^$TEST$ --output-on-failure -T Test" - ctest -R ^$TEST$ --output-on-failure -T Test + CTEST_COMMAND="ctest -R '^$TEST$' --output-on-failure -T 'Test'" + echo "$ $CTEST_COMMAND" + eval $CTEST_COMMAND EXIT_STATUS=$? echo "Done running $TEST." else @@ -60,4 +62,4 @@ if [[ ! -z "$(pgrep mongod)" ]]; then echo "+++ $([[ "$BUILDKITE" == 'true' ]] && echo ':leaves: ')Killing MongoDB" $(pgrep mongod | xargs kill -9) || : fi -exit $EXIT_STATUS \ No newline at end of file +exit $EXIT_STATUS