From fa89026f54a994f1ec18a7c428522b2d6ac6255a Mon Sep 17 00:00:00 2001 From: Keke Li Date: Wed, 5 Aug 2020 11:50:01 -0700 Subject: [PATCH 1/7] Add script support for CentOS 8 --- scripts/eosio_build.sh | 3 +-- scripts/eosio_build_centos.sh | 9 +++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/eosio_build.sh b/scripts/eosio_build.sh index 793376f7193..980271b25a9 100755 --- a/scripts/eosio_build.sh +++ b/scripts/eosio_build.sh @@ -118,8 +118,7 @@ echo "$( date -u )" echo "User: ${CURRENT_USER}" # echo "git head id: %s" "$( cat .git/refs/heads/master )" echo "Current branch: $( execute git rev-parse --abbrev-ref HEAD 2>/dev/null )" - -( [[ ! $NAME == "Ubuntu" ]] && [[ ! $ARCH == "Darwin" ]] ) && set -i # Ubuntu doesn't support interactive mode since it uses dash + Some folks are having this issue on Darwin; colors aren't supported yet anyway +( [[ ! $NAME == "Ubuntu" ]] && [[ ! $ARCH == "Darwin" ]] && [[ ! $NAME == "CentOS Linux" ]]) && set -i # Ubuntu doesn't support interactive mode since it uses dash + Some folks are having this issue on Darwin; colors aren't supported yet anyway # Ensure sudo is available (only if not using the root user) ensure-sudo diff --git a/scripts/eosio_build_centos.sh b/scripts/eosio_build_centos.sh index e6720e51241..7abf36163a9 100755 --- a/scripts/eosio_build_centos.sh +++ b/scripts/eosio_build_centos.sh @@ -12,6 +12,7 @@ echo "Disk space available: ${DISK_AVAIL}G" echo "" +if [[ "$(echo ${VERSION} | sed 's/ .*//g')" == 7 ]]; then # Repo necessary for rh-python3, devtoolset-8 and llvm-toolset-7.0 ensure-scl # GCC8 for Centos / Needed for CMAKE install even if we're pinning @@ -23,6 +24,14 @@ if [[ -d /opt/rh/devtoolset-8 ]]; then fi # Ensure packages exist ensure-yum-packages "${REPO_ROOT}/scripts/eosio_build_centos7_deps" +fi + +if [[ "$(echo ${VERSION} | sed 's/ .*//g')" == 8 ]]; then + echo "Install Development Tools ..." + dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm + dnf group install -y "Development Tools" +fi + export PYTHON3PATH="/opt/rh/rh-python36" if $DRYRUN || [ -d $PYTHON3PATH ]; then echo "${COLOR_CYAN}[Enabling python36]${COLOR_NC}" From 1fdf4c1c6403b0d119cd4f5a4d4a26748a639fb3 Mon Sep 17 00:00:00 2001 From: Keke Li Date: Thu, 6 Aug 2020 11:58:02 -0700 Subject: [PATCH 2/7] Add group-install-package to use yum groupinstall And use this new function relpace dnf command --- scripts/eosio_build_centos.sh | 4 ++-- scripts/helpers/general.sh | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/eosio_build_centos.sh b/scripts/eosio_build_centos.sh index 7abf36163a9..5b77b087793 100755 --- a/scripts/eosio_build_centos.sh +++ b/scripts/eosio_build_centos.sh @@ -28,8 +28,8 @@ fi if [[ "$(echo ${VERSION} | sed 's/ .*//g')" == 8 ]]; then echo "Install Development Tools ..." - dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm - dnf group install -y "Development Tools" + install-package https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm + group-install-package 'Development Tools' fi export PYTHON3PATH="/opt/rh/rh-python36" diff --git a/scripts/helpers/general.sh b/scripts/helpers/general.sh index c57908b6f76..02be712d010 100755 --- a/scripts/helpers/general.sh +++ b/scripts/helpers/general.sh @@ -90,6 +90,19 @@ function install-package() { true # Required; Weird behavior without it } +function group-install-package() { + if [[ $ARCH == "Linux" ]]; then + EXECUTION_FUNCTION="execute" + [[ $2 == "WETRUN" ]] && EXECUTION_FUNCTION="execute-always" + ( [[ $2 =~ "--" ]] || [[ $3 =~ "--" ]] ) && OPTIONS="${2}${3}" + # Can't use $SUDO_COMMAND: https://askubuntu.com/questions/953485/where-do-i-find-the-sudo-command-environment-variable + [[ $CURRENT_USER != "root" ]] && [[ ! -z $SUDO_LOCATION ]] && NEW_SUDO_COMMAND="$SUDO_LOCATION -E" + ( [[ $NAME =~ "Amazon Linux" ]] || [[ $NAME == "CentOS Linux" ]] ) && eval $EXECUTION_FUNCTION $NEW_SUDO_COMMAND $YUM $OPTIONS groupinstall -y '$1' + ( [[ $NAME =~ "Ubuntu" ]] ) && eval $EXECUTION_FUNCTION $NEW_SUDO_COMMAND $APTGET $OPTIONS install -y $1 + fi + true # Required; Weird behavior without it +} + function uninstall-package() { if [[ $ARCH == "Linux" ]]; then EXECUTION_FUNCTION="execute" From d3cdce8e708af5da320462577fe6cce6313ac2b9 Mon Sep 17 00:00:00 2001 From: Keke Li Date: Thu, 6 Aug 2020 14:24:50 -0700 Subject: [PATCH 3/7] Add install openssl-devel as Matt said this package is also needed to build eos. --- scripts/eosio_build_centos.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/eosio_build_centos.sh b/scripts/eosio_build_centos.sh index 5b77b087793..9f683f334d3 100755 --- a/scripts/eosio_build_centos.sh +++ b/scripts/eosio_build_centos.sh @@ -30,6 +30,7 @@ if [[ "$(echo ${VERSION} | sed 's/ .*//g')" == 8 ]]; then echo "Install Development Tools ..." install-package https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm group-install-package 'Development Tools' + install-package openssl-devel fi export PYTHON3PATH="/opt/rh/rh-python36" From 36907e855873006660ad24fb2e1fdbc8e1deb11a Mon Sep 17 00:00:00 2001 From: Keke Li Date: Thu, 3 Sep 2020 19:09:34 -0700 Subject: [PATCH 4/7] Add dockerfiles and change build script for pipeline build. --- .cicd/build-scripts.yml | 26 ++++++++- .../pinned/centos-8-pinned.dockerfile | 56 +++++++++++++++++++ .../unpinned/centos-8-unpinned.dockerfile | 39 +++++++++++++ 3 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 .cicd/platforms/pinned/centos-8-pinned.dockerfile create mode 100644 .cicd/platforms/unpinned/centos-8-unpinned.dockerfile diff --git a/.cicd/build-scripts.yml b/.cicd/build-scripts.yml index d75aefc8ad4..581639e5b09 100644 --- a/.cicd/build-scripts.yml +++ b/.cicd/build-scripts.yml @@ -21,7 +21,18 @@ steps: command: - "./scripts/eosio_build.sh -P -y" timeout: 180 - + + - label: ":centos: CentOS 8 - Build Pinned" + plugins: + - docker#v3.3.0: + image: "centos:8" + always-pull: true + agents: + queue: "automation-eks-eos-builder-fleet" + command: + - "./scripts/eosio_build.sh -P -y" + timeout: 180 + - label: ":darwin: macOS 10.14 - Build Pinned" env: REPO: "git@github.com:EOSIO/eos.git" @@ -124,6 +135,17 @@ steps: command: - "./scripts/eosio_build.sh -y" timeout: 180 + + - label: ":centos: CentOS 8 - Build UnPinned" + plugins: + - docker#v3.3.0: + image: "centos:8" + always-pull: true + agents: + queue: "automation-eks-eos-builder-fleet" + command: + - "./scripts/eosio_build.sh -y" + timeout: 180 - label: ":darwin: macOS 10.14 - Build UnPinned" env: @@ -189,4 +211,4 @@ steps: command: - "apt update && apt upgrade -y && apt install -y git" - "./scripts/eosio_build.sh -y" - timeout: 180 \ No newline at end of file + timeout: 180 diff --git a/.cicd/platforms/pinned/centos-8-pinned.dockerfile b/.cicd/platforms/pinned/centos-8-pinned.dockerfile new file mode 100644 index 00000000000..6b497b5e92a --- /dev/null +++ b/.cicd/platforms/pinned/centos-8-pinned.dockerfile @@ -0,0 +1,56 @@ +FROM centos:8 +ENV VERSION 1 +#install dependencies +RUN yum update -y && \ + yum install -y net-tools && \ + yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 && \ + yum --enablerepo=extras install -y graphviz bzip2-devel openssl-devel gmp-devel && \ + yum --enablerepo=extras install -y file libusbx-devel && \ + yum --enablerepo=extras install -y libcurl-devel patch vim-common jq && \ + yum install -y python3 +RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ + dnf group install -y "Development Tools" +# cmake3.18.0 +RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0.tar.gz && \ + tar -xzf cmake-3.18.0.tar.gz && \ + cd cmake-3.18.0 && \ + ./bootstrap --prefix=/usr/local && \ + make -j$(nproc) && make install && \ + rm -rf cmake-3.18.0.tar.gz cmake-3.18.2 +# clang10.0.0 +RUN git clone --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project clang10 && \ + mkdir /clang10/build && cd /clang10/build && \ + cmake -G 'Unix Makefiles' -DCMAKE_INSTALL_PREFIX='/usr/local' -DLLVM_ENABLE_PROJECTS='lld;polly;clang;clang-tools-extra;libcxx;libcxxabi;libunwind;compiler-rt' -DLLVM_BUILD_LLVM_DYLIB=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_DOCS=OFF -DLLVM_TARGETS_TO_BUILD=host -DCMAKE_BUILD_TYPE=Release ../llvm && \ + make -j$(nproc) && make install && \ + cd / && \ + rm -rf /clang10 +COPY ./.cicd/helpers/clang.make /tmp/clang.cmake +#build llvm10 +RUN git clone --depth 1 --single-branch --branch llvmorg-10.0.0 https://github.com/llvm/llvm-project llvm && \ + cd llvm/llvm && \ + mkdir build && \ + cd build && \ + cmake -G 'Unix Makefiles' -DLLVM_TARGETS_TO_BUILD=host -DLLVM_BUILD_TOOLS=false -DLLVM_ENABLE_RTTI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE='/tmp/clang.cmake' -DCMAKE_EXE_LINKER_FLAGS=-pthread -DCMAKE_SHARED_LINKER_FLAGS=-pthread -DLLVM_ENABLE_PIC=NO -DLLVM_ENABLE_TERMINFO=OFF .. && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf /llvm +# build boost +RUN curl -LO https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.bz2 && \ + tar -xjf boost_1_72_0.tar.bz2 && \ + cd boost_1_72_0 && \ + ./bootstrap.sh --with-toolset=clang --prefix=/usr/local && \ + ./b2 toolset=clang cxxflags='-stdlib=libc++ -D__STRICT_ANSI__ -nostdinc++ -I/usr/local/include/c++/v1 -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fpie' linkflags='-stdlib=libc++ -pie' link=static threading=multi --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ + cd / && \ + rm -rf boost_1_72_0.tar.bz2 /boost_1_72_0 +# install nvm +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +# load nvm in non-interactive shells +RUN cp ~/.bashrc ~/.bashrc.bak && \ + cat ~/.bashrc.bak | tail -3 > ~/.bashrc && \ + cat ~/.bashrc.bak | head -n '-3' >> ~/.bashrc && \ + rm ~/.bashrc.bak +# install node 10 +RUN bash -c '. ~/.bashrc; nvm install --lts=dubnium' && \ + ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/node" /usr/local/bin/node +RUN yum install -y nodejs diff --git a/.cicd/platforms/unpinned/centos-8-unpinned.dockerfile b/.cicd/platforms/unpinned/centos-8-unpinned.dockerfile new file mode 100644 index 00000000000..649f87bd0e4 --- /dev/null +++ b/.cicd/platforms/unpinned/centos-8-unpinned.dockerfile @@ -0,0 +1,39 @@ +FROM centos:8 +ENV VERSION 1 +#install dependencies +RUN yum update -y && \ + yum install -y net-tools && \ + yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 && \ + yum --enablerepo=extras install -y graphviz bzip2-devel openssl-devel gmp-devel && \ + yum --enablerepo=extras install -y file libusbx-devel && \ + yum --enablerepo=extras install -y libcurl-devel patch vim-common jq && \ + yum install -y python3 +RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ + dnf group install -y "Development Tools" +# cmake3.18.0 +RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0.tar.gz && \ + tar -xzf cmake-3.18.0.tar.gz && \ + cd cmake-3.18.0 && \ + ./bootstrap --prefix=/usr/local && \ + make -j$(nproc) && make install && \ + rm -rf cmake-3.18.0.tar.gz cmake-3.18.2 +# build boost +RUN curl -LO https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.bz2 && \ + tar -xjf boost_1_72_0.tar.bz2 && \ + cd boost_1_72_0 && \ + ./bootstrap.sh --prefix=/usr/local && \ + ./b2 --with-iostreams --with-date_time --with-filesystem --with-system --with-program_options --with-chrono --with-test -q -j$(nproc) install && \ + cd / && \ + rm -rf boost_1_72_0.tar.bz2 /boost_1_72_0 +# install nvm +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash +# load nvm in non-interactive shells +RUN cp ~/.bashrc ~/.bashrc.bak && \ + cat ~/.bashrc.bak | tail -3 > ~/.bashrc && \ + cat ~/.bashrc.bak | head -n '-3' >> ~/.bashrc && \ + rm ~/.bashrc.bak +# install node 10 +RUN bash -c '. ~/.bashrc; nvm install --lts=dubnium' && \ + ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/node" /usr/local/bin/node +RUN yum install -y nodejs + From 01304c47b1e99bacb877d111eff49d2f6911e86d Mon Sep 17 00:00:00 2001 From: Keke Li Date: Thu, 3 Sep 2020 22:03:36 -0700 Subject: [PATCH 5/7] rh-python36 isn't on Centos8, use it only on Centos7 --- .cicd/build.sh | 4 ++-- scripts/eosio_build_centos.sh | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.cicd/build.sh b/.cicd/build.sh index 6f26fe570c4..714a478e988 100755 --- a/.cicd/build.sh +++ b/.cicd/build.sh @@ -38,7 +38,7 @@ else # Linux elif [[ "$IMAGE_TAG" == 'ubuntu-18.04-unpinned' ]]; then CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER='clang++-7' -DCMAKE_C_COMPILER='clang-7' -DLLVM_DIR='/usr/lib/llvm-7/lib/cmake/llvm'" fi - if [[ "$IMAGE_TAG" == centos* ]]; then + if [[ "$IMAGE_TAG" == centos-7.* ]]; then PRE_COMMANDS="$PRE_COMMANDS && source /opt/rh/rh-python36/enable" fi BUILD_COMMANDS="cmake $CMAKE_EXTRAS .. && make -j$JOBS" @@ -59,4 +59,4 @@ else # Linux COMMANDS="$PRE_COMMANDS && $COMMANDS" echo "$ docker run $ARGS $(buildkite-intrinsics) $FULL_TAG bash -c \"$COMMANDS\"" eval docker run $ARGS $(buildkite-intrinsics) $FULL_TAG bash -c \"$COMMANDS\" -fi \ No newline at end of file +fi diff --git a/scripts/eosio_build_centos.sh b/scripts/eosio_build_centos.sh index 9f683f334d3..94b790157ff 100755 --- a/scripts/eosio_build_centos.sh +++ b/scripts/eosio_build_centos.sh @@ -24,6 +24,13 @@ if [[ -d /opt/rh/devtoolset-8 ]]; then fi # Ensure packages exist ensure-yum-packages "${REPO_ROOT}/scripts/eosio_build_centos7_deps" +export PYTHON3PATH="/opt/rh/rh-python36" +if $DRYRUN || [ -d $PYTHON3PATH ]; then + echo "${COLOR_CYAN}[Enabling python36]${COLOR_NC}" + execute source $PYTHON3PATH/enable + echo " ${COLOR_GREEN}- Python36 successfully enabled!${COLOR_NC}" + echo "" +fi fi if [[ "$(echo ${VERSION} | sed 's/ .*//g')" == 8 ]]; then @@ -33,13 +40,6 @@ if [[ "$(echo ${VERSION} | sed 's/ .*//g')" == 8 ]]; then install-package openssl-devel fi -export PYTHON3PATH="/opt/rh/rh-python36" -if $DRYRUN || [ -d $PYTHON3PATH ]; then - echo "${COLOR_CYAN}[Enabling python36]${COLOR_NC}" - execute source $PYTHON3PATH/enable - echo " ${COLOR_GREEN}- Python36 successfully enabled!${COLOR_NC}" - echo "" -fi # Handle clang/compiler ensure-compiler # CMAKE Installation From d8ed2fb8f7e304f3ed93aeaab8e996b885c28bf6 Mon Sep 17 00:00:00 2001 From: Keke Li Date: Fri, 4 Sep 2020 15:54:02 -0700 Subject: [PATCH 6/7] Chang docker files, pull develop to keep update with develop --- .cicd/platforms/pinned/centos-8-pinned.dockerfile | 5 +++-- .cicd/platforms/unpinned/centos-8-unpinned.dockerfile | 7 ++++--- .gitmodules | 3 --- libraries/fc | 1 - 4 files changed, 7 insertions(+), 9 deletions(-) delete mode 160000 libraries/fc diff --git a/.cicd/platforms/pinned/centos-8-pinned.dockerfile b/.cicd/platforms/pinned/centos-8-pinned.dockerfile index 6b497b5e92a..b8cce7fa34c 100644 --- a/.cicd/platforms/pinned/centos-8-pinned.dockerfile +++ b/.cicd/platforms/pinned/centos-8-pinned.dockerfile @@ -2,14 +2,15 @@ FROM centos:8 ENV VERSION 1 #install dependencies RUN yum update -y && \ - yum install -y net-tools && \ + yum install -y epel-release && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 && \ yum --enablerepo=extras install -y graphviz bzip2-devel openssl-devel gmp-devel && \ yum --enablerepo=extras install -y file libusbx-devel && \ yum --enablerepo=extras install -y libcurl-devel patch vim-common jq && \ yum install -y python3 RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ - dnf group install -y "Development Tools" + dnf group install -y "Development Tools" && \ + dnf --enablerepo=PowerTools install -y doxygen ocaml # cmake3.18.0 RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0.tar.gz && \ tar -xzf cmake-3.18.0.tar.gz && \ diff --git a/.cicd/platforms/unpinned/centos-8-unpinned.dockerfile b/.cicd/platforms/unpinned/centos-8-unpinned.dockerfile index 649f87bd0e4..0f7fc620283 100644 --- a/.cicd/platforms/unpinned/centos-8-unpinned.dockerfile +++ b/.cicd/platforms/unpinned/centos-8-unpinned.dockerfile @@ -2,14 +2,15 @@ FROM centos:8 ENV VERSION 1 #install dependencies RUN yum update -y && \ - yum install -y net-tools && \ + yum install -y epel-release && \ yum --enablerepo=extras install -y which git autoconf automake libtool make bzip2 && \ yum --enablerepo=extras install -y graphviz bzip2-devel openssl-devel gmp-devel && \ yum --enablerepo=extras install -y file libusbx-devel && \ yum --enablerepo=extras install -y libcurl-devel patch vim-common jq && \ - yum install -y python3 + yum install -y python3 llvm-toolset RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ - dnf group install -y "Development Tools" + dnf group install -y "Development Tools" && \ + dnf --enablerepo=PowerTools install -y doxygen ocaml # cmake3.18.0 RUN curl -LO https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0.tar.gz && \ tar -xzf cmake-3.18.0.tar.gz && \ diff --git a/.gitmodules b/.gitmodules index f3d406ce8f4..02f297cb560 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,9 +19,6 @@ [submodule "libraries/amqp-cpp"] path = libraries/amqp-cpp url = https://github.com/CopernicaMarketingSoftware/AMQP-CPP -[submodule "libraries/fc"] - path = libraries/fc - url = https://github.com/eosio/fc [submodule "libraries/chainbase"] path = libraries/chainbase url = https://github.com/eosio/chainbase diff --git a/libraries/fc b/libraries/fc deleted file mode 160000 index 26f699fbd7d..00000000000 --- a/libraries/fc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 26f699fbd7d3567cf56a7fb5eb752f259ab4680d From 44f1dc9daf62bd55489114b47892802e2b41b6cd Mon Sep 17 00:00:00 2001 From: Keke Li Date: Fri, 4 Sep 2020 17:17:38 -0700 Subject: [PATCH 7/7] submodule keep same as develop --- .gitmodules | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitmodules b/.gitmodules index 02f297cb560..f3d406ce8f4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,6 +19,9 @@ [submodule "libraries/amqp-cpp"] path = libraries/amqp-cpp url = https://github.com/CopernicaMarketingSoftware/AMQP-CPP +[submodule "libraries/fc"] + path = libraries/fc + url = https://github.com/eosio/fc [submodule "libraries/chainbase"] path = libraries/chainbase url = https://github.com/eosio/chainbase