Add scripts to build Alpine APK packages and Python wheels (apache#7101)

hrsakai · May 31, 2020 · 7471786 · 7471786
1 parent c579d53
commit 7471786
Show file tree

Hide file tree

Showing 11 changed files with 443 additions and 6 deletions.
diff --git a/pulsar-client-cpp/CMakeLists.txt b/pulsar-client-cpp/CMakeLists.txt
@@ -169,7 +169,7 @@ if (BUILD_PYTHON_WRAPPER)
         list(GET PYTHONLIBS_VERSION_NO_LIST 1 PYTHONLIBS_VERSION_MINOR)
         set(BOOST_PYTHON_NAME_POSTFIX ${PYTHONLIBS_VERSION_MAJOR}${PYTHONLIBS_VERSION_MINOR})
         # For python3 the lib name is boost_python3
-        set(BOOST_PYTHON_NAME_LIST python3;python3-mt;python-py${BOOST_PYTHON_NAME_POSTFIX};python${BOOST_PYTHON_NAME_POSTFIX}-mt;python${BOOST_PYTHON_NAME_POSTFIX})
+        set(BOOST_PYTHON_NAME_LIST python36;python37;python38;python3;python3-mt;python-py${BOOST_PYTHON_NAME_POSTFIX};python${BOOST_PYTHON_NAME_POSTFIX}-mt;python${BOOST_PYTHON_NAME_POSTFIX})
     else ()
         # Regular boost_python
         set(BOOST_PYTHON_NAME_LIST python;python-mt;python-py27;python27-mt;python27)

diff --git a/pulsar-client-cpp/docker/alpine/Dockerfile b/pulsar-client-cpp/docker/alpine/Dockerfile
@@ -0,0 +1,97 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+FROM alpine:3.11
+
+RUN apk update \
+    && apk upgrade \
+    && apk add --no-cache \
+    cmake make build-base curl \
+    python3-dev \
+    libffi-dev \
+    perl linux-headers bash \
+    alpine-sdk
+
+####################################
+# These dependencies can be found in Ubuntu but they're not compiled with -fPIC,
+# so they cannot be statically linked into a shared library
+####################################
+
+# Download and compile boost
+RUN curl -O -L https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.gz && \
+    tar xfz boost_1_72_0.tar.gz && \
+    cd /boost_1_72_0 && \
+    ./bootstrap.sh --with-libraries=program_options,regex,system,python --with-python=/usr/bin/python3 && \
+    ./b2  cxxflags=-fPIC link=static threading=multi variant=release install && \
+    rm -rf /boost_1_72_0.tar.gz /boost_1_72_0
+
+# ZLib
+RUN curl -O -L https://zlib.net/zlib-1.2.11.tar.gz  && \
+    tar xfz zlib-1.2.11.tar.gz && \
+    cd zlib-1.2.11 && \
+    CFLAGS="-fPIC -O3" ./configure && \
+    make -j4 && make install && \
+    rm -rf /zlib-1.2.11.tar.gz /zlib-1.2.11
+
+# Compile OpenSSL
+RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_0j.tar.gz && \
+   tar xfz OpenSSL_1_1_0j.tar.gz && \
+   cd openssl-OpenSSL_1_1_0j/ && \
+   ./Configure -fPIC no-shared linux-x86_64 && \
+   make -j8 && make install && \
+   rm -rf /OpenSSL_1_1_0j.tar.gz /openssl-OpenSSL_1_1_0j
+
+# Download and copile protoubf
+RUN curl -O -L  https://github.com/google/protobuf/releases/download/v3.11.3/protobuf-cpp-3.11.3.tar.gz && \
+    tar xvfz protobuf-cpp-3.11.3.tar.gz && \
+    cd protobuf-3.11.3/ && \
+    CXXFLAGS=-fPIC ./configure && \
+    make -j8 && make install && \
+    rm -rf /protobuf-cpp-3.11.3.tar.gz /protobuf-3.11.3
+
+# LibCurl
+RUN curl -O -L  https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \
+    tar xvfz curl-7.61.0.tar.gz && \
+    cd curl-7.61.0 && \
+    CFLAGS=-fPIC ./configure && \
+    make -j8 && make install && \
+    rm -rf /curl-7.61.0.tar.gz /curl-7.61.0
+
+# Zstandard
+RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.3.7/zstd-1.3.7.tar.gz && \
+    tar xvfz zstd-1.3.7.tar.gz && \
+    cd zstd-1.3.7 && \
+    CFLAGS="-fPIC -O3" make -j8 && \
+    make install && \
+    rm -rf /zstd-1.3.7 /zstd-1.3.7.tar.gz
+
+# Snappy
+RUN curl -L https://github.com/google/snappy/archive/1.1.8.tar.gz --output snappy-1.1.8.tar.gz && \
+    tar xvfz snappy-1.1.8.tar.gz && \
+    cd snappy-1.1.8 && \
+    CXXFLAGS="-fPIC -O3" cmake . && \
+    make -j8 && make install && \
+    rm -rf /snappy-1.1.8 /snappy-1.1.8.tar.gz
+
+
+RUN pip3 install wheel twine
+
+RUN adduser pulsar -D && \
+    addgroup pulsar abuild
diff --git a/pulsar-client-cpp/docker/alpine/Dockerfile-alpine-3.8 b/pulsar-client-cpp/docker/alpine/Dockerfile-alpine-3.8
@@ -0,0 +1,97 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+FROM alpine:3.8
+
+RUN apk update \
+    && apk upgrade \
+    && apk add --no-cache \
+    cmake make build-base curl \
+    python3-dev \
+    libffi-dev \
+    perl linux-headers bash \
+    alpine-sdk
+
+####################################
+# These dependencies can be found in Ubuntu but they're not compiled with -fPIC,
+# so they cannot be statically linked into a shared library
+####################################
+
+# Download and compile boost
+RUN curl -O -L https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.gz && \
+    tar xfz boost_1_72_0.tar.gz && \
+    cd /boost_1_72_0 && \
+    ./bootstrap.sh --with-libraries=program_options,regex,system,python --with-python=/usr/bin/python3 && \
+    ./b2  cxxflags=" -fPIC -I/usr/include/python3.6m" link=static threading=multi variant=release install && \
+    rm -rf /boost_1_72_0.tar.gz /boost_1_72_0
+
+# ZLib
+RUN curl -O -L https://zlib.net/zlib-1.2.11.tar.gz  && \
+    tar xfz zlib-1.2.11.tar.gz && \
+    cd zlib-1.2.11 && \
+    CFLAGS="-fPIC -O3" ./configure && \
+    make -j4 && make install && \
+    rm -rf /zlib-1.2.11.tar.gz /zlib-1.2.11
+
+# Compile OpenSSL
+RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_0j.tar.gz && \
+   tar xfz OpenSSL_1_1_0j.tar.gz && \
+   cd openssl-OpenSSL_1_1_0j/ && \
+   ./Configure -fPIC no-shared linux-x86_64 && \
+   make -j8 && make install && \
+   rm -rf /OpenSSL_1_1_0j.tar.gz /openssl-OpenSSL_1_1_0j
+
+# Download and copile protoubf
+RUN curl -O -L  https://github.com/google/protobuf/releases/download/v3.11.3/protobuf-cpp-3.11.3.tar.gz && \
+    tar xvfz protobuf-cpp-3.11.3.tar.gz && \
+    cd protobuf-3.11.3/ && \
+    CXXFLAGS=-fPIC ./configure && \
+    make -j8 && make install && \
+    rm -rf /protobuf-cpp-3.11.3.tar.gz /protobuf-3.11.3
+
+# LibCurl
+RUN curl -O -L  https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \
+    tar xvfz curl-7.61.0.tar.gz && \
+    cd curl-7.61.0 && \
+    CFLAGS=-fPIC ./configure && \
+    make -j8 && make install && \
+    rm -rf /curl-7.61.0.tar.gz /curl-7.61.0
+
+# Zstandard
+RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.4.4/zstd-1.4.4.tar.gz && \
+    tar xvfz zstd-1.4.4.tar.gz && \
+    cd zstd-1.4.4 && \
+    CFLAGS="-fPIC -O3" make -j8 && \
+    make install && \
+    rm -rf /zstd-1.4.4 /zstd-1.4.4.tar.gz
+
+# Snappy
+RUN curl -L https://github.com/google/snappy/archive/1.1.8.tar.gz --output snappy-1.1.8.tar.gz && \
+    tar xvfz snappy-1.1.8.tar.gz && \
+    cd snappy-1.1.8 && \
+    CXXFLAGS="-fPIC -O3" cmake . && \
+    make -j8 && make install && \
+    rm -rf /snappy-1.1.8 /snappy-1.1.8.tar.gz
+
+
+RUN pip3 install wheel twine
+
+RUN adduser pulsar -D && \
+    addgroup pulsar abuild
diff --git a/pulsar-client-cpp/docker/alpine/build-alpine-image.sh b/pulsar-client-cpp/docker/alpine/build-alpine-image.sh
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Build Alpine Image with pulsar python3 and cpp client libraries
+
+set -e
+
+ROOT_DIR=$(git rev-parse --show-toplevel)
+
+IMAGE_NAME=${IMAGE_NAME:-apachepulsar/pulsar-build:alpine-3.11}
+
+echo "==== Building image $IMAGE_NAME"
+
+cd $ROOT_DIR/pulsar-client-cpp/docker/alpine
+docker build -t $IMAGE_NAME -f $ROOT_DIR/pulsar-client-cpp/docker/alpine/Dockerfile .
+
+echo "==== Successfully built image $IMAGE_NAME"
diff --git a/pulsar-client-cpp/docker/alpine/build-wheel-file-within-docker.sh b/pulsar-client-cpp/docker/alpine/build-wheel-file-within-docker.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e -x
+
+ROOT_DIR=$(git rev-parse --show-toplevel)
+cd $ROOT_DIR/pulsar-client-cpp
+PYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR:-/usr/include/python3.8}
+PYTHON_LIBRARY=${PYTHON_LIBRARY:-/usr/lib/python3.8}
+
+cmake .  -DBUILD_TESTS=OFF \
+          -DBUILD_PYTHON_WRAPPER=ON \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DLINK_STATIC=ON  \
+          -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR} \
+          -DPYTHON_LIBRARY=${PYTHON_LIBRARY}
+
+make -j2 _pulsar
+
+cd python
+python3 setup.py bdist_wheel
diff --git a/pulsar-client-cpp/docker/alpine/build-wheel.sh b/pulsar-client-cpp/docker/alpine/build-wheel.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+ROOT_DIR=$(git rev-parse --show-toplevel)
+PROJECT_VERSION=$(python $ROOT_DIR/src/get-project-version.py)
+IMAGE_NAME=${IMAGE_NAME:-apachepulsar/pulsar-build:alpine-3.11}
+
+ROOT_DIR=$(git rev-parse --show-toplevel)
+cd $ROOT_DIR
+
+echo "Using image: $IMAGE_NAME"
+VOLUME_OPTION=${VOLUME_OPTION:-"-v $ROOT_DIR:/pulsar"}
+COMMAND="/pulsar/pulsar-client-cpp/docker/alpine/build-wheel-file-within-docker.sh"
+DOCKER_CMD="docker run -i ${VOLUME_OPTION} ${IMAGE_NAME}"
+$DOCKER_CMD bash -c "${COMMAND}"
diff --git a/pulsar-client-cpp/lib/CMakeLists.txt b/pulsar-client-cpp/lib/CMakeLists.txt
@@ -58,7 +58,8 @@ if (WIN32)
 endif()
 
 add_library(pulsarShared SHARED ${PULSAR_SOURCES})
-set_target_properties(pulsarShared PROPERTIES OUTPUT_NAME ${LIB_NAME_SHARED} VERSION ${LIBRARY_VERSION})
+set_property(TARGET pulsarShared PROPERTY OUTPUT_NAME ${LIB_NAME_SHARED})
+set_property(TARGET pulsarShared PROPERTY VERSION ${LIBRARY_VERSION})
 target_link_libraries(pulsarShared ${COMMON_LIBS} ${CMAKE_DL_LIBS})
 
 
@@ -75,11 +76,13 @@ if (NOT ${RECORD_OPENSSL_CRYPTO_LIBRARY} MATCHES ".+\\.a$")
 endif ()
 
 add_library(pulsarSharedNossl SHARED ${PULSAR_SOURCES})
-set_target_properties(pulsarSharedNossl PROPERTIES OUTPUT_NAME ${LIB_NAME_SHARED}nossl VERSION ${LIBRARY_VERSION})
+set_property(TARGET pulsarSharedNossl PROPERTY OUTPUT_NAME ${LIB_NAME_SHARED}nossl)
+set_property(TARGET pulsarSharedNossl PROPERTY VERSION ${LIBRARY_VERSION})
 target_link_libraries(pulsarSharedNossl ${COMMON_LIBS_NOSSL} ${CMAKE_DL_LIBS})
 
 add_library(pulsarStatic STATIC ${PULSAR_SOURCES})
-set_target_properties(pulsarStatic PROPERTIES OUTPUT_NAME ${LIB_NAME} VERSION ${LIBRARY_VERSION})
+set_property(TARGET pulsarStatic PROPERTY OUTPUT_NAME ${LIB_NAME})
+set_property(TARGET pulsarStatic PROPERTY VERSION ${LIBRARY_VERSION})
 target_compile_definitions(pulsarStatic PRIVATE PULSAR_STATIC)
 
 if (MSVC)
@@ -92,7 +95,7 @@ endif()
 # required dependencies except ssl
 if (LINK_STATIC)
     if (MSVC)
-        
+
         # This function is to remove either "debug" or "optimized" library names
         # out of the COMMON_LIBS list and return the sanitized list of libraries
         function(remove_libtype LIBLIST LIBTYPE OUTLIST)
@@ -112,7 +115,10 @@ if (LINK_STATIC)
         target_include_directories(pulsarStaticWithDeps PRIVATE ${dlfcn-win32_INCLUDE_DIRS})
         remove_libtype("${COMMON_LIBS}" "optimized" DEBUG_STATIC_LIBS)
         remove_libtype("${COMMON_LIBS}" "debug" STATIC_LIBS)
-        set_target_properties(pulsarStaticWithDeps PROPERTIES STATIC_LIBRARY_FLAGS_DEBUG ${DEBUG_STATIC_LIBS} STATIC_LIBRARY_FLAGS_RELEASE ${STATIC_LIBS} OUTPUT_NAME ${LIB_NAME}WithDeps VERSION ${LIBRARY_VERSION})
+        set_property(TARGET pulsarStaticWithDeps PROPERTY STATIC_LIBRARY_FLAGS_DEBUG ${DEBUG_STATIC_LIBS})
+        set_property(TARGET pulsarStaticWithDeps PROPERTY STATIC_LIBRARY_FLAGS_RELEASE ${STATIC_LIBS})
+        set_property(TARGET pulsarStaticWithDeps PROPERTY OUTPUT_NAME ${LIB_NAME}WithDeps)
+        set_property(TARGET pulsarStaticWithDeps PROPERTY VERSION ${LIBRARY_VERSION})
         install(TARGETS pulsarStaticWithDeps DESTINATION lib)
     else()
         # Build a list of the requird .a libs (except ssl) to merge

diff --git a/pulsar-client-cpp/pkg/apk/.gitignore b/pulsar-client-cpp/pkg/apk/.gitignore
@@ -0,0 +1,4 @@
+pkg
+perf
+examples
+lib