From 7deb23e87f8e2e83ef83ae60e4fb032cfc6fd53a Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 10 Dec 2023 21:28:56 -0800 Subject: [PATCH 01/13] [improve][pip] PIP-324: Alpine image --- .github/workflows/pulsar-ci.yaml | 13 +- build/build_java_test_image.sh | 1 - build/docker/Dockerfile | 100 --------------- build/docker/README.md | 48 ------- build/docker/publish.sh | 57 --------- build/pulsar_ci_tool.sh | 48 +------ docker/pulsar/Dockerfile | 121 +++++++++--------- docker/pulsar/pom.xml | 3 - ...lsar-client.sh => install-glibc-compat.sh} | 21 +-- pulsar-io/kinesis/pom.xml | 2 +- .../docker-images/java-test-image/Dockerfile | 55 +------- tests/docker-images/java-test-image/pom.xml | 10 +- .../latest-version-image/Dockerfile | 29 +++-- tests/docker-images/pom.xml | 14 ++ .../integration/cli/HealthCheckTest.java | 6 +- .../offload/TestFileSystemOffload.java | 2 +- 16 files changed, 120 insertions(+), 410 deletions(-) delete mode 100644 build/docker/Dockerfile delete mode 100644 build/docker/README.md delete mode 100755 build/docker/publish.sh rename docker/pulsar/scripts/{install-pulsar-client.sh => install-glibc-compat.sh} (64%) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index d499b1ec64931..e2c7cb44e2e5e 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -508,15 +508,10 @@ jobs: cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - - name: Pick ubuntu mirror for the docker image build - run: | - # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror - - name: Build java-test-image docker image run: | # build docker image - mvn -B -am -pl tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \ + mvn -B -am -pl docker/pulsar,tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \ -Dmaven.test.skip=true -Ddocker.squash=true -DskipSourceReleaseAssembly=true \ -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true @@ -868,17 +863,11 @@ jobs: cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries - - name: Pick ubuntu mirror for the docker image build - run: | - # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable - $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror - - name: Build latest-version-image docker image run: | # build docker image # include building of Connectors, Offloaders and server distros mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \ - -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" -DIMAGE_JDK_MAJOR_VERSION="${IMAGE_JDK_MAJOR_VERSION}" \ -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \ -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true diff --git a/build/build_java_test_image.sh b/build/build_java_test_image.sh index 459bf26f98eff..0747e6dacb82a 100755 --- a/build/build_java_test_image.sh +++ b/build/build_java_test_image.sh @@ -27,6 +27,5 @@ if [[ "$(docker version -f '{{.Server.Experimental}}' 2>/dev/null)" == "true" ]] SQUASH_PARAM="-Ddocker.squash=true" fi mvn -am -pl tests/docker-images/java-test-image -Pcore-modules,-main,integrationTests,docker \ - -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \ -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true $SQUASH_PARAM \ "$@" install \ No newline at end of file diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile deleted file mode 100644 index ed787f7b85ae8..0000000000000 --- a/build/docker/Dockerfile +++ /dev/null @@ -1,100 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -FROM ubuntu:22.04 - -ARG JDK_MAJOR_VERSION=17 - -# prepare the directory for pulsar related files -RUN mkdir /pulsar - -RUN apt-get update && \ - apt-get install -y software-properties-common && \ - apt-get update && \ - apt-get install -y tig g++ cmake libssl-dev libcurl4-openssl-dev \ - liblog4cxx-dev google-mock libgtest-dev \ - libboost-dev libboost-program-options-dev libboost-system-dev libboost-python-dev \ - libxml2-utils wget apt-transport-https \ - curl doxygen clang-format \ - gnupg2 golang-go zip unzip libzstd-dev libsnappy-dev wireshark-dev - -# Install Eclipse Temurin Package -RUN mkdir -p /etc/apt/keyrings \ - && wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc \ - && echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list \ - && apt-get update \ - && apt-get -y dist-upgrade \ - && apt-get -y install temurin-${JDK_MAJOR_VERSION:-17}-jdk - -# Compile and install gtest & gmock -RUN cd /usr/src/googletest && \ - cmake . && \ - make && \ - make install - -# Include gtest parallel to speed up unit tests -RUN git clone https://github.com/google/gtest-parallel.git - -# Build protobuf 3.x.y from source since the default protobuf from Ubuntu's apt source is 2.x.y -RUN curl -O -L https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protobuf-cpp-3.17.3.tar.gz && \ - tar xvfz protobuf-cpp-3.17.3.tar.gz && \ - cd protobuf-3.17.3/ && \ - CXXFLAGS=-fPIC ./configure && \ - make -j8 && make install && \ - cd .. && rm -rf protobuf-3.17.3/ protobuf-cpp-3.17.3.tar.gz -ENV LD_LIBRARY_PATH /usr/local/lib - -## Website build dependencies - -# Install Ruby-2.4.1 -RUN (curl -sSL https://rvm.io/mpapis.asc | gpg --import -) && \ - (curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -) && \ - (curl -sSL https://get.rvm.io | bash -s stable) -ENV PATH "$PATH:/usr/local/rvm/bin" -RUN rvm install 2.4.1 - -# Install nodejs and yarn -RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - -RUN apt-get install -y nodejs -RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - -RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list -RUN apt-get update && apt-get install yarn - -# Install crowdin -RUN wget https://artifacts.crowdin.com/repo/deb/crowdin.deb -O crowdin.deb -RUN dpkg -i crowdin.deb - -# Install PIP -RUN curl https://bootstrap.pypa.io/get-pip.py | python3 - -RUN pip3 --no-cache-dir install pdoc -# -# Installation -ARG MAVEN_VERSION=3.6.3 -ARG MAVEN_FILENAME="apache-maven-${MAVEN_VERSION}-bin.tar.gz" -ARG MAVEN_HOME=/opt/maven -ARG MAVEN_URL="http://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${MAVEN_FILENAME}" -ARG MAVEN_TMP="/tmp/${MAVEN_FILENAME}" -RUN wget --no-verbose -O ${MAVEN_TMP} ${MAVEN_URL} - -# Cleanup -RUN tar xzf ${MAVEN_TMP} -C /opt/ \ - && ln -s /opt/apache-maven-${MAVEN_VERSION} ${MAVEN_HOME} \ - && ln -s ${MAVEN_HOME}/bin/mvn /usr/local/bin - -RUN unset MAVEN_VERSION diff --git a/build/docker/README.md b/build/docker/README.md deleted file mode 100644 index bc2fe532f74e0..0000000000000 --- a/build/docker/README.md +++ /dev/null @@ -1,48 +0,0 @@ - - -This folder contains a Docker image that can used to compile the Pulsar C++ client library -and website in a reproducible environment. - -```shell -docker build -t pulsar-build . -``` - -The image is already available at https://hub.docker.com/r/apachepulsar/pulsar-build - -Example: `apachepulsar/pulsar-build:ubuntu-16.04` - -## Build and Publish pulsar-build image - -> Only committers have permissions on publishing pulsar images to `apachepulsar` docker hub. - -### Build pulsar-build image - - -```shell -docker build -t apachepulsar/pulsar-build:ubuntu-16.04 . -``` - -### Publish pulsar-build image - -```shell -publish.sh -``` diff --git a/build/docker/publish.sh b/build/docker/publish.sh deleted file mode 100755 index 6bfa56bace6d8..0000000000000 --- a/build/docker/publish.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -ROOT_DIR=$(git rev-parse --show-toplevel) -cd $ROOT_DIR/build/docker - -if [ -z "$DOCKER_USER" ]; then - echo "Docker user in variable \$DOCKER_USER was not set. Skipping image publishing" - exit 1 -fi - -if [ -z "$DOCKER_PASSWORD" ]; then - echo "Docker password in variable \$DOCKER_PASSWORD was not set. Skipping image publishing" - exit 1 -fi - -DOCKER_ORG="${DOCKER_ORG:-apachepulsar}" - -docker login ${DOCKER_REGISTRY} -u="$DOCKER_USER" -p="$DOCKER_PASSWORD" -if [ $? -ne 0 ]; then - echo "Failed to loging to Docker Hub" - exit 1 -fi - -if [[ -z ${DOCKER_REGISTRY} ]]; then - docker_registry_org=${DOCKER_ORG} -else - docker_registry_org=${DOCKER_REGISTRY}/${DOCKER_ORG} - echo "Starting to push images to ${docker_registry_org}..." -fi - -set -x - -# Fail if any of the subsequent commands fail -set -e - -# Push all images and tags -docker push ${docker_registry_org}/pulsar-build:ubuntu-16.04 - -echo "Finished pushing images to ${docker_registry_org}" diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh index ae33c3e91667b..034b2ce60cf37 100755 --- a/build/pulsar_ci_tool.sh +++ b/build/pulsar_ci_tool.sh @@ -46,8 +46,7 @@ function ci_print_thread_dumps() { # runs maven function _ci_mvn() { - mvn -B -ntp -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \ - "$@" + mvn -B -ntp "$@" } # runs OWASP Dependency Check for all projects @@ -55,50 +54,6 @@ function ci_dependency_check() { _ci_mvn -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl '!pulsar-client-tools-test' "$@" } -# Finds fastest up-to-date ubuntu mirror based on download speed -function ci_find_fast_ubuntu_mirror() { - local ubuntu_release=${1:-"$(lsb_release -c 2>/dev/null | cut -f2 || echo "jammy")"} - local ubuntu_arch=${2:-"$(dpkg --print-architecture 2>/dev/null || echo "amd64")"} - { - # choose mirrors that are up-to-date by checking the Last-Modified header for - { - # randomly choose up to 10 mirrors using http:// protocol - # (https isn't supported in docker containers that don't have ca-certificates installed) - curl -s http://mirrors.ubuntu.com/mirrors.txt | grep '^http://' | shuf -n 10 - # also consider Azure's Ubuntu mirror - echo http://azure.archive.ubuntu.com/ubuntu/ - } | xargs -I {} sh -c "ubuntu_release=$ubuntu_release ubuntu_arch=$ubuntu_arch;"'echo "$(curl -m 5 -sI {}dists/${ubuntu_release}/Contents-${ubuntu_arch}.gz|sed s/\\r\$//|grep Last-Modified|awk -F": " "{ print \$2 }" | LANG=C date -f- -u +%s)" "{}"' | sort -rg | awk '{ if (NR==1) TS=$1; if ($1 == TS) print $2 }' - } | xargs -I {} sh -c 'echo `curl -r 0-102400 -m 5 -s -w %{speed_download} -o /dev/null {}ls-lR.gz` {}' \ - |sort -g -r |head -1| awk '{ print $2 }' -} - -function ci_pick_ubuntu_mirror() { - echo "Choosing fastest up-to-date ubuntu mirror based on download speed..." - UBUNTU_MIRROR=$(ci_find_fast_ubuntu_mirror) - if [ -z "$UBUNTU_MIRROR" ]; then - # fallback to no mirror - UBUNTU_MIRROR="http://archive.ubuntu.com/ubuntu/" - UBUNTU_SECURITY_MIRROR="http://security.ubuntu.com/ubuntu/" - else - UBUNTU_SECURITY_MIRROR="${UBUNTU_MIRROR}" - fi - OLD_MIRROR=$(cat /etc/apt/sources.list | grep '^deb ' | head -1 | awk '{ print $2 }') - echo "Picked '$UBUNTU_MIRROR'. Current mirror is '$OLD_MIRROR'." - if [[ "$OLD_MIRROR" != "$UBUNTU_MIRROR" ]]; then - sudo sed -i "s|$OLD_MIRROR|$UBUNTU_MIRROR|g" /etc/apt/sources.list - sudo apt-get update - fi - # set the chosen mirror also in the UBUNTU_MIRROR and UBUNTU_SECURITY_MIRROR environment variables - # that can be used by docker builds - export UBUNTU_MIRROR - export UBUNTU_SECURITY_MIRROR - # make environment variables available for later GitHub Actions steps - if [ -n "$GITHUB_ENV" ]; then - echo "UBUNTU_MIRROR=$UBUNTU_MIRROR" >> $GITHUB_ENV - echo "UBUNTU_SECURITY_MIRROR=$UBUNTU_SECURITY_MIRROR" >> $GITHUB_ENV - fi -} - # installs a tool executable if it's not found on the PATH function ci_install_tool() { local tool_executable=$1 @@ -108,7 +63,6 @@ function ci_install_tool() { echo "::group::Installing ${tool_package}" sudo apt-get -y install ${tool_package} >/dev/null || { echo "Installing the package failed. Switching the ubuntu mirror and retrying..." - ci_pick_ubuntu_mirror # retry after picking the ubuntu mirror sudo apt-get -y install ${tool_package} } diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 6a0dc0100e7fd..ec8296a6d49aa 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -26,13 +26,7 @@ ADD ${PULSAR_TARBALL} / RUN mv /apache-pulsar-* /pulsar RUN rm -rf /pulsar/bin/*.cmd -COPY scripts/apply-config-from-env.py /pulsar/bin -COPY scripts/apply-config-from-env-with-prefix.py /pulsar/bin -COPY scripts/gen-yml-from-env.py /pulsar/bin -COPY scripts/generate-zookeeper-config.sh /pulsar/bin -COPY scripts/pulsar-zookeeper-ruok.sh /pulsar/bin -COPY scripts/watch-znode.py /pulsar/bin -COPY scripts/install-pulsar-client.sh /pulsar/bin +COPY scripts/* /pulsar/bin/ # The final image needs to give the root group sufficient permission for Pulsar components # to write to specific directories within /pulsar @@ -40,72 +34,79 @@ COPY scripts/install-pulsar-client.sh /pulsar/bin # container when gid=0 is prohibited. In that case, the container must be run with uid 10000 with # any group id != 0 (for example 10001). # The file permissions are preserved when copying files from this builder image to the target image. -RUN for SUBDIRECTORY in conf data download logs; do \ +RUN for SUBDIRECTORY in conf data download logs instances/deps; do \ [ -d /pulsar/$SUBDIRECTORY ] || mkdir /pulsar/$SUBDIRECTORY; \ - chmod -R ug+w /pulsar/$SUBDIRECTORY; \ + chmod -R ug+rwx /pulsar/$SUBDIRECTORY; \ chown -R 10000:0 /pulsar/$SUBDIRECTORY; \ done -### Create 2nd stage from Ubuntu image -### and add OpenJDK and Python dependencies (for Pulsar functions) +RUN chmod -R g+rx /pulsar/bin +RUN chmod -R o+rx /pulsar -FROM ubuntu:22.04 +## Create 2nd stage to build the Python dependencies +## Since it needs to have GCC available, we're doing it in a different layer +FROM alpine:3.19 AS python-deps + +RUN apk add --no-cache \ + bash \ + python3-dev \ + g++ \ + musl-dev \ + libffi-dev \ + py3-pip \ + py3-grpcio \ + py3-yaml + +RUN pip3 install --break-system-packages \ + kazoo + +ARG PULSAR_CLIENT_PYTHON_VERSION +RUN pip3 install --break-system-packages \ + pulsar-client[all]==${PULSAR_CLIENT_PYTHON_VERSION} -ARG DEBIAN_FRONTEND=noninteractive -ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu/ -ARG UBUNTU_SECURITY_MIRROR=http://security.ubuntu.com/ubuntu/ -ARG DEFAULT_USERNAME=pulsar -ARG JDK_MAJOR_VERSION=17 + +## Create final stage from Alpine image +## and add OpenJDK and Python dependencies (for Pulsar functions) +FROM alpine:3.19.1 +ENV LANG C.UTF-8 # Install some utilities -RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-http://archive.ubuntu.com/ubuntu/}|g" \ - -e "s|http://security\.ubuntu\.com/ubuntu/|${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}|g" /etc/apt/sources.list \ - && echo 'Acquire::http::Timeout "30";\nAcquire::http::ConnectionAttemptDelayMsec "2000";\nAcquire::https::Timeout "30";\nAcquire::https::ConnectionAttemptDelayMsec "2000";\nAcquire::ftp::Timeout "30";\nAcquire::ftp::ConnectionAttemptDelayMsec "2000";\nAcquire::Retries "15";' > /etc/apt/apt.conf.d/99timeout_and_retries \ - && apt-get update \ - && apt-get -y dist-upgrade \ - && apt-get -y install netcat dnsutils less procps iputils-ping \ - curl ca-certificates wget apt-transport-https \ - && apt-get -y install --no-install-recommends python3 python3-kazoo python3-pip - -# Install Eclipse Temurin Package -RUN mkdir -p /etc/apt/keyrings \ - && wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc \ - && echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list \ - && apt-get update \ - && apt-get -y dist-upgrade \ - && apt-get -y install temurin-${JDK_MAJOR_VERSION:-17}-jdk \ - && export ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' | awk '!/arm64/{$0="amd64"}1') \ - && echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/temurin-${JDK_MAJOR_VERSION:-17}-jdk-$ARCH/conf/security/java.security \ - && echo networkaddress.cache.negative.ttl=1 >> /usr/lib/jvm/temurin-${JDK_MAJOR_VERSION:-17}-jdk-$ARCH/conf/security/java.security \ - -# Cleanup apt -RUN apt-get -y --purge autoremove \ - && apt-get autoclean \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -RUN pip3 install pyyaml==6.0.1 - -# Pulsar currently writes to the below directories, assuming the default configuration. -# Note that number 4 is the reason that pulsar components need write access to the /pulsar directory. -# 1. /pulsar/data - both bookkeepers and zookeepers use this directory -# 2. /pulsar/logs - function workers write to this directory and pulsar-admin initializes this directory -# 3. /pulsar/download - functions write to this directory -# 4. /pulsar - hadoop writes to this directory -RUN mkdir /pulsar && chmod g+w /pulsar +RUN apk add --no-cache \ + bash \ + python3 \ + py3-pip \ + ca-certificates + +RUN wget -O /etc/apk/keys/amazoncorretto.rsa.pub https://apk.corretto.aws/amazoncorretto.rsa.pub && \ + echo "https://apk.corretto.aws" >> /etc/apk/repositories && \ + apk add --no-cache amazon-corretto-21 binutils && \ + # Use JLink to create a slimmer JDK distribution (see: https://adoptium.net/blog/2021/10/jlink-to-produce-own-runtime/) + # This still includes all JDK modules, though in the future we could compile a list of required modules + /usr/lib/jvm/default-jvm/bin/jlink --add-modules ALL-MODULE-PATH --compress zip-9 --no-man-pages --no-header-files --strip-debug --output /opt/corretto-slim && \ + apk del binutils amazon-corretto-21 && \ + mkdir -p /usr/lib/jvm/ && \ + mv /opt/corretto-slim /usr/lib/jvm/java-21-amazon-corretto && \ + ln -sfn /usr/lib/jvm/java-21-amazon-corretto /usr/lib/jvm/default-jvm + +ENV JAVA_HOME=/usr/lib/jvm/default-jvm + +RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/default-jvm/conf/security/java.security + +# Copy Python depedencies from the other stage +COPY --from=python-deps /usr/lib/python3.11/site-packages /usr/lib/python3.11/site-packages ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE COPY --from=pulsar /pulsar /pulsar -WORKDIR /pulsar -ARG PULSAR_CLIENT_PYTHON_VERSION -ENV PULSAR_CLIENT_PYTHON_VERSION ${PULSAR_CLIENT_PYTHON_VERSION} +## Netty TC native and Kinesis producer library are based on a C/C++ binary linked with Glibc +## Install compatibility library +RUN /pulsar/bin/install-glibc-compat.sh -# This script is intentionally run as the root user to make the dependencies available for all UIDs. -RUN chmod +x /pulsar/bin/install-pulsar-client.sh -RUN /pulsar/bin/install-pulsar-client.sh +WORKDIR /pulsar +ENV PATH=$PATH:$JAVA_HOME/bin:/pulsar/bin # The UID must be non-zero. Otherwise, it is arbitrary. No logic should rely on its specific value. -RUN useradd ${DEFAULT_USERNAME} -u 10000 -g 0 -USER 10000 +ARG DEFAULT_USERNAME=pulsar +RUN adduser ${DEFAULT_USERNAME} -u 10000 -G root -D +USER ${DEFAULT_USERNAME} diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 50d1867257a98..79ff4bd33b10c 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -82,9 +82,6 @@ target/pulsar-server-distribution-${project.version}-bin.tar.gz ${pulsar.client.python.version} - ${UBUNTU_MIRROR} - ${UBUNTU_SECURITY_MIRROR} - ${IMAGE_JDK_MAJOR_VERSION} ${project.basedir} diff --git a/docker/pulsar/scripts/install-pulsar-client.sh b/docker/pulsar/scripts/install-glibc-compat.sh similarity index 64% rename from docker/pulsar/scripts/install-pulsar-client.sh rename to docker/pulsar/scripts/install-glibc-compat.sh index 0951b2aec1b60..f871847ddd655 100755 --- a/docker/pulsar/scripts/install-pulsar-client.sh +++ b/docker/pulsar/scripts/install-glibc-compat.sh @@ -18,13 +18,18 @@ # under the License. # -set -x +set -e -x -# TODO: remove these lines once grpcio doesn't need to compile from source on ARM64 platform -ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' | awk '!/arm64/{$0="amd64"}1') -if [ "${ARCH}" == "arm64" ]; then - apt update - apt -y install build-essential python3-dev +ARCH=$(uname -m) +if [ "$ARCH" == "x86_64" ] +then + cd /tmp + echo 'https://storage.sev.monster/alpine/edge/testing' | tee -a /etc/apk/repositories + wget https://storage.sev.monster/alpine/edge/testing/x86_64/sevmonster-keys-1-r0.apk + apk add --no-cache --allow-untrusted ./sevmonster-keys-1-r0.apk + apk update + apk add gcompat libuuid + rm /lib/ld-linux-x86-64.so.2 + apk add --no-cache --force-overwrite glibc glibc-bin + rm *.apk fi - -pip3 install pulsar-client[all]==${PULSAR_CLIENT_PYTHON_VERSION} diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index 2301ffea44ad5..7580905382f57 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -32,7 +32,7 @@ 2.2.8 - 0.14.0 + 0.14.13 0.13.0 1.9.0 2.3.0 diff --git a/tests/docker-images/java-test-image/Dockerfile b/tests/docker-images/java-test-image/Dockerfile index 5a1bbf15e93b5..805f20a0570db 100644 --- a/tests/docker-images/java-test-image/Dockerfile +++ b/tests/docker-images/java-test-image/Dockerfile @@ -17,51 +17,16 @@ # under the License. # -FROM ubuntu:22.04 +ARG PULSAR_IMAGE +FROM $PULSAR_IMAGE -RUN groupadd -g 10001 pulsar -RUN adduser -u 10000 --gid 10001 --disabled-login --disabled-password --gecos '' pulsar - -ARG PULSAR_TARBALL=target/pulsar-server-distribution-bin.tar.gz -ADD ${PULSAR_TARBALL} / -RUN mv /apache-pulsar-* /pulsar -RUN chown -R root:root /pulsar +# Base pulsar image is designed not be modified, though we need to add more scripts +USER root COPY target/scripts /pulsar/bin RUN chmod a+rx /pulsar/bin/* -WORKDIR /pulsar - -ARG DEBIAN_FRONTEND=noninteractive -ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu/ -ARG UBUNTU_SECURITY_MIRROR=http://security.ubuntu.com/ubuntu/ -ARG JDK_MAJOR_VERSION=17 - -RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-http://archive.ubuntu.com/ubuntu/}|g" \ - -e "s|http://security\.ubuntu\.com/ubuntu/|${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}|g" /etc/apt/sources.list \ - && echo 'Acquire::http::Timeout "30";\nAcquire::http::ConnectionAttemptDelayMsec "2000";\nAcquire::https::Timeout "30";\nAcquire::https::ConnectionAttemptDelayMsec "2000";\nAcquire::ftp::Timeout "30";\nAcquire::ftp::ConnectionAttemptDelayMsec "2000";\nAcquire::Retries "15";' > /etc/apt/apt.conf.d/99timeout_and_retries \ - && apt-get update \ - && apt-get -y dist-upgrade \ - && apt-get -y install ca-certificates wget apt-transport-https - -# Install Eclipse Temurin Package -RUN mkdir -p /etc/apt/keyrings \ - && wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | tee /etc/apt/keyrings/adoptium.asc \ - && echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list \ - && apt-get update \ - && apt-get -y dist-upgrade \ - && apt-get -y install temurin-${JDK_MAJOR_VERSION:-17}-jdk \ - && export ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' | awk '!/arm64/{$0="amd64"}1') \ - && echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/temurin-${JDK_MAJOR_VERSION:-17}-jdk-$ARCH/conf/security/java.security - -# /pulsar/bin/watch-znode.py requires python3-kazoo -# /pulsar/bin/pulsar-managed-ledger-admin requires python3-protobuf -# gen-yml-from-env.py requires python3-yaml -# make python3 the default -RUN apt-get install -y python3-kazoo python3-protobuf python3-yaml \ - && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 - -RUN apt-get install -y supervisor procps curl less netcat dnsutils iputils-ping +RUN apk add --no-cache supervisor RUN mkdir -p /var/log/pulsar \ && mkdir -p /var/run/supervisor/ \ @@ -74,13 +39,3 @@ RUN mv /etc/supervisord/conf.d/supervisord.conf /etc/supervisord.conf COPY target/certificate-authority /pulsar/certificate-authority/ COPY target/java-test-functions.jar /pulsar/examples/ - -ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE - -RUN chown -R pulsar:0 /pulsar && chmod -R g=u /pulsar - -# cleanup -RUN apt-get -y --purge autoremove \ - && apt-get autoclean \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* diff --git a/tests/docker-images/java-test-image/pom.xml b/tests/docker-images/java-test-image/pom.xml index a980e334114ef..61d8c9668e5f4 100644 --- a/tests/docker-images/java-test-image/pom.xml +++ b/tests/docker-images/java-test-image/pom.xml @@ -33,12 +33,6 @@ docker - - target/pulsar-server-distribution-bin.tar.gz - ${UBUNTU_MIRROR} - ${UBUNTU_SECURITY_MIRROR} - ${IMAGE_JDK_MAJOR_VERSION} - integrationTests @@ -149,12 +143,16 @@ package build + tag ${docker.organization}/java-test-image + + ${docker.organization}/pulsar:${project.version}-${git.commit.id.abbrev} + ${project.basedir} latest diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile index f019af5c9269a..c23341c0748a2 100644 --- a/tests/docker-images/latest-version-image/Dockerfile +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -19,7 +19,7 @@ # build go lang examples first in a separate layer -FROM golang:1.21 as pulsar-function-go +FROM golang:1.21-alpine as pulsar-function-go COPY target/pulsar-function-go/ /go/src/github.com/apache/pulsar/pulsar-function-go RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go && go install ./... @@ -40,9 +40,7 @@ FROM apachepulsar/pulsar:latest # However, any processes exec'ing into the containers will run as root, by default. USER root -RUN rm -rf /var/lib/apt/lists/* && apt update - -RUN apt-get clean && apt-get update && apt-get install -y supervisor vim procps curl +RUN apk add --no-cache supervisor procps curl RUN mkdir -p /var/log/pulsar && mkdir -p /var/run/supervisor/ @@ -63,6 +61,7 @@ COPY python-examples/exclamation.zip /pulsar/examples/python-examples/ COPY python-examples/producer_schema.py /pulsar/examples/python-examples/ COPY python-examples/consumer_schema.py /pulsar/examples/python-examples/ COPY python-examples/exception_function.py /pulsar/examples/python-examples/ +RUN chmod g+rx /pulsar/examples/python-examples/ # copy java test examples COPY target/java-test-functions.jar /pulsar/examples/ @@ -93,16 +92,20 @@ COPY --from=pulsar-all /pulsar/connectors/pulsar-io-kinesis-*.nar /pulsar/connec # download Oracle JDBC driver for Oracle Debezium Connector tests RUN mkdir -p META-INF/bundled-dependencies -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/ojdbc8/19.3.0.0/ojdbc8-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/ucp/19.3.0.0/ucp-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/oraclepki/19.3.0.0/oraclepki-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/osdt_cert/19.3.0.0/osdt_cert-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/osdt_core/19.3.0.0/osdt_core-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/simplefan/19.3.0.0/simplefan-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/orai18n/19.3.0.0/orai18n-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/xdb/19.3.0.0/xdb-19.3.0.0.jar -RUN cd META-INF/bundled-dependencies && curl -sSLO https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/xmlparserv2/19.3.0.0/xmlparserv2-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/ojdbc8/19.3.0.0/ojdbc8-19.3.0.0.jar -o ojdbc8-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/ucp/19.3.0.0/ucp-19.3.0.0.jar -o ucp-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/oraclepki/19.3.0.0/oraclepki-19.3.0.0.jar -o oraclepki-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/osdt_cert/19.3.0.0/osdt_cert-19.3.0.0.jar -o osdt_cert-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/osdt_core/19.3.0.0/osdt_core-19.3.0.0.jar -o osdt_core-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/simplefan/19.3.0.0/simplefan-19.3.0.0.jar -o simplefan-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/orai18n/19.3.0.0/orai18n-19.3.0.0.jar -o orai18n-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/xdb/19.3.0.0/xdb-19.3.0.0.jar -o xdb-19.3.0.0.jar +RUN cd META-INF/bundled-dependencies && curl -sSL https://search.maven.org/remotecontent?filepath=com/oracle/ojdbc/xmlparserv2/19.3.0.0/xmlparserv2-19.3.0.0.jar -o xmlparserv2-19.3.0.0.jar RUN jar uf connectors/pulsar-io-debezium-oracle-*.nar META-INF/bundled-dependencies/ojdbc8-19.3.0.0.jar META-INF/bundled-dependencies/ucp-19.3.0.0.jar META-INF/bundled-dependencies/oraclepki-19.3.0.0.jar META-INF/bundled-dependencies/osdt_cert-19.3.0.0.jar META-INF/bundled-dependencies/osdt_core-19.3.0.0.jar META-INF/bundled-dependencies/simplefan-19.3.0.0.jar META-INF/bundled-dependencies/orai18n-19.3.0.0.jar META-INF/bundled-dependencies/xdb-19.3.0.0.jar META-INF/bundled-dependencies/xmlparserv2-19.3.0.0.jar +# Fix permissions for filesystem offloader +RUN mkdir -p pulsar +RUN chmod g+rwx pulsar + CMD bash diff --git a/tests/docker-images/pom.xml b/tests/docker-images/pom.xml index 1a0eeb1424b8a..770b490cdd390 100644 --- a/tests/docker-images/pom.xml +++ b/tests/docker-images/pom.xml @@ -52,6 +52,20 @@ latest-version-image java-test-image + + + + pl.project13.maven + git-commit-id-plugin + + false + true + true + false + + + + diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/HealthCheckTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/HealthCheckTest.java index 1670d970f09b1..6de44574610ab 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/HealthCheckTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/cli/HealthCheckTest.java @@ -95,7 +95,7 @@ private void assertHealthcheckFailure() throws Exception { @Test public void testZooKeeperDown() throws Exception { - pulsarCluster.getZooKeeper().execCmd("pkill", "-STOP", "-f", "QuorumPeerMain"); + pulsarCluster.getZooKeeper().execCmd("pkill", "-STOP", "java"); assertHealthcheckFailure(); } @@ -103,7 +103,7 @@ public void testZooKeeperDown() throws Exception { // @Test // public void testBrokerDown() throws Exception { // for (BrokerContainer b : pulsarCluster.getBrokers()) { - // b.execCmd("pkill", "-STOP", "-f", "PulsarBrokerStarter"); + // b.execCmd("pkill", "-STOP", "java"); // } // assertHealthcheckFailure(); // } @@ -111,7 +111,7 @@ public void testZooKeeperDown() throws Exception { @Test public void testBookKeeperDown() throws Exception { for (BKContainer b : pulsarCluster.getBookies()) { - b.execCmd("pkill", "-STOP", "-f", "Main"); + b.execCmd("pkill", "-STOP", "java"); } assertHealthcheckFailure(); } diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestFileSystemOffload.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestFileSystemOffload.java index 4504b58ca920b..a58ec92bafe3f 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestFileSystemOffload.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestFileSystemOffload.java @@ -49,7 +49,7 @@ protected Map getEnv() { result.put("managedLedgerMaxEntriesPerLedger", String.valueOf(getNumEntriesPerLedger())); result.put("managedLedgerMinLedgerRolloverTimeMinutes", "0"); result.put("managedLedgerOffloadDriver", "filesystem"); - result.put("fileSystemURI", "file:///"); + result.put("fileSystemURI", "file:///tmp"); return result; } From c510df343d93f44f72fb23965405f617709f4346 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 14 Feb 2024 14:53:57 -0800 Subject: [PATCH 02/13] Build jvm in a separate docker stage --- docker/pulsar/Dockerfile | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index ec8296a6d49aa..ef02156b1f41a 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -35,7 +35,7 @@ COPY scripts/* /pulsar/bin/ # any group id != 0 (for example 10001). # The file permissions are preserved when copying files from this builder image to the target image. RUN for SUBDIRECTORY in conf data download logs instances/deps; do \ - [ -d /pulsar/$SUBDIRECTORY ] || mkdir /pulsar/$SUBDIRECTORY; \ + mkdir -p /pulsar/$SUBDIRECTORY; \ chmod -R ug+rwx /pulsar/$SUBDIRECTORY; \ chown -R 10000:0 /pulsar/$SUBDIRECTORY; \ done @@ -65,6 +65,19 @@ RUN pip3 install --break-system-packages \ pulsar-client[all]==${PULSAR_CLIENT_PYTHON_VERSION} +### Create one stage to include JVM distribution +FROM alpine AS jvm + +RUN wget -O /etc/apk/keys/amazoncorretto.rsa.pub https://apk.corretto.aws/amazoncorretto.rsa.pub +RUN echo "https://apk.corretto.aws" >> /etc/apk/repositories +RUN apk add --no-cache amazon-corretto-21 binutils + +# Use JLink to create a slimmer JDK distribution (see: https://adoptium.net/blog/2021/10/jlink-to-produce-own-runtime/) +# This still includes all JDK modules, though in the future we could compile a list of required modules +RUN /usr/lib/jvm/default-jvm/bin/jlink --add-modules ALL-MODULE-PATH --compress zip-9 --no-man-pages --no-header-files --strip-debug --output /opt/jvm +RUN echo networkaddress.cache.ttl=1 >> /opt/jvm/conf/security/java.security + + ## Create final stage from Alpine image ## and add OpenJDK and Python dependencies (for Pulsar functions) FROM alpine:3.19.1 @@ -77,20 +90,8 @@ RUN apk add --no-cache \ py3-pip \ ca-certificates -RUN wget -O /etc/apk/keys/amazoncorretto.rsa.pub https://apk.corretto.aws/amazoncorretto.rsa.pub && \ - echo "https://apk.corretto.aws" >> /etc/apk/repositories && \ - apk add --no-cache amazon-corretto-21 binutils && \ - # Use JLink to create a slimmer JDK distribution (see: https://adoptium.net/blog/2021/10/jlink-to-produce-own-runtime/) - # This still includes all JDK modules, though in the future we could compile a list of required modules - /usr/lib/jvm/default-jvm/bin/jlink --add-modules ALL-MODULE-PATH --compress zip-9 --no-man-pages --no-header-files --strip-debug --output /opt/corretto-slim && \ - apk del binutils amazon-corretto-21 && \ - mkdir -p /usr/lib/jvm/ && \ - mv /opt/corretto-slim /usr/lib/jvm/java-21-amazon-corretto && \ - ln -sfn /usr/lib/jvm/java-21-amazon-corretto /usr/lib/jvm/default-jvm - -ENV JAVA_HOME=/usr/lib/jvm/default-jvm - -RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/default-jvm/conf/security/java.security +COPY --from=jvm /opt/jvm /opt/jvm +ENV JAVA_HOME=/opt/jvm # Copy Python depedencies from the other stage COPY --from=python-deps /usr/lib/python3.11/site-packages /usr/lib/python3.11/site-packages From 5bb935d9464acf21caf4f9272f4f049520d95b85 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 16 Feb 2024 08:11:12 -0800 Subject: [PATCH 03/13] Re-insert config for DNS negative cache --- docker/pulsar/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index ef02156b1f41a..74eabdceb76e1 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -76,6 +76,7 @@ RUN apk add --no-cache amazon-corretto-21 binutils # This still includes all JDK modules, though in the future we could compile a list of required modules RUN /usr/lib/jvm/default-jvm/bin/jlink --add-modules ALL-MODULE-PATH --compress zip-9 --no-man-pages --no-header-files --strip-debug --output /opt/jvm RUN echo networkaddress.cache.ttl=1 >> /opt/jvm/conf/security/java.security +RUN echo networkaddress.cache.negative.ttl=1 >> /opt/jvm/conf/security/java.security ## Create final stage from Alpine image From d6053b6ab7722bbc6ea01e3d3627d55162b5462c Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 4 Mar 2024 12:04:00 -0800 Subject: [PATCH 04/13] Added scripts to build glibc packages for Alpine --- docker/glibc-package/Dockerfile | 70 +++++++++++++++++++ docker/glibc-package/README.md | 39 +++++++++++ docker/glibc-package/scripts/APKBUILD | 50 +++++++++++++ .../scripts/glibc-bin.trigger} | 18 +---- docker/glibc-package/scripts/ld.so.conf | 23 ++++++ docker/pulsar/Dockerfile | 10 +-- 6 files changed, 190 insertions(+), 20 deletions(-) create mode 100644 docker/glibc-package/Dockerfile create mode 100644 docker/glibc-package/README.md create mode 100644 docker/glibc-package/scripts/APKBUILD rename docker/{pulsar/scripts/install-glibc-compat.sh => glibc-package/scripts/glibc-bin.trigger} (62%) create mode 100644 docker/glibc-package/scripts/ld.so.conf diff --git a/docker/glibc-package/Dockerfile b/docker/glibc-package/Dockerfile new file mode 100644 index 0000000000000..935162c113a4f --- /dev/null +++ b/docker/glibc-package/Dockerfile @@ -0,0 +1,70 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +ARG GLIBC_VERSION=2.38 + +FROM ubuntu:22.04 as build +ARG GLIBC_VERSION + +RUN apt-get -q update \ + && apt-get -qy install \ + bison \ + build-essential \ + gawk \ + gettext \ + openssl \ + python3 \ + texinfo \ + wget + +# Build GLibc +RUN wget -qO- https://ftpmirror.gnu.org/libc/glibc-${GLIBC_VERSION}.tar.gz | tar zxf - +RUN mkdir /glibc-build +WORKDIR /glibc-build +RUN /glibc-${GLIBC_VERSION}/configure \ + --prefix=/usr/glibc-compat \ + --libdir=/usr/glibc-compat/lib \ + --libexecdir=/usr/glibc-compat/lib \ + --enable-multi-arch \ + --enable-stack-protector=strong +RUN make -j$(nproc) +RUN make install +RUN tar --dereference --hard-dereference -zcf /glibc-bin.tar.gz /usr/glibc-compat + + +################################################ +## Build the APK package +FROM alpine:3.19 +ARG GLIBC_VERSION + +RUN apk add abuild sudo build-base + +RUN mkdir /build +WORKDIR build + +COPY --from=build /glibc-bin.tar.gz /build + +COPY ./scripts /build + +RUN echo "pkgver=\"${GLIBC_VERSION}\"" >> /build/APKBUILD +RUN echo "sha512sums=\"$(sha512sum glibc-bin.tar.gz ld.so.conf)\"" >> /build/APKBUILD + +RUN abuild-keygen -a -i -n +RUN abuild -F -c -r diff --git a/docker/glibc-package/README.md b/docker/glibc-package/README.md new file mode 100644 index 0000000000000..ee1f643705ad2 --- /dev/null +++ b/docker/glibc-package/README.md @@ -0,0 +1,39 @@ + + +# GLibc compatibility package + +This directory includes the Docker scripts to build an image with GLibc compiled for Alpine Linux. + +This is used to ensure plugins that are going to be used in the Pulsar image and that are depeding on GLibc, will +still be working correctly in the Alpine Image. (eg: Netty Tc-Native and Kinesis Producer Library). + +This image only needs to be re-created when we want to upgrade to a newer version of GLibc. + +# Steps + +1. Change the version in the Dockerfile for this directory. +2. Rebuild the image and push it to Docker Hub: +``` +docker buildx build --platform=linux/amd64,linux/arm64 -t apachepulsar/glibc-base:2.38 . --push +``` + +The image tag is then used in `docker/pulsar/Dockerfile`. diff --git a/docker/glibc-package/scripts/APKBUILD b/docker/glibc-package/scripts/APKBUILD new file mode 100644 index 0000000000000..484bdbf34e9f0 --- /dev/null +++ b/docker/glibc-package/scripts/APKBUILD @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +pkgname="glibc" +pkgrel="0" +pkgdesc="GNU C Library compatibility layer" +arch="all" +url="https:/pulsar.apache.org" +license="LGPL" +options="!check" +source="glibc-bin.tar.gz +ld.so.conf" +subpackages="${pkgname}-bin ${pkgname}-dev" +triggers="glibc-bin.trigger=/lib:/usr/lib:/usr/glibc-compat/lib" +depends="gcompat libuuid" + +package() { + mkdir -p $pkgdir/lib $pkgdir/usr/glibc-compat/lib/locale $pkgdir/usr/glibc-compat/lib64 $pkgdir/etc $pkgdir/usr/glibc-compat/etc/ + cp -a $srcdir/usr $pkgdir + cp $srcdir/ld.so.conf $pkgdir/usr/glibc-compat/etc/ld.so.conf + cd $pkgdir/usr/glibc-compat + rm -rf etc/rpc bin sbin lib/gconv lib/getconf lib/audit share var include + ln -s /usr/glibc-compat/lib/ld-linux-$(uname -m).so.1 $pkgdir/lib + ln -s /usr/glibc-compat/lib/ld-linux-$(uname -m).so.1 $pkgdir/usr/glibc-compat/lib64 + ln -s /usr/glibc-compat/etc/ld.so.cache $pkgdir/etc/ld.so.cache +} + +bin() { + depends="$pkgname bash libc6-compat libgcc" + mkdir -p $subpkgdir/usr/glibc-compat + cp -a $srcdir/usr/glibc-compat/bin $subpkgdir/usr/glibc-compat + cp -a $srcdir/usr/glibc-compat/sbin $subpkgdir/usr/glibc-compat +} + diff --git a/docker/pulsar/scripts/install-glibc-compat.sh b/docker/glibc-package/scripts/glibc-bin.trigger similarity index 62% rename from docker/pulsar/scripts/install-glibc-compat.sh rename to docker/glibc-package/scripts/glibc-bin.trigger index f871847ddd655..5bae5d7ca2bda 100755 --- a/docker/pulsar/scripts/install-glibc-compat.sh +++ b/docker/glibc-package/scripts/glibc-bin.trigger @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -18,18 +18,4 @@ # under the License. # -set -e -x - -ARCH=$(uname -m) -if [ "$ARCH" == "x86_64" ] -then - cd /tmp - echo 'https://storage.sev.monster/alpine/edge/testing' | tee -a /etc/apk/repositories - wget https://storage.sev.monster/alpine/edge/testing/x86_64/sevmonster-keys-1-r0.apk - apk add --no-cache --allow-untrusted ./sevmonster-keys-1-r0.apk - apk update - apk add gcompat libuuid - rm /lib/ld-linux-x86-64.so.2 - apk add --no-cache --force-overwrite glibc glibc-bin - rm *.apk -fi +/usr/glibc-compat/sbin/ldconfig \ No newline at end of file diff --git a/docker/glibc-package/scripts/ld.so.conf b/docker/glibc-package/scripts/ld.so.conf new file mode 100644 index 0000000000000..6548b9300bb9c --- /dev/null +++ b/docker/glibc-package/scripts/ld.so.conf @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +/usr/local/lib +/usr/glibc-compat/lib +/usr/lib +/lib diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 74eabdceb76e1..83d1647f8f137 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -79,6 +79,8 @@ RUN echo networkaddress.cache.ttl=1 >> /opt/jvm/conf/security/java.security RUN echo networkaddress.cache.negative.ttl=1 >> /opt/jvm/conf/security/java.security +FROM merlimat/glibc-base:2.38 as glibc + ## Create final stage from Alpine image ## and add OpenJDK and Python dependencies (for Pulsar functions) FROM alpine:3.19.1 @@ -91,6 +93,10 @@ RUN apk add --no-cache \ py3-pip \ ca-certificates +# Install GLibc compatibility library +COPY --from=glibc /root/packages /root/packages +RUN apk add --allow-untrusted --force-overwrite /root/packages/$(uname -m)/glibc-${GLIBC_VERSION}-r0.apk /root/packages/$(uname -m)/glibc-bin-${GLIBC_VERSION}-r0.apk + COPY --from=jvm /opt/jvm /opt/jvm ENV JAVA_HOME=/opt/jvm @@ -101,10 +107,6 @@ ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE COPY --from=pulsar /pulsar /pulsar -## Netty TC native and Kinesis producer library are based on a C/C++ binary linked with Glibc -## Install compatibility library -RUN /pulsar/bin/install-glibc-compat.sh - WORKDIR /pulsar ENV PATH=$PATH:$JAVA_HOME/bin:/pulsar/bin From 52dd22a5f676fb1830df24044d9d6e6a62ddaae6 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 4 Mar 2024 12:06:10 -0800 Subject: [PATCH 05/13] Always use numeric user id instead of name --- docker/pulsar/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 83d1647f8f137..0df2ce565c05f 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -113,4 +113,4 @@ ENV PATH=$PATH:$JAVA_HOME/bin:/pulsar/bin # The UID must be non-zero. Otherwise, it is arbitrary. No logic should rely on its specific value. ARG DEFAULT_USERNAME=pulsar RUN adduser ${DEFAULT_USERNAME} -u 10000 -G root -D -USER ${DEFAULT_USERNAME} +USER 10000 From 6cd2c20b280df9fbe0ae144ce88c48af6ff9c73d Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 4 Mar 2024 12:27:00 -0800 Subject: [PATCH 06/13] Added TODO disclaimer --- docker/pulsar/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 0df2ce565c05f..426d2daf58fb8 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -79,6 +79,7 @@ RUN echo networkaddress.cache.ttl=1 >> /opt/jvm/conf/security/java.security RUN echo networkaddress.cache.negative.ttl=1 >> /opt/jvm/conf/security/java.security +## TODO: use apachepulsar/glibc-base:2.38 once it's finalized FROM merlimat/glibc-base:2.38 as glibc ## Create final stage from Alpine image From 87c3f93c88f40d2ea8136281128f199d9eaf77a6 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 4 Mar 2024 13:52:59 -0800 Subject: [PATCH 07/13] Fixed use of non-defined variable --- docker/pulsar/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 426d2daf58fb8..49fbc811ad051 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -96,7 +96,7 @@ RUN apk add --no-cache \ # Install GLibc compatibility library COPY --from=glibc /root/packages /root/packages -RUN apk add --allow-untrusted --force-overwrite /root/packages/$(uname -m)/glibc-${GLIBC_VERSION}-r0.apk /root/packages/$(uname -m)/glibc-bin-${GLIBC_VERSION}-r0.apk +RUN apk add --allow-untrusted --force-overwrite /root/packages/$(uname -m)/glibc-2.38-r0.apk /root/packages/$(uname -m)/glibc-bin-2.38-r0.apk COPY --from=jvm /opt/jvm /opt/jvm ENV JAVA_HOME=/opt/jvm From 61e379f79c7e64e982b174e35c3e25d5b5b998ee Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 4 Mar 2024 21:36:43 -0800 Subject: [PATCH 08/13] Fixed linking in APK build --- docker/glibc-package/scripts/APKBUILD | 11 +++++++---- docker/pulsar/Dockerfile | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docker/glibc-package/scripts/APKBUILD b/docker/glibc-package/scripts/APKBUILD index 484bdbf34e9f0..a14744da47a6b 100644 --- a/docker/glibc-package/scripts/APKBUILD +++ b/docker/glibc-package/scripts/APKBUILD @@ -28,7 +28,7 @@ source="glibc-bin.tar.gz ld.so.conf" subpackages="${pkgname}-bin ${pkgname}-dev" triggers="glibc-bin.trigger=/lib:/usr/lib:/usr/glibc-compat/lib" -depends="gcompat libuuid" +depends="libuuid libgcc" package() { mkdir -p $pkgdir/lib $pkgdir/usr/glibc-compat/lib/locale $pkgdir/usr/glibc-compat/lib64 $pkgdir/etc $pkgdir/usr/glibc-compat/etc/ @@ -36,13 +36,16 @@ package() { cp $srcdir/ld.so.conf $pkgdir/usr/glibc-compat/etc/ld.so.conf cd $pkgdir/usr/glibc-compat rm -rf etc/rpc bin sbin lib/gconv lib/getconf lib/audit share var include - ln -s /usr/glibc-compat/lib/ld-linux-$(uname -m).so.1 $pkgdir/lib - ln -s /usr/glibc-compat/lib/ld-linux-$(uname -m).so.1 $pkgdir/usr/glibc-compat/lib64 + + FILENAME=$(ls $pkgdir/usr/glibc-compat/lib/ld-linux-*.so.*) + LIBNAME=$(basename $FILENAME) + ln -s /usr/glibc-compat/lib/$LIBNAME $pkgdir/lib/$LIBNAME + ln -s /usr/glibc-compat/lib/$LIBNAME $pkgdir/usr/glibc-compat/lib64/$LIBNAME ln -s /usr/glibc-compat/etc/ld.so.cache $pkgdir/etc/ld.so.cache } bin() { - depends="$pkgname bash libc6-compat libgcc" + depends="$pkgname" mkdir -p $subpkgdir/usr/glibc-compat cp -a $srcdir/usr/glibc-compat/bin $subpkgdir/usr/glibc-compat cp -a $srcdir/usr/glibc-compat/sbin $subpkgdir/usr/glibc-compat diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 49fbc811ad051..2a8551bb4a435 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -96,7 +96,7 @@ RUN apk add --no-cache \ # Install GLibc compatibility library COPY --from=glibc /root/packages /root/packages -RUN apk add --allow-untrusted --force-overwrite /root/packages/$(uname -m)/glibc-2.38-r0.apk /root/packages/$(uname -m)/glibc-bin-2.38-r0.apk +RUN apk add --allow-untrusted --force-overwrite /root/packages/$(uname -m)/glibc-2.38-r0.apk /root/packages/$(uname -m)/glibc-bin-2.38-r0.apk COPY --from=jvm /opt/jvm /opt/jvm ENV JAVA_HOME=/opt/jvm From 4499edade23902068ba5b108393a70ed1831eb32 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 5 Mar 2024 09:38:25 -0800 Subject: [PATCH 09/13] Upgrade KPL --- pulsar-io/kinesis/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index 7580905382f57..016fb421c5993 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -32,7 +32,7 @@ 2.2.8 - 0.14.13 + 0.15.9 0.13.0 1.9.0 2.3.0 From 470353e025fea14904981c65bb77e0b357f5e334 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 5 Mar 2024 13:19:38 -0800 Subject: [PATCH 10/13] Fixed deps order again --- docker/glibc-package/scripts/APKBUILD | 2 +- docker/pulsar/Dockerfile | 1 + pulsar-io/kinesis/pom.xml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/glibc-package/scripts/APKBUILD b/docker/glibc-package/scripts/APKBUILD index a14744da47a6b..0545508f0a7d4 100644 --- a/docker/glibc-package/scripts/APKBUILD +++ b/docker/glibc-package/scripts/APKBUILD @@ -45,7 +45,7 @@ package() { } bin() { - depends="$pkgname" + depends="$pkgname libc6-compat" mkdir -p $subpkgdir/usr/glibc-compat cp -a $srcdir/usr/glibc-compat/bin $subpkgdir/usr/glibc-compat cp -a $srcdir/usr/glibc-compat/sbin $subpkgdir/usr/glibc-compat diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 2a8551bb4a435..6cadceadbebba 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -92,6 +92,7 @@ RUN apk add --no-cache \ bash \ python3 \ py3-pip \ + gcompat \ ca-certificates # Install GLibc compatibility library diff --git a/pulsar-io/kinesis/pom.xml b/pulsar-io/kinesis/pom.xml index 016fb421c5993..7580905382f57 100644 --- a/pulsar-io/kinesis/pom.xml +++ b/pulsar-io/kinesis/pom.xml @@ -32,7 +32,7 @@ 2.2.8 - 0.15.9 + 0.14.13 0.13.0 1.9.0 2.3.0 From a969158c6a5b0d43fdc75e9ef1f423f4ad5fb5cc Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 6 Mar 2024 16:59:20 -0800 Subject: [PATCH 11/13] Shrink glibc image and keep only needed apk packages --- docker/glibc-package/Dockerfile | 10 +++++++++- docker/pulsar/Dockerfile | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docker/glibc-package/Dockerfile b/docker/glibc-package/Dockerfile index 935162c113a4f..f882f63cebe76 100644 --- a/docker/glibc-package/Dockerfile +++ b/docker/glibc-package/Dockerfile @@ -51,7 +51,7 @@ RUN tar --dereference --hard-dereference -zcf /glibc-bin.tar.gz /usr/glibc-compa ################################################ ## Build the APK package -FROM alpine:3.19 +FROM alpine:3.19 as apk ARG GLIBC_VERSION RUN apk add abuild sudo build-base @@ -68,3 +68,11 @@ RUN echo "sha512sums=\"$(sha512sum glibc-bin.tar.gz ld.so.conf)\"" >> /build/APK RUN abuild-keygen -a -i -n RUN abuild -F -c -r + +################################################ +## Last stage - Only leaves the packages +FROM busybox +ARG GLIBC_VERSION + +COPY --from=apk /root/packages/*/glibc-${GLIBC_VERSION}-r0.apk /root/packages +COPY --from=apk /root/packages/*/glibc-bin-${GLIBC_VERSION}-r0.apk /root/packages diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 6cadceadbebba..fab819b7da4af 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -97,7 +97,7 @@ RUN apk add --no-cache \ # Install GLibc compatibility library COPY --from=glibc /root/packages /root/packages -RUN apk add --allow-untrusted --force-overwrite /root/packages/$(uname -m)/glibc-2.38-r0.apk /root/packages/$(uname -m)/glibc-bin-2.38-r0.apk +RUN apk add --allow-untrusted --force-overwrite /root/packages/glibc-*.apk COPY --from=jvm /opt/jvm /opt/jvm ENV JAVA_HOME=/opt/jvm From 8bc1ef9dba6c4a7e0effbf0190fe6c69dd784be7 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 6 Mar 2024 17:22:52 -0800 Subject: [PATCH 12/13] Forgot to pre-create the directory --- docker/glibc-package/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/glibc-package/Dockerfile b/docker/glibc-package/Dockerfile index f882f63cebe76..f9c238cbdfc55 100644 --- a/docker/glibc-package/Dockerfile +++ b/docker/glibc-package/Dockerfile @@ -74,5 +74,6 @@ RUN abuild -F -c -r FROM busybox ARG GLIBC_VERSION +RUN mkdir -p /root/packages COPY --from=apk /root/packages/*/glibc-${GLIBC_VERSION}-r0.apk /root/packages COPY --from=apk /root/packages/*/glibc-bin-${GLIBC_VERSION}-r0.apk /root/packages From 8e4afd85d0a7a654d971d40f0a46aeb2e09c82e5 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 8 Mar 2024 10:12:43 -0800 Subject: [PATCH 13/13] Pulling from the official docker org --- docker/pulsar/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index fab819b7da4af..17d59d367c3f3 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -79,8 +79,7 @@ RUN echo networkaddress.cache.ttl=1 >> /opt/jvm/conf/security/java.security RUN echo networkaddress.cache.negative.ttl=1 >> /opt/jvm/conf/security/java.security -## TODO: use apachepulsar/glibc-base:2.38 once it's finalized -FROM merlimat/glibc-base:2.38 as glibc +FROM apachepulsar/glibc-base:2.38 as glibc ## Create final stage from Alpine image ## and add OpenJDK and Python dependencies (for Pulsar functions)