Skip to content

Commit

Permalink
[improve][pip] PIP-324: Alpine image
Browse files Browse the repository at this point in the history
  • Loading branch information
merlimat committed Feb 14, 2024
1 parent beed0cf commit f2a91a1
Show file tree
Hide file tree
Showing 16 changed files with 120 additions and 416 deletions.
13 changes: 1 addition & 12 deletions .github/workflows/pulsar-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -506,15 +506,10 @@ jobs:
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
- name: Pick ubuntu mirror for the docker image build
run: |
# pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror
- name: Build java-test-image docker image
run: |
# build docker image
mvn -B -am -pl tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
mvn -B -am -pl docker/pulsar,tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
-Dmaven.test.skip=true -Ddocker.squash=true -DskipSourceReleaseAssembly=true \
-Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
Expand Down Expand Up @@ -866,17 +861,11 @@ jobs:
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
- name: Pick ubuntu mirror for the docker image build
run: |
# pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh pick_ubuntu_mirror
- name: Build latest-version-image docker image
run: |
# build docker image
# include building of Connectors, Offloaders and server distros
mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
-DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" -DIMAGE_JDK_MAJOR_VERSION="${IMAGE_JDK_MAJOR_VERSION}" \
-Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \
-Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
Expand Down
1 change: 0 additions & 1 deletion build/build_java_test_image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,5 @@ if [[ "$(docker version -f '{{.Server.Experimental}}' 2>/dev/null)" == "true" ]]
SQUASH_PARAM="-Ddocker.squash=true"
fi
mvn -am -pl tests/docker-images/java-test-image -Pcore-modules,-main,integrationTests,docker \
-DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \
-Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true $SQUASH_PARAM \
"$@" install
100 changes: 0 additions & 100 deletions build/docker/Dockerfile

This file was deleted.

48 changes: 0 additions & 48 deletions build/docker/README.md

This file was deleted.

57 changes: 0 additions & 57 deletions build/docker/publish.sh

This file was deleted.

48 changes: 1 addition & 47 deletions build/pulsar_ci_tool.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,59 +46,14 @@ function ci_print_thread_dumps() {

# runs maven
function _ci_mvn() {
mvn -B -ntp -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \
"$@"
mvn -B -ntp "$@"
}

# runs OWASP Dependency Check for all projects
function ci_dependency_check() {
_ci_mvn -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl '!pulsar-client-tools-test' "$@"
}

# Finds fastest up-to-date ubuntu mirror based on download speed
function ci_find_fast_ubuntu_mirror() {
local ubuntu_release=${1:-"$(lsb_release -c 2>/dev/null | cut -f2 || echo "jammy")"}
local ubuntu_arch=${2:-"$(dpkg --print-architecture 2>/dev/null || echo "amd64")"}
{
# choose mirrors that are up-to-date by checking the Last-Modified header for
{
# randomly choose up to 10 mirrors using http:// protocol
# (https isn't supported in docker containers that don't have ca-certificates installed)
curl -s http://mirrors.ubuntu.com/mirrors.txt | grep '^http://' | shuf -n 10
# also consider Azure's Ubuntu mirror
echo http://azure.archive.ubuntu.com/ubuntu/
} | xargs -I {} sh -c "ubuntu_release=$ubuntu_release ubuntu_arch=$ubuntu_arch;"'echo "$(curl -m 5 -sI {}dists/${ubuntu_release}/Contents-${ubuntu_arch}.gz|sed s/\\r\$//|grep Last-Modified|awk -F": " "{ print \$2 }" | LANG=C date -f- -u +%s)" "{}"' | sort -rg | awk '{ if (NR==1) TS=$1; if ($1 == TS) print $2 }'
} | xargs -I {} sh -c 'echo `curl -r 0-102400 -m 5 -s -w %{speed_download} -o /dev/null {}ls-lR.gz` {}' \
|sort -g -r |head -1| awk '{ print $2 }'
}

function ci_pick_ubuntu_mirror() {
echo "Choosing fastest up-to-date ubuntu mirror based on download speed..."
UBUNTU_MIRROR=$(ci_find_fast_ubuntu_mirror)
if [ -z "$UBUNTU_MIRROR" ]; then
# fallback to no mirror
UBUNTU_MIRROR="http://archive.ubuntu.com/ubuntu/"
UBUNTU_SECURITY_MIRROR="http://security.ubuntu.com/ubuntu/"
else
UBUNTU_SECURITY_MIRROR="${UBUNTU_MIRROR}"
fi
OLD_MIRROR=$(cat /etc/apt/sources.list | grep '^deb ' | head -1 | awk '{ print $2 }')
echo "Picked '$UBUNTU_MIRROR'. Current mirror is '$OLD_MIRROR'."
if [[ "$OLD_MIRROR" != "$UBUNTU_MIRROR" ]]; then
sudo sed -i "s|$OLD_MIRROR|$UBUNTU_MIRROR|g" /etc/apt/sources.list
sudo apt-get update
fi
# set the chosen mirror also in the UBUNTU_MIRROR and UBUNTU_SECURITY_MIRROR environment variables
# that can be used by docker builds
export UBUNTU_MIRROR
export UBUNTU_SECURITY_MIRROR
# make environment variables available for later GitHub Actions steps
if [ -n "$GITHUB_ENV" ]; then
echo "UBUNTU_MIRROR=$UBUNTU_MIRROR" >> $GITHUB_ENV
echo "UBUNTU_SECURITY_MIRROR=$UBUNTU_SECURITY_MIRROR" >> $GITHUB_ENV
fi
}

# installs a tool executable if it's not found on the PATH
function ci_install_tool() {
local tool_executable=$1
Expand All @@ -108,7 +63,6 @@ function ci_install_tool() {
echo "::group::Installing ${tool_package}"
sudo apt-get -y install ${tool_package} >/dev/null || {
echo "Installing the package failed. Switching the ubuntu mirror and retrying..."
ci_pick_ubuntu_mirror
# retry after picking the ubuntu mirror
sudo apt-get -y install ${tool_package}
}
Expand Down
Loading

0 comments on commit f2a91a1

Please sign in to comment.