fix: Actually pull from the BUILD_FROM_BRANCH branch when set.

also, don't push to the public docker.io registry by default.
elastic · Oct 10, 2024 · d66eb55 · d66eb55
1 parent 2c26641
commit d66eb55
Show file tree

Hide file tree

Showing 5 changed files with 78 additions and 28 deletions.
diff --git a/.buildkite/dev-docker/manifest.sh b/.buildkite/dev-docker/manifest.sh
@@ -7,7 +7,11 @@ source .buildkite/retry.sh
 set +x
 
 BUILD_FROM_BRANCH=$(buildkite-agent meta-data get BUILD_FROM_BRANCH)
+if [[ -z "${BUILD_FROM_BRANCH}" ]]; then
+    BUILD_FROM_BRANCH=${BUILDKITE_BRANCH}
+fi
 PUSH_LATEST=$(buildkite-agent meta-data get PUSH_LATEST)
+PUBLIC_DOCKER_REPO=$(buildkite-agent meta-data get PUBLIC_DOCKER_REPO)
 
 # login to docker registry
 DOCKER_PASSWORD=$(vault read -field token /secret/ci/elastic-rally/release/docker-hub-rally)
@@ -17,4 +21,4 @@ unset DOCKER_PASSWORD
 set -x
 export TERM=dumb
 export LC_ALL=en_US.UTF-8
-./build-dev-docker-manifest.sh "$BUILD_FROM_BRANCH" "$PUSH_LATEST"
+./build-dev-docker-manifest.sh "$BUILD_FROM_BRANCH" "$PUSH_LATEST" "$PUBLIC_DOCKER_REPO"
diff --git a/.buildkite/dev-docker/pipeline.yml b/.buildkite/dev-docker/pipeline.yml
@@ -4,8 +4,17 @@ steps:
     fields:
       - text: "BUILD_FROM_BRANCH"
         key: "BUILD_FROM_BRANCH"
-        default: "master"
-        hint: "The branch to build from e.g. 'master'."
+        default: ""
+        hint: "The branch to build from e.g. 'master'. Leave blank to build from the current branch: $BUILDKITE_BRANCH."
+      - select: "PUBLIC_DOCKER_REPO"
+        key: "PUBLIC_DOCKER_REPO"
+        hint: "Push the Docker image to the public Docker registry (default: No)."
+        default: "false"
+        options:
+          - label: "Yes"
+            value: "true"
+          - label: "No"
+            value: "false"
       - select: "PUSH_LATEST"
         key: "PUSH_LATEST"
         hint: "Push the <branch>-latest tag to the registry."

diff --git a/.buildkite/dev-docker/run.sh b/.buildkite/dev-docker/run.sh
@@ -7,7 +7,11 @@ source .buildkite/retry.sh
 set +x
 
 BUILD_FROM_BRANCH=$(buildkite-agent meta-data get BUILD_FROM_BRANCH)
+if [[ -z "${BUILD_FROM_BRANCH}" ]]; then
+    BUILD_FROM_BRANCH=${BUILDKITE_BRANCH}
+fi
 PUSH_LATEST=$(buildkite-agent meta-data get PUSH_LATEST)
+PUBLIC_DOCKER_REPO=$(buildkite-agent meta-data get PUBLIC_DOCKER_REPO)
 
 if [[ $# -ne 1 ]]; then
     echo "Usage: $0 <arch>"
@@ -17,9 +21,20 @@ fi
 ARCH="$1"
 
 # login to docker registry
-DOCKER_PASSWORD=$(vault read -field token /secret/ci/elastic-rally/release/docker-hub-rally)
-retry 5 docker login -u elasticmachine -p $DOCKER_PASSWORD
+if [[ $PUBLIC_DOCKER_REPO == "true" ]]; then
+    VAULT_PATH="secret/ci/elastic-rally/release/docker-hub-rally"
+    DOCKER_REGISTRY="docker.io"
+    DOCKER_PASSWORD=$(vault read -field token "${VAULT_PATH}")
+else
+    VAULT_PATH="/secret/ci/elastic-elasticsearch-benchmarks/employees/cloud/docker-registry-api-credentials"
+    DOCKER_REGISTRY="docker.elastic.co"
+    DOCKER_PASSWORD=$(vault read -field password "${VAULT_PATH}")
+fi
+
+DOCKER_USERNAME=$(vault read -field username "${VAULT_PATH}")
+retry 5 docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} ${DOCKER_USERNAME}
 unset DOCKER_PASSWORD
+unset DOCKER_USERNAME
 
 tmp_dir=$(mktemp --directory)
 pushd "$tmp_dir"
@@ -36,7 +51,7 @@ git --no-pager show
 set -x
 export TERM=dumb
 export LC_ALL=en_US.UTF-8
-./build-dev-docker.sh "$BUILD_FROM_BRANCH" "$ARCH" "$PUSH_LATEST"
+./build-dev-docker.sh "$BUILD_FROM_BRANCH" "$ARCH" "$PUSH_LATEST" "$PUBLIC_DOCKER_REPO"
 
 popd
 popd

diff --git a/build-dev-docker-manifest.sh b/build-dev-docker-manifest.sh
@@ -29,12 +29,19 @@ function push_failed {
 }
 
 if [[ $# -eq 0 ]] ; then
-    echo "ERROR: $0 requires the Rally version to push as a command line argument and you didn't supply it."
-    echo "For example: $0 master"
+    echo "ERROR: $0 requires the Rally branch to build from as a command line argument and you didn't supply it."
+    echo "For example: $0 master true"
     exit 1
 fi
 export RALLY_BRANCH=$1
 export PUSH_LATEST=$2
+export PUBLIC_DOCKER_REPO=$3
+if [[ $PUBLIC_DOCKER_REPO == "true" ]]; then
+    export DOCKER_IMAGE="elastic/rally"
+else
+    export DOCKER_IMAGE="docker.elastic.co/employees/es-perf/rally"
+fi
+
 export RALLY_LICENSE=$(awk 'FNR>=2 && FNR<=2' LICENSE | sed 's/^[ \t]*//')
 
 export GIT_SHA=$(git rev-parse --short HEAD)
@@ -53,22 +60,22 @@ echo "========================================================"
 echo "Pulling Docker images for Rally $RALLY_VERSION          "
 echo "========================================================"
 
-docker pull elastic/rally:${RALLY_VERSION}-amd64
-docker pull elastic/rally:${RALLY_VERSION}-arm64
+docker pull ${DOCKER_IMAGE}:${RALLY_VERSION}-amd64
+docker pull ${DOCKER_IMAGE}:${RALLY_VERSION}-arm64
 
 echo "======================================================="
 echo "Creating Docker manifest image for Rally $RALLY_VERSION"
 echo "======================================================="
 
-docker manifest create elastic/rally:${RALLY_VERSION} \
-    --amend elastic/rally:${RALLY_VERSION}-amd64 \
-    --amend elastic/rally:${RALLY_VERSION}-arm64
+docker manifest create ${DOCKER_IMAGE}:${RALLY_VERSION} \
+    --amend ${DOCKER_IMAGE}:${RALLY_VERSION}-amd64 \
+    --amend ${DOCKER_IMAGE}:${RALLY_VERSION}-arm64
 
 trap push_failed ERR
 echo "======================================================="
-echo "Publishing Docker image elastic/rally:$RALLY_VERSION   "
+echo "Publishing Docker image ${DOCKER_IMAGE}:$RALLY_VERSION   "
 echo "======================================================="
-docker manifest push elastic/rally:${RALLY_VERSION}
+docker manifest push ${DOCKER_IMAGE}:${RALLY_VERSION}
 
 trap - ERR
 
@@ -77,15 +84,15 @@ if [[ $PUSH_LATEST == "true" ]]; then
     echo "Creating Docker manifest image for Rally $DOCKER_TAG_LATEST"
     echo "======================================================="
 
-    docker manifest create elastic/rally:${DOCKER_TAG_LATEST} \
-        --amend elastic/rally:${DOCKER_TAG_LATEST}-amd64 \
-        --amend elastic/rally:${DOCKER_TAG_LATEST}-arm64
+    docker manifest create ${DOCKER_IMAGE}:${DOCKER_TAG_LATEST} \
+        --amend ${DOCKER_IMAGE}:${DOCKER_TAG_LATEST}-amd64 \
+        --amend ${DOCKER_IMAGE}:${DOCKER_TAG_LATEST}-arm64
 
     trap push_failed ERR
     echo "======================================================="
-    echo "Publishing Docker image elastic/rally:${DOCKER_TAG_LATEST}"
+    echo "Publishing Docker image ${DOCKER_IMAGE}:${DOCKER_TAG_LATEST}"
     echo "======================================================="
-    docker manifest push elastic/rally:${DOCKER_TAG_LATEST}
+    docker manifest push ${DOCKER_IMAGE}:${DOCKER_TAG_LATEST}
 fi
 
 trap - ERR
diff --git a/build-dev-docker.sh b/build-dev-docker.sh
@@ -28,15 +28,22 @@ function push_failed {
     echo "Error while pushing Docker image. Did you \`docker login\`?"
 }
 
-if [[ $# -ne 3 ]] ; then
+if [[ $# -ne 4 ]] ; then
     echo "ERROR: $0 requires the Rally branch to build, the architecture, and whether to push the latest \
         as command line arguments and they weren't supplied."
-    echo "For example: $0 master amd64 true"
+    echo "For example: $0 master amd64 true true"
     exit 1
 fi
 export RALLY_BRANCH=$1
 export ARCH=$2
 export PUSH_LATEST=$3
+export PUBLIC_DOCKER_REPO=$4
+
+if [[ $PUBLIC_DOCKER_REPO == "true" ]]; then
+    export DOCKER_IMAGE="elastic/rally"
+else
+    export DOCKER_IMAGE="docker.elastic.co/employees/es-perf/rally"
+fi
 
 export RALLY_LICENSE=$(awk 'FNR>=2 && FNR<=2' LICENSE | sed 's/^[ \t]*//')
 
@@ -53,11 +60,19 @@ else
     export DOCKER_TAG_LATEST="${RALLY_BRANCH}-latest-${ARCH}"
 fi
 
+# Make new temporary directory to checkout the `RALLY_BRANCH` branch
+tmp_dir=$(mktemp --directory)
+pushd "$tmp_dir"
+git clone https://github.com/elastic/rally
+popd
+rally_dir="${tmp_dir}/rally"
+
+
 echo "========================================================"
 echo "Building Docker image for Rally $RALLY_VERSION          "
 echo "========================================================"
 
-docker build -t elastic/rally:${RALLY_VERSION} --build-arg RALLY_VERSION --build-arg RALLY_LICENSE -f docker/Dockerfiles/dev/Dockerfile $PWD
+docker build -t ${DOCKER_IMAGE}:${RALLY_VERSION} --build-arg RALLY_VERSION --build-arg RALLY_LICENSE -f docker/Dockerfiles/dev/Dockerfile ${rally_dir}
 
 echo "======================================================="
 echo "Testing Docker image for Rally release $RALLY_VERSION  "
@@ -66,19 +81,19 @@ echo "======================================================="
 ./release-docker-test.sh dev
 
 echo "======================================================="
-echo "Publishing Docker image elastic/rally:$RALLY_VERSION   "
+echo "Publishing Docker image ${DOCKER_IMAGE}:$RALLY_VERSION   "
 echo "======================================================="
 
 trap push_failed ERR
-docker push elastic/rally:${RALLY_VERSION}
+docker push ${DOCKER_IMAGE}:${RALLY_VERSION}
 
 if [[ $PUSH_LATEST == "true" ]]; then
     echo "============================================"
-    echo "Publishing Docker image elastic/rally:${DOCKER_TAG_LATEST}"
+    echo "Publishing Docker image ${DOCKER_IMAGE}:${DOCKER_TAG_LATEST}"
     echo "============================================"
 
-    docker tag elastic/rally:${RALLY_VERSION} elastic/rally:${DOCKER_TAG_LATEST}
-    docker push elastic/rally:${DOCKER_TAG_LATEST}
+    docker tag ${DOCKER_IMAGE}:${RALLY_VERSION} ${DOCKER_IMAGE}:${DOCKER_TAG_LATEST}
+    docker push ${DOCKER_IMAGE}:${DOCKER_TAG_LATEST}
 fi
 
 trap - ERR