-
Notifications
You must be signed in to change notification settings - Fork 313
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Buildkite pipeline for building dev docker images [ES-9318] (#1873
) * feat: Change Dockerfile base images to Wolfi Running the docker image with a mounted `/rally/.rally` directory that already contains the configuration file didn't work. So I had to change how we parse the logging config and rally.ini files. Now we delay the parsing of the environment variables to read time, to make running from docker and outside of docker compatible with each other. Added buildkite jobs to build both release and development docker images
- Loading branch information
Showing
14 changed files
with
564 additions
and
76 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,53 @@ | ||
| steps: | ||
| - label: ":wave: Greetings" # Label (with rich emojis https://ela.st/bk-emoji). | ||
| command: "echo 'My first pipeline!'" # Command to run (evaluated by Bash). | ||
| - input: "Build parameters" | ||
| if: build.source != "schedule" | ||
| fields: | ||
| - text: "BUILD_FROM_BRANCH" | ||
| key: "BUILD_FROM_BRANCH" | ||
| default: "" | ||
| hint: "The branch to build from e.g. 'master'. Leave blank to build from the current branch: $BUILDKITE_BRANCH." | ||
| required: false | ||
| - select: "PUBLIC_DOCKER_REPO" | ||
| key: "PUBLIC_DOCKER_REPO" | ||
| hint: "Push the Docker image to the public Docker registry (default: No)." | ||
| default: "false" | ||
| options: | ||
| - label: "Yes" | ||
| value: "true" | ||
| - label: "No" | ||
| value: "false" | ||
| - select: "PUSH_LATEST" | ||
| key: "PUSH_LATEST" | ||
| hint: "Push the <branch>-latest tag to the registry." | ||
| default: "true" | ||
| options: | ||
| - label: "Yes" | ||
| value: "true" | ||
| - label: "No" | ||
| value: "false" | ||
| - wait | ||
| - label: ":docker: Build Docker Artifacts for Rally amd64" | ||
| command: bash .buildkite/dev-docker/run.sh build amd64 | ||
| key: "amd64" | ||
| agents: | ||
| machineType: "n2-standard-8" | ||
| image: family/core-ubuntu-2204 | ||
| zone: "us-central1-a" | ||
| provider: "gcp" | ||
| - label: ":docker: Build Docker Artifacts for Rally arm64" | ||
| command: bash .buildkite/dev-docker/run.sh build arm64 | ||
| key: "arm64" | ||
| agents: | ||
| machineType: "t2a-standard-8" | ||
| image: family/core-ubuntu-2204-aarch64 | ||
| zone: "us-central1-a" | ||
| provider: "gcp" | ||
| - label: ":docker: build docker manifest" | ||
| command: bash .buildkite/dev-docker/run.sh manifest both | ||
| key: "manifest" | ||
| depends_on: | ||
| - "amd64" | ||
| - "arm64" | ||
| agents: | ||
| zone: "us-central1-a" | ||
| provider: "gcp" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,14 +1,43 @@ | ||
| agents: | ||
| provider: "gcp" | ||
| zone: "us-central1-a" | ||
|
|
||
| steps: | ||
| - input: "Build parameters" | ||
| fields: | ||
| - text: "RELEASE_VERSION" | ||
| key: "RELEASE_VERSION" | ||
| default: "" | ||
| hint: "The version to release e.g. '2.8.0'." | ||
| - select: "PUSH_LATEST" | ||
| key: "PUSH_LATEST" | ||
| hint: "Update the latest tag in the registry." | ||
| # True, because we want the scheduled pipeline to update the latest tag, eventually | ||
| default: "true" | ||
| options: | ||
| - label: "Yes" | ||
| value: "true" | ||
| - label: "No" | ||
| value: "false" | ||
|
|
||
| - wait | ||
| - label: "Release Docker Artifacts for Rally" | ||
| command: bash .buildkite/release-docker/run.sh | ||
| command: bash .buildkite/release-docker/run.sh build amd64 | ||
| # Run on GCP to use `docker` | ||
| key: "amd64" | ||
| agents: | ||
| machineType: "n2-standard-8" | ||
| image: family/core-ubuntu-2204 | ||
| - label: "Release Docker Artifacts for Rally" | ||
| command: bash .buildkite/release-docker/run.sh build arm64 | ||
| # Run on GCP to use `docker` | ||
| key: "arm64" | ||
| agents: | ||
| provider: gcp | ||
| machineType: "t2a-standard-8" | ||
| image: family/core-ubuntu-2204-aarch64 | ||
| - label: ":docker: build docker manifest" | ||
| command: bash .buildkite/release-docker/run.sh manifest | ||
| key: "manifest" | ||
| depends_on: | ||
| - "amd64" | ||
| - "arm64" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Licensed to Elasticsearch B.V. under one or more contributor | ||
| # license agreements. See the NOTICE file distributed with | ||
| # this work for additional information regarding copyright | ||
| # ownership. Elasticsearch B.V. licenses this file to you under | ||
| # the Apache License, Version 2.0 (the "License"); you may | ||
| # not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
|
|
||
| # Prerequisites for releasing: | ||
|
|
||
| # Logged in on Docker Hub (docker login) | ||
|
|
||
| # fail this script immediately if any command fails with a non-zero exit code | ||
| set -eu | ||
|
|
||
| function push_failed { | ||
| echo "Error while pushing Docker image. Did you \`docker login\`?" | ||
| } | ||
|
|
||
| if [[ $# -eq 0 ]] ; then | ||
| echo "ERROR: $0 requires the Rally branch to build from as a command line argument and you didn't supply it." | ||
| echo "For example: $0 master true" | ||
| exit 1 | ||
| fi | ||
| export RALLY_BRANCH=$1 | ||
| export PUSH_LATEST=$2 | ||
| export PUBLIC_DOCKER_REPO=$3 | ||
| if [[ $PUBLIC_DOCKER_REPO == "true" ]]; then | ||
| export RALLY_DOCKER_IMAGE="elastic/rally" | ||
| else | ||
| export RALLY_DOCKER_IMAGE="docker.elastic.co/es-perf/rally" | ||
| fi | ||
|
|
||
| export RALLY_LICENSE=$(awk 'FNR>=2 && FNR<=2' LICENSE | sed 's/^[ \t]*//') | ||
|
|
||
| export GIT_SHA=$(git rev-parse --short HEAD) | ||
| export DATE=$(date +%Y%m%d) | ||
|
|
||
| export RALLY_VERSION="${RALLY_BRANCH}-${GIT_SHA}-${DATE}" | ||
| export MAIN_BRANCH=$(git remote show origin | sed -n '/HEAD branch/s/.*: //p') | ||
|
|
||
| if [[ $RALLY_BRANCH == $MAIN_BRANCH ]]; then | ||
| export DOCKER_TAG_LATEST="dev-latest" | ||
| else | ||
| export DOCKER_TAG_LATEST="${RALLY_BRANCH}-latest" | ||
| fi | ||
|
|
||
| echo "========================================================" | ||
| echo "Pulling Docker images for Rally $RALLY_VERSION " | ||
| echo "========================================================" | ||
|
|
||
| docker pull ${RALLY_DOCKER_IMAGE}:${RALLY_VERSION}-amd64 | ||
| docker pull ${RALLY_DOCKER_IMAGE}:${RALLY_VERSION}-arm64 | ||
|
|
||
| echo "=======================================================" | ||
| echo "Creating Docker manifest image for Rally $RALLY_VERSION" | ||
| echo "=======================================================" | ||
|
|
||
| docker manifest create ${RALLY_DOCKER_IMAGE}:${RALLY_VERSION} \ | ||
| --amend ${RALLY_DOCKER_IMAGE}:${RALLY_VERSION}-amd64 \ | ||
| --amend ${RALLY_DOCKER_IMAGE}:${RALLY_VERSION}-arm64 | ||
|
|
||
| trap push_failed ERR | ||
| echo "=======================================================" | ||
| echo "Publishing Docker image ${RALLY_DOCKER_IMAGE}:$RALLY_VERSION " | ||
| echo "=======================================================" | ||
| docker manifest push ${RALLY_DOCKER_IMAGE}:${RALLY_VERSION} | ||
|
|
||
| trap - ERR | ||
|
|
||
| if [[ $PUSH_LATEST == "true" ]]; then | ||
| echo "=======================================================" | ||
| echo "Creating Docker manifest image for Rally $DOCKER_TAG_LATEST" | ||
| echo "=======================================================" | ||
|
|
||
| docker manifest create ${RALLY_DOCKER_IMAGE}:${DOCKER_TAG_LATEST} \ | ||
| --amend ${RALLY_DOCKER_IMAGE}:${DOCKER_TAG_LATEST}-amd64 \ | ||
| --amend ${RALLY_DOCKER_IMAGE}:${DOCKER_TAG_LATEST}-arm64 | ||
|
|
||
| trap push_failed ERR | ||
| echo "=======================================================" | ||
| echo "Publishing Docker image ${RALLY_DOCKER_IMAGE}:${DOCKER_TAG_LATEST}" | ||
| echo "=======================================================" | ||
| docker manifest push ${RALLY_DOCKER_IMAGE}:${DOCKER_TAG_LATEST} | ||
| fi | ||
|
|
||
| trap - ERR |
Oops, something went wrong.