From 26b5a7ccd74802d94a770f70459261394f939c6a Mon Sep 17 00:00:00 2001 From: Charith Ellawala Date: Wed, 4 Nov 2020 11:25:38 +0000 Subject: [PATCH 1/3] Add Docker multi-arch support --- Dockerfile | 4 +++- Makefile | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2a1c5e6075..6aa826c4d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ # Build the operator binary FROM golang:1.15.4 as builder +ARG TARGETPLATFORM +ARG BUILDPLATFORM ARG GO_LDFLAGS ARG GO_TAGS WORKDIR /go/src/github.com/elastic/cloud-on-k8s @@ -15,7 +17,7 @@ COPY pkg/ pkg/ COPY cmd/ cmd/ # Build -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ +RUN CGO_ENABLED=0 GOOS=linux \ go build \ -mod readonly \ -ldflags "$GO_LDFLAGS" -tags="$GO_TAGS" -a \ diff --git a/Makefile b/Makefile index 36aec055f0..e6033f0def 100644 --- a/Makefile +++ b/Makefile @@ -344,10 +344,12 @@ switch-eks: ################################# docker-build: go-generate generate-config-file - docker build . \ + docker buildx build . \ --build-arg GO_LDFLAGS='$(GO_LDFLAGS)' \ --build-arg GO_TAGS='$(GO_TAGS)' \ --build-arg VERSION='$(VERSION)' \ + --platform linux/amd64,linux/arm64 \ + --push \ -t $(OPERATOR_IMAGE) docker-push: From 9cf04e4dd2e7f1c73484cd8f0d4b634be2a28bd6 Mon Sep 17 00:00:00 2001 From: Charith Ellawala Date: Thu, 5 Nov 2020 09:02:30 +0000 Subject: [PATCH 2/3] Add multi-arch build to nightly WIP WIP WIP Force docker login WIP Install buildx qemu Script steps WIP Increase timeout Multi-arch e2e image Increase timeout Avoid double build Plain progress Rebase Fix Docker script Add ARM build to nightly --- .ci/Dockerfile | 12 ++++-- .ci/Makefile | 2 +- Dockerfile | 2 +- Makefile | 36 +++++++++++++--- hack/{docker-push.sh => docker.sh} | 69 +++++++++++++++++++++++++++--- hack/manifest-gen/Makefile | 2 +- test/e2e/Dockerfile | 4 +- 7 files changed, 108 insertions(+), 19 deletions(-) rename hack/{docker-push.sh => docker.sh} (56%) diff --git a/.ci/Dockerfile b/.ci/Dockerfile index aa5caed432..74db362c65 100644 --- a/.ci/Dockerfile +++ b/.ci/Dockerfile @@ -6,7 +6,8 @@ ENV SHELLCHECK_VERSION=0.7.1 ENV KUBEBUILDER_VERSION=2.3.1 ENV GCLOUD_VERSION=297.0.1 ENV KUBECTL_VERSION=1.14.7 -ENV DOCKER_VERSION=19.03.9 +ENV DOCKER_VERSION=19.03.13 +ENV DOCKER_BUILDX_VERSION=0.4.2 ENV GOTESTSUM_VERSION=0.5.0 ENV KIND_VERSION=0.8.1 ENV OPENSHIFT_TOOLS_VERSION=4.3.19 @@ -43,9 +44,14 @@ RUN curl -fsSLO https://download.docker.com/linux/static/stable/x86_64/docker-${ tar xzf docker-${DOCKER_VERSION}.tgz --strip 1 -C /usr/local/bin docker/docker && \ rm docker-${DOCKER_VERSION}.tgz -# xz-utils to decompress shellcheck and unzip for aws-cli +# Docker buildx extension for building multi-arch images +RUN mkdir -p ~/.docker/cli-plugins && \ + curl -fsSLo ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v${DOCKER_BUILDX_VERSION}/buildx-v${DOCKER_BUILDX_VERSION}.linux-amd64 && \ + chmod a+x ~/.docker/cli-plugins/docker-buildx + +# xz-utils to decompress shellcheck, unzip for aws-cli, qemu-system-arm and qemu-user-static for multi-arch RUN apt-get update && apt-get --no-install-recommends -y install \ - unzip xz-utils && \ + unzip xz-utils qemu-system-arm qemu-user-static && \ apt-get clean && apt-get autoclean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/.ci/Makefile b/.ci/Makefile index 988551df3e..2a3bd1bfc0 100644 --- a/.ci/Makefile +++ b/.ci/Makefile @@ -67,7 +67,7 @@ ci-build-image: write-ci-docker-creds -t $(CI_IMAGE) \ --label "commit.hash=$(shell git rev-parse --short --verify HEAD)" \ $(ROOT_DIR) && \ - ../hack/docker-push.sh $(CI_IMAGE) \ + ../hack/docker.sh -l -p $(CI_IMAGE) \ ) # make Docker creds available from inside the CI container through the .registry.env file diff --git a/Dockerfile b/Dockerfile index 6aa826c4d2..3b727ca26f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the operator binary -FROM golang:1.15.4 as builder +FROM --platform=$BUILDPLATFORM golang:1.15.4 as builder ARG TARGETPLATFORM ARG BUILDPLATFORM diff --git a/Makefile b/Makefile index e6033f0def..2260b8d74c 100644 --- a/Makefile +++ b/Makefile @@ -197,6 +197,11 @@ build-operator-image: && echo "OK: image $(OPERATOR_IMAGE) already published" \ || $(MAKE) docker-build docker-push +build-operator-multiarch-image: + @ docker buildx imagetools inspect $(OPERATOR_IMAGE) | grep -q 'linux/arm64' 2>&1 >/dev/null \ + && echo "OK: image $(OPERATOR_IMAGE) already published" \ + || $(MAKE) docker-multiarch-build + # if the current k8s cluster is on GKE, GCLOUD_PROJECT must be set check-gke: ifneq ($(findstring gke_,$(KUBECTL_CLUSTER)),) @@ -342,9 +347,10 @@ switch-eks: ################################# ## -- Docker images -- ## ################################# - -docker-build: go-generate generate-config-file +docker-multiarch-build: go-generate generate-config-file + @ hack/docker.sh -l -m $(OPERATOR_IMAGE) docker buildx build . \ + --progress=plain \ --build-arg GO_LDFLAGS='$(GO_LDFLAGS)' \ --build-arg GO_TAGS='$(GO_TAGS)' \ --build-arg VERSION='$(VERSION)' \ @@ -352,8 +358,16 @@ docker-build: go-generate generate-config-file --push \ -t $(OPERATOR_IMAGE) +docker-build: go-generate generate-config-file + DOCKER_BUILDKIT=1 docker build . \ + --progress=plain \ + --build-arg GO_LDFLAGS='$(GO_LDFLAGS)' \ + --build-arg GO_TAGS='$(GO_TAGS)' \ + --build-arg VERSION='$(VERSION)' \ + -t $(OPERATOR_IMAGE) + docker-push: - @ hack/docker-push.sh $(OPERATOR_IMAGE) + @ hack/docker.sh -l -p $(OPERATOR_IMAGE) purge-gcr-images: @ for i in $(gcloud container images list-tags $(BASE_IMG) | tail +3 | awk '{print $$2}'); \ @@ -386,10 +400,20 @@ E2E_DEPLOY_CHAOS_JOB ?= false # clean to remove irrelevant/build-breaking generated public keys e2e-docker-build: clean - docker build --build-arg E2E_JSON=$(E2E_JSON) -t $(E2E_IMG) -f test/e2e/Dockerfile . + DOCKER_BUILDKIT=1 docker build --progress=plain --build-arg E2E_JSON=$(E2E_JSON) -t $(E2E_IMG) -f test/e2e/Dockerfile . e2e-docker-push: - @ hack/docker-push.sh $(E2E_IMG) + @ hack/docker.sh -l -p $(E2E_IMG) + +e2e-docker-multiarch-build: clean + @ hack/docker.sh -l -m $(E2E_IMG) + docker buildx build \ + --progress=plain \ + --file test/e2e/Dockerfile \ + --build-arg E2E_JSON=$(E2E_JSON) \ + --platform linux/amd64,linux/arm64 \ + --push \ + -t $(E2E_IMG) . e2e-run: @go run test/e2e/cmd/main.go run \ @@ -454,7 +478,7 @@ ci-build-operator-e2e-run: setup-e2e build-operator-image e2e-run run-deployer: build-deployer ./hack/deployer/deployer execute --plans-file hack/deployer/config/plans.yml --config-file deployer-config.yml -ci-release: clean ci-check build-operator-image +ci-release: clean ci-check build-operator-multiarch-image @ echo $(OPERATOR_IMAGE) was pushed! ########################## diff --git a/hack/docker-push.sh b/hack/docker.sh similarity index 56% rename from hack/docker-push.sh rename to hack/docker.sh index 3447f62424..b7203f4bd0 100755 --- a/hack/docker-push.sh +++ b/hack/docker.sh @@ -26,10 +26,10 @@ docker-login() { local image=$1 local registry=${image%%"/"*} - if grep -q "$registry" ~/.docker/config.json; then - # already logged in - return 0 - fi +# if grep -q "$registry" ~/.docker/config.json; then +# # already logged in +# return 0 +# fi case "$image" in @@ -60,5 +60,62 @@ docker-push() { docker push "$image" | grep -v -E 'Waiting|Layer already|Preparing|Pushing|Pushed' } -docker-login "$@" -docker-push "$@" +docker-multiarch-init() { + local BUILDER_NAME="eck-multi-arch" + docker buildx create --driver docker-container --name "$BUILDER_NAME" --platform linux/amd64,linux/arm64 --use >/dev/null 2>&1 || echo "$BUILDER_NAME already exists" + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes >/dev/null 2>&1 +} + +usage() { + echo "Usage: $0 <-l | -m | -p> image" + echo " -l Login to registry" + echo " -m Configure system for multi-arch build" + echo " -p Push to registry" + exit 2 +} + + +OPT_LOGIN="no" +OPT_PUSH="no" +OPT_MULTI_ARCH="no" + +while getopts ":lpm" OPT; do + case "$OPT" in + l) + OPT_LOGIN="yes" + ;; + m) + OPT_MULTI_ARCH="yes" + ;; + p) + OPT_PUSH="yes" + ;; + \?) + usage + ;; + *) + usage + ;; + esac +done + +shift $((OPTIND - 1)) + +if [[ ! $# -eq 1 ]]; then + usage +fi + +echo ">> Image == $1" + +if [[ "$OPT_MULTI_ARCH" == "yes" ]]; then + docker-multiarch-init +fi + +if [[ "$OPT_LOGIN" == "yes" ]]; then + docker-login "$1" +fi + +if [[ "$OPT_PUSH" == "yes" ]]; then + docker-push "$1" +fi + diff --git a/hack/manifest-gen/Makefile b/hack/manifest-gen/Makefile index ba40c41ee7..d36573edc5 100644 --- a/hack/manifest-gen/Makefile +++ b/hack/manifest-gen/Makefile @@ -20,7 +20,7 @@ docker-build: .PHONY: docker-push docker-push: - @ ../docker-push.sh $(DOCKER_IMAGE) + @ ../docker.sh -l -p $(DOCKER_IMAGE) .PHONY: docker-gen-global docker-gen-global: docker-build diff --git a/test/e2e/Dockerfile b/test/e2e/Dockerfile index 10d3eab8a1..15358953c8 100644 --- a/test/e2e/Dockerfile +++ b/test/e2e/Dockerfile @@ -1,6 +1,8 @@ # Docker image for the E2E tests runner -FROM golang:1.15.4 +FROM --platform=$BUILDPLATFORM golang:1.15.4 +ARG TARGETPLATFORM +ARG BUILDPLATFORM ARG E2E_JSON ENV E2E_JSON $E2E_JSON From 8ae33a24cbd9ba07af49f1d9db8d0a0b60cd3ff4 Mon Sep 17 00:00:00 2001 From: Charith Ellawala Date: Thu, 26 Nov 2020 13:42:47 +0000 Subject: [PATCH 3/3] Address CR comments --- dev-setup.md | 2 +- hack/check/check-requisites.sh | 19 +++++++++++++++++++ hack/docker.sh | 8 ++++---- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/dev-setup.md b/dev-setup.md index f47f13997a..f0138a5178 100644 --- a/dev-setup.md +++ b/dev-setup.md @@ -10,7 +10,7 @@ Before you start, install the following tools and packages: * [golangci-lint](https://github.com/golangci/golangci-lint) * [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) (>= 1.14) * [kubebuilder](https://github.com/kubernetes-sigs/kubebuilder) (>= 2.0.0) -* [docker](https://docs.docker.com/) +* [docker](https://docs.docker.com/) (>= 19.0.0 with optional `buildx` extension for multi-arch builds) * Kubernetes distribution such as [minikube](https://kubernetes.io/docs/tasks/tools/install-minikube/) or [kind](https://kind.sigs.k8s.io), or access to a hosted Kubernetes service such as [GKE](https://cloud.google.com/kubernetes-engine) or [AKS](https://azure.microsoft.com/en-us/services/kubernetes-service/) ### Get sources diff --git a/hack/check/check-requisites.sh b/hack/check/check-requisites.sh index 5461537619..7405741a49 100755 --- a/hack/check/check-requisites.sh +++ b/hack/check/check-requisites.sh @@ -10,6 +10,7 @@ set -eu MIN_GO_VERSION=13 MIN_KUBECTL_VERSION=14 +MIN_DOCKER_VERSION=19 green="\e[32m" red="\e[31m" @@ -81,6 +82,23 @@ check_kubectl_version() { printf "\n" } +check_docker_version() { + local major + major=$(docker version -f '{{.Client.Version}}' | sed -E 's|([0-9]+)\.[0-9]+\.[0-9]+.*|\1|') + local docker_version + docker_version=$(docker version -f '{{.Client.Version}}') + + printf "Checking for Docker >= %s.0.0... " "$MIN_DOCKER_VERSION" + if [[ "$major" -gt $MIN_DOCKER_VERSION ]]; then + printf "%bok%b (%s)" "${green}" "${reset}" "$docker_version" + else + printf "%bko$%b (%s)" "${red}" "${reset}" "$docker_version" + all_found=false + fi + printf "\n" +} + + check go check golangci-lint check kubectl @@ -88,6 +106,7 @@ check kubebuilder check_oneof gcloud minikube kind check_go_version check_kubectl_version +check_docker_version echo if [[ "$all_found" != "true" ]]; then diff --git a/hack/docker.sh b/hack/docker.sh index b7203f4bd0..7ffbaf1fcd 100755 --- a/hack/docker.sh +++ b/hack/docker.sh @@ -26,10 +26,10 @@ docker-login() { local image=$1 local registry=${image%%"/"*} -# if grep -q "$registry" ~/.docker/config.json; then -# # already logged in -# return 0 -# fi + if grep -q "$registry" ~/.docker/config.json; then + echo "Skipping Docker login" + return 0 + fi case "$image" in