Add multi-arch build to nightly

WIP

WIP

WIP

Force docker login

WIP

Install buildx

qemu

Script steps

WIP

Increase timeout

Multi-arch e2e image

Increase timeout

Avoid double build

Plain progress

Rebase

Fix Docker script

Add ARM build to nightly

.ci/Dockerfile

@@ -6,7 +6,8 @@ ENV SHELLCHECK_VERSION=0.7.1
 ENV KUBEBUILDER_VERSION=2.3.1
 ENV GCLOUD_VERSION=297.0.1
 ENV KUBECTL_VERSION=1.14.7
-ENV DOCKER_VERSION=19.03.9
+ENV DOCKER_VERSION=19.03.13
+ENV DOCKER_BUILDX_VERSION=0.4.2
 ENV GOTESTSUM_VERSION=0.5.0
 ENV KIND_VERSION=0.8.1
 ENV OPENSHIFT_TOOLS_VERSION=4.3.19
@@ -43,9 +44,14 @@ RUN curl -fsSLO https://download.docker.com/linux/static/stable/x86_64/docker-${
     tar xzf docker-${DOCKER_VERSION}.tgz --strip 1 -C /usr/local/bin docker/docker && \
     rm docker-${DOCKER_VERSION}.tgz
 
-# xz-utils to decompress shellcheck and unzip for aws-cli
+# Docker buildx extension for building multi-arch images
+RUN mkdir -p ~/.docker/cli-plugins && \ 
+    curl -fsSLo ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v${DOCKER_BUILDX_VERSION}/buildx-v${DOCKER_BUILDX_VERSION}.linux-amd64 && \
+    chmod a+x ~/.docker/cli-plugins/docker-buildx
+
+# xz-utils to decompress shellcheck, unzip for aws-cli, qemu-system-arm and qemu-user-static for multi-arch
 RUN apt-get update && apt-get --no-install-recommends -y install \
-    unzip xz-utils && \
+    unzip xz-utils qemu-system-arm qemu-user-static && \
     apt-get clean && apt-get autoclean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 

.ci/Makefile

@@ -67,7 +67,7 @@ ci-build-image: write-ci-docker-creds
 			-t $(CI_IMAGE) \
 			--label "commit.hash=$(shell git rev-parse --short --verify HEAD)" \
 			$(ROOT_DIR) && \
-		../hack/docker-push.sh $(CI_IMAGE) \
+		../hack/docker.sh -l -p $(CI_IMAGE) \
 	)
 
 # make Docker creds available from inside the CI container through the .registry.env file

Dockerfile

@@ -1,5 +1,5 @@
 # Build the operator binary
-FROM golang:1.15.4 as builder
+FROM --platform=$BUILDPLATFORM golang:1.15.4 as builder
 
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM

Makefile

@@ -197,6 +197,11 @@ build-operator-image:
 	&& echo "OK: image $(OPERATOR_IMAGE) already published" \
 	|| $(MAKE) docker-build docker-push
 
+build-operator-multiarch-image:
+	@ docker buildx imagetools inspect $(OPERATOR_IMAGE) | grep -q 'linux/arm64' 2>&1 >/dev/null \
+	&& echo "OK: image $(OPERATOR_IMAGE) already published" \
+	|| $(MAKE) docker-multiarch-build
+
 # if the current k8s cluster is on GKE, GCLOUD_PROJECT must be set
 check-gke:
 ifneq ($(findstring gke_,$(KUBECTL_CLUSTER)),)
@@ -342,18 +347,27 @@ switch-eks:
 #################################
 ##  --    Docker images    --  ##
 #################################
-
-docker-build: go-generate generate-config-file
+docker-multiarch-build: go-generate generate-config-file 
+	@ hack/docker.sh -l -m $(OPERATOR_IMAGE)
 	docker buildx build . \
+		--progress=plain \
 		--build-arg GO_LDFLAGS='$(GO_LDFLAGS)' \
 		--build-arg GO_TAGS='$(GO_TAGS)' \
 		--build-arg VERSION='$(VERSION)' \
 		--platform linux/amd64,linux/arm64 \
 		--push \
 		-t $(OPERATOR_IMAGE)
 
+docker-build: go-generate generate-config-file 
+	DOCKER_BUILDKIT=1 docker build . \
+		--progress=plain \
+		--build-arg GO_LDFLAGS='$(GO_LDFLAGS)' \
+		--build-arg GO_TAGS='$(GO_TAGS)' \
+		--build-arg VERSION='$(VERSION)' \
+		-t $(OPERATOR_IMAGE)
+
 docker-push:
-	@ hack/docker-push.sh $(OPERATOR_IMAGE)
+	@ hack/docker.sh -l -p $(OPERATOR_IMAGE)
 
 purge-gcr-images:
 	@ for i in $(gcloud container images list-tags $(BASE_IMG) | tail +3 | awk '{print $$2}'); \
@@ -386,10 +400,20 @@ E2E_DEPLOY_CHAOS_JOB       ?= false
 
 # clean to remove irrelevant/build-breaking generated public keys
 e2e-docker-build: clean
-	docker build --build-arg E2E_JSON=$(E2E_JSON) -t $(E2E_IMG) -f test/e2e/Dockerfile .
+	DOCKER_BUILDKIT=1 docker build --progress=plain --build-arg E2E_JSON=$(E2E_JSON) -t $(E2E_IMG) -f test/e2e/Dockerfile .
 
 e2e-docker-push:
-	@ hack/docker-push.sh $(E2E_IMG)
+	@ hack/docker.sh -l -p $(E2E_IMG)
+
+e2e-docker-multiarch-build: clean
+	@ hack/docker.sh -l -m $(E2E_IMG)
+	docker buildx build \
+		--progress=plain \
+		--file test/e2e/Dockerfile \
+		--build-arg E2E_JSON=$(E2E_JSON) \
+		--platform linux/amd64,linux/arm64 \
+		--push \
+		-t $(E2E_IMG) .
 
 e2e-run:
 	@go run test/e2e/cmd/main.go run \
@@ -454,7 +478,7 @@ ci-build-operator-e2e-run: setup-e2e build-operator-image e2e-run
 run-deployer: build-deployer
 	./hack/deployer/deployer execute --plans-file hack/deployer/config/plans.yml --config-file deployer-config.yml
 
-ci-release: clean ci-check build-operator-image
+ci-release: clean ci-check build-operator-multiarch-image
 	@ echo $(OPERATOR_IMAGE) was pushed!
 
 ##########################

hack/docker-push.sh → hack/docker.sh

@@ -26,10 +26,10 @@ docker-login() {
     local image=$1
     local registry=${image%%"/"*}
 
-    if grep -q "$registry" ~/.docker/config.json; then
-        # already logged in
-        return 0
-    fi
+#    if grep -q "$registry" ~/.docker/config.json; then
+#        # already logged in
+#        return 0
+#    fi
 
     case "$image" in
 
@@ -60,5 +60,62 @@ docker-push() {
         docker push "$image" | grep -v -E 'Waiting|Layer already|Preparing|Pushing|Pushed'
 }
 
-docker-login "$@"
-docker-push  "$@"
+docker-multiarch-init() {
+    local BUILDER_NAME="eck-multi-arch"
+    docker buildx create --driver docker-container --name "$BUILDER_NAME" --platform linux/amd64,linux/arm64 --use >/dev/null 2>&1 || echo "$BUILDER_NAME already exists"
+    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes >/dev/null 2>&1 
+}
+
+usage() {
+    echo "Usage: $0 <-l | -m | -p> image"
+    echo "  -l   Login to registry"
+    echo "  -m   Configure system for multi-arch build"
+    echo "  -p   Push to registry"
+    exit 2
+}
+
+
+OPT_LOGIN="no"
+OPT_PUSH="no"
+OPT_MULTI_ARCH="no"
+
+while getopts ":lpm" OPT; do
+    case "$OPT" in
+        l)
+            OPT_LOGIN="yes"
+            ;;
+        m)
+            OPT_MULTI_ARCH="yes"
+            ;;
+        p)
+            OPT_PUSH="yes"
+            ;;
+        \?) 
+            usage
+            ;;
+        *)
+            usage
+            ;;
+    esac
+done
+
+shift $((OPTIND - 1))
+
+if [[ ! $# -eq 1 ]]; then
+    usage
+fi
+
+echo ">> Image == $1"
+
+if [[ "$OPT_MULTI_ARCH" == "yes" ]]; then
+    docker-multiarch-init
+fi
+
+if [[ "$OPT_LOGIN" == "yes" ]]; then
+    docker-login "$1"
+fi
+
+if [[ "$OPT_PUSH" == "yes" ]]; then
+    docker-push "$1"
+fi
+

hack/manifest-gen/Makefile

@@ -20,7 +20,7 @@ docker-build:
 
 .PHONY: docker-push
 docker-push:
-	@ ../docker-push.sh $(DOCKER_IMAGE)
+	@ ../docker.sh -l -p $(DOCKER_IMAGE)
 
 .PHONY: docker-gen-global
 docker-gen-global: docker-build

test/e2e/Dockerfile

@@ -1,6 +1,8 @@
 # Docker image for the E2E tests runner
-FROM golang:1.15.4
+FROM --platform=$BUILDPLATFORM golang:1.15.4 
 
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
 ARG E2E_JSON
 ENV E2E_JSON $E2E_JSON