From a133b2d9293963edef9362714a6861939df93a0c Mon Sep 17 00:00:00 2001
From: Elina Akhmanova <elinka@google.com>
Date: Wed, 17 May 2023 08:51:44 +0000
Subject: [PATCH] Use hack script for e2e test image building

The e2e image needs to be built with docker buildx
because it may be built in a cross-compiled way
and it has not only pre-copied binaries but also
some dependencies (sh script, curl, gcloud-cli).

Signed-off-by: Elina Akhmanova <elinka@google.com>
---
 cloudbuild.yaml        | 21 ++++++++++++++-------
 hack/push-multiarch.sh | 16 ++++++++++------
 2 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/cloudbuild.yaml b/cloudbuild.yaml
index 6a723f16c4..1e18bc2cdb 100644
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@@ -5,33 +5,40 @@ options:
   machineType: E2_HIGHCPU_8
 steps:
 - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20230206-8160eea68e
-  id: build-binaries
+  id: push-images
   entrypoint: bash
   env:
   - DOCKER_CLI_EXPERIMENTAL=enabled
+  - REGISTRY=gcr.io/k8s-ingress-image-push
   - VERSION=$_PULL_BASE_REF
   - VERBOSE=1
   - HOME=/root
   - USER=root
-  args: 
+  args:
   - -c
   - |
-    make all-build ALL_ARCH="amd64 arm64"
+    # Build and push every image except e2e-test 
+    make all-push ALL_ARCH="amd64 arm64" \
+      CONTAINER_BINARIES="glbc 404-server 404-server-with-metrics echo fuzzer"
 - name: gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20230206-8160eea68e
-  id: push-images
-  waitFor: ["build-binaries"]
+  id: push-e2e-test-image
   entrypoint: bash
   env:
   - DOCKER_CLI_EXPERIMENTAL=enabled
-  - REGISTRY="gcr.io/k8s-ingress-image-push"
+  - REGISTRY=gcr.io/k8s-ingress-image-push
   - VERSION=$_PULL_BASE_REF
   - VERBOSE=1
   - HOME=/root
   - USER=root
+  - BINARIES=e2e-test
   args:
   - -c
   - |
-    make all-push ALL_ARCH="amd64 arm64"
+    # Build and push e2e-test image 
+    # This image is built separately because it has
+    # additional dependencies (curl, gcloud-cli), and
+    # requires `docker buildx` for multiarch building process
+    ./hack/push-multiarch.sh
 substitutions:
   _GIT_TAG: "12345"
   _PULL_BASE_REF: "main"
diff --git a/hack/push-multiarch.sh b/hack/push-multiarch.sh
index 468d6bf8b5..d0b65deb1d 100755
--- a/hack/push-multiarch.sh
+++ b/hack/push-multiarch.sh
@@ -1,6 +1,10 @@
 #!/bin/bash
 
 # Run commands that build and push images
+
+# This script is used instead of build/rules.mk
+# whenever you need to build multiarch image with
+# docker buildx build --platform=smth
 # (see Dockerfiles in the root directory)
 
 # ENV variables: 
@@ -32,14 +36,14 @@ docker buildx create --use
 for binary in ${BINARIES}
 do
     MULTIARCH_IMAGE="${REGISTRY}/ingress-gce-${binary}"
-    DOCKER_PARAMETERS=""
+    MANIFEST_PARAMETERS=""
     for arch in ${ALL_ARCH}
     do
         echo "building & pushing a docker image for ${binary} (${arch}).."
         # creates arch dependant dockerfiles for every binary
         sed                                     \
-            -e 's|ARG_ARCH|${arch}|g' \
-            -e 's|ARG_BIN|${binary}|g' \
+            -e "s|ARG_ARCH|${arch}|g" \
+            -e "s|ARG_BIN|${binary}|g" \
             Dockerfile.${binary} > .dockerfile-${arch}.${binary}
 
         # buildx builds and pushes images for any arch
@@ -47,12 +51,12 @@ do
         docker buildx build --platform=linux/$arch \
             -f .dockerfile-${arch}.${binary} \
             -t ${IMAGE_NAME}:${VERSION} --push .
-        docker pull ${IMAGE_NAME}:${VERSION}
-        DOCKER_PARAMETERS="$DOCKER_PARAMETERS ${IMAGE_NAME}:${VERSION}"
+        MANIFEST_PARAMETERS="$MANIFEST_PARAMETERS ${IMAGE_NAME}:${VERSION}"
     done
 
     echo "creating a multiatch manifest (${MULTIARCH_IMAGE}) from a list of images.."
-    docker buildx imagetools create -t "${MULTIARCH_IMAGE}:${VERSION}" ${DOCKER_PARAMETERS}
+    docker manifest create "${MULTIARCH_IMAGE}:${VERSION}" ${MANIFEST_PARAMETERS}
+    docker manifest push "${MULTIARCH_IMAGE}:${VERSION}"
 
     echo "done, a result image: ${MULTIARCH_IMAGE}:${VERSION}." 
 done