build in parallel containers

Signed-off-by: YujiOshima <yuji.oshima0x3fd@gmail.com>
kubeflow · May 15, 2018 · ddd2ba2 · ddd2ba2
1 parent 0a95175
commit ddd2ba2
Show file tree

Hide file tree

Showing 4 changed files with 111 additions and 41 deletions.
diff --git a/test/scripts/build-earlystoppings.sh b/test/scripts/build-earlystoppings.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Copyright 2018 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This shell script is used to build an image from our argo workflow
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH}
+REGISTRY="${GCP_REGISTRY}"
+PROJECT="${GCP_PROJECT}"
+GO_DIR=${GOPATH}/src/github.com/${REPO_OWNER}/${REPO_NAME}
+VERSION=$(git describe --tags --always --dirty)
+
+echo "Activating service-account"
+gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
+
+echo "Copy source to GOPATH"
+mkdir -p ${GO_DIR}
+cp -r cmd ${GO_DIR}/cmd
+cp -r pkg ${GO_DIR}/pkg
+cp -r vendor ${GO_DIR}/vendor
+
+cd ${GO_DIR}
+
+cp cmd/earlystopping/medianstopping/Dockerfile .
+gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/earlystopping-medianstopping:${VERSION} --project=${PROJECT} &
diff --git a/test/scripts/build-manager.sh b/test/scripts/build-manager.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2018 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This shell script is used to build an image from our argo workflow
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export PATH=${GOPATH}/bin:/usr/local/go/bin:${PATH}
+REGISTRY="${GCP_REGISTRY}"
+PROJECT="${GCP_PROJECT}"
+GO_DIR=${GOPATH}/src/github.com/${REPO_OWNER}/${REPO_NAME}
+VERSION=$(git describe --tags --always --dirty)
+
+echo "Activating service-account"
+gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
+
+echo "Copy source to GOPATH"
+mkdir -p ${GO_DIR}
+cp -r cmd ${GO_DIR}/cmd
+cp -r pkg ${GO_DIR}/pkg
+cp -r vendor ${GO_DIR}/vendor
+cp -r modeldb ${GO_DIR}/modeldb
+
+cd ${GO_DIR}
+cp cmd/manager/Dockerfile .
+gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/vizier-core:${VERSION} --project=${PROJECT} &
+
+cp modeldb/Dockerfile .
+gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/katib-frontend:${VERSION} --project=${PROJECT} &
+
+go build -o bin/katib github.com/kubeflow/katib/cmd/cli
diff --git a/test/scripts/build.sh → test/scripts/build-suggestions.sh b/test/scripts/build.sh → test/scripts/build-suggestions.sh
@@ -29,53 +29,22 @@ VERSION=$(git describe --tags --always --dirty)
 echo "Activating service-account"
 gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
 
-echo "Create symlink to GOPATH"
-mkdir -p ${GOPATH}/src/github.com/${REPO_OWNER}
-ln -s ${PWD} ${GO_DIR}
+echo "Copy source to GOPATH"
+mkdir -p ${GO_DIR}
+cp -r cmd ${GO_DIR}/cmd
+cp -r pkg ${GO_DIR}/pkg
+cp -r vendor ${GO_DIR}/vendor
 
 cd ${GO_DIR}
-#echo "building container in gcloud"
-#gcloud version
-# gcloud components update -q
-
-pids=()
-cp cmd/manager/Dockerfile .
-gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/vizier-core:${VERSION} --project=${PROJECT} &
-pids+=($!)
-sleep 30 # wait for copy code to gcloud
 
 cp cmd/suggestion/random/Dockerfile .
 gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-random:${VERSION} --project=${PROJECT} &
-pids+=($!)
-sleep 30 # wait for copy code to gcloud
 
 cp cmd/suggestion/grid/Dockerfile .
 gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-grid:${VERSION} --project=${PROJECT} &
-pids+=($!)
-sleep 30 # wait for copy code to gcloud
 
 #cp cmd/suggestion/hyperband/Dockerfile .
 #gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-hyperband:${VERSION} --project=${PROJECT} &
-#pids+=($!)
-#sleep 30 # wait for copy code to gcloud
 
 cp cmd/suggestion/bayesianoptimization/Dockerfile .
 gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/suggestion-bayesianoptimization:${VERSION} --project=${PROJECT} &
-pids+=($!)
-sleep 30 # wait for copy code to gcloud
-
-cp cmd/earlystopping/medianstopping/Dockerfile .
-gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/earlystopping-medianstopping:${VERSION} --project=${PROJECT} &
-pids+=($!)
-sleep 30 # wait for copy code to gcloud
-
-cp modeldb/Dockerfile .
-gcloud container builds submit . --tag=${REGISTRY}/${REPO_NAME}/katib-frontend:${VERSION} --project=${PROJECT} &
-pids+=($!)
-
-for pid in ${pids[@]}; do
-  wait $pid
-  if [ $? -ne 0 ]; then
-    exit 1
-  fi
-done
diff --git a/test/workflows/components/workflows.libsonnet b/test/workflows/components/workflows.libsonnet
@@ -203,8 +203,16 @@
                 }],
                 [
                   {
-                    name: "build",
-                    template: "build",
+                    name: "build-manager",
+                    template: "build-manager",
+                  },
+                  {
+                    name: "build-suggestions",
+                    template: "build-suggestions",
+                  },
+                  {
+                    name: "build-earlystoppings",
+                    template: "build-earlystoppings",
                   },
                   {
                     name: "create-pr-symlink",
@@ -291,9 +299,15 @@
               "copy_artifacts",
               "--bucket=" + bucket,
             ]),  // copy-artifacts
-            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("build", testWorkerImage, [
-              "test/scripts/build.sh",
-            ]),  // build
+            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("build-manager", testWorkerImage, [
+              "test/scripts/build-manager.sh",
+            ]),  // build-manager
+            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("build-suggestions", testWorkerImage, [
+              "test/scripts/build-suggestions.sh",
+            ]),  // build-suggestions
+            $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("build-earlystoppings", testWorkerImage, [
+              "test/scripts/build-earlystoppings.sh",
+            ]),  // build-earlystoppings
             $.parts(namespace, name).e2e(prow_env, bucket).buildTemplate("unit-test", testWorkerImage, [
               "test/scripts/unit-test.sh",
             ]),  // unit test