From 882bf21cfd542846b7df71812b8c4dfb4c7ae351 Mon Sep 17 00:00:00 2001
From: Dan Crankshaw <dscrankshaw@gmail.com>
Date: Sun, 3 Sep 2017 11:46:05 -0700
Subject: [PATCH 1/8] Propagage kube test creds and add k8s integration test to
 CI

---
 bin/ci_checks.sh             | 26 ++++++++++++-
 bin/construct_kube_config.py | 72 ++++++++++++++++++++++++++++++++++++
 bin/run_ci.sh                |  7 +++-
 bin/run_unittests.sh         |  4 +-
 4 files changed, 103 insertions(+), 6 deletions(-)
 create mode 100644 bin/construct_kube_config.py

diff --git a/bin/ci_checks.sh b/bin/ci_checks.sh
index 1426f68b9..f6e0c4193 100755
--- a/bin/ci_checks.sh
+++ b/bin/ci_checks.sh
@@ -4,6 +4,13 @@ set -e
 set -u
 set -o pipefail
 
+function clean_up {
+    # Clean up credentials
+    rm $KUBECONFIG
+    exit
+}
+
+
 unset CDPATH
 # one-liner from http://stackoverflow.com/a/246128
 # Determines absolute path of the directory containing
@@ -16,8 +23,23 @@ tag=$(<VERSION.txt)
 cd -
 
 # Test docker login
-docker pull 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:$tag
+# docker pull 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:$tag
+
+# Set up credentials for K8s testing cluster.
+export KUBECONFIG=~/kubeconfig_$(date +"%Y%m%d%H%M%S")
+
+# We only need to trap exit starting here, because before this line
+# KUBECONFIG hasn't been set yet and we haven't created any resources
+# that need to be cleaned up.
+trap clean_up SIGHUP SIGINT SIGTERM EXIT
+
+# NOTE: This script will fail if the following environment variables are not set
+# + CLIPPER_K8S_CERT_AUTH
+# + CLIPPER_K8S_CLIENT_CERT
+# + CLIPPER_K8S_CLIENT_KEY
+# + CLIPPER_K8S_PASSWORD
+python $DIR/construct_kube_config.py $KUBECONFIG
+kubectl get nodes
 
-# Let the user start this script from anywhere in the filesystem.
 $DIR/check_format.sh
 $DIR/run_unittests.sh
diff --git a/bin/construct_kube_config.py b/bin/construct_kube_config.py
new file mode 100644
index 000000000..ef494e127
--- /dev/null
+++ b/bin/construct_kube_config.py
@@ -0,0 +1,72 @@
+from __future__ import print_function, absolute_import
+import os
+import yaml
+import sys
+
+cert_auth_env_var = "CLIPPER_K8S_CERT_AUTH"
+client_cert_env_var = "CLIPPER_K8S_CLIENT_CERT"
+client_key_env_var = "CLIPPER_K8S_CLIENT_KEY"
+pw_env_var = "CLIPPER_K8S_PASSWORD"
+required_vars = [cert_auth_env_var, client_cert_env_var, client_key_env_var, pw_env_var]
+
+
+def write_config(dest_path):
+
+    for v in required_vars:
+        if v not in os.environ:
+            print("{} not defined. Cannot construct kube config".format(v))
+            sys.exit(1)
+
+    cert_auth_data = os.environ[cert_auth_env_var]
+    client_cert_data = os.environ[client_cert_env_var]
+    client_key_data = os.environ[client_key_env_var]
+    pw_data = os.environ[pw_env_var]
+
+    base_config = {
+        'kind': 'Config',
+        'preferences': {},
+        'current-context': 'cluster.clipper-k8s-testing.com',
+        'contexts': [
+            {
+                'name': 'cluster.clipper-k8s-testing.com',
+                'context': {
+                    'cluster': 'cluster.clipper-k8s-testing.com',
+                    'user': 'cluster.clipper-k8s-testing.com'}}],
+        'clusters': [
+            {
+                'cluster': {
+                    'certificate-authority-data': cert_auth_data,
+                    'server': 'https://api.cluster.clipper-k8s-testing.com'
+                },
+                'name': 'cluster.clipper-k8s-testing.com'
+            }
+        ],
+        'apiVersion': 'v1',
+        'users': [
+            {
+                'name': 'cluster.clipper-k8s-testing.com',
+                'user': {
+                    'username': 'admin',
+                    'password': pw_data,
+                    'client-key-data': client_key_data,
+                    'client-certificate-data': client_cert_data
+                }
+            },
+            {
+                'name': 'cluster.clipper-k8s-testing.com-basic-auth',
+                'user': {
+                    'username': 'admin',
+                    'password': pw_data
+                }
+            }
+        ]
+    }
+    with open(dest_path, "w") as f:
+        yaml.dump(base_config, f)
+
+
+if __name__ == '__main__':
+    config_path = sys.argv[1]
+    config_path = os.path.abspath(os.path.expanduser(config_path))
+    print("Writing kube config to {}".format(config_path))
+    write_config(config_path)
diff --git a/bin/run_ci.sh b/bin/run_ci.sh
index 46d9ed9aa..a6ac062bf 100755
--- a/bin/run_ci.sh
+++ b/bin/run_ci.sh
@@ -27,4 +27,9 @@ docker push 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/management_fron
 
 # Run tests
 docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp \
-    -v /home/jenkins/.docker:/root/.docker clipper/unittests:$tag
+    -v /home/jenkins/.docker:/root/.docker \
+    -e CLIPPER_K8S_CERT_AUTH=$CLIPPER_K8S_CERT_AUTH \
+    -e CLIPPER_K8S_CLIENT_CERT=$CLIPPER_K8S_CLIENT_CERT \
+    -e CLIPPER_K8S_CLIENT_KEY=$CLIPPER_K8S_CLIENT_KEY \
+    -e CLIPPER_K8S_PASSWORD=$CLIPPER_K8S_PASSWORD \
+    clipper/unittests:$tag
diff --git a/bin/run_unittests.sh b/bin/run_unittests.sh
index fc9a79a4e..766e2a07e 100755
--- a/bin/run_unittests.sh
+++ b/bin/run_unittests.sh
@@ -130,9 +130,7 @@ function run_integration_tests {
   python ../integration-tests/many_apps_many_models.py 2 3
   python ../integration-tests/deploy_pyspark_models.py
   python ../integration-tests/deploy_pyspark_pipeline_models.py
-  # TODO: K8s tests still have some kinks that need to be worked
-  # out on the cluster.
-  # python ../integration-tests/kubernetes_integration_test.py
+  python ../integration-tests/kubernetes_integration_test.py
 }
 
 function run_all_tests {

From 351a2d7b55dc2151321dc81d9dc31b018c7e900b Mon Sep 17 00:00:00 2001
From: Dan Crankshaw <dscrankshaw@gmail.com>
Date: Sun, 3 Sep 2017 11:46:48 -0700
Subject: [PATCH 2/8] format code

---
 bin/construct_kube_config.py | 69 ++++++++++++++++++------------------
 1 file changed, 35 insertions(+), 34 deletions(-)

diff --git a/bin/construct_kube_config.py b/bin/construct_kube_config.py
index ef494e127..407e9e55a 100644
--- a/bin/construct_kube_config.py
+++ b/bin/construct_kube_config.py
@@ -7,7 +7,9 @@
 client_cert_env_var = "CLIPPER_K8S_CLIENT_CERT"
 client_key_env_var = "CLIPPER_K8S_CLIENT_KEY"
 pw_env_var = "CLIPPER_K8S_PASSWORD"
-required_vars = [cert_auth_env_var, client_cert_env_var, client_key_env_var, pw_env_var]
+required_vars = [
+    cert_auth_env_var, client_cert_env_var, client_key_env_var, pw_env_var
+]
 
 
 def write_config(dest_path):
@@ -23,43 +25,42 @@ def write_config(dest_path):
     pw_data = os.environ[pw_env_var]
 
     base_config = {
-        'kind': 'Config',
+        'kind':
+        'Config',
         'preferences': {},
-        'current-context': 'cluster.clipper-k8s-testing.com',
-        'contexts': [
-            {
-                'name': 'cluster.clipper-k8s-testing.com',
-                'context': {
-                    'cluster': 'cluster.clipper-k8s-testing.com',
-                    'user': 'cluster.clipper-k8s-testing.com'}}],
-        'clusters': [
-            {
-                'cluster': {
-                    'certificate-authority-data': cert_auth_data,
-                    'server': 'https://api.cluster.clipper-k8s-testing.com'
-                },
-                'name': 'cluster.clipper-k8s-testing.com'
+        'current-context':
+        'cluster.clipper-k8s-testing.com',
+        'contexts': [{
+            'name': 'cluster.clipper-k8s-testing.com',
+            'context': {
+                'cluster': 'cluster.clipper-k8s-testing.com',
+                'user': 'cluster.clipper-k8s-testing.com'
             }
-        ],
-        'apiVersion': 'v1',
-        'users': [
-            {
-                'name': 'cluster.clipper-k8s-testing.com',
-                'user': {
-                    'username': 'admin',
-                    'password': pw_data,
-                    'client-key-data': client_key_data,
-                    'client-certificate-data': client_cert_data
-                }
+        }],
+        'clusters': [{
+            'cluster': {
+                'certificate-authority-data': cert_auth_data,
+                'server': 'https://api.cluster.clipper-k8s-testing.com'
             },
-            {
-                'name': 'cluster.clipper-k8s-testing.com-basic-auth',
-                'user': {
-                    'username': 'admin',
-                    'password': pw_data
-                }
+            'name': 'cluster.clipper-k8s-testing.com'
+        }],
+        'apiVersion':
+        'v1',
+        'users': [{
+            'name': 'cluster.clipper-k8s-testing.com',
+            'user': {
+                'username': 'admin',
+                'password': pw_data,
+                'client-key-data': client_key_data,
+                'client-certificate-data': client_cert_data
             }
-        ]
+        }, {
+            'name': 'cluster.clipper-k8s-testing.com-basic-auth',
+            'user': {
+                'username': 'admin',
+                'password': pw_data
+            }
+        }]
     }
     with open(dest_path, "w") as f:
         yaml.dump(base_config, f)

From 40693204acf91e040954cec4b084856e08a65de1 Mon Sep 17 00:00:00 2001
From: Dan Crankshaw <dscrankshaw@gmail.com>
Date: Sun, 3 Sep 2017 12:06:47 -0700
Subject: [PATCH 3/8] added comment

---
 bin/ci_checks.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/ci_checks.sh b/bin/ci_checks.sh
index f6e0c4193..52eee5e11 100755
--- a/bin/ci_checks.sh
+++ b/bin/ci_checks.sh
@@ -39,6 +39,8 @@ trap clean_up SIGHUP SIGINT SIGTERM EXIT
 # + CLIPPER_K8S_CLIENT_KEY
 # + CLIPPER_K8S_PASSWORD
 python $DIR/construct_kube_config.py $KUBECONFIG
+
+# Test K8s cluster access
 kubectl get nodes
 
 $DIR/check_format.sh

From 82b61b33cb4fc2ac2f523ce08b65186ee3455c68 Mon Sep 17 00:00:00 2001
From: Dan Crankshaw <dscrankshaw@gmail.com>
Date: Sun, 3 Sep 2017 12:17:29 -0700
Subject: [PATCH 4/8] add kubectl to unittests dockerfile

---
 dockerfiles/ClipperTestsDockerfile | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/dockerfiles/ClipperTestsDockerfile b/dockerfiles/ClipperTestsDockerfile
index 10ca2f50c..80d27bd5b 100644
--- a/dockerfiles/ClipperTestsDockerfile
+++ b/dockerfiles/ClipperTestsDockerfile
@@ -41,6 +41,10 @@ ENV MAVEN_HOME /usr/share/maven
 ENV MAVEN_CONFIG /root/.m2
 ENV JZMQ_HOME /usr/local/lib/
 
+# Install kubectl
+RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \
+      && chmod +x ./kubectl \
+      mv ./kubectl /usr/localbin/kubectl
 
 COPY ./ /clipper
 

From a99424dead6a0070dc2fa46e4c124696c2dbde6f Mon Sep 17 00:00:00 2001
From: Dan Crankshaw <dscrankshaw@gmail.com>
Date: Sun, 3 Sep 2017 12:27:17 -0700
Subject: [PATCH 5/8] fix typo

---
 dockerfiles/ClipperTestsDockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dockerfiles/ClipperTestsDockerfile b/dockerfiles/ClipperTestsDockerfile
index 80d27bd5b..06022b84f 100644
--- a/dockerfiles/ClipperTestsDockerfile
+++ b/dockerfiles/ClipperTestsDockerfile
@@ -44,7 +44,7 @@ ENV JZMQ_HOME /usr/local/lib/
 # Install kubectl
 RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \
       && chmod +x ./kubectl \
-      mv ./kubectl /usr/localbin/kubectl
+      mv ./kubectl /usr/local/bin/kubectl
 
 COPY ./ /clipper
 

From 8959363694d050127c6f5aa9615f794915727f2f Mon Sep 17 00:00:00 2001
From: Dan Crankshaw <dscrankshaw@gmail.com>
Date: Sun, 3 Sep 2017 12:43:39 -0700
Subject: [PATCH 6/8] fix unittests dockerfile

---
 dockerfiles/ClipperTestsDockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dockerfiles/ClipperTestsDockerfile b/dockerfiles/ClipperTestsDockerfile
index 06022b84f..8b2bba1d9 100644
--- a/dockerfiles/ClipperTestsDockerfile
+++ b/dockerfiles/ClipperTestsDockerfile
@@ -43,8 +43,8 @@ ENV JZMQ_HOME /usr/local/lib/
 
 # Install kubectl
 RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \
-      && chmod +x ./kubectl \
-      mv ./kubectl /usr/local/bin/kubectl
+      && chmod +x kubectl \
+      mv kubectl /usr/local/bin/
 
 COPY ./ /clipper
 

From 54f777492196d2bc805a8ac26657b6a0f8bac446 Mon Sep 17 00:00:00 2001
From: Dan Crankshaw <dscrankshaw@gmail.com>
Date: Sun, 3 Sep 2017 13:13:57 -0700
Subject: [PATCH 7/8] fixed dockerfile

---
 dockerfiles/ClipperTestsDockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dockerfiles/ClipperTestsDockerfile b/dockerfiles/ClipperTestsDockerfile
index 8b2bba1d9..0c1b37a6e 100644
--- a/dockerfiles/ClipperTestsDockerfile
+++ b/dockerfiles/ClipperTestsDockerfile
@@ -44,7 +44,7 @@ ENV JZMQ_HOME /usr/local/lib/
 # Install kubectl
 RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \
       && chmod +x kubectl \
-      mv kubectl /usr/local/bin/
+      && mv kubectl /usr/local/bin/
 
 COPY ./ /clipper
 

From 09f0ec8531f061f8aa97176e892cbae4696b50bd Mon Sep 17 00:00:00 2001
From: Dan Crankshaw <dscrankshaw@gmail.com>
Date: Sun, 3 Sep 2017 16:22:04 -0700
Subject: [PATCH 8/8] address comments

---
 bin/ci_checks.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/ci_checks.sh b/bin/ci_checks.sh
index 52eee5e11..5a1c1552d 100755
--- a/bin/ci_checks.sh
+++ b/bin/ci_checks.sh
@@ -23,7 +23,7 @@ tag=$(<VERSION.txt)
 cd -
 
 # Test docker login
-# docker pull 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:$tag
+docker pull 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:$tag
 
 # Set up credentials for K8s testing cluster.
 export KUBECONFIG=~/kubeconfig_$(date +"%Y%m%d%H%M%S")