Pick up kube credentials from env variables for k8s integration test (#…

…284)

* Propagage kube test creds and add k8s integration test to CI

* format code

* added comment

* add kubectl to unittests dockerfile

* fix typo

* fix unittests dockerfile

* fixed dockerfile

* address comments

bin/ci_checks.sh

@@ -4,6 +4,13 @@ set -e
 set -u
 set -o pipefail
 
+function clean_up {
+    # Clean up credentials
+    rm $KUBECONFIG
+    exit
+}
+
+
 unset CDPATH
 # one-liner from http://stackoverflow.com/a/246128
 # Determines absolute path of the directory containing
@@ -18,6 +25,23 @@ cd -
 # Test docker login
 docker pull 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/query_frontend:$tag
 
-# Let the user start this script from anywhere in the filesystem.
+# Set up credentials for K8s testing cluster.
+export KUBECONFIG=~/kubeconfig_$(date +"%Y%m%d%H%M%S")
+
+# We only need to trap exit starting here, because before this line
+# KUBECONFIG hasn't been set yet and we haven't created any resources
+# that need to be cleaned up.
+trap clean_up SIGHUP SIGINT SIGTERM EXIT
+
+# NOTE: This script will fail if the following environment variables are not set
+# + CLIPPER_K8S_CERT_AUTH
+# + CLIPPER_K8S_CLIENT_CERT
+# + CLIPPER_K8S_CLIENT_KEY
+# + CLIPPER_K8S_PASSWORD
+python $DIR/construct_kube_config.py $KUBECONFIG
+
+# Test K8s cluster access
+kubectl get nodes
+
 $DIR/check_format.sh
 $DIR/run_unittests.sh

bin/construct_kube_config.py

@@ -0,0 +1,73 @@
+from __future__ import print_function, absolute_import
+import os
+import yaml
+import sys
+
+cert_auth_env_var = "CLIPPER_K8S_CERT_AUTH"
+client_cert_env_var = "CLIPPER_K8S_CLIENT_CERT"
+client_key_env_var = "CLIPPER_K8S_CLIENT_KEY"
+pw_env_var = "CLIPPER_K8S_PASSWORD"
+required_vars = [
+    cert_auth_env_var, client_cert_env_var, client_key_env_var, pw_env_var
+]
+
+
+def write_config(dest_path):
+
+    for v in required_vars:
+        if v not in os.environ:
+            print("{} not defined. Cannot construct kube config".format(v))
+            sys.exit(1)
+
+    cert_auth_data = os.environ[cert_auth_env_var]
+    client_cert_data = os.environ[client_cert_env_var]
+    client_key_data = os.environ[client_key_env_var]
+    pw_data = os.environ[pw_env_var]
+
+    base_config = {
+        'kind':
+        'Config',
+        'preferences': {},
+        'current-context':
+        'cluster.clipper-k8s-testing.com',
+        'contexts': [{
+            'name': 'cluster.clipper-k8s-testing.com',
+            'context': {
+                'cluster': 'cluster.clipper-k8s-testing.com',
+                'user': 'cluster.clipper-k8s-testing.com'
+            }
+        }],
+        'clusters': [{
+            'cluster': {
+                'certificate-authority-data': cert_auth_data,
+                'server': 'https://api.cluster.clipper-k8s-testing.com'
+            },
+            'name': 'cluster.clipper-k8s-testing.com'
+        }],
+        'apiVersion':
+        'v1',
+        'users': [{
+            'name': 'cluster.clipper-k8s-testing.com',
+            'user': {
+                'username': 'admin',
+                'password': pw_data,
+                'client-key-data': client_key_data,
+                'client-certificate-data': client_cert_data
+            }
+        }, {
+            'name': 'cluster.clipper-k8s-testing.com-basic-auth',
+            'user': {
+                'username': 'admin',
+                'password': pw_data
+            }
+        }]
+    }
+    with open(dest_path, "w") as f:
+        yaml.dump(base_config, f)
+
+
+if __name__ == '__main__':
+    config_path = sys.argv[1]
+    config_path = os.path.abspath(os.path.expanduser(config_path))
+    print("Writing kube config to {}".format(config_path))
+    write_config(config_path)

bin/run_ci.sh

@@ -27,4 +27,9 @@ docker push 568959175238.dkr.ecr.us-west-1.amazonaws.com/clipper/management_fron
 
 # Run tests
 docker run --rm --network=host -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp \
-    -v /home/jenkins/.docker:/root/.docker clipper/unittests:$tag
+    -v /home/jenkins/.docker:/root/.docker \
+    -e CLIPPER_K8S_CERT_AUTH=$CLIPPER_K8S_CERT_AUTH \
+    -e CLIPPER_K8S_CLIENT_CERT=$CLIPPER_K8S_CLIENT_CERT \
+    -e CLIPPER_K8S_CLIENT_KEY=$CLIPPER_K8S_CLIENT_KEY \
+    -e CLIPPER_K8S_PASSWORD=$CLIPPER_K8S_PASSWORD \
+    clipper/unittests:$tag

bin/run_unittests.sh

@@ -130,9 +130,7 @@ function run_integration_tests {
   python ../integration-tests/many_apps_many_models.py 2 3
   python ../integration-tests/deploy_pyspark_models.py
   python ../integration-tests/deploy_pyspark_pipeline_models.py
-  # TODO: K8s tests still have some kinks that need to be worked
-  # out on the cluster.
-  # python ../integration-tests/kubernetes_integration_test.py
+  python ../integration-tests/kubernetes_integration_test.py
 }
 
 function run_all_tests {

dockerfiles/ClipperTestsDockerfile

@@ -41,6 +41,10 @@ ENV MAVEN_HOME /usr/share/maven
 ENV MAVEN_CONFIG /root/.m2
 ENV JZMQ_HOME /usr/local/lib/
 
+# Install kubectl
+RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl \
+      && chmod +x kubectl \
+      && mv kubectl /usr/local/bin/
 
 COPY ./ /clipper