[CI] Several CI/Infrastructure Improvement & Fix

.gitignore

@@ -1,5 +1,6 @@
 *.swp
 .color_coded
+.vscode
 .ycm_p
 .ycm_extra_conf.py
 *.pyc

.travis.yml

@@ -0,0 +1,45 @@
+# Maintainer: Simon Mo <xmo@berkeley.edu>
+# - Clipper use Travis CI to run its Kubernetes test. 
+# - We will spin up minikube and start a docker registry at localhost:5000
+# - Once all environment is ready, it will run shipyard to generate the Makefile
+# - Travis will wait for all docker images related to k8s test to finish building
+# - Then travis will run the test sequentially with retry-once logic. 
+
+# Travis will start the build in a VM when sudo is required
+sudo: required
+
+language: python
+
+env:
+  matrix:
+  # This will make sure .minikube config resides in current user folder
+  - CHANGE_MINIKUBE_NONE_USER=true
+
+before_script:
+# Download kubectl and minikube
+- curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.9.0/bin/linux/amd64/kubectl
+  && chmod +x kubectl && sudo mv kubectl /usr/local/bin/
+- curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.25.2/minikube-linux-amd64
+  && chmod +x minikube && sudo mv minikube /usr/local/bin/
+
+# Setup minikube and docker registry
+- sudo minikube start --vm-driver=none --kubernetes-version=v1.9.0 --insecure-registry localhost:5000
+- docker run -d -p 5000:5000 --restart=always  registry:2
+- minikube update-context
+- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}';
+  until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
+  sleep 1; done
+- kubectl cluster-info
+- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}';
+  until kubectl -n kube-system get pods -lcomponent=kube-addon-manager -o jsonpath="$JSONPATH"
+  2>&1 | grep -q "Ready=True"; do sleep 1;echo "waiting for kube-addon-manager to
+  be available"; kubectl get pods --all-namespaces; done
+- JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}';
+  until kubectl -n kube-system get pods -lk8s-app=kube-dns -o jsonpath="$JSONPATH"
+  2>&1 | grep -q "Ready=True"; do sleep 1;echo "waiting for kube-dns to be available";
+  kubectl get pods --all-namespaces; done
+
+script:
+- cd clipper_admin; pip install -q -e .; cd ..
+- bash ./bin/shipyard.sh
+- bash ./bin/run_travis.sh

CMakeLists.txt

@@ -38,7 +38,10 @@ set_property(TARGET cityhash PROPERTY
         INTERFACE_LINK_LIBRARIES ${CITYHASH_LIBRARIES})
 
 # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -pedantic -Wextra -Werror -Wno-deprecated-declarations -Wno-unsupported-friend")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -pedantic -Wextra -Wno-deprecated-declarations")
+# Note(simon):
+#   https://github.com/ucbrise/clipper/pull/595
+#   pedantic is turned off for now until we figure out a way to turn off dlib warnings
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wno-deprecated-declarations")
 if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unsupported-friend")
 endif()

bin/README.md

@@ -1,10 +1,23 @@
 # Description of CI process
 
-The wonderful AMPlab Jenkins is responsible for running our integration test.
+The wonderful AMPlab Jenkins and Travis CI is responsible for running our integration test.
 
-## How does the CI process work
+## How does the CI process work (version 2.0)
+- There are three components of Clipper CI:
+    1. Shipyard: generate Makefiles for both building and testing.
+    2. Jenkins: build, push, and test containers. It will run all tests except kubernetes test. 
+    3. Travis: run kubernetes test. Travis will wait for Jenkins finish building it's required containers, using `wait_kubernetes_containers` Makefile target. 
+- Rationale:
+    1. We want to build the images in parallel as much as we can, however, our images have complex dependencies. Makefile is our best bet. However, one does not simply write complex Makefile. We built our in-house, light-weight makefile generator _shipyard_ to provide a declarative and pythonic way to generate the file. 
+    2. Jenkins is our main CI worker. It simply set up the environment and run whatever makefile provides. 
+    3. One downside of Jenkins is that it only runs docker, not VM. Minikube is a scarce resource. For this reason, Travis is used because it provides a VM. 
+- Other tooling:
+    1. `colorize_output.py` script in the `bin/` directory is used to tag each line of the log output. 
+    1. [A webapp](http://clipper-jenkins-viewer.herokuapp.com/) deployed at heroku can be used to disaggregate the logs and visualize Makefile dag. 
 
 
+## How does the CI process work (version 1.0)
+
 0. Jenkins pull the PR.
 1. Jenkins inject environment variables configured in admin page. Currently, we set the following environment variables 
 in Pull Request Builder:
@@ -24,6 +37,6 @@ in Pull Request Builder:
     - It runs `run_unnitests.sh` to run all tests. 
         - (Only in Python3) It will only run the integration test part, which contains all the python tests and R tests
 
-## Note on Minikube (WIP)
+## Note on Minikube
 - We are under the process of moving away from AWS EKS to Minikube in our CI process. Once the PR is in, there will be 
-more detail here. 
+more detail here. 

bin/cleanup_jenkins.sh

@@ -0,0 +1,14 @@
+# This script will delete all testing containers and images
+# available on this machine. 
+
+CLIPPER_REGISTRY=clippertesting
+
+# best effort, ignore all errors
+try_cleanup() {
+    (docker ps | grep "$@" | awk '{ print $1 }' | xargs docker kill) || true
+    (docker container ls | grep "$@" | awk '{ print $3 }' | xargs docker container rm) || true
+    (docker image ls | grep "$@" | awk '{ print $3 }' | xargs docker image rm -f) || true
+}
+
+try_cleanup CLIPPER_REGISTRY > /dev/null
+try_cleanup shipyard > /dev/null

bin/colorize_output.py

@@ -0,0 +1,43 @@
+from random import shuffle
+import sys
+import argparse
+import re
+
+parser = argparse.ArgumentParser(description="Colorize stdin; (optionally) add a tag.")
+parser.add_argument("--tag", type=str, help="Optional tag")
+parser.add_argument("--no-color", action="store_true", help="Flag to force the output to be no color.")
+
+args = parser.parse_args()
+tag = "[{}]".format(args.tag) if args.tag else ""
+
+ALL_COLORS = [
+    "\u001b[30m", # Black
+    "\u001b[31m",  # Red
+    "\u001b[32m",  # Green
+    "\u001b[33m",  # Yellow
+    "\u001b[34m",  # Blue
+    "\u001b[35m",  # Magenta
+    "\u001b[36m",  # Cyan
+]
+RESET = "\u001b[0m"
+
+shuffle(ALL_COLORS)
+COLOR = ALL_COLORS[0]
+
+if args.no_color:
+    COLOR = ""
+    RESET = ""
+
+# https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python
+ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
+
+for line in sys.stdin:
+    print(
+        "{begin_color} {tag} {line} {end_color}".format(
+            begin_color=COLOR,
+            tag=tag,
+            line=ansi_escape.sub('', line.strip()),
+            end_color=RESET,  # Reset
+        )
+    )
+    sys.stdout.flush()

bin/deprecated/construct_kube_config.py

@@ -0,0 +1,72 @@
+from __future__ import print_function, absolute_import
+import os
+import yaml
+import sys
+
+cert_auth_env_var = "CLIPPER_K8S_CERT_AUTH"
+client_cert_env_var = "CLIPPER_K8S_CLIENT_CERT"
+client_key_env_var = "CLIPPER_K8S_CLIENT_KEY"
+pw_env_var = "CLIPPER_K8S_PASSWORD"
+required_vars = [cert_auth_env_var, client_cert_env_var, client_key_env_var, pw_env_var]
+
+
+def write_config(dest_path):
+
+    for v in required_vars:
+        if v not in os.environ:
+            print("{} not defined. Cannot construct kube config".format(v))
+            sys.exit(1)
+
+    cert_auth_data = os.environ[cert_auth_env_var]
+    client_cert_data = os.environ[client_cert_env_var]
+    client_key_data = os.environ[client_key_env_var]
+    pw_data = os.environ[pw_env_var]
+
+    base_config = {
+        "kind": "Config",
+        "preferences": {},
+        "current-context": "jenkins.clipper-k8s-testing.com",
+        "contexts": [
+            {
+                "name": "jenkins.clipper-k8s-testing.com",
+                "context": {
+                    "cluster": "jenkins.clipper-k8s-testing.com",
+                    "user": "jenkins.clipper-k8s-testing.com",
+                },
+            }
+        ],
+        "clusters": [
+            {
+                "cluster": {
+                    "certificate-authority-data": cert_auth_data,
+                    "server": "https://api.jenkins.clipper-k8s-testing.com",
+                },
+                "name": "jenkins.clipper-k8s-testing.com",
+            }
+        ],
+        "apiVersion": "v1",
+        "users": [
+            {
+                "name": "jenkins.clipper-k8s-testing.com",
+                "user": {
+                    "username": "admin",
+                    "password": pw_data,
+                    "client-key-data": client_key_data,
+                    "client-certificate-data": client_cert_data,
+                },
+            },
+            {
+                "name": "jenkins.clipper-k8s-testing.com-basic-auth",
+                "user": {"username": "admin", "password": pw_data},
+            },
+        ],
+    }
+    with open(dest_path, "w") as f:
+        yaml.dump(base_config, f)
+
+
+if __name__ == "__main__":
+    config_path = sys.argv[1]
+    config_path = os.path.abspath(os.path.expanduser(config_path))
+    print("Writing kube config to {}".format(config_path))
+    write_config(config_path)