From fd5e60437cefdbc73f3c95e6e74700dea1666bed Mon Sep 17 00:00:00 2001
From: Matt Schallert <matt@chronosphere.io>
Date: Fri, 14 Feb 2020 10:39:18 -0500
Subject: [PATCH] [ci] Run e2e integration tests in Kind (#191)

---
 .buildkite/hooks/pre-exit           | 15 ++++++
 .buildkite/pipeline.yml             | 10 ++++
 Makefile                            |  9 +++-
 integration/manifests/operator.yaml | 10 ++--
 scripts/kind-create-cluster.sh      | 72 +++++++++++++++++++++++++++++
 scripts/run_e2e_tests.sh            |  7 ++-
 6 files changed, 117 insertions(+), 6 deletions(-)
 create mode 100755 .buildkite/hooks/pre-exit
 create mode 100755 scripts/kind-create-cluster.sh

diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
new file mode 100755
index 00000000..8bfa0400
--- /dev/null
+++ b/.buildkite/hooks/pre-exit
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# If there was a kind cluster created during CI, tear it down.
+KIND="$HOME/bin/kind"
+if [[ -f "$KIND" ]]; then
+  echo "--- :kubernetes: deleting kind cluster"
+    $KIND get clusters -q | while read -r CLUSTER; do
+    $KIND delete cluster --name "$CLUSTER"
+  done
+fi
+
+echo "--- :git: cleaning checkout"
+git clean -dffx
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 2ffff107..b8889384 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -27,3 +27,13 @@ steps:
         config: .buildkite/docker-compose.yml
         workdir: /go/src/github.com/m3db/m3db-operator
     <<: *common
+  # NB(schallert): skipped until OSS CI stack upgraded
+  # - name: ":kubernetes: e2e tests"
+  #   command: make clean test-e2e
+  #   env:
+  #     CGO_ENABLED: 0
+  #     GIMME_GO_VERSION: 1.13.x
+  #   plugins:
+  #     gopath-checkout#v1.0.1:
+  #       import: github.com/m3db/m3db-operator
+  #   <<: *common
diff --git a/Makefile b/Makefile
index 4e17395f..e975ad5b 100644
--- a/Makefile
+++ b/Makefile
@@ -125,10 +125,15 @@ test-no-deps: test-base
 	@echo "--- $@"
 	@$(tools_bin_path)/gocov convert $(coverfile) | $(tools_bin_path)/gocov report
 
+.PHONY: kind-create-cluster
+kind-create-cluster:
+	@echo "--- Starting KIND cluster"
+	@./scripts/kind-create-cluster.sh
+
 .PHONY: test-e2e
-test-e2e:
+test-e2e: kind-create-cluster
 	@echo "--- $@"
-	$(SELF_DIR)/scripts/run_e2e_tests.sh
+	PATH=$(HOME)/bin:$(PATH) $(SELF_DIR)/scripts/run_e2e_tests.sh
 
 .PHONY: testhtml
 testhtml: test-base
diff --git a/integration/manifests/operator.yaml b/integration/manifests/operator.yaml
index 1a46a7ef..39d33dee 100644
--- a/integration/manifests/operator.yaml
+++ b/integration/manifests/operator.yaml
@@ -26,7 +26,7 @@ rules:
   verbs: ["create", "get", "deletecollection", "delete"]
 - apiGroups: [""]
   resources: ["pods"]
-  verbs: ["list", "get", "watch", "update"]
+  verbs: ["list", "get", "watch", "update", "patch"]
 - apiGroups: ["apps"]
   resources: ["statefulsets", "deployments"]
   verbs: ["*"]
@@ -68,8 +68,12 @@ spec:
       labels:
         name: m3db-operator
     spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        runAsGroup: 65534
       containers:
         - name: m3db-operator
-          image: quay.io/m3db/m3db-operator:latest
-          imagePullPolicy: Always
+          image: m3db-operator-kind
+          imagePullPolicy: Never
       serviceAccount: operator-test-sa
diff --git a/scripts/kind-create-cluster.sh b/scripts/kind-create-cluster.sh
new file mode 100755
index 00000000..2fa5980a
--- /dev/null
+++ b/scripts/kind-create-cluster.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+set -exo pipefail
+
+echo "--- :kubernetes: Installing kind"
+
+KUBE_VERSION=${KUBE_VERSION:-v1.15.7}
+CLUSTER_NAME=kind
+L_UNAME=$(uname | tr "[:upper:]" "[:lower:]")
+
+mkdir -p "$HOME/bin"
+
+if [[ ! -x "$HOME/bin/kind" || "$BUILDKITE" == "true" ]]; then
+  curl -sL -o "$HOME/bin/kind" "https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-${L_UNAME}-amd64"
+fi
+
+if [[ ! -x "$HOME/bin/kubectl" || "$BUILDKITE" == "true" ]]; then
+  curl -sL -o "$HOME/bin/kubectl" "https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/${L_UNAME}/amd64/kubectl"
+fi
+
+chmod +x "$HOME/bin/kind" "$HOME/bin/kubectl"
+export PATH="$HOME/bin:$PATH"
+
+echo "--- :kubernetes: Deleting existing kind clusters"
+kind get clusters -q | while read -r CLUSTER; do
+  kind delete cluster --name "$CLUSTER"
+done
+
+echo "--- :kubernetes: Creating kind cluster"
+
+cat > cluster.yaml <<EOF
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+  kubeadmConfigPatches:
+  - |
+    apiVersion: kubeadm.k8s.io/v1beta2
+    kind: JoinConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "failure-domain.beta.kubernetes.io/zone=us-east1-b"
+- role: worker
+  kubeadmConfigPatches:
+  - |
+    apiVersion: kubeadm.k8s.io/v1beta2
+    kind: JoinConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "failure-domain.beta.kubernetes.io/zone=us-east1-c"
+- role: worker
+  kubeadmConfigPatches:
+  - |
+    apiVersion: kubeadm.k8s.io/v1beta2
+    kind: JoinConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "failure-domain.beta.kubernetes.io/zone=us-east1-d"
+EOF
+
+kind create cluster --image "kindest/node:$KUBE_VERSION" --name "$CLUSTER_NAME" --config cluster.yaml
+
+kubectl get nodes
+kubectl cluster-info
+
+echo "--- :kubernetes: Kind cluster created"
+
+echo "--- Waiting for cluster to be ready"
+while kubectl get nodes | grep -q NotReady; do
+  sleep 5
+done
diff --git a/scripts/run_e2e_tests.sh b/scripts/run_e2e_tests.sh
index 9ba2f609..f6031d7c 100755
--- a/scripts/run_e2e_tests.sh
+++ b/scripts/run_e2e_tests.sh
@@ -30,7 +30,7 @@ function main() {
   done
 
   # If a kubectl proxy process is running, delete and wait until it's dead
-  if pgrep -fq 'kubectl proxy'; then
+  if pgrep -f 'kubectl proxy'; then
     pkill -f 'kubectl proxy'
     local PROXYCOUNT=0
     while [[ $PROXYCOUNT -lt 10 ]]; do
@@ -46,6 +46,11 @@ function main() {
 
   trap cleanup EXIT
 
+  if [[ -z "$SKIP_DOCKER_BUILD" ]]; then
+    docker build -t m3db-operator-kind .
+    kind load docker-image m3db-operator-kind
+  fi
+
   kubectl proxy &
   go clean -testcache
   go test -v -tags integration ./integration/e2e