Switch to prow for e2e testing. Fix intermittent test failures.

.travis.yml

@@ -9,32 +9,12 @@ go_import_path: github.com/jetstack/navigator
 services:
 - docker
 
-jobs:
-  include:
-    - stage: test
-      env:
-      - KUBERNETES_VERSION=v1.8.0
-      before_script:
-      - ./hack/install-e2e-dependencies.sh
-      script:
-      - make BUILD_TAG=latest build e2e-test
-
-    - stage: test
-      env:
-      - KUBERNETES_VERSION=v1.7.0
-      before_script:
-      - ./hack/install-e2e-dependencies.sh
-      script:
-      - make BUILD_TAG=latest build e2e-test
-
-    - stage: test
-      script:
-      - make verify
-
-    - stage: build
-      script:
-      - make go_build docker_build
-      - if [ "${TRAVIS_PULL_REQUEST}" = "false" ] && [ "${TRAVIS_BRANCH}" = "master" ]; then
-          mkdir -p ~/.docker && echo "${DOCKER_AUTH_CONFIG}" > ~/.docker/config.json && chmod 600 ~/.docker/config.json;
-          make docker_push IMAGE_TAGS="${TRAVIS_COMMIT} latest";
-        fi
+- make docker_build
+- if [ "${TRAVIS_PULL_REQUEST}" = "false" ] && [ "${TRAVIS_BRANCH}" = "master" ]; then
+    mkdir -p ~/.docker && echo "${DOCKER_AUTH_CONFIG}" > ~/.docker/config.json && chmod 600 ~/.docker/config.json;
+    make docker_push IMAGE_TAGS="${TRAVIS_COMMIT} latest";
+  fi
+
+branches:
+  only:
+  - master

Makefile

@@ -1,3 +1,4 @@
+SHELL := /bin/bash
 BINDIR        ?= bin
 HACK_DIR     ?= hack
 NAVIGATOR_PKG = github.com/jetstack/navigator
@@ -8,6 +9,7 @@ REGISTRY := jetstackexperimental
 IMAGE_NAME := navigator
 BUILD_TAG := build
 IMAGE_TAGS := canary
+CHART_VALUES := ${HACK_DIR}/testdata/values.yaml
 
 BUILD_IMAGE_DIR := hack/builder
 BUILD_IMAGE_NAME := navigator/builder
@@ -32,11 +34,15 @@ all: verify build docker_build
 
 test: go_test
 
-.hack_e2e:
-	@${HACK_DIR}/prepare-e2e.sh
-	@${HACK_DIR}/e2e.sh
+.run_e2e:
+	export CHART_VALUES=${CHART_VALUES}; \
+	${HACK_DIR}/prepare-e2e.sh; \
+	${HACK_DIR}/e2e.sh
 
-e2e-test: docker_build .hack_e2e
+.e2e_init:
+	${HACK_DIR}/install-e2e-dependencies.sh
+
+e2e-test: .e2e_init build docker_build .run_e2e
 
 build: $(CMDS)
 
@@ -58,6 +64,7 @@ verify: .hack_verify go_verify
 DOCKER_BUILD_TARGETS = $(addprefix docker_build_, $(CMDS))
 $(DOCKER_BUILD_TARGETS):
 	$(eval DOCKER_BUILD_CMD := $(subst docker_build_,,$@))
+	eval $$(minikube docker-env --profile $$HOSTNAME --shell sh); \
 	docker build -t $(REGISTRY)/$(IMAGE_NAME)-$(DOCKER_BUILD_CMD):$(BUILD_TAG) -f Dockerfile.$(DOCKER_BUILD_CMD) .
 docker_build: $(DOCKER_BUILD_TARGETS)
 
@@ -66,6 +73,7 @@ $(DOCKER_PUSH_TARGETS):
 	$(eval DOCKER_PUSH_CMD := $(subst docker_push_,,$@))
 	set -e; \
 		for tag in $(IMAGE_TAGS); do \
+		eval $$(minikube docker-env --profile $$HOSTNAME --shell sh); \
 		docker tag $(REGISTRY)/$(IMAGE_NAME)-$(DOCKER_PUSH_CMD):$(BUILD_TAG) $(REGISTRY)/$(IMAGE_NAME)-$(DOCKER_PUSH_CMD):$${tag} ; \
 		docker push $(REGISTRY)/$(IMAGE_NAME)-$(DOCKER_PUSH_CMD):$${tag}; \
 	done

contrib/charts/navigator/templates/apiserver.yaml

@@ -36,11 +36,7 @@ spec:
           args:
           - navigator-apiserver
           - --etcd-servers=http://localhost:2379
-          - --requestheader-client-ca-file=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-          - --requestheader-username-headers=X-Remote-User
-          - --requestheader-group-headers=X-Remote-Group
-          - --requestheader-extra-headers-prefix=X-Remote-Extra
-          - --v={{ .Values.apiserver.logLevel }}
+{{ toYaml .Values.apiserver.extraArgs | indent 10 }}
           resources:
 {{ toYaml .Values.resources | indent 12 }}
         - name: etcd

contrib/charts/navigator/templates/rbac.yaml

@@ -87,7 +87,7 @@ items:
     name: "{{ template "fullname" . }}:controller"
   rules:
   - apiGroups: ["navigator.jetstack.io"]
-    resources: ["elasticsearchclusters", "pilots"]
+    resources: ["elasticsearchclusters", "pilots", "elasticsearchclusters/status", "pilots/status"]
     verbs:     ["get", "list", "watch", "update", "create", "delete"]
   - apiGroups: [""]
     resources: ["services", "configmaps", "serviceaccounts", "pods"]

contrib/charts/navigator/values.yaml

@@ -11,6 +11,13 @@ apiserver:
   ## which require cluster admin access to deploy.
   rbacDisabled: false
 
+  ## Extra arguments to pass to the navigator-apiserver
+  extraArgs:
+#  - --requestheader-client-ca-file=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+#  - --requestheader-username-headers=X-Remote-User
+#  - --requestheader-group-headers=X-Remote-Group
+#  - --requestheader-extra-headers-prefix=X-Remote-Extra  - --proxy-client-cert-file="${CERT_DIR}/client-auth-proxy.crt"
+
   ## Optional: if not set, a service account will be automatically created
   # serviceAccount: "apiserver-svc-acct"
   image:

hack/e2e.sh

@@ -19,12 +19,14 @@ source "${SCRIPT_DIR}/libe2e.sh"
 helm delete --purge "${RELEASE_NAME}" || true
 kube_delete_namespace_and_wait "${USER_NAMESPACE}"
 
+if [ "${CHART_VALUES}" == "" ]; then
+    echo "CHART_VALUES must be set";
+    exit 1
+fi
+
 echo "Installing navigator..."
 helm install --wait --name "${RELEASE_NAME}" contrib/charts/navigator \
-        --set apiserver.image.pullPolicy=Never \
-        --set apiserver.logLevel=100 \
-        --set controller.image.pullPolicy=Never \
-        --set controller.logLevel=100
+        --values ${CHART_VALUES}
 
 # Wait for navigator API to be ready
 function navigator_ready() {

hack/install-e2e-dependencies.sh

@@ -6,30 +6,14 @@ set -eux
 SCRIPT_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
 source "${SCRIPT_DIR}/libe2e.sh"
 
-curl -Lo helm.tar.gz \
-     https://storage.googleapis.com/kubernetes-helm/helm-v2.6.1-linux-amd64.tar.gz
-tar xvf helm.tar.gz
-sudo mv linux-amd64/helm /usr/local/bin
-
-curl -Lo kubectl \
-     https://storage.googleapis.com/kubernetes-release/release/$KUBERNETES_VERSION/bin/linux/amd64/kubectl
-chmod +x kubectl
-sudo mv kubectl /usr/local/bin/
-
-curl -Lo minikube \
-     https://storage.googleapis.com/minikube/releases/v0.23.0/minikube-linux-amd64
-chmod +x minikube
-sudo mv minikube /usr/local/bin/
-
-docker run -v /usr/local/bin:/hostbin quay.io/jetstack/ubuntu-nsenter cp /nsenter /hostbin/nsenter
-
 # Create a cluster. We do this as root as we are using the 'docker' driver.
-# We enable RBAC on the cluster too, to test the RBAC in Navigators chart
-sudo -E CHANGE_MINIKUBE_NONE_USER=true minikube start \
+# The kubeadm bootstrapper enables RBAC by default.
+minikube start \
      -v 100 \
-     --vm-driver=none \
+     --vm-driver=kvm \
      --kubernetes-version="$KUBERNETES_VERSION" \
-     --extra-config=apiserver.Authorization.Mode=RBAC
+     --bootstrapper=kubeadm \
+     --profile="$HOSTNAME"
 
 echo "Waiting up to 5 minutes for Kubernetes to be ready..."
 if ! retry TIMEOUT=300 kubectl get nodes; then

hack/prepare-e2e.sh

@@ -32,33 +32,7 @@ items:
     kind: ServiceAccount
     name: tiller
     namespace: kube-system
-### Generic ###
-# Create a ClusterRole to work with ElasticsearchCluster resources
-- apiVersion: rbac.authorization.k8s.io/v1beta1
-  kind: ClusterRole
-  metadata:
-    name: navigator:authenticated
-  # this rule defined on the role for specifically the
-  # namespace-lifecycle admission-controller
-  rules:
-  - apiGroups: ["navigator.jetstack.io"]
-    resources: ["elasticsearchclusters", "pilots"]
-    verbs:     ["get", "list", "watch", "create", "update", "delete"]
-- apiVersion: rbac.authorization.k8s.io/v1beta1
-  kind: ClusterRoleBinding
-  metadata:
-    name: "navigator:authenticated"
-  roleRef:
-    apiGroup: rbac.authorization.k8s.io
-    kind: ClusterRole
-    name: navigator:authenticated
-  subjects:
-  - kind: Group
-    name: system:authenticated
-    apiGroup: rbac.authorization.k8s.io
-  - kind: Group
-    name: system:unauthenticated
-    apiGroup: rbac.authorization.k8s.io
+
 EOF
 helm init --service-account=tiller
 

hack/testdata/values.yaml

@@ -0,0 +1,38 @@
+# Default values for navigator.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+createAPIService: true
+
+rbac:
+  enabled: true
+
+apiserver:
+  ## Set to true to skip deploying the apiserver components RBAC policies,
+  ## which require cluster admin access to deploy.
+  rbacDisabled: false
+
+  extraArgs:
+  - --v=100
+
+  ## Optional: if not set, a service account will be automatically created
+  # serviceAccount: "apiserver-svc-acct"
+  image:
+    repository: jetstackexperimental/navigator-apiserver
+    tag: build
+    pullPolicy: Never
+
+controller:
+  ## Optional: namespace to watch for resources in. This can be used when RBAC
+  ## restricts you to a single namespace.
+  # namespace: default
+  ## Optional: if not set, a service account will be automatically created
+  # serviceAccount: "controller-svc-acct"
+  image:
+    repository: jetstackexperimental/navigator-controller
+    tag: build
+    pullPolicy: Never
+
+resources:
+  requests:
+    cpu: 50m
+    memory: 64Mi