feat: CI improvements and RBAC fixes

.github/workflows/ci.yml

@@ -20,3 +20,60 @@ jobs:
         with:
           version: latest
           args: release --snapshot --rm-dist --skip-publish
+      - name: Setup kind
+        uses: engineerd/setup-kind@v0.5.0
+        with:
+          config: "ci/kind-config.yaml"
+      - name: Show cluster info and switch to kube-system
+        run: |
+          kubectl cluster-info
+          echo "current-context:" $(kubectl config current-context)
+          kubectl config set-context --current --namespace kube-system
+      - name: Deploy ingress-nginx
+        timeout-minutes: 2
+        run: |
+          kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/kind/deploy.yaml
+          kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=90s
+      - name: Import image into kind
+        run: kind load docker-image postfinance/kubenurse:latest
+      - name: Deploy example setup
+        run: |
+          kubectl apply -k ci/deployment/
+      - name: Wait for pods
+        timeout-minutes: 1
+        run: |
+          sleep 15 # wait for the scheduler to create pods
+          kubectl wait pods -l app=kubenurse --for=condition=Ready
+          kubectl get pods -l app=kubenurse
+      - name: Check state of pods
+        timeout-minutes: 2
+        run: |
+          sleep 60 # Wait to generate some checks etc.
+          test $(kubectl get pods -l app=kubenurse | wc -l) -eq 4 # Test for 3 Pods + header
+          echo "Number of kubenurses is ok"
+          test $(kubectl logs -l app=kubenurse | grep -v "I'm ready to help you" | wc -l) -eq 0 # Test that there are no unexpected log lines present
+          echo "Logs are ok"
+          kubectl get pods -l app=kubenurse -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\n"}{end}' | (! grep -v 0) # Validate for 0 restarts
+          echo "Restart count is ok"
+          curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/metrics | grep "kubenurse_request_" > /dev/null
+          echo "Metrics contain kubenurse specific data"
+          curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/metrics | grep "kubenurse_request_" | grep 'type="path_' > /dev/null
+          echo "Metrics contains neighbours"
+          curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/metrics | (! grep "kubenurse_errors_total")
+          echo "Metrics contains no errors"
+          curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/alive | grep '"neighbourhood_state": "ok"' > /dev/null
+          echo "Neighbourhood state ok"
+          curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/alive | grep '"neighbourhood": \[' > /dev/null # If no others are discovered, this is null
+          echo "Discovery ok: Status page includes neighbours"
+      - name: Show kubenurse status
+        run: |
+          curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/alive
+        if: ${{ always() }}
+      - name: Describe resources on failure
+        run: |
+          kubectl get pods -o wide
+          kubectl logs -l app=kubenurse
+          kubectl describe pods -l app=kubenurse
+          kubectl describe daemonsets -l app=kubenurse
+          kubectl get events
+        if: ${{ failure() }}

README.md

@@ -124,6 +124,8 @@ Neighbours are discovered by querying the kube-apiserver for every Pod in the
 `KUBENURSE_NAMESPACE` with label `KUBENURSE_NEIGHBOUR_FILTER`.
 The request is done directly to the Pod-IP and the metric types contains the prefix
 `path_` and the hostname of the kubelet on which the neighbour kubenurse should run.
+Only kubenurses on nodes that are schedulable are considered as neighbours,
+this can be changed by setting `KUBENURSE_ALLOW_UNSCHEDULABLE="true"`.
 
 Metric type: `path_$KUBELET_HOSTNAME`
 

ci/deployment/conf.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kubenurse
+spec:
+  template:
+    spec:
+      containers:
+      - name: kubenurse
+        env:
+        - name: KUBENURSE_INSECURE
+          value: "true"
+        - name: KUBENURSE_INGRESS_URL
+          value: https://ingress-nginx-controller.ingress-nginx.svc.cluster.local
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: kubenurse
+spec:
+  rules:
+  - host: ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    http:
+      paths:
+      - backend:
+          serviceName: kubenurse
+          servicePort: 8080
+  tls:
+  - hosts:
+    - ingress-nginx-controller.ingress-nginx.svc.cluster.local

ci/deployment/kustomization.yaml

@@ -0,0 +1,7 @@
+bases:
+- ../../examples/
+patchesStrategicMerge:
+- conf.yaml
+images:
+- name: postfinance/kubenurse
+  newTag: latest

ci/kind-config.yaml

@@ -0,0 +1,19 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: InitConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "ingress-ready=true"
+  extraPortMappings: # required for ingress-nginx
+  - containerPort: 80
+    hostPort: 80
+    protocol: TCP
+  - containerPort: 443
+    hostPort: 443
+    protocol: TCP
+- role: worker
+- role: worker

examples/daemonset.yaml

@@ -20,7 +20,7 @@ spec:
         prometheus.io/scheme: "http"
         prometheus.io/scrape: "true"
     spec:
-      serviceAccountName: nurse
+      serviceAccountName: kubenurse
       containers:
       - name: kubenurse
         env:
@@ -32,7 +32,7 @@ spec:
           value: kube-system
         - name: KUBENURSE_NEIGHBOUR_FILTER
           value: "app=kubenurse"
-        image: "postfinance/kubenurse:v1.2.0"
+        image: "postfinance/kubenurse:v1.3.2"
         ports:
         - containerPort: 8080
           protocol: TCP

examples/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+- daemonset.yaml
+- ingress.yaml
+- rbac.yaml
+- serviceaccount.yaml
+- service.yaml

examples/rbac.yaml

@@ -2,13 +2,57 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: nurse
+  name: kubenurse
   namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kubenurse
+subjects:
+- kind: ServiceAccount
+  name: kubenurse
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kubenurse
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+---
+# This resource is not needed if KUBENURSE_ALLOW_UNSCHEDULABLE=true
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubenurse
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: view
+  name: kubenurse
 subjects:
 - kind: ServiceAccount
-  name: nurse
+  name: kubenurse
   namespace: kube-system
+---
+# This resource is not needed if KUBENURSE_ALLOW_UNSCHEDULABLE=true
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubenurse
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - get
+  - watch

examples/serviceaccount.yaml

@@ -2,5 +2,5 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: nurse
+  name: kubenurse
   namespace: kube-system