From 394daf190c0813a38c0849c29aed63ea09ec4199 Mon Sep 17 00:00:00 2001 From: Boris Djurdjevic Date: Tue, 20 Apr 2021 13:12:57 +0200 Subject: [PATCH] feat: CI improvements and RBAC fixes --- .github/workflows/ci.yml | 57 ++++++++++++++++++++++++++++++++ README.md | 2 ++ ci/deployment/conf.yaml | 30 +++++++++++++++++ ci/deployment/kustomization.yaml | 7 ++++ ci/kind-config.yaml | 19 +++++++++++ examples/daemonset.yaml | 4 +-- examples/kustomization.yaml | 6 ++++ examples/rbac.yaml | 50 ++++++++++++++++++++++++++-- examples/serviceaccount.yaml | 2 +- 9 files changed, 171 insertions(+), 6 deletions(-) create mode 100644 ci/deployment/conf.yaml create mode 100644 ci/deployment/kustomization.yaml create mode 100644 ci/kind-config.yaml create mode 100644 examples/kustomization.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b8d73f8..6d23f5f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,3 +20,60 @@ jobs: with: version: latest args: release --snapshot --rm-dist --skip-publish + - name: Setup kind + uses: engineerd/setup-kind@v0.5.0 + with: + config: "ci/kind-config.yaml" + - name: Show cluster info and switch to kube-system + run: | + kubectl cluster-info + echo "current-context:" $(kubectl config current-context) + kubectl config set-context --current --namespace kube-system + - name: Deploy ingress-nginx + timeout-minutes: 2 + run: | + kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/kind/deploy.yaml + kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=90s + - name: Import image into kind + run: kind load docker-image postfinance/kubenurse:latest + - name: Deploy example setup + run: | + kubectl apply -k ci/deployment/ + - name: Wait for pods + timeout-minutes: 1 + run: | + sleep 15 # wait for the scheduler to create pods + kubectl wait pods -l app=kubenurse --for=condition=Ready + kubectl get pods -l app=kubenurse + - name: Check state of pods + timeout-minutes: 2 + run: | + sleep 60 # Wait to generate some checks etc. + test $(kubectl get pods -l app=kubenurse | wc -l) -eq 4 # Test for 3 Pods + header + echo "Number of kubenurses is ok" + test $(kubectl logs -l app=kubenurse | grep -v "I'm ready to help you" | wc -l) -eq 0 # Test that there are no unexpected log lines present + echo "Logs are ok" + kubectl get pods -l app=kubenurse -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\n"}{end}' | (! grep -v 0) # Validate for 0 restarts + echo "Restart count is ok" + curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/metrics | grep "kubenurse_request_" > /dev/null + echo "Metrics contain kubenurse specific data" + curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/metrics | grep "kubenurse_request_" | grep 'type="path_' > /dev/null + echo "Metrics contains neighbours" + curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/metrics | (! grep "kubenurse_errors_total") + echo "Metrics contains no errors" + curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/alive | grep '"neighbourhood_state": "ok"' > /dev/null + echo "Neighbourhood state ok" + curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/alive | grep '"neighbourhood": \[' > /dev/null # If no others are discovered, this is null + echo "Discovery ok: Status page includes neighbours" + - name: Show kubenurse status + run: | + curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/alive + if: ${{ always() }} + - name: Describe resources on failure + run: | + kubectl get pods -o wide + kubectl logs -l app=kubenurse + kubectl describe pods -l app=kubenurse + kubectl describe daemonsets -l app=kubenurse + kubectl get events + if: ${{ failure() }} diff --git a/README.md b/README.md index 53658257..063ee9ae 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,8 @@ Neighbours are discovered by querying the kube-apiserver for every Pod in the `KUBENURSE_NAMESPACE` with label `KUBENURSE_NEIGHBOUR_FILTER`. The request is done directly to the Pod-IP and the metric types contains the prefix `path_` and the hostname of the kubelet on which the neighbour kubenurse should run. +Only kubenurses on nodes that are schedulable are considered as neighbours, +this can be changed by setting `KUBENURSE_ALLOW_UNSCHEDULABLE="true"`. Metric type: `path_$KUBELET_HOSTNAME` diff --git a/ci/deployment/conf.yaml b/ci/deployment/conf.yaml new file mode 100644 index 00000000..7c714f5e --- /dev/null +++ b/ci/deployment/conf.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kubenurse +spec: + template: + spec: + containers: + - name: kubenurse + env: + - name: KUBENURSE_INSECURE + value: "true" + - name: KUBENURSE_INGRESS_URL + value: https://ingress-nginx-controller.ingress-nginx.svc.cluster.local +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: kubenurse +spec: + rules: + - host: ingress-nginx-controller.ingress-nginx.svc.cluster.local + http: + paths: + - backend: + serviceName: kubenurse + servicePort: 8080 + tls: + - hosts: + - ingress-nginx-controller.ingress-nginx.svc.cluster.local diff --git a/ci/deployment/kustomization.yaml b/ci/deployment/kustomization.yaml new file mode 100644 index 00000000..3dd2d4b1 --- /dev/null +++ b/ci/deployment/kustomization.yaml @@ -0,0 +1,7 @@ +bases: +- ../../examples/ +patchesStrategicMerge: +- conf.yaml +images: +- name: postfinance/kubenurse + newTag: latest diff --git a/ci/kind-config.yaml b/ci/kind-config.yaml new file mode 100644 index 00000000..42f73dc7 --- /dev/null +++ b/ci/kind-config.yaml @@ -0,0 +1,19 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: # required for ingress-nginx + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP +- role: worker +- role: worker diff --git a/examples/daemonset.yaml b/examples/daemonset.yaml index f6f3db88..0d02fddc 100644 --- a/examples/daemonset.yaml +++ b/examples/daemonset.yaml @@ -20,7 +20,7 @@ spec: prometheus.io/scheme: "http" prometheus.io/scrape: "true" spec: - serviceAccountName: nurse + serviceAccountName: kubenurse containers: - name: kubenurse env: @@ -32,7 +32,7 @@ spec: value: kube-system - name: KUBENURSE_NEIGHBOUR_FILTER value: "app=kubenurse" - image: "postfinance/kubenurse:v1.2.0" + image: "postfinance/kubenurse:v1.3.2" ports: - containerPort: 8080 protocol: TCP diff --git a/examples/kustomization.yaml b/examples/kustomization.yaml new file mode 100644 index 00000000..03d8c680 --- /dev/null +++ b/examples/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- daemonset.yaml +- ingress.yaml +- rbac.yaml +- serviceaccount.yaml +- service.yaml diff --git a/examples/rbac.yaml b/examples/rbac.yaml index 43ef54e9..028f4554 100644 --- a/examples/rbac.yaml +++ b/examples/rbac.yaml @@ -2,13 +2,57 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: nurse + name: kubenurse namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubenurse +subjects: +- kind: ServiceAccount + name: kubenurse + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubenurse + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +--- +# This resource is not needed if KUBENURSE_ALLOW_UNSCHEDULABLE=true +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubenurse roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: view + name: kubenurse subjects: - kind: ServiceAccount - name: nurse + name: kubenurse namespace: kube-system +--- +# This resource is not needed if KUBENURSE_ALLOW_UNSCHEDULABLE=true +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubenurse +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - get + - watch diff --git a/examples/serviceaccount.yaml b/examples/serviceaccount.yaml index c907374b..b9b140fb 100644 --- a/examples/serviceaccount.yaml +++ b/examples/serviceaccount.yaml @@ -2,5 +2,5 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: nurse + name: kubenurse namespace: kube-system