Merge pull request #6 from mjtrangoni/add-cilium-lrp

Add possibility to use cilium LocalRedirectPolicy CRD
lablabs · Nov 29, 2023 · ff7850f · ff7850f
2 parents 5eb0a1d + a418582
commit ff7850f
Show file tree

Hide file tree

Showing 10 changed files with 210 additions and 23 deletions.
diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml
@@ -11,19 +11,35 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Run chart-testing (lint)
-        id: lint
-        uses: helm/chart-testing-action@v2.4.0
+      - name: Set up Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: v3.12.1
+
+      - uses: actions/setup-python@v4
         with:
-          command: lint
-          config: ct.yaml
+          python-version: '3.9'
+          check-latest: true
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2.4.0
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }})
+          if [[ -n "$changed" ]]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Run chart-testing (lint)
+        if: steps.list-changed.outputs.changed == 'true'
+        run: ct lint --target-branch ${{ github.event.repository.default_branch }}
 
       - name: Create kind cluster
+        if: steps.list-changed.outputs.changed == 'true'
         uses: helm/kind-action@v1.8.0
-        if: steps.lint.outputs.changed == 'true'
 
       - name: Run chart-testing (install)
-        uses: helm/chart-testing-action@v2.4.0
-        with:
-          command: install
-          config: ct.yaml
+        if: steps.list-changed.outputs.changed == 'true'
+        run: ct install --target-branch ${{ github.event.repository.default_branch }}
diff --git a/charts/node-local-dns/Chart.yaml b/charts/node-local-dns/Chart.yaml
@@ -1,8 +1,8 @@
 ---
 apiVersion: v2
 name: node-local-dns
-version: 1.4.0-rc.1
-appVersion: 1.22.9
+version: 1.4.0-rc.2
+appVersion: 1.22.24
 home: https://github.com/lablabs/k8s-nodelocaldns-helm
 description: NodeLocal DNS Cache helm chart
 keywords:

diff --git a/charts/node-local-dns/README.md b/charts/node-local-dns/README.md
@@ -7,6 +7,12 @@ helm repo add k8s-nodelocaldns-helm  https://lablabs.github.io/k8s-nodelocaldns-
 helm install k8s-nodelocaldns-helm/node-local-dns
 ```
 
+### Testing
+
+```console
+helm test node-local-dns
+```
+
 ## Configuration
 
 This chart deploys NodeLocal DNSCache Daemon set according to <https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/>.
@@ -15,13 +21,50 @@ It is designed to work both with iptables and IPVS setup.
 
 Latest available `node-local-dns` image can be found at [node-local-dns google container repository](https://console.cloud.google.com/gcr/images/google-containers/GLOBAL/k8s-dns-node-cache)
 
+### Cilium
+
+For clusters running [cilium](https://cilium.io/), there is a CRD,
+[local-redirect-policy](https://docs.cilium.io/en/stable/network/kubernetes/local-redirect-policy/),
+which needs be extra enabled via `--set localRedirectPolicy=true`.
+It enables pod traffic destined to an IP address and port/protocol tuple or Kubernetes service to be redirected
+locally to backend pod(s) within a node, using eBPF.
+The namespace of backend pod(s) need to match with that of the policy.
+
+For using this feature, values should provides the following extra configuration,
+
+For getting the `CLUSTER_DNS_IP`,
+
+```console
+kubectl get svc kube-dns -n kube-system -o jsonpath={.spec.clusterIP}
+```
+
+```yaml
+config:
+  localDnsIp: CLUSTER_DNS_IP
+  cilium:
+    clusterDNSService: kube-dns
+    clusterDNSNamespace: kube-system
+    udp:
+      enabled: true
+      portName: dns
+    tcp:
+      enabled: true
+      portName: dns-tcp
+```
+
+#### RKE2
+
+As this feature heavily depends on the Cluster DNS implementation, for a [Rancher Kubernetes Engine 2](https://docs.rke2.io/) cluster,
+`clusterDNSService` should be `rke2-coredns-rke2-coredns`, and port names,
+`udp-53` and `tcp-53` respectively.
+
 ### Values
 
 The following table lists the configurable parameters of the Node-local-dns chart and their default values.
 
 | Parameter                | Description             | Default        |
 | ------------------------ | ----------------------- | -------------- |
-| `image.repository` |  | `"k8s.gcr.io/dns/k8s-dns-node-cache"` |
+| `image.repository` |  | `"registry.k8s.io/dns/k8s-dns-node-cache"` |
 | `image.pullPolicy` |  | `"IfNotPresent"` |
 | `image.tag` |  | `"1.22.9"` |
 | `image.args.skipTeardown` |  | `true` |
@@ -34,6 +77,12 @@ The following table lists the configurable parameters of the Node-local-dns char
 | `image.args.quiet` | `false` |
 | `imagePullSecrets` |  | `[]` |
 | `config.localDnsIp` |  | `"169.254.20.11"` |
+| `config.cilium.clusterDNSService` | Cluster DNS service name | `"kube-dns"` |
+| `config.cilium.clusterDNSNamespace` | Cluster DNS namespace | `"kube-system"` |
+| `config.cilium.udp.enabled` | Enable UDP port mapping upstream DNS service | `false` |
+| `config.cilium.udp.portName` | UDP port name upstream DNS service | `"dns"` |
+| `config.cilium.tcp.enabled` | Enable TCP port mapping upstream DNS service | `false` |
+| `config.cilium.tcp.portName` | TCP port name upstream DNS service | `"dns-tcp"` |
 | `config.zones` |  | `[{".:53":{"plugins":{"errors":true,"reload":true,"debug":false,"log":{"format":"combined","classes":"all"},"cache":{"parameters":30,"denial":{},"success":{},"prefetch":{},"serve_stale":false},"forward":{"parameters":"__PILLAR__UPSTREAM__SERVERS__","force_tcp":false,"prefer_udp":false,"policy":"","max_fails":"","expire":"","health_check":"","except":""},"prometheus":true,"health":{"port":8080}}}},{"ip6.arpa:53":{"plugins":{"errors":true,"reload":true,"debug":false,"log":{"format":"combined","classes":"all"},"cache":{"parameters":30},"forward":{"parameters":"__PILLAR__UPSTREAM__SERVERS__","force_tcp":false},"prometheus":true,"health":{"port":8080}}}},{"in-addr.arpa:53":{"plugins":{"errors":true,"reload":true,"debug":false,"log":{"format":"combined","classes":"all"},"cache":{"parameters":30},"forward":{"parameters":"__PILLAR__UPSTREAM__SERVERS__","force_tcp":false},"prometheus":true,"health":{"port":8080}}}}]` |
 | `useHostNetwork` |  | `true` |
 | `updateStrategy.rollingUpdate.maxUnavailable` |  | `"10%"` |
@@ -65,7 +114,7 @@ config:
           reload: true
           debug: false
           log:
-            fomat: common
+            format: common
             classes: all
           cache:
             parameters: 30

diff --git a/charts/node-local-dns/templates/ciliumlocalredirectpolicy.yaml b/charts/node-local-dns/templates/ciliumlocalredirectpolicy.yaml
@@ -0,0 +1,33 @@
+{{- if hasKey .Values.config "cilium" }}
+---
+apiVersion: "cilium.io/v2"
+kind: CiliumLocalRedirectPolicy
+metadata:
+  name: "node-local-dns"
+  labels:
+    {{- include "node-local-dns.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  redirectFrontend:
+    serviceMatcher:
+      serviceName: {{ .Values.config.cilium.clusterDNSService | default "kube-dns" }}
+      namespace: {{ .Values.config.cilium.clusterDNSNamespace | default "kube-system" }}
+  redirectBackend:
+    localEndpointSelector:
+      matchLabels:
+        {{- include "node-local-dns.selectorLabels" . | nindent 8 }}
+    toPorts:
+      {{- if .Values.config.cilium.udp.enabled }}
+      - port: "53"
+        name: {{ .Values.config.cilium.udp.portName | default "dns" }}
+        protocol: UDP
+     {{- end }}
+     {{- if .Values.config.cilium.tcp.enabled }}
+      - port: "53"
+        name: {{ .Values.config.cilium.tcp.portName | default "dns-tcp" }}
+        protocol: TCP
+     {{- end }}
+{{- end }}
diff --git a/charts/node-local-dns/templates/configmap.yaml b/charts/node-local-dns/templates/configmap.yaml
@@ -9,6 +9,7 @@ data:
   Corefile: |-
     {{- $localDnsIp := .Values.config.localDnsIp -}}
     {{- $metricsPort := .Values.metrics.port -}}
+    {{- $ciliumConfig := ternary true false (hasKey .Values.config "cilium") -}}
 
     {{- range $k, $v := .Values.config.zones }}
     {{ $k }} {
@@ -42,7 +43,11 @@ data:
       debug
       {{- end }}
       loop
+      {{- if not $ciliumConfig }}
       bind {{ $localDnsIp }}
+      {{- else }}
+      bind 0.0.0.0
+      {{- end }}
       forward . {{ $v.plugins.forward.parameters }} {
       {{- if $v.plugins.forward.policy }}
         policy {{ $v.plugins.forward.policy }}
@@ -70,7 +75,11 @@ data:
       prometheus :{{ $metricsPort }}
       {{- end }}
       {{- if $v.plugins.health }}
+      {{- if not $ciliumConfig }}
       health {{ $localDnsIp }}:{{ $v.plugins.health.port }}
+      {{- else }}
+      health
+      {{- end }}
       {{- end }}
     {{- end }}
     }

diff --git a/charts/node-local-dns/templates/daemonset.yaml b/charts/node-local-dns/templates/daemonset.yaml
@@ -29,7 +29,9 @@ spec:
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       priorityClassName: {{ .Values.priorityClassName }}
+      {{- if not (hasKey .Values.config "cilium") }}
       hostNetwork: {{ .Values.useHostNetwork }}
+      {{- end }}
       dnsPolicy: Default
       containers:
         - name: {{ .Chart.Name }}
@@ -39,16 +41,22 @@ spec:
             - -localip
             - "{{ .Values.config.localDnsIp }}"
             {{- if .Values.image.args.skipTeardown }}
-            - -skipteardown
+            - -skipteardown=true
+            {{- else }}
+            - -skipteardown=false
             {{- end }}
             {{- if .Values.image.args.setupEptables }}
             - -setupeptables
             {{- end }}
             {{- if .Values.image.args.setupInterface }}
-            - -setupinterface
+            - -setupinterface=true
+            {{- else }}
+            - -setupinterface=false
             {{- end }}
             {{- if .Values.image.args.setupIptables }}
-            - -setupiptables
+            - -setupiptables=true
+            {{- else }}
+            - -setupiptables=false
             {{- end }}
             {{- if .Values.image.args.quiet }}
             - -quiet
@@ -65,19 +73,35 @@ spec:
             - /etc/Corefile
             - -syncinterval
             - {{ .Values.image.args.syncInterval }}
+            {{- if .Values.image.args.setupInterface }}
             - -interfacename
             - {{ .Values.image.args.interfaceName }}
+            {{- end }}
             - -metrics-listen-address
             - "0.0.0.0:{{ add .Values.metrics.port 100 }}"
           ports:
             - name: metrics
               containerPort: {{ .Values.metrics.port }}
               protocol: TCP
+          {{- if (hasKey .Values.config "cilium") }}
+          {{- if .Values.config.cilium.udp.enabled }}
+            - name: {{ .Values.config.cilium.udp.portName | default "dns" }}
+              containerPort: 53
+              protocol: UDP
+          {{- end }}
+          {{- if .Values.config.cilium.tcp.enabled }}
+            - name: {{ .Values.config.cilium.tcp.portName | default "dns-tcp" }}
+              containerPort: 53
+              protocol: TCP
+          {{- end }}
+          {{- end }}
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
           livenessProbe:
             httpGet:
+              {{- if not (hasKey .Values.config "cilium") }}
               host: {{ .Values.config.localDnsIp }}
+              {{- end }}
               path: /health
               port: {{ default "8080" .Values.image.args.healthPort }}
             initialDelaySeconds: 60

diff --git a/charts/node-local-dns/templates/service.yaml b/charts/node-local-dns/templates/service.yaml
@@ -0,0 +1,31 @@
+{{- if hasKey .Values.config "cilium" }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.image.args.upstreamSvc }}
+  labels:
+    {{- include "node-local-dns.labels" . | nindent 4 }}
+    k8s-app: kube-dns
+    kubernetes.io/name: "KubeDNSUpstream"
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  ports:
+  {{- if .Values.config.cilium.udp.enabled }}
+  - name: {{ .Values.config.cilium.udp.portName | default "dns" }}
+    port: 53
+    protocol: UDP
+    targetPort: 53
+  {{- end }}
+  {{- if .Values.config.cilium.tcp.enabled }}
+  - name: {{ .Values.config.cilium.tcp.portName | default "dns-tcp" }}
+    port: 53
+    protocol: TCP
+    targetPort: 53
+  {{- end }}
+  selector:
+    k8s-app: kube-dns
+{{- end }}
diff --git a/charts/node-local-dns/templates/serviceaccount.yaml b/charts/node-local-dns/templates/serviceaccount.yaml
@@ -1,5 +1,5 @@
----
 {{- if .Values.serviceAccount.create }}
+---
 apiVersion: v1
 kind: ServiceAccount
 metadata:

diff --git a/charts/node-local-dns/templates/tests/test-dns-resolution.yaml b/charts/node-local-dns/templates/tests/test-dns-resolution.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "node-local-dns.fullname" . }}-dns-test"
+  labels:
+    {{- include "node-local-dns.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: dns-test
+      image: tutum/dnsutils
+      command: ['dig']
+      args: ['google.com']
+  restartPolicy: Never
diff --git a/charts/node-local-dns/values.yaml b/charts/node-local-dns/values.yaml
@@ -2,21 +2,30 @@
 image:
   repository: registry.k8s.io/dns/k8s-dns-node-cache
   pullPolicy: IfNotPresent
-  tag: 1.22.9
+  tag: 1.22.24
   args:
     interfaceName: nodelocaldns
     healthPort: 8080
     skipTeardown: true
     syncInterval: 1ns
-#    setupIptables: false
-#    setupEbtables: false
-#    quiet: false
-#    upstreamSvc: kube-dns
+    setupIptables: true
+    setupEbtables: true
+    quiet: false
+    upstreamSvc: kube-dns
 
 imagePullSecrets: []
 
 config:
   localDnsIp: 169.254.20.11
+#  cilium:
+#    clusterDNSService: kube-dns
+#    clusterDNSNamespace: kube-system
+#    udp:
+#      enabled: true
+#      portName: dns
+#    tcp:
+#      enabled: true
+#      portName: dns-tcp
   zones:
     .:53:
       plugins:
@@ -84,6 +93,7 @@ config:
         health:
           port: 8080
 
+# useHostNetwork is always false when using cilium
 useHostNetwork: true
 
 updateStrategy: