Merge pull request #238 from larivierec/feature/support-tls-grpc-routes

feat(gateway-api): Add initial support for tls and grpc routes

README.md

@@ -1,6 +1,6 @@
 # k8s_gateway
 
-A CoreDNS plugin that is very similar to [k8s_external](https://coredns.io/plugins/k8s_external/) but supporting all types of Kubernetes external resources - Ingress, Service of type LoadBalancer and HTTPRoutes from the [Gateway API project](https://gateway-api.sigs.k8s.io/). 
+A CoreDNS plugin that is very similar to [k8s_external](https://coredns.io/plugins/k8s_external/) but supporting all types of Kubernetes external resources - Ingress, Service of type LoadBalancer, HTTPRoutes, TLSRoutes, GRPCRoutes from the [Gateway API project](https://gateway-api.sigs.k8s.io/).
 
 This plugin relies on its own connection to the k8s API server and doesn't share any code with the existing [kubernetes](https://coredns.io/plugins/kubernetes/) plugin. The assumption is that this plugin can now be deployed as a separate instance (alongside the internal kube-dns) and act as a single external DNS interface into your Kubernetes cluster(s).
 
@@ -10,14 +10,14 @@ This plugin relies on its own connection to the k8s API server and doesn't share
 
 | Kind | Matching Against | External IPs are from | 
 | ---- | ---------------- | -------- |
-| HTTPRoute<sup>[1](#foot1)</sup> | all FQDNs from `spec.hostnames` matching configured zones | `gateway.status.addresses`<sup>[2](#foot2)</sup> |
+| HTTPRoute/TLSRoute/GRPCRoute<sup>[1](#foot1)</sup> | all FQDNs from `spec.hostnames` matching configured zones | `gateway.status.addresses`<sup>[2](#foot2)</sup> |
 | Ingress | all FQDNs from `spec.rules[*].host` matching configured zones | `.status.loadBalancer.ingress` |
 | Service<sup>[3](#foot3)</sup> | `name.namespace` + any of the configured zones OR any string consisting of lower case alphanumeric characters, '-' or '.', specified in the `coredns.io/hostname` or `external-dns.alpha.kubernetes.io/hostname` annotations (see [this](https://github.com/ori-edge/k8s_gateway/blob/master/test/service-annotation.yml#L8) for an example) | `.status.loadBalancer.ingress` |
 | VirtualServer<sup>[4](#foot4)</sup> | `spec.host` | `.status.externalEnpoints.ip` |
 
 
-<a name="f1">1</a>: Currently supported version of GatewayAPI CRDs is v0.4.0.</br>
-<a name="f2">2</a>: Gateway is a separate resource specified in the `spec.parentRefs` of HTTPRoute.</br>
+<a name="f1">1</a>: Currently supported version of GatewayAPI CRDs is v0.8.1+.</br>
+<a name="f2">2</a>: Gateway is a separate resource specified in the `spec.parentRefs` of HTTPRoute|TLSRoute|GRPCRoute.</br>
 <a name="f3">3</a>: Only resolves service of type LoadBalancer</br>
 <a name="f4">4</a>: Currently supported version of [nginxinc kubernetes-ingress](https://github.com/nginxinc/kubernetes-ingress) is 1.12.3</br>
 
@@ -63,7 +63,7 @@ k8s_gateway ZONE
 ```
 
 
-* `resources` a subset of supported Kubernetes resources to watch. By default all supported resources are monitored. Available options are `[ Ingress | Service | HTTPRoute | VirtualServer ]`.
+* `resources` a subset of supported Kubernetes resources to watch. By default all supported resources are monitored. Available options are `[ Ingress | Service | HTTPRoute | TLSRoute | GRPCRoute | VirtualServer ]`.
 * `ttl` can be used to override the default TTL value of 60 seconds.
 * `apex` can be used to override the default apex record value of `{ReleaseName}-k8s-gateway.{Namespace}`
 * `secondary` can be used to specify the optional apex record value of a peer nameserver running in the cluster (see `Dual Nameserver Deployment` section below).
@@ -228,6 +228,73 @@ To cleanup local environment do:
 make nuke
 ```
 
+## Apple Silicon Development
+
+Developing with apple silicon requires lima/colima installed on your machine. It sadly, did not work at all with kind.
+Below, you'll find the `yaml` used for developing with Cilium CNI and k3s.
+
+Colima version at the time: [v0.5.6](https://github.com/abiosoft/colima/releases/tag/v0.5.6)
+
+```yaml
+cpu: 6
+disk: 60
+memory: 16
+arch: host
+runtime: containerd
+kubernetes:
+  enabled: true
+  version: v1.28.2+k3s1
+  k3sArgs:
+    - --flannel-backend=none
+    - --disable=servicelb
+    - --disable=traefik
+    - --disable-network-policy
+    - --disable-kube-proxy
+autoActivate: true
+network:
+  address: false
+  dns: []
+  dnsHosts:
+    host.docker.internal: host.lima.internal
+  driver: slirp
+forwardAgent: false
+docker:
+  insecure-registries:
+    - localhost:5000
+    - host.docker.internal:5000
+vmType: vz
+rosetta: true
+mountType: virtiofs
+mountInotify: false
+cpuType: host
+layer: false
+provision:
+  - mode: system
+    script: |
+      set -e
+
+      # needed for cilium
+      mount bpffs -t bpf /sys/fs/bpf
+      mount --make-shared /sys/fs/bpf
+
+      mkdir -p /run/cilium/cgroupv2
+      mount -t cgroup2 none /run/cilium/cgroupv2
+      mount --make-shared /run/cilium/cgroupv2/
+      ln -s /opt/cni/bin/cilium-cni /usr/libexec/cni/cilium-cni
+sshConfig: true
+mounts: []
+env: {}
+cgroupsV2: false
+```
+
+### Steps
+
+1. In `Tiltfile.nerdctl`
+2. `colima start` with above configuration
+3. `tilt up -f Tiltfile.nerdctl` + space bar for the environment to trigger.
+
+The stacks should deploy and you'll have a proper stack that builds `k8s-gateway` with `coredns` and deploys to `kube-system` namespace.
+
 ## Also see
 
 [Blogpost](https://medium.com/from-the-edge/a-self-hosted-external-dns-resolver-for-kubernetes-111a27d6fc2c)  

Tiltfile

@@ -1,74 +1,84 @@
+allow_k8s_contexts('colima')
+allow_k8s_contexts('local')
+
+# using others with the makefile
 load('ext://restart_process', 'docker_build_with_restart')
 load('ext://helm_remote', 'helm_remote')
 
 IMG = 'localhost:5000/coredns'
 
 def binary():
-    return "CGO_ENABLED=0  GOOS=linux GOATCH=amd64 GO111MODULE=on go build cmd/coredns.go"
+    return "CGO_ENABLED=0  GOOS=linux GOARCH=amd64 GO111MODULE=on go build cmd/coredns.go"
 
 local_resource('recompile', binary(), deps=['cmd', 'gateway.go', 'kubernetes.go', 'setup.go', 'apex.go'])
 
-docker_build_with_restart(IMG, '.', 
-    dockerfile='tilt.Dockerfile', 
+
+docker_build_with_restart(IMG, '.',
+    dockerfile='tilt.Dockerfile',
     entrypoint=['/coredns'], 
-    only=['./coredns'], 
     live_update=[
         sync('./coredns', '/coredns'),
         ]
-    )
+)
 
+# Cilium CNI
+helm_remote('cilium',
+            version="1.15.0-pre.1",
+            namespace="kube-system",
+            repo_name='cilium',
+            values=['./test/cilium/helm-values.yaml'],
+            repo_url='https://helm.cilium.io')
+k8s_yaml('./test/cilium/dual-stack/crd-values.yaml')
 
-k8s_kind("kind")
 
 # CoreDNS with updated RBAC
 k8s_yaml(helm(
     './charts/k8s-gateway',
     namespace="kube-system",
     name='excoredns',
-    values=['./test/k8s-gateway-values.yaml'],
+    values=['./test/dual-stack/k8s-gateway-values.yaml'],
     )
 )
 
 # Baremetal ingress controller (nodeport-based)
 helm_remote('ingress-nginx',
-            version="4.0.15",
+            version="4.8.3",
             repo_name='ingress-nginx',
             set=['controller.admissionWebhooks.enabled=false'],
             repo_url='https://kubernetes.github.io/ingress-nginx')
 
-# Backend deployment for testing
-k8s_yaml('./test/backend.yml')
-
-# Metallb
-helm_remote('metallb',
-            version="0.12.1",
-            repo_name='metallb',
-            values=['./test/metallb-values.yaml'],
-            repo_url='https://metallb.github.io/metallb')
-
 # Nginxinc kubernetes-ingress
-k8s_kind('VirtualServer', api_version='k8s.nginx.org/v1')
 helm_remote('nginx-ingress',
-            version="0.12.0",
+            version="1.0.1",
             release_name="nginxinc",
-            repo_name='nginx-stable',
-            values=['./test/nginxinc-kubernetes-ingress/values.yaml'],
-            repo_url='https://helm.nginx.com/stable')
-
-
-# Gateway API
-k8s_kind('HTTPRoute', api_version='gateway.networking.k8s.io/v1beta1')
-k8s_kind('Gateway', api_version='gateway.networking.k8s.io/v1beta1')
-k8s_yaml('./test/gateway-api/crds.yml')
-
+            repo_name='oci://ghcr.io/nginxinc/charts',
+            values=['./test/nginxinc-kubernetes-ingress/values.yaml']
+            )
+k8s_kind('VirtualServer', api_version='k8s.nginx.org/v1')
 
 helm_remote('istiod',
-            version="1.12.1",
+            version="1.19.3",
             repo_name='istio',
             set=['global.istioNamespace=default', 'base.enableIstioConfigCRDs=false', 'telemetry.enabled=false'],
             repo_url='https://istio-release.storage.googleapis.com/charts')
 helm_remote('gateway',
-            version="1.12.1",
+            version="1.19.3",
             repo_name='istio',
             namespace='default',
             repo_url='https://istio-release.storage.googleapis.com/charts')
+
+# Backend deployment for testing
+k8s_yaml('./test/backend.yml')
+
+# gateway-apis
+k8s_yaml('./test/gateway-api/crds.yml')
+
+# Gateway API
+k8s_kind('HTTPRoute', api_version='gateway.networking.k8s.io/v1beta1')
+k8s_kind('TLSRoute', api_version='gateway.networking.k8s.io/v1alpha2')
+k8s_kind('GRPCRoute', api_version='gateway.networking.k8s.io/v1alpha2')
+k8s_kind('Gateway', api_version='gateway.networking.k8s.io/v1beta1')
+k8s_yaml('./test/gateway-api/resources.yml')
+k8s_yaml('./test/gatewayclasses.yaml')
+k8s_yaml('./test/dual-stack/service-annotation.yml')
+k8s_yaml('./test/dual-stack/ingress-services.yml')

Tiltfile.nerdctl

@@ -0,0 +1,83 @@
+allow_k8s_contexts('colima')
+allow_k8s_contexts('local')
+
+# using lima
+load('ext://nerdctl', 'nerdctl_build')
+load('ext://helm_remote', 'helm_remote')
+
+IMG = 'localhost:5000/coredns'
+
+def binary():
+    return "CGO_ENABLED=0  GOOS=linux GOARCH=arm64 GO111MODULE=on go build cmd/coredns.go"
+
+local_resource('recompile', binary(), deps=['cmd', 'gateway.go', 'kubernetes.go', 'setup.go', 'apex.go'])
+
+# if using mac m1+, you'll need to use colima
+nerdctl_build(IMG, '.',
+    dockerfile='tilt.Dockerfile', 
+    entrypoint=['/coredns'], 
+    live_update=[
+        sync('./coredns', '/coredns'),
+        ]
+)
+
+# Cilium CNI
+helm_remote('cilium',
+            version="1.15.0-pre.1",
+            namespace="kube-system",
+            repo_name='cilium',
+            values=['./test/cilium/helm-values.yaml'],
+            repo_url='https://helm.cilium.io')
+k8s_yaml('./test/cilium/single-stack/crd-values.yaml')
+
+# CoreDNS with updated RBAC
+k8s_yaml(helm(
+    './charts/k8s-gateway',
+    namespace="kube-system",
+    name='excoredns',
+    values=['./test/single-stack/k8s-gateway-values.yaml'],
+    )
+)
+
+# Baremetal ingress controller (nodeport-based)
+helm_remote('ingress-nginx',
+            version="4.8.3",
+            repo_name='ingress-nginx',
+            set=['controller.admissionWebhooks.enabled=false'],
+            repo_url='https://kubernetes.github.io/ingress-nginx')
+
+# Nginxinc kubernetes-ingress
+helm_remote('nginx-ingress',
+            version="1.0.1",
+            release_name="nginxinc",
+            repo_name='oci://ghcr.io/nginxinc/charts',
+            values=['./test/nginxinc-kubernetes-ingress/values.yaml']
+            )
+k8s_kind('VirtualServer', api_version='k8s.nginx.org/v1')
+
+helm_remote('istiod',
+            version="1.19.3",
+            repo_name='istio',
+            set=['global.istioNamespace=default', 'base.enableIstioConfigCRDs=false', 'telemetry.enabled=false'],
+            repo_url='https://istio-release.storage.googleapis.com/charts')
+helm_remote('gateway',
+            version="1.19.3",
+            repo_name='istio',
+            namespace='default',
+            repo_url='https://istio-release.storage.googleapis.com/charts')
+
+# Backend deployment for testing
+k8s_yaml('./test/backend.yml')
+
+# gateway-apis
+k8s_yaml('./test/gateway-api/crds.yml')
+
+# Gateway API
+k8s_kind('HTTPRoute', api_version='gateway.networking.k8s.io/v1beta1')
+k8s_kind('TLSRoute', api_version='gateway.networking.k8s.io/v1alpha2')
+k8s_kind('GRPCRoute', api_version='gateway.networking.k8s.io/v1alpha2')
+k8s_kind('Gateway', api_version='gateway.networking.k8s.io/v1beta1')
+k8s_yaml('./test/gateway-api/resources.yml')
+k8s_yaml('./test/gatewayclasses.yaml')
+k8s_yaml('./test/single-stack/service-annotation.yml')
+k8s_yaml('./test/single-stack/ingress-services.yml')

apex_dual_test.go

@@ -13,10 +13,15 @@ import (
 )
 
 func setupEmptyLookupFuncs() {
-
 	if resource := lookupResource("HTTPRoute"); resource != nil {
 		resource.lookup = func(_ []string) []netip.Addr { return []netip.Addr{} }
 	}
+	if resource := lookupResource("TLSRoute"); resource != nil {
+		resource.lookup = func(_ []string) []netip.Addr { return []netip.Addr{} }
+	}
+	if resource := lookupResource("GRPCRoute"); resource != nil {
+		resource.lookup = func(_ []string) []netip.Addr { return []netip.Addr{} }
+	}
 	if resource := lookupResource("Ingress"); resource != nil {
 		resource.lookup = func(_ []string) []netip.Addr { return []netip.Addr{} }
 	}
@@ -26,7 +31,6 @@ func setupEmptyLookupFuncs() {
 }
 
 func TestDualNS(t *testing.T) {
-
 	ctrl := &KubeController{hasSynced: true}
 	gw := newGateway()
 	gw.Zones = []string{"example.com."}