From 53f61c9231aa70d313db2fb754c7873fceeaf8cd Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 1 Aug 2023 19:58:12 +0200 Subject: [PATCH] Add and enable two egressAllowRules to ensure DNS access --- jupyterhub/templates/_helpers-netpol.tpl | 29 ++++++-- jupyterhub/values.schema.yaml | 71 +++++++++++++++++-- jupyterhub/values.yaml | 8 +++ tools/templates/lint-and-validate-values.yaml | 2 + 4 files changed, 99 insertions(+), 11 deletions(-) diff --git a/jupyterhub/templates/_helpers-netpol.tpl b/jupyterhub/templates/_helpers-netpol.tpl index 5adbd3d275..4bf2596f5e 100644 --- a/jupyterhub/templates/_helpers-netpol.tpl +++ b/jupyterhub/templates/_helpers-netpol.tpl @@ -30,20 +30,35 @@ {{- define "jupyterhub.networkPolicy.renderEgressRules" -}} {{- $root := index . 0 }} {{- $netpol := index . 1 }} -{{- if $netpol.egressAllowRules.dnsPortsPrivateIPs }} -# Allow outbound connections to the DNS port in the private IP ranges +{{- if or (or $netpol.egressAllowRules.dnsPortsCloudMetadataServer $netpol.egressAllowRules.dnsPortsKubeSystemNamespace) $netpol.egressAllowRules.dnsPortsPrivateIPs }} - ports: - - protocol: UDP - port: 53 - - protocol: TCP - port: 53 + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP to: + {{- if $netpol.egressAllowRules.dnsPortsCloudMetadataServer }} + # Allow outbound connections to DNS ports on the cloud metadata server + - ipBlock: + cidr: {{ $root.Values.singleuser.cloudMetadata.ip }}/32 + {{- end }} + {{- if $netpol.egressAllowRules.dnsPortsKubeSystemNamespace }} + # Allow outbound connections to DNS ports on pods in the kube-system + # namespace + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + {{- end }} + {{- if $netpol.egressAllowRules.dnsPortsPrivateIPs }} + # Allow outbound connections to DNS ports on destinations in the private IP + # ranges - ipBlock: cidr: 10.0.0.0/8 - ipBlock: cidr: 172.16.0.0/12 - ipBlock: cidr: 192.168.0.0/16 + {{- end }} {{- end }} {{- if $netpol.egressAllowRules.nonPrivateIPs }} @@ -53,7 +68,7 @@ cidr: 0.0.0.0/0 except: # As part of this rule, don't: - # - allow outbound connections to private IP + # - allow outbound connections to private IPs - 10.0.0.0/8 - 172.16.0.0/12 - 192.168.0.0/16 diff --git a/jupyterhub/values.schema.yaml b/jupyterhub/values.schema.yaml index f2756d8880..2c45084c40 100644 --- a/jupyterhub/values.schema.yaml +++ b/jupyterhub/values.schema.yaml @@ -667,6 +667,49 @@ properties: IP ranges but makes an exception for the cloud metadata server, leaving this as the definitive configuration to allow access to the cloud metadata server. + dnsPortsCloudMetadataServer: + type: boolean + description: | + Defaults to `true` for all network policies. + + When enabled this rule allows the respective pod(s) to + establish outbound connections to the cloud metadata server + via port 53. + + Relying on this rule should go hand in hand with disabling + [`singleuser.cloudMetadata.blockWithIptables`](schema_singleuser.cloudMetadata.blockWithIptables). + + Known situations when this rule can be relevant: + + - In GKE clusters with Cloud DNS that is reached at the + cloud metadata server's non-private IP. + + ```{note} + This chart doesn't know how to identify the DNS server that + pods will rely on due to variations between how k8s clusters + have been setup. Due to that, multiple rules are enabled by + default to ensure DNS connectivity. + ``` + dnsPortsKubeSystemNamespace: + type: boolean + description: | + Defaults to `true` for all network policies. + + When enabled this rule allows the respective pod(s) to + establish outbound connections to pods in the kube-system + namespace via port 53. + + Known situations when this rule can be relevant: + + - GKE, EKS, AKS, and other clusters relying directly on + `kube-dns` or `coredns` pods in the `kube-system` namespace. + + ```{note} + This chart doesn't know how to identify the DNS server that + pods will rely on due to variations between how k8s clusters + have been setup. Due to that, multiple rules are enabled by + default to ensure DNS connectivity. + ``` dnsPortsPrivateIPs: type: boolean description: | @@ -675,10 +718,23 @@ properties: When enabled this rule allows the respective pod(s) to establish outbound connections to private IPs via port 53. - Note that we can't reliably identify the k8s internal DNS - server due to variations between k8s clusters. Due to that, - this rule which is critical for core functionality, can be - disabled for a more refined custom rule. + Known situations when this rule can be relevant: + + - GKE clusters relying on a DNS server indirectly via a a node + local DNS cache at an unknown private IP. + + ```{note} + This chart doesn't know how to identify the DNS server that + pods will rely on due to variations between how k8s clusters + have been setup. Due to that, multiple rules are enabled by + default to ensure DNS connectivity. + + ```{warning} + This rule is not expected to work in clusters relying on + Cilium to enforce the NetworkPolicy rules (includes GKE + clusters with Dataplane v2), this is due to a [known + limitation](https://github.com/cilium/cilium/issues/9209). + ``` nonPrivateIPs: type: boolean description: | @@ -713,6 +769,13 @@ properties: If possible, try to avoid setting this to true as it gives broad permissions that could be specified more directly via the [`.egress`](schema_singleuser.networkPolicy.egress). + + ```{warning} + This rule is not expected to work in clusters relying on + Cilium to enforce the NetworkPolicy rules (includes GKE + clusters with Dataplane v2), this is due to a [known + limitation](https://github.com/cilium/cilium/issues/9209). + ``` interNamespaceAccessLabels: enum: [accept, ignore] description: | diff --git a/jupyterhub/values.yaml b/jupyterhub/values.yaml index 4433d27269..364af89847 100644 --- a/jupyterhub/values.yaml +++ b/jupyterhub/values.yaml @@ -107,6 +107,8 @@ hub: egress: [] egressAllowRules: cloudMetadataServer: true + dnsPortsCloudMetadataServer: true + dnsPortsKubeSystemNamespace: true dnsPortsPrivateIPs: true nonPrivateIPs: true privateIPs: true @@ -231,6 +233,8 @@ proxy: egress: [] egressAllowRules: cloudMetadataServer: true + dnsPortsCloudMetadataServer: true + dnsPortsKubeSystemNamespace: true dnsPortsPrivateIPs: true nonPrivateIPs: true privateIPs: true @@ -278,6 +282,8 @@ proxy: egress: [] egressAllowRules: cloudMetadataServer: true + dnsPortsCloudMetadataServer: true + dnsPortsKubeSystemNamespace: true dnsPortsPrivateIPs: true nonPrivateIPs: true privateIPs: true @@ -354,6 +360,8 @@ singleuser: egress: [] egressAllowRules: cloudMetadataServer: false + dnsPortsCloudMetadataServer: true + dnsPortsKubeSystemNamespace: true dnsPortsPrivateIPs: true nonPrivateIPs: true privateIPs: false diff --git a/tools/templates/lint-and-validate-values.yaml b/tools/templates/lint-and-validate-values.yaml index 694f6f37ce..811d5ec818 100644 --- a/tools/templates/lint-and-validate-values.yaml +++ b/tools/templates/lint-and-validate-values.yaml @@ -177,6 +177,8 @@ hub: enabled: true egressAllowRules: cloudMetadataServer: true + dnsPortsCloudMetadataServer: true + dnsPortsKubeSystemNamespace: true dnsPortsPrivateIPs: true nonPrivateIPs: true privateIPs: false