fix(kuma-dp): fix conntrack collisions (#3459)

* fix(kuma-dp): fix conntrack collisions

Vendors this fix (istio/istio#33572) from Istio

Signed-off-by: John Harris <john.harris@konghq.com>

* chore(*): adjust changes to kuma environment

As our e2e tests for universal are done from inside of a docker
container, to make the networking work, we are bridging docker
network to the host, which results in additional iptables rules
inside every container within this network. Problem with these
rules is that it's doing some NAT'ing for DNS udp datagrams with
addition of randomly picked (during the container startup) port.
It's problematic for this conntrack change as it works inside
the `raw` table and among others `PREROUTING` chain and expects
the datagrams from known port (53), which the earlier described
NAT'ing is changing. This probably could be fixed by rethinking
some of the rules, but as it's an edge case, after consultation
with the team I decided it's not worth the time needed to properly
solve it and instead I introduced to a `--skip-dns-conntrack-zone-split`
flag for `kumactl install transparent-proxy`, which allows us to
skip attaching the conntrack-collision iptables rules. This change
was necessary for making some of the tests to work (univeral).

I also fixed the code to include our flag for capturing all dns
traffic and instead of hardcoding port `15053`, to use the one
from the configuration.

Signed-off-by: Bart Smykla <bartek@smykla.com>

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Bart Smykla <bartek@smykla.com>

app/kumactl/cmd/completion/testdata/bash.golden

@@ -3259,6 +3259,8 @@ _kumactl_install_transparent-proxy()
     two_word_flags+=("--redirect-outbound-port")
     local_nonpersistent_flags+=("--redirect-outbound-port")
     local_nonpersistent_flags+=("--redirect-outbound-port=")
+    flags+=("--skip-dns-conntrack-zone-split")
+    local_nonpersistent_flags+=("--skip-dns-conntrack-zone-split")
     flags+=("--skip-resolv-conf")
     local_nonpersistent_flags+=("--skip-resolv-conf")
     flags+=("--store-firewalld")

app/kumactl/cmd/install/install_transparent_proxy.go

@@ -22,46 +22,48 @@ import (
 )
 
 type transparentProxyArgs struct {
-	DryRun                 bool
-	Verbose                bool
-	RedirectPortOutBound   string
-	RedirectInbound        bool
-	RedirectPortInBound    string
-	RedirectPortInBoundV6  string
-	ExcludeInboundPorts    string
-	ExcludeOutboundPorts   string
-	UID                    string
-	User                   string
-	RedirectDNS            bool
-	RedirectAllDNSTraffic  bool
-	AgentDNSListenerPort   string
-	DNSUpstreamTargetChain string
-	SkipResolvConf         bool
-	StoreFirewalld         bool
-	KumaCpIP               net.IP
+	DryRun                    bool
+	Verbose                   bool
+	RedirectPortOutBound      string
+	RedirectInbound           bool
+	RedirectPortInBound       string
+	RedirectPortInBoundV6     string
+	ExcludeInboundPorts       string
+	ExcludeOutboundPorts      string
+	UID                       string
+	User                      string
+	RedirectDNS               bool
+	RedirectAllDNSTraffic     bool
+	AgentDNSListenerPort      string
+	DNSUpstreamTargetChain    string
+	SkipResolvConf            bool
+	StoreFirewalld            bool
+	KumaCpIP                  net.IP
+	SkipDNSConntrackZoneSplit bool
 }
 
 var defaultCpIP = net.IPv4(0, 0, 0, 0)
 
 func newInstallTransparentProxy() *cobra.Command {
 	args := transparentProxyArgs{
-		DryRun:                 false,
-		Verbose:                false,
-		RedirectPortOutBound:   "15001",
-		RedirectInbound:        true,
-		RedirectPortInBound:    "15006",
-		RedirectPortInBoundV6:  "15010",
-		ExcludeInboundPorts:    "",
-		ExcludeOutboundPorts:   "",
-		UID:                    "",
-		User:                   "",
-		RedirectDNS:            false,
-		RedirectAllDNSTraffic:  false,
-		AgentDNSListenerPort:   "15053",
-		DNSUpstreamTargetChain: "RETURN",
-		SkipResolvConf:         false,
-		StoreFirewalld:         false,
-		KumaCpIP:               defaultCpIP,
+		DryRun:                    false,
+		Verbose:                   false,
+		RedirectPortOutBound:      "15001",
+		RedirectInbound:           true,
+		RedirectPortInBound:       "15006",
+		RedirectPortInBoundV6:     "15010",
+		ExcludeInboundPorts:       "",
+		ExcludeOutboundPorts:      "",
+		UID:                       "",
+		User:                      "",
+		RedirectDNS:               false,
+		RedirectAllDNSTraffic:     false,
+		AgentDNSListenerPort:      "15053",
+		DNSUpstreamTargetChain:    "RETURN",
+		SkipResolvConf:            false,
+		StoreFirewalld:            false,
+		KumaCpIP:                  defaultCpIP,
+		SkipDNSConntrackZoneSplit: false,
 	}
 	cmd := &cobra.Command{
 		Use:   "transparent-proxy",
@@ -175,6 +177,7 @@ runuser -u kuma-dp -- \
 	cmd.Flags().BoolVar(&args.SkipResolvConf, "skip-resolv-conf", args.SkipResolvConf, "skip modifying the host `/etc/resolv.conf`")
 	cmd.Flags().BoolVar(&args.StoreFirewalld, "store-firewalld", args.StoreFirewalld, "store the iptables changes with firewalld")
 	cmd.Flags().IPVar(&args.KumaCpIP, "kuma-cp-ip", args.KumaCpIP, "the IP address of the Kuma CP which exposes the DNS service on port 53.")
+	cmd.Flags().BoolVar(&args.SkipDNSConntrackZoneSplit, "skip-dns-conntrack-zone-split", args.SkipDNSConntrackZoneSplit, "skip applying conntrack zone splitting iptables rules")
 
 	return cmd
 }
@@ -217,20 +220,21 @@ func modifyIpTables(cmd *cobra.Command, args *transparentProxyArgs) error {
 		_, _ = cmd.OutOrStdout().Write([]byte("kumactl is about to apply the iptables rules that will enable transparent proxying on the machine. The SSH connection may drop. If that happens, just reconnect again.\n"))
 	}
 	output, err := tp.Setup(&config.TransparentProxyConfig{
-		DryRun:                 args.DryRun,
-		Verbose:                args.Verbose,
-		RedirectPortOutBound:   args.RedirectPortOutBound,
-		RedirectInBound:        args.RedirectInbound,
-		RedirectPortInBound:    args.RedirectPortInBound,
-		RedirectPortInBoundV6:  args.RedirectPortInBoundV6,
-		ExcludeInboundPorts:    args.ExcludeInboundPorts,
-		ExcludeOutboundPorts:   args.ExcludeOutboundPorts,
-		UID:                    uid,
-		GID:                    gid,
-		RedirectDNS:            args.RedirectDNS,
-		RedirectAllDNSTraffic:  args.RedirectAllDNSTraffic,
-		AgentDNSListenerPort:   args.AgentDNSListenerPort,
-		DNSUpstreamTargetChain: args.DNSUpstreamTargetChain,
+		DryRun:                    args.DryRun,
+		Verbose:                   args.Verbose,
+		RedirectPortOutBound:      args.RedirectPortOutBound,
+		RedirectInBound:           args.RedirectInbound,
+		RedirectPortInBound:       args.RedirectPortInBound,
+		RedirectPortInBoundV6:     args.RedirectPortInBoundV6,
+		ExcludeInboundPorts:       args.ExcludeInboundPorts,
+		ExcludeOutboundPorts:      args.ExcludeOutboundPorts,
+		UID:                       uid,
+		GID:                       gid,
+		RedirectDNS:               args.RedirectDNS,
+		RedirectAllDNSTraffic:     args.RedirectAllDNSTraffic,
+		AgentDNSListenerPort:      args.AgentDNSListenerPort,
+		DNSUpstreamTargetChain:    args.DNSUpstreamTargetChain,
+		SkipDNSConntrackZoneSplit: args.SkipDNSConntrackZoneSplit,
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to setup transparent proxy")

app/kumactl/cmd/install/install_transparent_proxy_test.go

@@ -69,7 +69,7 @@ var _ = Describe("kumactl install tracing", func() {
 			},
 			goldenFile: "install-transparent-proxy.defaults.golden.txt",
 		}),
-		Entry("should generate defaults with user id and DNS redirected ", testCase{
+		Entry("should generate defaults with user id and DNS redirected", testCase{
 			extraArgs: []string{
 				"--kuma-dp-uid", "0",
 				"--kuma-cp-ip", "1.2.3.4",
@@ -80,6 +80,18 @@ var _ = Describe("kumactl install tracing", func() {
 			},
 			goldenFile: "install-transparent-proxy.dns.golden.txt",
 		}),
+		Entry("should generate defaults with user id and DNS redirected without conntrack zone splitting", testCase{
+			extraArgs: []string{
+				"--kuma-dp-uid", "0",
+				"--kuma-cp-ip", "1.2.3.4",
+				"--skip-resolv-conf",
+				"--redirect-all-dns-traffic",
+				"--redirect-dns-port", "12345",
+				"--redirect-dns-upstream-target-chain", "DOCKER_OUTPUT",
+				"--skip-dns-conntrack-zone-split",
+			},
+			goldenFile: "install-transparent-proxy.dns.golden.txt",
+		}),
 		Entry("should generate defaults with overrides", testCase{
 			extraArgs: []string{
 				"--kuma-dp-user", "root",

docs/cmd/kumactl/kumactl_install_transparent-proxy.md

@@ -79,6 +79,7 @@ kumactl install transparent-proxy [flags]
       --redirect-inbound-port networking.transparentProxying.redirectPortInbound        inbound port redirected to Envoy, as specified in dataplane's networking.transparentProxying.redirectPortInbound (default "15006")
       --redirect-inbound-port-v6 networking.transparentProxying.redirectPortInboundV6   IPv6 inbound port redirected to Envoy, as specified in dataplane's networking.transparentProxying.redirectPortInboundV6 (default "15010")
       --redirect-outbound-port networking.transparentProxying.redirectPortOutbound      outbound port redirected to Envoy, as specified in dataplane's networking.transparentProxying.redirectPortOutbound (default "15001")
+      --skip-dns-conntrack-zone-split                                                   skip applying conntrack zone splitting iptables rules
       --skip-resolv-conf /etc/resolv.conf                                               skip modifying the host /etc/resolv.conf
       --store-firewalld                                                                 store the iptables changes with firewalld
       --verbose                                                                         verbose

pkg/plugins/runtime/k8s/webhooks/injector/injector.go

@@ -284,6 +284,7 @@ func (i *KumaInjector) NewInitContainer(pod *kube_core.Pod) (kube_core.Container
 			Capabilities: &kube_core.Capabilities{
 				Add: []kube_core.Capability{
 					kube_core.Capability("NET_ADMIN"),
+					kube_core.Capability("NET_RAW"),
 				},
 			},
 		},

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.01.golden.yaml

@@ -150,6 +150,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.02.golden.yaml

@@ -158,6 +158,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.03.golden.yaml

@@ -212,6 +212,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   nodeSelector:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.04.golden.yaml

@@ -150,6 +150,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.05.golden.yaml

@@ -143,6 +143,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
 status: {}

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.06.golden.yaml

@@ -151,6 +151,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.07.golden.yaml

@@ -150,6 +150,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.08.golden.yaml

@@ -153,6 +153,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.09.golden.yaml

@@ -152,6 +152,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.11.golden.yaml

@@ -151,6 +151,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.12.golden.yaml

@@ -150,6 +150,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.13.golden.yaml

@@ -150,6 +150,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.14.golden.yaml

@@ -162,6 +162,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.15.golden.yaml

@@ -162,6 +162,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.16.golden.yaml

@@ -162,6 +162,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.17.golden.yaml

@@ -159,6 +159,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.18.golden.yaml

@@ -159,6 +159,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.19.golden.yaml

@@ -159,6 +159,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.21.golden.yaml

@@ -162,6 +162,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.22.golden.yaml

@@ -162,6 +162,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.23.golden.yaml

@@ -167,6 +167,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.24.golden.yaml

@@ -163,6 +163,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.25.golden.yaml

@@ -174,6 +174,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.26.golden.yaml

@@ -175,6 +175,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes:

pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.27.golden.yaml

@@ -174,6 +174,7 @@ spec:
       capabilities:
         add:
         - NET_ADMIN
+        - NET_RAW
       runAsGroup: 0
       runAsUser: 0
   volumes: