chore(transparentproxy) default to modify resolv.conf (#1828)

Signed-off-by: Jakub Dyszkiewicz <jakub.dyszkiewicz@gmail.com> Co-authored-by: Nikolay Nikolaev <nikolay.nikolaev@konghq.com> Co-authored-by: Jakub Dyszkiewicz <jakub.dyszkiewicz@gmail.com>
kumahq · Apr 16, 2021 · 907fd2f · 907fd2f
1 parent 77ab443
commit 907fd2f
Show file tree

Hide file tree

Showing 39 changed files with 154 additions and 51 deletions.
diff --git a/app/kumactl/cmd/completion/testdata/bash.golden b/app/kumactl/cmd/completion/testdata/bash.golden
@@ -2394,8 +2394,6 @@ _kumactl_install_transparent-proxy()
     local_nonpersistent_flags+=("--kuma-dp-user=")
     flags+=("--modify-iptables")
     local_nonpersistent_flags+=("--modify-iptables")
-    flags+=("--modify-resolv-conf")
-    local_nonpersistent_flags+=("--modify-resolv-conf")
     flags+=("--redirect-dns")
     local_nonpersistent_flags+=("--redirect-dns")
     flags+=("--redirect-dns-port=")
@@ -2415,6 +2413,8 @@ _kumactl_install_transparent-proxy()
     flags+=("--redirect-outbound-port=")
     two_word_flags+=("--redirect-outbound-port")
     local_nonpersistent_flags+=("--redirect-outbound-port=")
+    flags+=("--skip-resolv-conf")
+    local_nonpersistent_flags+=("--skip-resolv-conf")
     flags+=("--store-firewalld")
     local_nonpersistent_flags+=("--store-firewalld")
     flags+=("--config-file=")

diff --git a/app/kumactl/cmd/completion/testdata/zsh.golden b/app/kumactl/cmd/completion/testdata/zsh.golden
@@ -974,14 +974,14 @@ function _kumactl_install_transparent-proxy {
     '--kuma-dp-uid[the UID of the user that will run kuma-dp]:' \
     '--kuma-dp-user[the user that will run kuma-dp]:' \
     '--modify-iptables[modify the host iptables to redirect the traffic to Envoy]' \
-    '--modify-resolv-conf[skip modifying the host `/etc/resolv.conf`]' \
     '--redirect-dns[redirect the DNS requests to a specified port]' \
     '--redirect-dns-port[the port where the DNS agent is listening]:' \
     '--redirect-dns-upstream-target-chain[(optional) the iptables chain where the upstream DNS requests should be directed to. Use with care.]:' \
     '--redirect-inbound[redirect the inbound traffic to the Envoy. Should be disabled for Gateway data plane proxies.]' \
     '--redirect-inbound-port[inbound port redirected to Envoy, as specified in dataplane'\''s `networking.transparentProxying.redirectPortInbound`]:' \
     '--redirect-inbound-port-v6[IPv6 inbound port redirected to Envoy, as specified in dataplane'\''s `networking.transparentProxying.redirectPortInboundV6`]:' \
     '--redirect-outbound-port[outbound port redirected to Envoy, as specified in dataplane'\''s `networking.transparentProxying.redirectPortOutbound`]:' \
+    '--skip-resolv-conf[skip modifying the host `/etc/resolv.conf`]' \
     '--store-firewalld[store the iptables changes with firewalld]' \
     '--config-file[path to the configuration file to use]:' \
     '--log-level[log level: one of off|info|debug]:' \

diff --git a/app/kumactl/cmd/install/install_transparent_proxy.go b/app/kumactl/cmd/install/install_transparent_proxy.go
@@ -33,7 +33,7 @@ type transparenProxyArgs struct {
 	RedirectDNS            bool
 	AgentDNSListenerPort   string
 	DNSUpstreamTargetChain string
-	ModifyResolvConf       bool
+	SkipResolvConf         bool
 	StoreFirewalld         bool
 	KumaCpIP               net.IP
 }
@@ -55,7 +55,7 @@ func newInstallTransparentProxy() *cobra.Command {
 		RedirectDNS:            false,
 		AgentDNSListenerPort:   "15053",
 		DNSUpstreamTargetChain: "RETURN",
-		ModifyResolvConf:       false,
+		SkipResolvConf:         false,
 		StoreFirewalld:         false,
 		KumaCpIP:               defaultCpIP,
 	}
@@ -123,12 +123,12 @@ runuser -u kuma-dp -- \
 				return errors.Errorf("--kuma-dp-user or --kuma-dp-uid should be supplied")
 			}
 
-			if args.RedirectDNS && args.ModifyResolvConf {
-				return errors.Errorf("please supply only one of --redirect-dns or --modify-resolv-conf")
+			if args.RedirectDNS && !args.SkipResolvConf {
+				return errors.Errorf("please set --skip-resolv-conf when using --redirect-dns")
 			}
 
-			if args.ModifyResolvConf && args.KumaCpIP.String() == defaultCpIP.String() {
-				return errors.Errorf("please supply a valid `--kuma-cp-ip`")
+			if !args.SkipResolvConf && args.KumaCpIP.String() == defaultCpIP.String() {
+				return errors.Errorf("please supply a valid --kuma-cp-ip")
 			}
 
 			if args.ModifyIptables {
@@ -137,7 +137,7 @@ runuser -u kuma-dp -- \
 				}
 			}
 
-			if args.ModifyResolvConf {
+			if !args.SkipResolvConf {
 				if err := modifyResolvConf(cmd, &args); err != nil {
 					return err
 				}
@@ -160,8 +160,8 @@ runuser -u kuma-dp -- \
 	cmd.Flags().StringVar(&args.UID, "kuma-dp-uid", args.UID, "the UID of the user that will run kuma-dp")
 	cmd.Flags().BoolVar(&args.RedirectDNS, "redirect-dns", args.RedirectDNS, "redirect the DNS requests to a specified port")
 	cmd.Flags().StringVar(&args.AgentDNSListenerPort, "redirect-dns-port", args.AgentDNSListenerPort, "the port where the DNS agent is listening")
-	cmd.Flags().StringVar(&args.DNSUpstreamTargetChain, "redirect-dns-upstream-target-chain", args.DNSUpstreamTargetChain, "(optional) the iptables chain where the upstream DNS requests should be directed to. Use with care.")
-	cmd.Flags().BoolVar(&args.ModifyResolvConf, "modify-resolv-conf", args.ModifyResolvConf, "skip modifying the host `/etc/resolv.conf`")
+	cmd.Flags().StringVar(&args.DNSUpstreamTargetChain, "redirect-dns-upstream-target-chain", args.DNSUpstreamTargetChain, "(optional) the iptables chain where the upstream DNS requests should be directed to. It is only applied for IP V4. Use with care.")
+	cmd.Flags().BoolVar(&args.SkipResolvConf, "skip-resolv-conf", args.SkipResolvConf, "skip modifying the host `/etc/resolv.conf`")
 	cmd.Flags().BoolVar(&args.StoreFirewalld, "store-firewalld", args.StoreFirewalld, "store the iptables changes with firewalld")
 	cmd.Flags().IPVar(&args.KumaCpIP, "kuma-cp-ip", args.KumaCpIP, "the IP address of the Kuma CP which exposes the DNS service on port 53.")
 

diff --git a/app/kumactl/cmd/install/install_transparent_proxy_test.go b/app/kumactl/cmd/install/install_transparent_proxy_test.go
@@ -74,6 +74,7 @@ var _ = Describe("kumactl install tracing", func() {
 			extraArgs: []string{
 				"--kuma-dp-uid", "0",
 				"--kuma-cp-ip", "1.2.3.4",
+				"--skip-resolv-conf",
 				"--redirect-dns",
 				"--redirect-dns-port", "12345",
 				"--redirect-dns-upstream-target-chain", "DOCKER_OUTPUT",

diff --git a/mk/build.mk b/mk/build.mk
@@ -24,7 +24,7 @@ export PATH := $(BUILD_KUMACTL_DIR):$(PATH)
 GO_BUILD := GOOS=${GOOS} GOARCH=${GOARCH} CGO_ENABLED=0 go build -v $(GOFLAGS) $(LD_FLAGS)
 GO_BUILD_COREDNS := GOOS=${GOOS} GOARCH=${GOARCH} CGO_ENABLED=0 go build -v
 
-COREDNS_GIT_REPOSITORY ?= git@github.com:coredns/coredns.git
+COREDNS_GIT_REPOSITORY ?= https://github.com/coredns/coredns.git
 COREDNS_VERSION ?= v1.8.3
 COREDNS_TMP_DIRECTORY ?= $(BUILD_DIR)/coredns
 COREDNS_PLUGIN_CFG_PATH ?= $(TOP)/tools/builds/coredns/templates/plugin.cfg
@@ -50,6 +50,7 @@ build/coredns:
 	git clone --branch $(COREDNS_VERSION) --depth 1 $(COREDNS_GIT_REPOSITORY) $(COREDNS_TMP_DIRECTORY)
 	cp $(COREDNS_PLUGIN_CFG_PATH) $(COREDNS_TMP_DIRECTORY)
 	cd $(COREDNS_TMP_DIRECTORY) && \
+		GOOS= GOARCH= go generate coredns.go && \
 		$(GO_BUILD_COREDNS) -ldflags="-s -w -X github.com/coredns/coredns/coremain.GitCommit=$(shell git describe --dirty --always)" -o $(BUILD_ARTIFACTS_DIR)/coredns/coredns
 	rm -rf "$(COREDNS_TMP_DIRECTORY)"
 

diff --git a/pkg/plugins/runtime/k8s/controllers/pod_controller.go b/pkg/plugins/runtime/k8s/controllers/pod_controller.go
@@ -2,6 +2,7 @@ package controllers
 
 import (
 	"context"
+	"strconv"
 
 	core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
 	"github.com/kumahq/kuma/pkg/dns/vips"
@@ -54,6 +55,8 @@ type PodReconciler struct {
 	Persistence       *vips.Persistence
 	ResourceConverter k8s_common.Converter
 	SystemNamespace   string
+	UseBuiltinDNS     bool
+	BuiltinDNSPort    uint32
 }
 
 func (r *PodReconciler) Reconcile(req kube_ctrl.Request) (kube_ctrl.Result, error) {
@@ -131,6 +134,19 @@ func (r *PodReconciler) Reconcile(req kube_ctrl.Request) (kube_ctrl.Result, erro
 		return kube_ctrl.Result{}, err
 	}
 
+	if r.UseBuiltinDNS {
+		if pod.Annotations == nil {
+			pod.Annotations = map[string]string{}
+		}
+		// will be used by the CNI to enable the relevant iptables settings
+		pod.Annotations[metadata.KumaBuiltinDNS] = metadata.AnnotationEnabled
+		pod.Annotations[metadata.KumaBuiltinDNSPort] = strconv.FormatInt(int64(r.BuiltinDNSPort), 10)
+
+		if err = r.Update(ctx, pod); err != nil {
+			return kube_ctrl.Result{}, errors.Wrapf(err, "unable to update annotations on pod %s", pod.Name)
+		}
+	}
+
 	return kube_ctrl.Result{}, nil
 }
 

diff --git a/pkg/plugins/runtime/k8s/metadata/annotations.go b/pkg/plugins/runtime/k8s/metadata/annotations.go
@@ -55,6 +55,10 @@ const (
 	// KumaMetricsPrometheusPath to override `Mesh`-wide default path
 	KumaMetricsPrometheusPath = "prometheus.metrics.kuma.io/path"
 
+	// KumaBuiltinDNS the sidecar will use its builtin DNS
+	KumaBuiltinDNS     = "kuma.io/builtindns"
+	KumaBuiltinDNSPort = "kuma.io/builtindnsport"
+
 	KumaTrafficExcludeInboundPorts  = "traffic.kuma.io/exclude-inbound-ports"
 	KumaTrafficExcludeOutboundPorts = "traffic.kuma.io/exclude-outbound-ports"
 )

diff --git a/pkg/plugins/runtime/k8s/plugin.go b/pkg/plugins/runtime/k8s/plugin.go
@@ -160,6 +160,8 @@ func addPodReconciler(mgr kube_ctrl.Manager, rt core_runtime.Runtime, converter
 		ResourceConverter: converter,
 		Persistence:       vips.NewPersistence(rt.ResourceManager(), rt.ConfigManager()),
 		SystemNamespace:   rt.Config().Store.Kubernetes.SystemNamespace,
+		UseBuiltinDNS:     rt.Config().Runtime.Kubernetes.Injector.BuiltinDNS.Enabled,
+		BuiltinDNSPort:    rt.Config().Runtime.Kubernetes.Injector.BuiltinDNS.Port,
 	}
 	return reconciler.SetupWithManager(mgr)
 }

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/injector.go b/pkg/plugins/runtime/k8s/webhooks/injector/injector.go
@@ -420,7 +420,9 @@ func (i *KumaInjector) NewInitContainer(pod *kube_core.Pod) (kube_core.Container
 	excludeInboundPorts, _ := metadata.Annotations(pod.Annotations).GetString(metadata.KumaTrafficExcludeInboundPorts)
 	excludeOutboundPorts, _ := metadata.Annotations(pod.Annotations).GetString(metadata.KumaTrafficExcludeOutboundPorts)
 
-	dnsArg := []string{}
+	dnsArg := []string{
+		"--skip-resolv-conf",
+	}
 
 	if i.cfg.BuiltinDNS.Enabled {
 		dnsArg = append(dnsArg,

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.01.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.01.golden.yaml
@@ -126,6 +126,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.02.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.02.golden.yaml
@@ -134,6 +134,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.03.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.03.golden.yaml
@@ -188,6 +188,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.04.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.04.golden.yaml
@@ -126,6 +126,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.05.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.05.golden.yaml
@@ -119,6 +119,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.06.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.06.golden.yaml
@@ -127,6 +127,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.07.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.07.golden.yaml
@@ -126,6 +126,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.08.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.08.golden.yaml
@@ -129,6 +129,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.09.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.09.golden.yaml
@@ -128,6 +128,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.11.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.11.golden.yaml
@@ -127,6 +127,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.12.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.12.golden.yaml
@@ -126,6 +126,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.13.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.13.golden.yaml
@@ -126,6 +126,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.14.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.14.golden.yaml
@@ -138,6 +138,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.15.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.15.golden.yaml
@@ -138,6 +138,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.16.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.16.golden.yaml
@@ -138,6 +138,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.17.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.17.golden.yaml
@@ -135,6 +135,7 @@ spec:
     - 1234,1235
     - --exclude-outbound-ports
     - "1236"
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.18.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.18.golden.yaml
@@ -135,6 +135,7 @@ spec:
     - 1234,5678
     - --exclude-outbound-ports
     - 4321,7654
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.19.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.19.golden.yaml
@@ -135,6 +135,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.21.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.21.golden.yaml
@@ -138,6 +138,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.22.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.22.golden.yaml
@@ -138,6 +138,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.23.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.23.golden.yaml
@@ -143,6 +143,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.24.golden.yaml b/pkg/plugins/runtime/k8s/webhooks/injector/testdata/inject.24.golden.yaml
@@ -139,6 +139,7 @@ spec:
     - ""
     - --exclude-outbound-ports
     - ""
+    - --skip-resolv-conf
     command:
     - /usr/bin/kumactl
     - install

diff --git a/test/dockerfiles/Dockerfile.universal b/test/dockerfiles/Dockerfile.universal
@@ -24,7 +24,7 @@ RUN echo "# use this file to override default configuration of \`kuma-cp\`" > /k
 RUN apt update \
   && apt dist-upgrade -y \
   && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-  tzdata openssh-server curl ncat vim net-tools iptables iproute2 dnsutils tmux \
+  tzdata openssh-server curl ncat vim net-tools iptables iproute2 dnsutils tmux strace tcpdump \
   && rm -rf /var/lib/apt/lists/*
 
 RUN ssh-keygen -A \