fix(kuma-cp) builtin DNS resolve alias with dots (#2208)

Signed-off-by: Jakub Dyszkiewicz <jakub.dyszkiewicz@gmail.com>
kumahq · Jun 25, 2021 · 8a58c35 · 8a58c35
1 parent 0f9deba
commit 8a58c35
Show file tree

Hide file tree

Showing 21 changed files with 70 additions and 32 deletions.
diff --git a/pkg/xds/generator/dns_generator.go b/pkg/xds/generator/dns_generator.go
@@ -1,6 +1,8 @@
 package generator
 
 import (
+	"strings"
+
 	"github.com/asaskevich/govalidator"
 
 	mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
@@ -52,6 +54,7 @@ func (g DNSGenerator) computeVIPs(ctx xds_context.Context, proxy *core_xds.Proxy
 		if domain, ok := domainsByIPs[outbound.Address]; ok {
 			// add regular .mesh domain
 			meshedVips[domain+"."+ctx.ControlPlane.DNSResolver.GetDomain()] = outbound.Address
+			meshedVips[strings.ReplaceAll(domain, "_", ".")+"."+ctx.ControlPlane.DNSResolver.GetDomain()] = outbound.Address
 			// add hostname from address in external service
 			endpoints := proxy.Routing.OutboundTargets[outbound.Tags[mesh_proto.ServiceTag]]
 			for _, endpoint := range endpoints {

diff --git a/pkg/xds/generator/dns_generator_test.go b/pkg/xds/generator/dns_generator_test.go
@@ -34,8 +34,8 @@ var _ = Describe("DNSGenerator", func() {
 
 			dnsResolver := resolver.NewDNSResolver("mesh")
 			dnsResolver.SetVIPs(map[string]string{
-				"backend": "240.0.0.0",
-				"httpbin": "240.0.0.1",
+				"backend_test-ns_svc_8080": "240.0.0.0",
+				"httpbin":                  "240.0.0.1",
 			})
 			ctx := xds_context.Context{
 				ConnectionInfo: xds_context.ConnectionInfo{

diff --git a/pkg/xds/generator/testdata/dns/1-dataplane.input.yaml b/pkg/xds/generator/testdata/dns/1-dataplane.input.yaml
@@ -3,7 +3,7 @@ networking:
     - port: 80
       address: 240.0.0.0
       tags:
-        kuma.io/service: backend
+        kuma.io/service: backend_test-ns_svc_8080
     - port: 80
       address: 240.0.0.1
       tags:

diff --git a/pkg/xds/generator/testdata/dns/1-envoy-config.golden.yaml b/pkg/xds/generator/testdata/dns/1-envoy-config.golden.yaml
@@ -29,7 +29,13 @@ resources:
                 addressList:
                   address:
                   - 240.0.0.0
-              name: backend.mesh
+              name: backend.test-ns.svc.8080.mesh
+            - answerTtl: 30s
+              endpoint:
+                addressList:
+                  address:
+                  - 240.0.0.0
+              name: backend_test-ns_svc_8080.mesh
             - answerTtl: 30s
               endpoint:
                 addressList:

diff --git a/test/e2e/deploy/kuma_deploy_global_zone.go b/test/e2e/deploy/kuma_deploy_global_zone.go
@@ -90,7 +90,6 @@ spec:
 
 		err = NewClusterSetup().
 			Install(Kuma(core.Zone, optsZone...)).
-			Install(KumaDNS()).
 			Install(YamlK8s(namespaceWithSidecarInjection(TestNamespace))).
 			Install(DemoClientK8s("default")).
 			Install(EchoServerK8s("default")).

diff --git a/test/e2e/externalservices/externalservices_kubernetes.go b/test/e2e/externalservices/externalservices_kubernetes.go
@@ -91,9 +91,6 @@ metadata:
 
 		// Global
 		cluster = clusters.GetCluster(Kuma1)
-		deployOptsFuncs = append(deployOptsFuncs,
-			WithEnv("KUMA_RUNTIME_KUBERNETES_INJECTOR_BUILTIN_DNS_ENABLED", "true"))
-
 		err = NewClusterSetup().
 			Install(Kuma(core.Standalone, deployOptsFuncs...)).
 			Install(YamlK8s(namespaceWithSidecarInjection(TestNamespace))).

diff --git a/test/e2e/externalservices/externalservices_multizone_universal.go b/test/e2e/externalservices/externalservices_multizone_universal.go
@@ -90,7 +90,7 @@ networking:
 
 		err = NewClusterSetup().
 			Install(Kuma(core.Zone, optsZone1...)).
-			Install(DemoClientUniversal(AppModeDemoClient, defaultMesh, demoClientToken, WithTransparentProxy(true), WithBuiltinDNS(true))).
+			Install(DemoClientUniversal(AppModeDemoClient, defaultMesh, demoClientToken, WithTransparentProxy(true))).
 			Setup(zone1)
 		Expect(err).ToNot(HaveOccurred())
 		err = zone1.VerifyKuma()
@@ -104,7 +104,7 @@ networking:
 
 		err = NewClusterSetup().
 			Install(Kuma(core.Zone, optsZone2...)).
-			Install(DemoClientUniversal(AppModeDemoClient, defaultMesh, demoClientToken, WithTransparentProxy(true), WithBuiltinDNS(true))).
+			Install(DemoClientUniversal(AppModeDemoClient, defaultMesh, demoClientToken, WithTransparentProxy(true))).
 			Setup(zone2)
 		Expect(err).ToNot(HaveOccurred())
 		err = zone2.VerifyKuma()

diff --git a/test/e2e/externalservices/externalservices_universal.go b/test/e2e/externalservices/externalservices_universal.go
@@ -70,7 +70,7 @@ networking:
 		err = NewClusterSetup().
 			Install(externalservice.Install(externalservice.HttpServer, externalservice.UniversalAppEchoServer)).
 			Install(externalservice.Install(externalservice.HttpsServer, externalservice.UniversalAppHttpsEchoServer)).
-			Install(DemoClientUniversal(AppModeDemoClient, "default", demoClientToken, WithTransparentProxy(true), WithBuiltinDNS(true))).
+			Install(DemoClientUniversal(AppModeDemoClient, "default", demoClientToken, WithTransparentProxy(true))).
 			Setup(cluster)
 		Expect(err).ToNot(HaveOccurred())
 

diff --git a/test/e2e/healthcheck/hybrid/healthcheck_hybrid.go b/test/e2e/healthcheck/hybrid/healthcheck_hybrid.go
@@ -67,7 +67,6 @@ metadata:
 		zoneK8s = k8sClusters.GetCluster(Kuma2)
 		err = NewClusterSetup().
 			Install(Kuma(core.Zone, optsZoneK8s...)).
-			Install(KumaDNS()).
 			Install(YamlK8s(namespaceWithSidecarInjection(TestNamespace))).
 			Install(DemoClientK8s("default")).
 			Setup(zoneK8s)

diff --git a/test/e2e/healthcheck/universal/policy_http.go b/test/e2e/healthcheck/universal/policy_http.go
@@ -59,7 +59,7 @@ conf:
 		Expect(err).ToNot(HaveOccurred())
 
 		err = DemoClientUniversal("dp-demo-client", "default", demoClientToken,
-			WithTransparentProxy(true), WithBuiltinDNS(true))(cluster)
+			WithTransparentProxy(true))(cluster)
 		Expect(err).ToNot(HaveOccurred())
 		err = TestServerUniversal("test-server", "default", testServerToken,
 			WithArgs([]string{"health-check", "http"}),

diff --git a/test/e2e/healthcheck/universal/policy_tcp.go b/test/e2e/healthcheck/universal/policy_tcp.go
@@ -63,7 +63,7 @@ conf:
 		Expect(err).ToNot(HaveOccurred())
 
 		err = DemoClientUniversal("dp-demo-client", "default", demoClientToken,
-			WithTransparentProxy(true), WithBuiltinDNS(true))(cluster)
+			WithTransparentProxy(true))(cluster)
 		Expect(err).ToNot(HaveOccurred())
 		err = TestServerUniversal("test-server", "default", testServerToken,
 			WithArgs([]string{"health-check", "tcp"}),

diff --git a/test/e2e/helm/kuma_helm_deploy_autoscaling.go b/test/e2e/helm/kuma_helm_deploy_autoscaling.go
@@ -45,7 +45,6 @@ func ControlPlaneAutoscalingWithHelmChart() {
 
 		err = NewClusterSetup().
 			Install(Kuma(core.Standalone, deployOptsFuncs...)).
-			Install(KumaDNS()).
 			Setup(cluster)
 		Expect(err).ToNot(HaveOccurred())
 	})

diff --git a/test/e2e/helm/kuma_helm_deploy_global_zone.go b/test/e2e/helm/kuma_helm_deploy_global_zone.go
@@ -79,7 +79,6 @@ metadata:
 
 		err = NewClusterSetup().
 			Install(Kuma(core.Zone, optsZone...)).
-			Install(KumaDNS()).
 			Install(YamlK8s(namespaceWithSidecarInjection(TestNamespace))).
 			Install(DemoClientK8s("default")).
 			Install(EchoServerK8s("default")).

diff --git a/test/e2e/helm/kuma_helm_deploy_multi_apps.go b/test/e2e/helm/kuma_helm_deploy_multi_apps.go
@@ -63,7 +63,6 @@ metadata:
 
 		err = NewClusterSetup().
 			Install(Kuma(core.Standalone, deployOptsFuncs...)).
-			Install(KumaDNS()).
 			Install(YamlK8s(defaultMesh)).
 			Setup(cluster)
 		Expect(err).ToNot(HaveOccurred())
@@ -112,5 +111,11 @@ metadata:
 				"curl", "-v", "-m", "3", "--fail", "echo-server_kuma-test_svc_80.mesh")
 			return stderr, err
 		}, "10s", "1s").Should(ContainSubstring("HTTP/1.1 200 OK"))
+
+		Eventually(func() (string, error) { // should access a service with . instead of _
+			_, stderr, err := cluster.ExecWithRetries(TestNamespace, clientPod.GetName(), "demo-client",
+				"curl", "-v", "-m", "3", "--fail", "echo-server.kuma-test.svc.80.mesh")
+			return stderr, err
+		}, "10s", "1s").Should(ContainSubstring("HTTP/1.1 200 OK"))
 	})
 }
diff --git a/test/e2e/hybrid/kuma_hybrid.go b/test/e2e/hybrid/kuma_hybrid.go
@@ -79,7 +79,7 @@ metadata:
 			WithIngress(),
 			WithGlobalAddress(globalCP.GetKDSServerAddress()),
 			WithCNI(),
-			WithEnv("KUMA_RUNTIME_KUBERNETES_INJECTOR_BUILTIN_DNS_ENABLED", "true"))
+			WithEnv("KUMA_RUNTIME_KUBERNETES_INJECTOR_BUILTIN_DNS_ENABLED", "false")) // check if old resolving still works
 
 		err = NewClusterSetup().
 			Install(Kuma(core.Zone, optsZone1...)).
@@ -95,7 +95,8 @@ metadata:
 		zone2 = k8sClusters.GetCluster(Kuma2)
 		optsZone2 = append(optsZone2,
 			WithIngress(),
-			WithGlobalAddress(globalCP.GetKDSServerAddress()))
+			WithGlobalAddress(globalCP.GetKDSServerAddress()),
+			WithEnv("KUMA_RUNTIME_KUBERNETES_INJECTOR_BUILTIN_DNS_ENABLED", "false"))
 
 		err = NewClusterSetup().
 			Install(Kuma(core.Zone, optsZone2...)).
@@ -117,8 +118,8 @@ metadata:
 
 		err = NewClusterSetup().
 			Install(Kuma(core.Zone, optsZone3...)).
-			Install(EchoServerUniversal(AppModeEchoServer, nonDefaultMesh, "universal", echoServerToken, WithTransparentProxy(true))).
-			Install(DemoClientUniversal(AppModeDemoClient, nonDefaultMesh, demoClientToken, WithTransparentProxy(true))).
+			Install(EchoServerUniversal(AppModeEchoServer, nonDefaultMesh, "universal", echoServerToken, WithTransparentProxy(true), WithBuiltinDNS(false))).
+			Install(DemoClientUniversal(AppModeDemoClient, nonDefaultMesh, demoClientToken, WithTransparentProxy(true), WithBuiltinDNS(false))).
 			Install(IngressUniversal(ingressTokenKuma3)).
 			Setup(zone3)
 		Expect(err).ToNot(HaveOccurred())

diff --git a/test/e2e/kic/kubernetes/kong_ingress.go b/test/e2e/kic/kubernetes/kong_ingress.go
@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/retry"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -63,18 +64,18 @@ metadata:
 		Expect(err).ToNot(HaveOccurred())
 		// Global
 		kubernetes = k8sClusters.GetCluster(Kuma1)
-		kubernetesOps = append(kubernetesOps, WithEnv("KUMA_RUNTIME_KUBERNETES_INJECTOR_BUILTIN_DNS_ENABLED", "true"))
 		err = NewClusterSetup().
 			Install(Kuma(config_core.Standalone, kubernetesOps...)).
 			Install(YamlK8s(namespaceWithSidecarInjection(TestNamespace))).
 			Install(EchoServerK8s("default")).
-			Install(EchoServerK8sIngress()).
 			Setup(kubernetes)
 		Expect(err).ToNot(HaveOccurred())
 		err = kubernetes.VerifyKuma()
 		Expect(err).ToNot(HaveOccurred())
 	})
 	E2EAfterEach(func() {
+		err := k8s.RunKubectlE(kubernetes.GetTesting(), kubernetes.GetKubectlOptions(), "delete", "ingress", "--all", "-n", "kuma-test")
+		Expect(err).ToNot(HaveOccurred())
 		Expect(kubernetes.DeleteNamespace(ingressNamespace)).To(Succeed())
 	})
 	E2EAfterSuite(func() {
@@ -88,6 +89,7 @@ metadata:
 		err := NewClusterSetup().
 			Install(kic.KongIngressController()).
 			Install(kic.KongIngressNodePort()).
+			Install(EchoServerK8sIngress()).
 			Setup(kubernetes)
 		Expect(err).ToNot(HaveOccurred())
 
@@ -109,6 +111,7 @@ metadata:
 		err := NewClusterSetup().
 			Install(kic.KongIngressController(kic.WithNamespace(ingressNamespace))).
 			Install(kic.KongIngressNodePort(kic.WithNamespace(ingressNamespace))).
+			Install(EchoServerK8sIngressWithMeshDNS()).
 			Setup(kubernetes)
 		Expect(err).ToNot(HaveOccurred())
 

diff --git a/test/e2e/tracing/tracing_kubernetes.go b/test/e2e/tracing/tracing_kubernetes.go
@@ -65,12 +65,8 @@ spec:
 		c, err := NewK8SCluster(NewTestingT(), Kuma1, Silent)
 		Expect(err).ToNot(HaveOccurred())
 		cluster = c
-		deployOptsFuncs = append(deployOptsFuncs,
-			WithEnv("KUMA_RUNTIME_KUBERNETES_INJECTOR_BUILTIN_DNS_ENABLED", "true"))
-
 		err = NewClusterSetup().
 			Install(Kuma(core.Standalone, deployOptsFuncs...)).
-			Install(KumaDNS()).
 			Install(YamlK8s(namespaceWithSidecarInjection(TestNamespace))).
 			Install(DemoClientK8s("default")).
 			Install(EchoServerK8s("default")).

diff --git a/test/e2e/trafficpermission/hybrid/traffic_permission_hybrid.go b/test/e2e/trafficpermission/hybrid/traffic_permission_hybrid.go
@@ -93,7 +93,6 @@ spec:
 
 		err = NewClusterSetup().
 			Install(Kuma(config_core.Zone, optsZoneKube...)).
-			Install(KumaDNS()).
 			Install(YamlK8s(namespaceWithSidecarInjection(TestNamespace))).
 			Install(DemoClientK8s("default")).
 			Setup(zoneKube)

diff --git a/test/framework/interface.go b/test/framework/interface.go
@@ -48,7 +48,7 @@ type deployOptions struct {
 	appArgs        []string
 	token          string
 	transparent    bool
-	builtindns     bool
+	builtindns     *bool // true by default
 	protocol       string
 	serviceName    string
 	serviceVersion string
@@ -251,7 +251,7 @@ func WithTransparentProxy(transparent bool) DeployOptionsFunc {
 
 func WithBuiltinDNS(builtindns bool) DeployOptionsFunc {
 	return func(o *deployOptions) {
-		o.builtindns = builtindns
+		o.builtindns = &builtindns
 	}
 }
 

diff --git a/test/framework/setup.go b/test/framework/setup.go
@@ -204,6 +204,37 @@ spec:
 	return YamlK8s(ingress)
 }
 
+func EchoServerK8sIngressWithMeshDNS() InstallFunc {
+	ingress := `
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: echo-server-externalname
+  namespace: kuma-test
+spec:
+  type: ExternalName
+  externalName: echo-server.kuma-test.svc.80.mesh
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  namespace: kuma-test
+  name: k8s-ingress-dot-mesh
+  annotations:
+    kubernetes.io/ingress.class: kong
+spec:
+  rules:
+  - http:
+      paths:
+      - path: /
+        backend:
+          serviceName: echo-server-externalname
+          servicePort: 80
+`
+	return YamlK8s(ingress)
+}
+
 func EchoServerK8s(mesh string) InstallFunc {
 	const name = "echo-server"
 	service := `

diff --git a/test/framework/universal_cluster.go b/test/framework/universal_cluster.go
@@ -207,8 +207,9 @@ func (c *UniversalCluster) DeployApp(fs ...DeployOptionsFunc) error {
 		return err
 	}
 
+	builtindns := opts.builtindns == nil || *opts.builtindns
 	if transparent {
-		app.setupTransparent(c.apps[AppModeCP].ip, opts.builtindns)
+		app.setupTransparent(c.apps[AppModeCP].ip, builtindns)
 	}
 
 	ip := app.ip
@@ -219,7 +220,7 @@ func (c *UniversalCluster) DeployApp(fs ...DeployOptionsFunc) error {
 		}
 	}
 
-	err = c.CreateDP(app, opts.name, ip, dpyaml, token, opts.builtindns)
+	err = c.CreateDP(app, opts.name, ip, dpyaml, token, builtindns)
 	if err != nil {
 		return err
 	}