feat(kuma-cp) Add kuma.io/ignore annotation (#3142)

This annotation ignores a kubernetes resource when generating kuma configuration It currently only works for service and it's meant to be used when migrating to kuma where you might have services that are used outside of kuma and that will be removed once migration is complete Signed-off-by: Charly Molter <charly.molter@konghq.com>
kumahq · Nov 12, 2021 · 1c8405d · 1c8405d
1 parent 95d176b
commit 1c8405d
Show file tree

Hide file tree

Showing 7 changed files with 131 additions and 14 deletions.
diff --git a/pkg/plugins/runtime/k8s/controllers/pod_controller.go b/pkg/plugins/runtime/k8s/controllers/pod_controller.go
@@ -157,7 +157,7 @@ func (r *PodReconciler) findMatchingServices(ctx context.Context, pod *kube_core
 	}
 
 	// only consider Services that match this Pod
-	matchingServices := util_k8s.FindServices(allServices, util_k8s.AnySelector(), util_k8s.MatchServiceThatSelectsPod(pod))
+	matchingServices := util_k8s.FindServices(allServices, util_k8s.Not(util_k8s.Ignored()), util_k8s.AnySelector(), util_k8s.MatchServiceThatSelectsPod(pod))
 
 	return matchingServices, nil
 }

diff --git a/pkg/plugins/runtime/k8s/controllers/pod_controller_test.go b/pkg/plugins/runtime/k8s/controllers/pod_controller_test.go
@@ -27,6 +27,7 @@ import (
 	mesh_k8s "github.com/kumahq/kuma/pkg/plugins/resources/k8s/native/api/v1alpha1"
 	"github.com/kumahq/kuma/pkg/plugins/resources/memory"
 	. "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/controllers"
+	"github.com/kumahq/kuma/pkg/plugins/runtime/k8s/metadata"
 )
 
 var _ = Describe("PodReconciler", func() {
@@ -182,6 +183,30 @@ var _ = Describe("PodReconciler", func() {
 						"app": "sample",
 					},
 				},
+			},
+			&kube_core.Service{
+				ObjectMeta: kube_meta.ObjectMeta{
+					Namespace: "demo",
+					Name:      "ignored-service",
+					Annotations: map[string]string{
+						metadata.KumaIgnoreAnnotation: metadata.AnnotationTrue,
+					},
+				},
+				Spec: kube_core.ServiceSpec{
+					ClusterIP: "192.168.0.1",
+					Ports: []kube_core.ServicePort{
+						{
+							Port: 85,
+							TargetPort: kube_intstr.IntOrString{
+								Type:   kube_intstr.Int,
+								IntVal: 8080,
+							},
+						},
+					},
+					Selector: map[string]string{
+						"app": "sample",
+					},
+				},
 			}).Build()
 
 		reconciler = &PodReconciler{

diff --git a/pkg/plugins/runtime/k8s/controllers/service_controller.go b/pkg/plugins/runtime/k8s/controllers/service_controller.go
@@ -55,10 +55,19 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req kube_ctrl.Request
 	}
 
 	log.Info("annotating service which is part of the mesh", "annotation", fmt.Sprintf("%s=%s", metadata.IngressServiceUpstream, metadata.AnnotationTrue))
-	if svc.Annotations == nil {
-		svc.Annotations = map[string]string{}
+	annotations := metadata.Annotations(svc.Annotations)
+	if annotations == nil {
+		annotations = metadata.Annotations{}
 	}
-	svc.Annotations[metadata.IngressServiceUpstream] = metadata.AnnotationTrue
+	ignored, _, err := annotations.GetBool(metadata.KumaIgnoreAnnotation)
+	if err != nil {
+		return kube_ctrl.Result{}, errors.Wrapf(err, "unable to retrieve %s annotation for %s", metadata.KumaIgnoreAnnotation, svc.Name)
+	}
+	if ignored {
+		return kube_ctrl.Result{}, nil
+	}
+	annotations[metadata.IngressServiceUpstream] = metadata.AnnotationTrue
+	svc.Annotations = annotations
 
 	if err = r.Update(ctx, svc); err != nil {
 		return kube_ctrl.Result{}, errors.Wrapf(err, "unable to update ingress service upstream annotation on service %s", svc.Name)

diff --git a/pkg/plugins/runtime/k8s/controllers/service_controller_test.go b/pkg/plugins/runtime/k8s/controllers/service_controller_test.go
@@ -51,6 +51,26 @@ var _ = Describe("ServiceReconciler", func() {
 				},
 				Spec: kube_core.ServiceSpec{},
 			},
+			&kube_core.Service{
+				ObjectMeta: kube_meta.ObjectMeta{
+					Namespace: "non-system-ns-with-sidecar-injection",
+					Name:      "ignored",
+					Annotations: map[string]string{
+						metadata.KumaIgnoreAnnotation: metadata.AnnotationTrue,
+					},
+				},
+				Spec: kube_core.ServiceSpec{},
+			},
+			&kube_core.Service{
+				ObjectMeta: kube_meta.ObjectMeta{
+					Namespace: "non-system-ns-with-sidecar-injection",
+					Name:      "non-ignored",
+					Annotations: map[string]string{
+						metadata.KumaIgnoreAnnotation: metadata.AnnotationFalse,
+					},
+				},
+				Spec: kube_core.ServiceSpec{},
+			},
 			&kube_core.Service{
 				ObjectMeta: kube_meta.ObjectMeta{
 					Namespace: "non-system-ns-without-sidecar-injection",
@@ -138,4 +158,45 @@ var _ = Describe("ServiceReconciler", func() {
 		Expect(svc.GetAnnotations()[metadata.IngressServiceUpstream]).To(Equal(metadata.AnnotationTrue))
 	})
 
+	It("should ignore service in an annotated namespace with ignored annotation", func() {
+		// given
+		req := kube_ctrl.Request{
+			NamespacedName: kube_types.NamespacedName{Namespace: "non-system-ns-with-sidecar-injection", Name: "ignored"},
+		}
+
+		// when
+		result, err := reconciler.Reconcile(context.Background(), req)
+
+		// then
+		Expect(err).ToNot(HaveOccurred())
+		Expect(result).To(BeZero())
+
+		// and service is annotated
+		svc := &kube_core.Service{}
+		err = kubeClient.Get(context.Background(), req.NamespacedName, svc)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(svc.GetAnnotations()).ToNot(HaveKey(metadata.IngressServiceUpstream))
+	})
+
+	It("should ignore service in an annotated namespace with ignored annotation", func() {
+		// given
+		req := kube_ctrl.Request{
+			NamespacedName: kube_types.NamespacedName{Namespace: "non-system-ns-with-sidecar-injection", Name: "non-ignored"},
+		}
+
+		// when
+		result, err := reconciler.Reconcile(context.Background(), req)
+
+		// then
+		Expect(err).ToNot(HaveOccurred())
+		Expect(result).To(BeZero())
+
+		// and service is annotated
+		svc := &kube_core.Service{}
+		err = kubeClient.Get(context.Background(), req.NamespacedName, svc)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(svc.GetAnnotations()).To(HaveKey(metadata.IngressServiceUpstream))
+		Expect(svc.GetAnnotations()[metadata.IngressServiceUpstream]).To(Equal(metadata.AnnotationTrue))
+		Expect(svc.GetAnnotations()[metadata.KumaIgnoreAnnotation]).To(Equal(metadata.AnnotationFalse))
+	})
 })
diff --git a/pkg/plugins/runtime/k8s/metadata/annotations.go b/pkg/plugins/runtime/k8s/metadata/annotations.go
@@ -72,6 +72,7 @@ const (
 // Annotations that are being automatically set by the Kuma Sidecar Injector.
 const (
 	KumaSidecarInjectedAnnotation                  = "kuma.io/sidecar-injected"
+	KumaIgnoreAnnotation                           = "kuma.io/ignore"
 	KumaSidecarUID                                 = "kuma.io/sidecar-uid"
 	KumaTransparentProxyingAnnotation              = "kuma.io/transparent-proxying"
 	KumaTransparentProxyingInboundPortAnnotation   = "kuma.io/transparent-proxying-inbound-port"

diff --git a/pkg/plugins/runtime/k8s/util/util.go b/pkg/plugins/runtime/k8s/util/util.go
@@ -7,6 +7,8 @@ import (
 	kube_core "k8s.io/api/core/v1"
 	kube_labels "k8s.io/apimachinery/pkg/labels"
 	kube_intstr "k8s.io/apimachinery/pkg/util/intstr"
+
+	"github.com/kumahq/kuma/pkg/plugins/runtime/k8s/metadata"
 )
 
 type ServicePredicate func(*kube_core.Service) bool
@@ -31,13 +33,32 @@ func AnySelector() ServicePredicate {
 	}
 }
 
+func Not(predicate ServicePredicate) ServicePredicate {
+	return func(svc *kube_core.Service) bool {
+		return !predicate(svc)
+	}
+}
+
+func Ignored() ServicePredicate {
+	return func(svc *kube_core.Service) bool {
+		if svc.Annotations == nil {
+			return false
+		}
+		ignore, _, _ := metadata.Annotations(svc.Annotations).GetBool(metadata.KumaIgnoreAnnotation)
+		return ignore
+	}
+}
+
 func FindServices(svcs *kube_core.ServiceList, predicates ...ServicePredicate) []*kube_core.Service {
 	matching := make([]*kube_core.Service, 0)
 	for i := range svcs.Items {
 		svc := &svcs.Items[i]
 		allMatched := true
 		for _, predicate := range predicates {
-			allMatched = allMatched && predicate(svc)
+			if !predicate(svc) {
+				allMatched = false
+				break
+			}
 		}
 		if allMatched {
 			matching = append(matching, svc)

diff --git a/pkg/plugins/runtime/k8s/util/util_test.go b/pkg/plugins/runtime/k8s/util/util_test.go
@@ -10,7 +10,7 @@ import (
 	kube_core "k8s.io/api/core/v1"
 	kube_meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	. "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/util"
+	"github.com/kumahq/kuma/pkg/plugins/runtime/k8s/util"
 )
 
 var _ = Describe("Util", func() {
@@ -35,7 +35,7 @@ var _ = Describe("Util", func() {
 			}
 
 			// when
-			predicate := MatchServiceThatSelectsPod(pod)
+			predicate := util.MatchServiceThatSelectsPod(pod)
 			// then
 			Expect(predicate(svc)).To(BeTrue())
 		})
@@ -60,7 +60,7 @@ var _ = Describe("Util", func() {
 			}
 
 			// when
-			predicate := MatchServiceThatSelectsPod(pod)
+			predicate := util.MatchServiceThatSelectsPod(pod)
 			// then
 			Expect(predicate(svc)).To(BeFalse())
 		})
@@ -70,7 +70,7 @@ var _ = Describe("Util", func() {
 	DescribeTable("FindServices",
 		func(pod *kube_core.Pod, svcs *kube_core.ServiceList, matchSvcNames []string) {
 			// when
-			matchingServices := FindServices(svcs, AnySelector(), MatchServiceThatSelectsPod(pod))
+			matchingServices := util.FindServices(svcs, util.AnySelector(), util.MatchServiceThatSelectsPod(pod))
 			// then
 			Expect(matchingServices).To(WithTransform(func(svcs []*kube_core.Service) []string {
 				var res []string
@@ -184,10 +184,10 @@ var _ = Describe("Util", func() {
 
 	Describe("CopyStringMap", func() {
 		It("should return nil if input is nil", func() {
-			Expect(CopyStringMap(nil)).To(BeNil())
+			Expect(util.CopyStringMap(nil)).To(BeNil())
 		})
 		It("should return empty map if input is empty map", func() {
-			Expect(CopyStringMap(map[string]string{})).To(Equal(map[string]string{}))
+			Expect(util.CopyStringMap(map[string]string{})).To(Equal(map[string]string{}))
 		})
 		It("should return a copy if input map is not empty", func() {
 			// given
@@ -197,7 +197,7 @@ var _ = Describe("Util", func() {
 			}
 
 			// when
-			copy := CopyStringMap(original)
+			copy := util.CopyStringMap(original)
 			// then
 			Expect(copy).To(Equal(original))
 			Expect(copy).ToNot(BeIdenticalTo(original))
@@ -229,7 +229,7 @@ var _ = Describe("Util", func() {
 					Expect(err).ToNot(HaveOccurred())
 
 					// when
-					actual, _, err := FindPort(&pod, &svcPort)
+					actual, _, err := util.FindPort(&pod, &svcPort)
 					// then
 					Expect(err).ToNot(HaveOccurred())
 					// and
@@ -410,7 +410,7 @@ var _ = Describe("Util", func() {
 					Expect(err).ToNot(HaveOccurred())
 
 					// when
-					actual, _, err := FindPort(&pod, &svcPort)
+					actual, _, err := util.FindPort(&pod, &svcPort)
 					// then
 					Expect(err).To(HaveOccurred())
 					// and