Add possibility to use multiple loadbalancerClasses and use service.spec.loadBalancerClass

README.md

@@ -252,6 +252,7 @@ metadata:
     # Reference the name of the SSH key provided to OpenStack for debugging .
     yawol.stackit.cloud/debugsshkey: "OS-keyName"
     # Allows filtering services in cloud-controller.
+    # Deprecated: Use service.spec.loadBalancerClass instead.
     yawol.stackit.cloud/className: "test"
     # Specify the number of LoadBalancer machines to deploy (default 1).
     yawol.stackit.cloud/replicas: "3"

api/v1beta1/loadbalancer_types.go

@@ -32,6 +32,7 @@ const (
 	// ServiceDebugSSHKey set an sshkey
 	ServiceDebugSSHKey = "yawol.stackit.cloud/debugsshkey"
 	// ServiceClassName for filtering services in cloud-controller
+	// Deprecated: use .spec.loadBalancerClass instead
 	ServiceClassName = "yawol.stackit.cloud/className"
 	// ServiceReplicas for setting loadbalancer replicas in cloud-controller
 	ServiceReplicas = "yawol.stackit.cloud/replicas"

cmd/yawol-cloud-controller/main.go

@@ -5,6 +5,7 @@ import (
 	"flag"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -18,6 +19,7 @@ import (
 	yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1"
 	"github.com/stackitcloud/yawol/controllers/yawol-cloud-controller/controlcontroller"
 	"github.com/stackitcloud/yawol/controllers/yawol-cloud-controller/targetcontroller"
+	"github.com/stackitcloud/yawol/internal/helper"
 	"go.uber.org/zap/zapcore"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -33,6 +35,8 @@ var (
 	setupLog = ctrl.Log.WithName("setup")
 )
 
+type loadbalancerClassNames []string
+
 const (
 	// Namespace in for LoadBalancer CRs
 	EnvClusterNamespace = "CLUSTER_NAMESPACE"
@@ -72,7 +76,8 @@ func main() {
 	var targetEnableLeaderElection bool
 	var targetKubeconfig string
 	var controlKubeconfig string
-	var className string
+	var classNames loadbalancerClassNames
+	var emptyClassName bool
 	// settings for leases
 	var leasesDurationInt int
 	var leasesRenewDeadlineInt int
@@ -94,9 +99,12 @@ func main() {
 		"K8s credentials for watching the Service resources.")
 	flag.StringVar(&controlKubeconfig, "control-kubeconfig", "",
 		"K8s credentials for deploying the LoadBalancer resources.")
-	flag.StringVar(&className, "classname", "",
-		"Only listen to Services with the given className. "+
-			"Default is empty and listen to all services with out className annotation")
+	flag.Var(&classNames, "classname",
+		"Only listen to Services with the given className. Can be set multiple times. "+
+			"If no classname is set it will defaults to "+helper.DefaultLoadbalancerClass+" "+
+			"and services without class. See also --empty-classname.")
+	flag.BoolVar(&emptyClassName, "empty-classname", true,
+		"Listen to services without a loadBalancerClass. Default is true.")
 	flag.IntVar(&leasesDurationInt, "leases-duration", 60,
 		"Is the time in seconds a non-leader will wait until forcing to acquire leadership.")
 	flag.IntVar(&leasesRenewDeadlineInt, "leases-renew-deadline", 50,
@@ -113,6 +121,13 @@ func main() {
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
 
+	if len(classNames) == 0 {
+		classNames = append(classNames, helper.DefaultLoadbalancerClass)
+	}
+	if emptyClassName {
+		classNames = append(classNames, "")
+	}
+
 	leasesDuration = time.Duration(leasesDurationInt) * time.Second
 	leasesRenewDeadline = time.Duration(leasesRenewDeadlineInt) * time.Second
 	leasesRetryPeriod = time.Duration(leasesRetryPeriodInt) * time.Second
@@ -178,7 +193,7 @@ func main() {
 		Log:                    ctrl.Log.WithName("controller").WithName("Service"),
 		Scheme:                 targetMgr.GetScheme(),
 		Recorder:               targetMgr.GetEventRecorderFor("yawol-cloud-controller"),
-		ClassName:              className,
+		ClassNames:             classNames,
 	}).SetupWithManager(targetMgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Service")
 		os.Exit(1)
@@ -352,3 +367,12 @@ func getInfrastructureDefaultsFromEnvOrDie() targetcontroller.InfrastructureDefa
 		InternalLB:       pointer.Bool(internalLb),
 	}
 }
+
+func (i *loadbalancerClassNames) String() string {
+	return strings.Join(*i, ",")
+}
+
+func (i *loadbalancerClassNames) Set(value string) error {
+	*i = append(*i, value)
+	return nil
+}

controllers/yawol-cloud-controller/targetcontroller/service_controller.go

@@ -41,7 +41,7 @@ type ServiceReconciler struct {
 	Log                    logr.Logger
 	Scheme                 *runtime.Scheme
 	Recorder               record.EventRecorder
-	ClassName              string
+	ClassNames             []string
 }
 
 // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch
@@ -57,14 +57,9 @@ func (r *ServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
-	className, ok := svc.Annotations[yawolv1beta1.ServiceClassName]
-	if !ok {
-		className = ""
-	}
-
 	infraDefaults := GetMergedInfrastructureDetails(r.InfrastructureDefaults, svc)
 
-	if className != r.ClassName {
+	if !helper.CheckLoadBalancerClasses(svc, r.ClassNames) {
 		r.Log.WithValues("service", req.NamespacedName).Info("service and controller classname does not match")
 		if err := r.ControlClient.Get(ctx, types.NamespacedName{
 			Namespace: *infraDefaults.Namespace,
@@ -560,11 +555,7 @@ func (r *ServiceReconciler) deletionRoutine(
 	}
 
 	if apierrors.IsNotFound(err) {
-		className, ok := svc.Annotations[yawolv1beta1.ServiceClassName]
-		if !ok {
-			className = ""
-		}
-		if r.ClassName != className {
+		if !helper.CheckLoadBalancerClasses(svc, r.ClassNames) {
 			return ctrl.Result{}, nil
 		}
 

controllers/yawol-cloud-controller/targetcontroller/service_controller_test.go

@@ -472,11 +472,11 @@ var _ = Describe("Check loadbalancer reconcile", Serial, Ordered, func() {
 			}, time.Second*5, time.Millisecond*500).Should(Succeed())
 		})
 
-		It("create service with wrong className", func() {
+		It("create service with wrong className in annotation", func() {
 			By("create service")
 			service := v1.Service{
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      "service-test8",
+					Name:      "class-name-service-test1",
 					Namespace: "default",
 					Annotations: map[string]string{
 						yawolv1beta1.ServiceClassName: "foo",
@@ -498,19 +498,19 @@ var _ = Describe("Check loadbalancer reconcile", Serial, Ordered, func() {
 
 			By("check for LB creation")
 			Consistently(func() error {
-				err := k8sClient.Get(ctx, types.NamespacedName{Name: "default--service-test8", Namespace: "default"}, &lb)
+				err := k8sClient.Get(ctx, types.NamespacedName{Name: "default--class-name-service-test1", Namespace: "default"}, &lb)
 				if err != nil {
 					return client.IgnoreNotFound(err)
 				}
 				return helper.ErrInvalidClassname
 			}, time.Second*5, time.Millisecond*500).Should(Succeed())
 		})
 
-		It("create service with correct classname", func() {
+		It("create service with correct classname in annotation", func() {
 			By("create service")
 			service := v1.Service{
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      "service-test15",
+					Name:      "class-name-service-test2",
 					Namespace: "default",
 					Annotations: map[string]string{
 						yawolv1beta1.ServiceClassName: "",
@@ -533,7 +533,7 @@ var _ = Describe("Check loadbalancer reconcile", Serial, Ordered, func() {
 			By("check creation of LB")
 			Eventually(func() error {
 				err := k8sClient.Get(ctx, types.NamespacedName{
-					Name:      "default--service-test15",
+					Name:      "default--class-name-service-test2",
 					Namespace: "default",
 				}, &lb)
 				return err
@@ -547,7 +547,136 @@ var _ = Describe("Check loadbalancer reconcile", Serial, Ordered, func() {
 					return err
 				}
 				for _, event := range eventList.Items {
-					if event.InvolvedObject.Name == "service-test15" &&
+					if event.InvolvedObject.Name == "class-name-service-test2" &&
+						event.InvolvedObject.Kind == "Service" &&
+						strings.Contains(event.Message, "LoadBalancer is in creation") {
+						return nil
+					}
+				}
+				return helper.ErrNoEventFound
+			}, time.Second*5, time.Millisecond*500).Should(Succeed())
+		})
+
+		It("create service with wrong className in spec", func() {
+			By("create service")
+			service := v1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "class-name-service-test3",
+					Namespace: "default",
+				},
+				Spec: v1.ServiceSpec{
+					LoadBalancerClass: pointer.String("foo"),
+					Ports: []v1.ServicePort{
+						{
+							Name:       "port1",
+							Protocol:   v1.ProtocolTCP,
+							Port:       65030,
+							TargetPort: intstr.IntOrString{IntVal: 12345},
+							NodePort:   30133,
+						},
+					},
+					Type: "LoadBalancer",
+				}}
+			Expect(k8sClient.Create(ctx, &service)).Should(Succeed())
+
+			By("check for LB creation")
+			Consistently(func() error {
+				err := k8sClient.Get(ctx, types.NamespacedName{Name: "default--class-name-service-test1", Namespace: "default"}, &lb)
+				if err != nil {
+					return client.IgnoreNotFound(err)
+				}
+				return helper.ErrInvalidClassname
+			}, time.Second*5, time.Millisecond*500).Should(Succeed())
+		})
+
+		It("create service with correct classname in spec", func() {
+			By("create service")
+			service := v1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "class-name-service-test4",
+					Namespace: "default",
+				},
+				Spec: v1.ServiceSpec{
+					LoadBalancerClass: pointer.String(helper.DefaultLoadbalancerClass),
+					Ports: []v1.ServicePort{
+						{
+							Name:       "port1",
+							Protocol:   v1.ProtocolTCP,
+							Port:       12345,
+							TargetPort: intstr.IntOrString{IntVal: 12345},
+							NodePort:   30335,
+						},
+					},
+					Type: "LoadBalancer",
+				}}
+			Expect(k8sClient.Create(ctx, &service)).Should(Succeed())
+
+			By("check creation of LB")
+			Eventually(func() error {
+				err := k8sClient.Get(ctx, types.NamespacedName{
+					Name:      "default--class-name-service-test4",
+					Namespace: "default",
+				}, &lb)
+				return err
+			}, time.Second*5, time.Millisecond*500).Should(Succeed())
+
+			By("Check Event for creation")
+			Eventually(func() error {
+				eventList := v1.EventList{}
+				err := k8sClient.List(ctx, &eventList)
+				if err != nil {
+					return err
+				}
+				for _, event := range eventList.Items {
+					if event.InvolvedObject.Name == "class-name-service-test4" &&
+						event.InvolvedObject.Kind == "Service" &&
+						strings.Contains(event.Message, "LoadBalancer is in creation") {
+						return nil
+					}
+				}
+				return helper.ErrNoEventFound
+			}, time.Second*5, time.Millisecond*500).Should(Succeed())
+		})
+
+		It("create service without classname", func() {
+			By("create service")
+			service := v1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "class-name-service-test5",
+					Namespace: "default",
+				},
+				Spec: v1.ServiceSpec{
+					Ports: []v1.ServicePort{
+						{
+							Name:       "port1",
+							Protocol:   v1.ProtocolTCP,
+							Port:       12345,
+							TargetPort: intstr.IntOrString{IntVal: 12345},
+							NodePort:   30360,
+						},
+					},
+					Type: "LoadBalancer",
+				}}
+			Expect(k8sClient.Create(ctx, &service)).Should(Succeed())
+
+			By("check creation of LB")
+			Eventually(func() error {
+				err := k8sClient.Get(ctx, types.NamespacedName{
+					Name:      "default--class-name-service-test5",
+					Namespace: "default",
+				}, &lb)
+				return err
+			}, time.Second*5, time.Millisecond*500).Should(Succeed())
+
+			By("Check Event for creation")
+			Eventually(func() error {
+				eventList := v1.EventList{}
+				err := k8sClient.List(ctx, &eventList)
+				if err != nil {
+					return err
+				}
+				for _, event := range eventList.Items {
+					if event.InvolvedObject.Name == "class-name-service-test5" &&
 						event.InvolvedObject.Kind == "Service" &&
 						strings.Contains(event.Message, "LoadBalancer is in creation") {
 						return nil

controllers/yawol-cloud-controller/targetcontroller/suite_test.go

@@ -17,6 +17,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1"
+	"github.com/stackitcloud/yawol/internal/helper"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -101,7 +102,7 @@ var _ = BeforeSuite(func() {
 		Log:                    ctrl.Log.WithName("controllers").WithName("Service"),
 		Scheme:                 k8sManager.GetScheme(),
 		Recorder:               k8sManager.GetEventRecorderFor("Loadbalancer"),
-		ClassName:              "",
+		ClassNames:             []string{"", helper.DefaultLoadbalancerClass},
 	}).SetupWithManager(k8sManager)
 	Expect(err).ToNot(HaveOccurred())
 

docs/development.md

@@ -104,7 +104,6 @@ The controllers are using the default kubeconfig ($KUBECONFIG, InCluster or
    # or
    kubectl create deployment --image nginx:latest nginx --replicas 1
    kubectl expose deployment --port 80 --type LoadBalancer nginx --name loadbalancer
-   kubectl annotate service loadbalancer yawol.stackit.cloud/className=test # annotation needs to match the value of the `classname` flag from `run-ycc.sh`
    ```
 
 2. Check if the yawol-cloud-controller created a new `LoadBalancer` object

example-setup/yawol-cloud-controller/service.yaml

@@ -15,6 +15,7 @@ metadata:
     # yawol.stackit.cloud/tcpProxyProtocol: "false"
     # yawol.stackit.cloud/tcpProxyProtocolPortsFilter: ""
 spec:
+  loadBalancerClass: "test"
   type: LoadBalancer
   selector:
     app: nginx

internal/helper/const.go

@@ -12,4 +12,5 @@ const (
 	LoadBalancerKind             = "LoadBalancer"
 	VRRPInstanceName             = "ENVOY"
 	HasKeepalivedMaster          = "HasKeepalivedMaster"
+	DefaultLoadbalancerClass     = "stackit.cloud/yawol"
 )

internal/helper/service.go

@@ -13,6 +13,7 @@ import (
 	coreV1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -164,6 +165,22 @@ func GetLoadBalancerNameFromService(service *coreV1.Service) string {
 	return service.Namespace + "--" + service.Name
 }
 
+func getLoadBalancerClass(service *coreV1.Service) string {
+	if className := service.Annotations[yawolv1beta1.ServiceClassName]; className != "" {
+		return className
+	}
+
+	if service.Spec.LoadBalancerClass != nil {
+		return *service.Spec.LoadBalancerClass
+	}
+
+	return ""
+}
+
+func CheckLoadBalancerClasses(service *coreV1.Service, validClasses []string) bool {
+	return sets.New(validClasses...).Has(getLoadBalancerClass(service))
+}
+
 // ValidateService checks if the service is valid
 func ValidateService(svc *coreV1.Service) error {
 	for _, port := range svc.Spec.Ports {