add support for kubernetes 1.24.x

api/v1beta1/loadbalancermachine_types.go

@@ -85,6 +85,9 @@ type LoadBalancerMachineStatus struct {
 	// ServiceAccountName contains the namespacedName from the ServiceAccount for a LoadBalancerMachine.
 	// +optional
 	ServiceAccountName *string `json:"serviceAccountName,omitempty"`
+	// SecretName contains the namespacedName from the Secret which belongs to the Serviceaccount.
+	// +optional
+	ServiceAccountSecretName *string `json:"serviceAccountSecretName,omitempty"`
 	// RoleName contains the namespacedName from the Role for a LoadBalancerMachine.
 	// +optional
 	RoleName *string `json:"roleName,omitempty"`

charts/yawol-controller/crds/yawol.stackit.cloud_loadbalancermachines.yaml

@@ -307,6 +307,10 @@ spec:
                 description: ServiceAccountName contains the namespacedName from the
                   ServiceAccount for a LoadBalancerMachine.
                 type: string
+              serviceAccountSecretName:
+                description: SecretName contains the namespacedName from the Secret
+                  which belongs to the Serviceaccount.
+                type: string
             type: object
         type: object
     served: true

charts/yawol-controller/templates/rbac-yawol-controller.yaml

@@ -17,9 +17,13 @@ rules:
     resources:
       - "secrets"
     verbs:
-    - "get"
-    - "list"
-    - "watch"
+    - get
+    - list
+    - watch
+    - create
+    - update
+    - patch
+    - delete
   - apiGroups: [""]
     resources:
       - "serviceaccounts"

cmd/yawol-controller/main.go

@@ -19,6 +19,7 @@ import (
 	helpermetrics "github.com/stackitcloud/yawol/internal/metrics"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	discovery "k8s.io/client-go/discovery"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 
@@ -122,10 +123,11 @@ func main() {
 	var loadBalancerMgr manager.Manager
 	var loadBalancerSetMgr manager.Manager
 	var loadBalancerMachineMgr manager.Manager
+	cfg := ctrl.GetConfigOrDie()
 
 	// Controller 2
 	if lbController {
-		loadBalancerMgr, err = ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		loadBalancerMgr, err = ctrl.NewManager(cfg, ctrl.Options{
 			Scheme:                     scheme,
 			MetricsBindAddress:         metricsAddrLb,
 			Port:                       9443,
@@ -168,7 +170,7 @@ func main() {
 
 	// Controller 3
 	if lbSetController {
-		loadBalancerSetMgr, err = ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		loadBalancerSetMgr, err = ctrl.NewManager(cfg, ctrl.Options{
 			Scheme:                     scheme,
 			MetricsBindAddress:         metricsAddrLbs,
 			Port:                       9444,
@@ -214,7 +216,9 @@ func main() {
 			panic("could not read env " + EnvAPIEndpoint)
 		}
 
-		loadBalancerMachineMgr, err = ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(cfg)
+
+		loadBalancerMachineMgr, err = ctrl.NewManager(cfg, ctrl.Options{
 			Scheme:                     scheme,
 			MetricsBindAddress:         metricsAddrLbm,
 			Port:                       9445,
@@ -243,6 +247,7 @@ func main() {
 			APIEndpoint:      apiEndpoint,
 			Metrics:          &helpermetrics.LoadBalancerMachineMetrics,
 			OpenstackTimeout: openstackTimeout,
+			DiscoveryClient:  discoveryClient,
 		}).SetupWithManager(loadBalancerMachineMgr); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "LoadBalancerMachine")
 			os.Exit(1)

controllers/yawol-controller/loadbalancermachine/loadbalancermachine_controller.go

@@ -27,6 +27,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/tools/clientcmd/api/latest"
 	"k8s.io/client-go/tools/record"
@@ -38,8 +39,9 @@ import (
 
 const (
 	// ServiceFinalizer Name of finalizer for controller4
-	ServiceFinalizer   = "yawol.stackit.cloud/controller4"
-	DefaultRequeueTime = 10 * time.Millisecond
+	ServiceFinalizer             = "yawol.stackit.cloud/controller4"
+	DefaultRequeueTime           = 10 * time.Millisecond
+	ServiceAccountNameAnnotation = "kubernetes.io/service-account.name"
 )
 
 var ipv4Regex = `^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4})`
@@ -59,6 +61,7 @@ type LoadBalancerMachineReconciler struct { //nolint:revive // naming from kubeb
 	getOsClientForIni os.GetOSClientFunc
 	WorkerCount       int
 	OpenstackTimeout  time.Duration
+	DiscoveryClient   *discovery.DiscoveryClient
 }
 
 // Reconcile Reconciles a LoadBalancerMachine
@@ -153,6 +156,11 @@ func (r *LoadBalancerMachineReconciler) Reconcile(ctx context.Context, req ctrl.
 		return res, err
 	}
 
+	// Reconcile ServiceAccountSecret for yawollet access
+	if err := r.reconcileSecret(ctx, loadBalancerMachine, sa); err != nil {
+		return res, err
+	}
+
 	// Reconcile Role for yawollet access
 	if err := r.reconcileRole(ctx, loadBalancerMachine, loadbalancer); err != nil {
 		return ctrl.Result{}, err
@@ -183,7 +191,7 @@ func (r *LoadBalancerMachineReconciler) Reconcile(ctx context.Context, req ctrl.
 		return ctrl.Result{}, err
 	}
 
-	if err := r.reconcileServer(ctx, osClient, loadbalancer, loadBalancerMachine, sa, vip); err != nil {
+	if err := r.reconcileServer(ctx, osClient, loadbalancer, loadBalancerMachine, vip); err != nil {
 		return ctrl.Result{}, err
 	}
 
@@ -230,7 +238,7 @@ func (r *LoadBalancerMachineReconciler) reconcileSA(
 		},
 	}
 
-	err := r.Client.Get(ctx, client.ObjectKey{Name: sa.Name, Namespace: sa.Namespace}, &sa)
+	err := r.Client.Get(ctx, client.ObjectKeyFromObject(&sa), &sa)
 	if err != nil {
 		if errors2.IsNotFound(err) {
 			if err = r.Client.Create(ctx, &sa); err != nil {
@@ -254,6 +262,81 @@ func (r *LoadBalancerMachineReconciler) reconcileSA(
 	return sa, nil
 }
 
+func (r *LoadBalancerMachineReconciler) reconcileSecret(
+	ctx context.Context,
+	loadBalancerMachine *yawolv1beta1.LoadBalancerMachine,
+	sa v1.ServiceAccount,
+) error {
+	r.Log.Info("Check Secret", "loadBalancerMachineName", loadBalancerMachine.Name)
+	if loadBalancerMachine.Status.ServiceAccountSecretName != nil {
+		// no need to check if the secret is still present
+		// on false secret, the machine will delete itself
+		return nil
+	}
+
+	// use pre 1.24 created secret
+	if len(sa.Secrets) > 0 {
+		r.Log.Info("Use Secret from ServiceAccount", "loadBalancerMachineName", loadBalancerMachine.Name)
+		namespacedName := types.NamespacedName{
+			Name: sa.Secrets[0].Name,
+			// the referenced secret has no namespace
+			// since it is the same as the service account
+			Namespace: sa.Namespace,
+		}.String()
+
+		// return since secret is already created
+		return helper.PatchLBMStatus(ctx, r.Client.Status(), loadBalancerMachine, yawolv1beta1.LoadBalancerMachineStatus{
+			ServiceAccountSecretName: &namespacedName,
+		})
+	}
+
+	// check if kubernetes handles secret creation
+	version, err := kubernetes.GetVersion(r.DiscoveryClient)
+	if err != nil {
+		return err
+	}
+
+	if version.IsLower(&kubernetes.Version{Major: 1, Minor: 24}) {
+		// secret not created by kubernetes
+		// requeue until created
+		return helper.ErrSecretNotFound
+	}
+
+	r.Log.Info("Create Secret", "loadBalancerMachineName", loadBalancerMachine.Name)
+	// secret does not exist, create a new one
+	secret := v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      loadBalancerMachine.Name,
+			Namespace: loadBalancerMachine.Namespace,
+			Annotations: map[string]string{
+				ServiceAccountNameAnnotation: sa.Name,
+			},
+		},
+		Type: v1.SecretTypeServiceAccountToken,
+	}
+
+	err = r.Client.Get(ctx, client.ObjectKeyFromObject(&secret), &v1.Secret{})
+	if client.IgnoreNotFound(err) != nil {
+		return err
+	}
+
+	if errors2.IsNotFound(err) {
+		if err := r.Client.Create(ctx, &secret); err != nil {
+			return err
+		}
+	}
+
+	namespacedNameString := types.NamespacedName{
+		Name:      secret.Name,
+		Namespace: secret.Namespace,
+	}.String()
+
+	// patch secretName in status
+	return helper.PatchLBMStatus(ctx, r.Client.Status(), loadBalancerMachine, yawolv1beta1.LoadBalancerMachineStatus{
+		ServiceAccountSecretName: &namespacedNameString,
+	})
+}
+
 func (r *LoadBalancerMachineReconciler) reconcileRole(
 	ctx context.Context,
 	loadBalancerMachine *yawolv1beta1.LoadBalancerMachine,
@@ -568,7 +651,6 @@ func (r *LoadBalancerMachineReconciler) reconcileServer(
 	osClient os.Client,
 	loadbalancer *yawolv1beta1.LoadBalancer,
 	loadBalancerMachine *yawolv1beta1.LoadBalancerMachine,
-	serviceAccount v1.ServiceAccount,
 	vip string,
 ) error {
 	var srvClient os.ServerClient
@@ -581,7 +663,7 @@ func (r *LoadBalancerMachineReconciler) reconcileServer(
 
 	// get kubeconfig which will be passed to VM user-data for yawollet access
 	var kubeconfig string
-	if kubeconfig, err = r.getKubeConfigForServiceAccount(ctx, loadBalancerMachine.Namespace, serviceAccount); err != nil {
+	if kubeconfig, err = r.getKubeConfigForServiceAccount(ctx, loadBalancerMachine.Status.ServiceAccountSecretName); err != nil {
 		return err
 	}
 
@@ -967,17 +1049,20 @@ func (r *LoadBalancerMachineReconciler) waitForServerStatus(
 
 func (r *LoadBalancerMachineReconciler) getKubeConfigForServiceAccount(
 	ctx context.Context,
-	namespace string,
-	sa v1.ServiceAccount,
+	namespacedName *string,
 ) (string, error) {
-	if len(sa.Secrets) < 1 {
-		return "", fmt.Errorf("%w for serviceAccount %s", helper.ErrSecretNotFound, sa.Name)
+	if namespacedName == nil {
+		return "", fmt.Errorf("%w NamespacedName is nil", helper.ErrSecretNotFound)
 	}
 
 	sec := v1.Secret{}
-	var err error
-	if err = r.Client.Get(ctx, types.NamespacedName{Name: sa.Secrets[0].Name, Namespace: namespace}, &sec); err != nil {
-		return "", fmt.Errorf("%w cloud not getting %s", helper.ErrSecretNotFound, sec.Name)
+	nn, err := kubernetes.ToNamespacedName(*namespacedName)
+	if err != nil {
+		return "", err
+	}
+
+	if err := r.Client.Get(ctx, nn, &sec); err != nil {
+		return "", fmt.Errorf("%w could not get %s", helper.ErrSecretNotFound, nn.String())
 	}
 
 	var token, ca []byte
@@ -1005,7 +1090,7 @@ func (r *LoadBalancerMachineReconciler) getKubeConfigForServiceAccount(
 		Contexts: map[string]*api.Context{"context": {
 			Cluster:   "default-cluster",
 			AuthInfo:  "user",
-			Namespace: namespace,
+			Namespace: nn.Namespace,
 		}},
 		CurrentContext: "context",
 	}

controllers/yawol-controller/loadbalancermachine/loadbalancermachine_controller_test.go

@@ -100,16 +100,21 @@ var _ = Describe("load balancer machine", func() {
 			Eventually(func(g Gomega) {
 				var actualRole rbac.Role
 				g.Expect(k8sClient.Get(ctx, lbmNN, &actualRole)).To(Succeed())
-
-				// TODO somehow the arrays are not equal even though they are
 				g.Expect(len(actualRole.Rules)).To(Equal(len(getPolicyRules(lb, lbm))))
 			}, timeout, interval).Should(Succeed())
 
-			By("checking sa secret")
+			By("checking service account")
 			Eventually(func(g Gomega) {
 				g.Expect(k8sClient.Get(ctx, lbmNN, &v1.ServiceAccount{})).To(Succeed())
 			}, timeout, interval).Should(Succeed())
 
+			By("checking secret")
+			Eventually(func(g Gomega) {
+				secret := &v1.Secret{}
+				g.Expect(k8sClient.Get(ctx, lbmNN, secret)).To(Succeed())
+				g.Expect(secret.Annotations).To(HaveKeyWithValue(ServiceAccountNameAnnotation, lbmNN.Name))
+			}, timeout, interval).Should(Succeed())
+
 			By("checking rolebinding")
 			Eventually(func(g Gomega) {
 				var roleBinding rbac.RoleBinding
@@ -124,6 +129,19 @@ var _ = Describe("load balancer machine", func() {
 			}, timeout, interval).Should(Succeed())
 		})
 
+		It("should update lbm status", func() {
+			lbmNN := runtimeClient.ObjectKeyFromObject(lbm)
+
+			By("checking that the rbac role gets created")
+			Eventually(func(g Gomega) {
+				var updatedLBM LBM
+				g.Expect(k8sClient.Get(ctx, lbmNN, &updatedLBM)).To(Succeed())
+				g.Expect(updatedLBM.Status.ServiceAccountName).ToNot(BeNil())
+				g.Expect(updatedLBM.Status.ServiceAccountSecretName).ToNot(BeNil())
+				g.Expect(updatedLBM.Status.RoleName).ToNot(BeNil())
+			}, timeout, interval).Should(Succeed())
+		})
+
 		It("should create openstack resources", func() {
 			lbmNN := runtimeClient.ObjectKeyFromObject(lbm)
 

controllers/yawol-controller/loadbalancermachine/suite_test.go

@@ -12,6 +12,7 @@ import (
 	. "github.com/onsi/gomega"
 	yawolv1beta1 "github.com/stackitcloud/yawol/api/v1beta1"
 	helpermetrics "github.com/stackitcloud/yawol/internal/metrics"
+	discovery "k8s.io/client-go/discovery"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -95,14 +96,18 @@ var _ = BeforeSuite(func() {
 	}
 	Expect(k8sClient.Create(context.Background(), &secret)).Should(Succeed())
 
+	discoveryClient, err := discovery.NewDiscoveryClientForConfig(cfg)
+	Expect(err).ToNot(HaveOccurred())
+
 	loadBalancerMachineReconciler = &LoadBalancerMachineReconciler{
-		APIEndpoint: "https://lala.com",
-		Client:      k8sManager.GetClient(),
-		Log:         ctrl.Log.WithName("controllers").WithName("LoadBalancerMachine"),
-		Scheme:      k8sManager.GetScheme(),
-		Recorder:    k8sManager.GetEventRecorderFor("LoadBalancerMachine"),
-		RecorderLB:  k8sManager.GetEventRecorderFor("yawol-service"),
-		Metrics:     &helpermetrics.LoadBalancerMachineMetrics,
+		APIEndpoint:     "https://example.com",
+		Client:          k8sManager.GetClient(),
+		Log:             ctrl.Log.WithName("controllers").WithName("LoadBalancerMachine"),
+		Scheme:          k8sManager.GetScheme(),
+		Recorder:        k8sManager.GetEventRecorderFor("LoadBalancerMachine"),
+		RecorderLB:      k8sManager.GetEventRecorderFor("yawol-service"),
+		Metrics:         &helpermetrics.LoadBalancerMachineMetrics,
+		DiscoveryClient: discoveryClient,
 	}
 
 	err = loadBalancerMachineReconciler.SetupWithManager(k8sManager)