F5Networks · arzzon · Oct 3, 2024 · Oct 3, 2024
diff --git a/docs/RELEASE-NOTES.rst b/docs/RELEASE-NOTES.rst
@@ -118,7 +118,7 @@ Added Functionality
 ```````````````````
 **What's new:**
     * Multi Cluster
-        * Support Alternate backend and cluster Ratio for Transport Server. See `Example <https://github.com/F5Networks/k8s-bigip-ctlr/blob/2.x-master/docs/config_examples/multicluster/customResource/transportServer/ts-with-weight-extended-service.yaml>`_.
+        * Support Alternate backend and cluster Ratio for Transport Server. See `Example <https://github.com/F5Networks/k8s-bigip-ctlr/blob/2.x-master/docs/config_examples/multicluster/customResource/transportServer/>`_.
     * CRD
         * `Issue 3337 <https://github.com/F5Networks/k8s-bigip-ctlr/issues/3337>`_: Support for access profile and per request policy in policy CRD and VS CRD. See `Example <https://github.com/F5Networks/k8s-bigip-ctlr/blob/2.x-master/docs/config_examples/customResource/Policy/policy-with-profileAccess.yaml>`_.
         * `Issue 3352 <https://github.com/F5Networks/k8s-bigip-ctlr/issues/3352>`_: Add support for alternate backend,weight and ratio for transport server. See `Example <https://github.com/F5Networks/k8s-bigip-ctlr/blob/2.x-master/docs/config_examples/customResource/TransportServer/ts-with-weight-alternate-backend/ts-with-weight-alternate-backend.yaml>`_.
@@ -183,7 +183,7 @@ Added Functionality
     * Multi Cluster
         * `Issue 3284 <https://github.com/F5Networks/k8s-bigip-ctlr/issues/3284>`_: Add support to avoid service pool creation for clusters under maintenance. See `Example <https://github.com/F5Networks/k8s-bigip-ctlr/blob/2.x-master/docs/config_examples/multicluster/extendedConfigmap/>`_
         * Streamline the naming convention for extended service references and multi cluster references annotations.
-        * See `Example with the updated field names for extendedServiceReferences in VS CRD: <https://github.com/F5Networks/k8s-bigip-ctlr/blob/2.x-master/docs/config_examples/multicluster/customResource/virtualServer/vs-with-extended-services.yaml>`_
+        * See `Example with the updated field names for extendedServiceReferences in VS CRD: <https://github.com/F5Networks/k8s-bigip-ctlr/blob/2.x-master/docs/config_examples/multicluster/customResource/virtualServer/>`_
         * See `Example the updated field names for multiClusterServices annotation in NextGenRoutes: <https://github.com/F5Networks/k8s-bigip-ctlr/blob/2.x-master/docs/config_examples/multicluster/routes/route-with-multicluster-service-annotation.yaml>`_
     * CRD
         * `Issue 3225 <https://github.com/F5Networks/k8s-bigip-ctlr/issues/3225>`_: Support for Host Persistence to configure and disable the Persistence in VS Policy Rule action based on host in VirtualServer. See `Example <https://github.com/F5Networks/k8s-bigip-ctlr/blob/2.x-master/docs/config_examples/customResource/VirtualServer/virtual-server-with-hostPersistence/>`_

diff --git a/docs/config_examples/customResourceDefinitions/customresourcedefinitions.yml b/docs/config_examples/customResourceDefinitions/customresourcedefinitions.yml
@@ -370,7 +370,7 @@ spec:
                         maximum: 65535
                       serviceDownAction:
                         type: string
-                      multiClusterServices:
+                      extendedServiceReferences:
                         type: array
                         items:
                           type: object
@@ -766,7 +766,7 @@ spec:
                       maximum: 65535
                     serviceDownAction:
                       type: string
-                    multiClusterServices:
+                    extendedServiceReferences:
                       type: array
                       items:
                         type: object

diff --git a/docs/config_examples/multicluster/README.md b/docs/config_examples/multicluster/README.md
@@ -365,7 +365,7 @@ Health probe parameters are provided in highAvailabilityCIS in extended configma
 
 
 ### Route Annotation for Multi-ClusterServices
-Services running in any other OpenShift clusters, apart from the HA cluster pair, can be referenced in the route annotations as mentioned below:
+Services running in any other OpenShift clusters, as mentioned below:
 ```
 virtual-server.f5.com/multiClusterServices: 
 '[
@@ -426,30 +426,12 @@ Following is the sample deployment for primary CIS deployment:
 
 
 ### Virutal Server Pool with Multi-ClusterServices
-Services running in any other OpenShift/Kubernetes clusters, apart from the HA cluster pair, can be referenced in the VS Pool as mentioned below:
-```
-  pools:
-  - path: /tea
-    serviceNamespace: tea
-    service: svc-2
-    servicePort: 80
-    multiClusterServices:
-    - clusterName: cluster2
-      namespace: ns1
-      servicePort: 8080
-      service: svc-1
-    - clusterName: cluster3
-      namespace: ns2
-      servicePort: 80
-      service: svc-ext-1
-```
+This is not supported as of now. It will be supported soon.
 
 ### Transport Server Pool with Multi-ClusterServices
-Services running in any other OpenShift/Kubernetes clusters, apart from the HA cluster pair, can be referenced in the TS Pool as mentioned below:
+Services running in any other OpenShift/Kubernetes clusters those are monitored by CIS, can be referenced in the TS Pool as mentioned below:
 ```
   pool:
-    service: svc-1
-    servicePort: 8181
     multiClusterServices:
     - clusterName: cluster2
       service: svc-1
@@ -663,11 +645,6 @@ Ok[root@cluster-1-worker0 ~]#
 where 10.244.1.213 is the CIS PodIP.
 
 
-### How multiClusterServices is different from multiClusterServices?
-multiClusterServices is applicable for Virtual Server CR or Transport Server CR and multiClusterServices is applicable for NextGen Routes.
-multiClusterServices is used to refer the services running in any other OpenShift/Kubernetes clusters, apart from the HA cluster pair, in the VS Pool or TS Pool.
-multiClusterServices is used to refer the services running in any other OpenShift/Kubernetes clusters, apart from the HA cluster pair, in the Route annotation.
-
 ### Which services can be provided as multiClusterServices?
 Any service running in any OpenShift/Kubernetes clusters which are part of the multiCluster setup can be provided as multiClusterServices.
 
@@ -690,13 +667,9 @@ where cluster2 is the external cluster apart from the HA cluster pair.
 Note: External Clusters doesn't need to install CIS
 
 ### How to configure multiClusterServices in Virtual Server CR or Transport Server CR?
-multiClusterServices is a field in Virtual Server CR or Transport Server CR. Below is the sample Virtual Server CR with multiClusterServices:
+multiClusterServices is not supported in VirutalServer CR yet. It's supported in Transport Server CR only when CIS is running in "default" mode. Below is the sample Transport Server CR with multiClusterServices:
 ```
   pools:
-  - path: /tea
-    serviceNamespace: tea
-    service: svc-2
-    servicePort: 80
     multiClusterServices:
     - clusterName: cluster3
       namespace: ns1
@@ -708,7 +681,6 @@ multiClusterServices is a field in Virtual Server CR or Transport Server CR. Bel
       service: svc-ext-1
 ```
 where clusterName is the name of the cluster where the service is running, namespace is the namespace where the service is running, servicePort is the port of the service and service is the name of the service.
-where cluster3 and cluster4 are the external clusters apart from the HA cluster pair.
 Note: External Clusters doesn't need to install CIS
 
 ### Can I specify the services running in CIS HA cluster in multiClusterServices?

diff --git a/...ver/ts-with-extended-service-ha-svcs.yaml → ...tServer/ts-with-multiClusterServices.yaml b/...ver/ts-with-extended-service-ha-svcs.yaml → ...tServer/ts-with-multiClusterServices.yaml
diff --git a/...rver/ts-with-weight-extended-service.yaml → ...-with-weights-in-pool-service-and-ab.yaml b/...rver/ts-with-weight-extended-service.yaml → ...-with-weights-in-pool-service-and-ab.yaml
@@ -15,17 +15,9 @@ spec:
       - service: svc-1-external-service
         serviceNamespace: default
         weight: 20
-    multiClusterServices:
-      - clusterName: cluster3
-        namespace: default
-        service: svc-1-external-service
-        servicePort: 1344
-        weight: 70
-      - clusterName: cluster4
-        namespace: default
-        service: svc-1-external-service
-        servicePort: 1344
-        weight: 70
+      - service: svc-2-external-service
+        serviceNamespace: default
+        weight: 60
     monitor:
       interval: 20
       timeout: 10

diff --git a/...portServer/ts-with-external-services.yaml → ...rver/ts-without-multiClusterServices.yaml b/...portServer/ts-with-external-services.yaml → ...rver/ts-without-multiClusterServices.yaml
@@ -17,12 +17,3 @@ spec:
       type: tcp
       interval: 10
       timeout: 10
-    multiClusterServices:
-    - clusterName: cluster2
-      service: svc-1
-      namespace: ns1
-      servicePort: 8181
-    - clusterName: cluster3
-      service: svc-ext-1
-      namespace: ns2
-      servicePort: 8282
diff --git a/.../config_examples/multicluster/customResource/virtualServer/vs-with-extended-services.yaml b/.../config_examples/multicluster/customResource/virtualServer/vs-with-extended-services.yaml
diff --git a/...ver/vs-with-extended-services-and-ab.yaml → ...alServer/vs-with-pool-service-and-ab.yaml b/...ver/vs-with-extended-services-and-ab.yaml → ...alServer/vs-with-pool-service-and-ab.yaml
@@ -16,14 +16,5 @@ spec:
       alternateBackends:
         - service: svc-edge-b
           weight: 30
-      multiClusterServices:
-        - clusterName: cluster2
-          namespace: ns1
-          servicePort: 80
-          service: svc-1
-          weight: 20
-        - clusterName: cluster3
-          namespace: ns2
-          servicePort: 80
-          service: svc-ext-1
-          weight: 30
+        - service: svc-edge-c
+          weight: 20
diff --git a/...ples/multicluster/extendedConfigmap/global-spec-config-for-multicluster-default-mode.yaml b/...ples/multicluster/extendedConfigmap/global-spec-config-for-multicluster-default-mode.yaml
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    f5nr: "true"
+  name: extended-spec-config
+  namespace: kube-system
+data:
+  extendedSpec: |
+    mode: default
+    highAvailabilityCIS:
+      primaryEndPoint: http://10.145.72.114:8001
+      probeInterval: 30
+      retryInterval: 3
+      primaryCluster:
+        clusterName: cluster1
+        secret: default/kubeconfig1
+      secondaryCluster:
+        clusterName: cluster2
+        secret: default/kubeconfig2
+    externalClustersConfig:
+    - clusterName: cluster3
+      secret: default/kubeconfig3
+    - clusterName: cluster4
+      secret: default/kubeconfig4
+    - clusterName: cluster5
+      secret: default/kubeconfig5
diff --git a/pkg/controller/informers.go b/pkg/controller/informers.go
@@ -1127,21 +1127,12 @@ func (ctlr *Controller) enqueueUpdatedService(obj, cur interface{}, clusterName
 		}
 	}
 
-	// Check partition update for LoadBalancer service
-	partitionUpdate := false
-	if svc.Spec.Type == corev1.ServiceTypeLoadBalancer {
-		oldPartition, _ := svc.Annotations[LBServicePartitionAnnotation]
-		newPartition, _ := curSvc.Annotations[LBServicePartitionAnnotation]
-		if oldPartition != newPartition {
-			partitionUpdate = true
-		}
-	}
-
 	if (svc.Spec.Type != curSvc.Spec.Type && svc.Spec.Type == corev1.ServiceTypeLoadBalancer) ||
 		(svc.Spec.Type == corev1.ServiceTypeLoadBalancer && (svc.Annotations[LBServiceIPAnnotation] != curSvc.Annotations[LBServiceIPAnnotation] || svc.Annotations[LBServiceHostAnnotation] != curSvc.Annotations[LBServiceHostAnnotation])) ||
 		(svc.Annotations[LBServiceIPAMLabelAnnotation] != curSvc.Annotations[LBServiceIPAMLabelAnnotation]) ||
 		!reflect.DeepEqual(svc.Labels, curSvc.Labels) || !reflect.DeepEqual(svc.Spec.Ports, curSvc.Spec.Ports) ||
-		!reflect.DeepEqual(svc.Spec.Selector, curSvc.Spec.Selector) || partitionUpdate {
+		!reflect.DeepEqual(svc.Spec.Selector, curSvc.Spec.Selector) ||
+		(svc.Annotations[LBServicePartitionAnnotation] != curSvc.Annotations[LBServicePartitionAnnotation]) {
 		log.Debugf("Enqueueing Old Service: %v %v", svc, getClusterLog(clusterName))
 		key := &rqKey{
 			namespace:   svc.ObjectMeta.Namespace,

diff --git a/pkg/controller/resourceConfig.go b/pkg/controller/resourceConfig.go
@@ -2316,14 +2316,7 @@ func (ctlr *Controller) prepareRSConfigFromTransportServer(
 	} else {
 		ctlr.updateMultiClusterResourceServiceMap(rsCfg, rsRef, vs.Spec.Pool.Service, vs.Spec.Pool.Path, pool, vs.Spec.Pool.ServicePort, "")
 	}
-	if ctlr.multiClusterMode != "" && ctlr.discoveryMode == DefaultMode {
-		if vs.Spec.Pool.Service != "" {
-			log.Warning("Base service is ignored for default discovery mode")
-		}
-		if len(vs.Spec.Pool.AlternateBackends) > 0 {
-			log.Warning("alternate backends are ignored for default discovery mode")
-		}
-	}
+
 	if ctlr.isSinglePoolRatioEnabled(vs) {
 		defaultWeight := 100
 		pool.Balance = PoolLBMemberRatio

diff --git a/pkg/controller/validate.go b/pkg/controller/validate.go
@@ -93,14 +93,18 @@ func (ctlr *Controller) checkValidVirtualServer(
 		}
 	}
 	for _, pool := range vsResource.Spec.Pools {
-		if pool.MultiClusterServices == nil {
-			continue
+		if pool.MultiClusterServices != nil {
+			err = fmt.Sprintf("%v MultiClusterServices is currently not supported for VS CR. Consider removing "+
+				"it from the virtual server %s", ctlr.getMultiClusterLog(), vsName)
+			log.Errorf(err)
+			ctlr.updateResourceStatus(VirtualServer, vsResource, "", "", errors.New(err))
+			return false
 		}
 		for _, mcs := range pool.MultiClusterServices {
 			err := ctlr.checkValidMultiClusterService(mcs, true)
 			if err != nil {
-				// In case of invalid extendedServiceReference, just log the error and proceed
-				log.Errorf("[MultiCluster] invalid extendedServiceReference: %v for VS: %s: %v", mcs, vsName, err)
+				// In case of invalid multiClusterServices, just log the error and proceed
+				log.Errorf("[MultiCluster] invalid multiClusterServices: %v for VS: %s: %v", mcs, vsName, err)
 				continue
 			}
 		}
@@ -114,7 +118,22 @@ func (ctlr *Controller) checkValidTransportServer(
 ) bool {
 
 	// Check if the required fields are set as per the recommendations
-	if ctlr.multiClusterMode == "" {
+	// Validation for multiCluster setup with default mode
+	if ctlr.discoveryMode == DefaultMode {
+		if tsResource.Spec.Pool.MultiClusterServices == nil {
+			err := fmt.Sprintf("[MultiCluster] MultiClusterServices is not provided for TransportServer %s/%s but "+
+				"CIS is running with default mode", tsResource.ObjectMeta.Namespace, tsResource.ObjectMeta.Name)
+			log.Errorf(err)
+			ctlr.updateResourceStatus(TransportServer, tsResource, "", "", errors.New(err))
+			return false
+		}
+		if tsResource.Spec.Pool.Service != "" || tsResource.Spec.Pool.ServicePort != (intstr.IntOrString{}) ||
+			tsResource.Spec.Pool.Weight != nil || tsResource.Spec.Pool.AlternateBackends != nil {
+			log.Warningf("[MultiCluster] Ignoring Pool Service/ServicePort/Weight/AlternateBackends provided for "+
+				"TransportServer %s as these are not supported in default mode", tsResource.ObjectMeta.Name)
+		}
+	} else {
+		// validation for non multiCluster case
 		if tsResource.Spec.Pool.MultiClusterServices != nil {
 			err := fmt.Sprintf("MultiClusterServices is set for TransportServer %s/%s but CIS is not running in "+
 				"multiCluster mode", tsResource.ObjectMeta.Namespace, tsResource.ObjectMeta.Name)
@@ -129,19 +148,6 @@ func (ctlr *Controller) checkValidTransportServer(
 			ctlr.updateResourceStatus(TransportServer, tsResource, "", "", errors.New(err))
 			return false
 		}
-	} else if ctlr.discoveryMode == DefaultMode {
-		if tsResource.Spec.Pool.MultiClusterServices == nil {
-			err := fmt.Sprintf("[MultiCluster] MultiClusterServices is not provided for TransportServer %s/%s but "+
-				"CIS is running with default mode", tsResource.ObjectMeta.Namespace, tsResource.ObjectMeta.Name)
-			log.Errorf(err)
-			ctlr.updateResourceStatus(TransportServer, tsResource, "", "", errors.New(err))
-			return false
-		}
-		if tsResource.Spec.Pool.Service != "" || tsResource.Spec.Pool.ServicePort != (intstr.IntOrString{}) ||
-			tsResource.Spec.Pool.Weight != nil || tsResource.Spec.Pool.AlternateBackends != nil {
-			log.Warningf("[MultiCluster] Ignoring Pool Service/ServicePort/Weight/AlternateBackends provided for "+
-				"TransportServer %s as these are not supported in default mode", tsResource.ObjectMeta.Name)
-		}
 	}
 	vsNamespace := tsResource.ObjectMeta.Namespace
 	vsName := tsResource.ObjectMeta.Name

diff --git a/pkg/controller/worker.go b/pkg/controller/worker.go
@@ -253,10 +253,6 @@ func (ctlr *Controller) processResources() bool {
 		if ctlr.mode != OpenShiftMode {
 			break
 		}
-		if ctlr.discoveryMode == DefaultMode {
-			log.Errorf("Routes are not supported with multiCluster mode: %v", ctlr.discoveryMode)
-			break
-		}
 		route := rKey.rsc.(*routeapi.Route)
 		// processRoutes knows when to delete a VS (in the event of global config update and route delete)
 		// so should not trigger delete from here
@@ -307,10 +303,6 @@ func (ctlr *Controller) processResources() bool {
 		if ctlr.mode == OpenShiftMode || ctlr.mode == KubernetesMode {
 			break
 		}
-		if ctlr.discoveryMode == DefaultMode {
-			log.Errorf("Virtual servers are not supported with multiCluster mode: %v", ctlr.discoveryMode)
-			break
-		}
 		virtual := rKey.rsc.(*cisapiv1.VirtualServer)
 		rscRefKey := resourceRef{
 			kind:      VirtualServer,
@@ -3007,7 +2999,6 @@ func (ctlr *Controller) processTransportServers(
 
 	rsMap := ctlr.resources.getPartitionResourceMap(partition)
 	rsMap[rsName] = rsCfg
-	ctlr.updateResourceStatus(TransportServer, virtual, ip, "", nil)
 	if len(rsCfg.MetaData.hosts) > 0 {
 		ctlr.ProcessAssociatedExternalDNS(rsCfg.MetaData.hosts)
 	}