Support for static routes via central manager (#3245)

config/apis/cis/v1/types.go

@@ -469,9 +469,10 @@ type DeployConfigSpec struct {
 }
 
 type BaseConfig struct {
-	NamespaceLabel string `json:"namespaceLabel,omitempty"`
-	NodeLabel      string `json:"nodeLabel,omitempty"`
-	RouteLabel     string `json:"routeLabel,omitempty"`
+	NamespaceLabel       string `json:"namespaceLabel,omitempty"`
+	NodeLabel            string `json:"nodeLabel,omitempty"`
+	RouteLabel           string `json:"routeLabel,omitempty"`
+	ControllerIdentifier string `json:"controllerIdentifier"`
 }
 
 type NetworkConfig struct {

docs/cis-3.x/deploy-config/cis-deploy-config-cr.yaml

@@ -10,6 +10,7 @@ spec:
   baseConfig:
     namespaceLabel: controller=cis
     nodeLabel: controller=cis
+    controllerIdentifier: cluster-1
   networkConfig:
     orchestrationCNI: ovn-static
     metaData:

docs/cis-3.x/install/k8s/sample-k8s-bigip-ctlr.yaml

@@ -57,8 +57,6 @@ spec:
             # See the k8s-bigip-ctlr documentation for information about
             # all config options
             # https://clouddocs.f5.com/containers/latest/
-            "--cm-url=<ip_address-or-hostname>",
-            "--orchestration-cni=nodeport",
             "--deploy-config-cr=kube-system/cis-config",
             "--manage-custom-resources=true",
             "--credentials-directory=/tmp/creds",

docs/config_examples/customResourceDefinitions/incubator/customresourcedefinitions.yml

@@ -1125,13 +1125,20 @@ spec:
                   type: object
                 baseConfig:
                   properties:
+                    controllerIdentifier:
+                      type: string
+                      x-kubernetes-validations:
+                        - message: "Controller identifier can not be changed. Please delete and recreate the CIS controller and deploy config CR."
+                          rule: self == oldSelf
                     namespaceLabel:
                       type: string
                       pattern: '^[a-zA-Z0-9][-A-Za-z0-9_.\/]{0,61}[a-zA-Z0-9]=[a-zA-Z0-9][-A-Za-z0-9_.]{0,61}[a-zA-Z0-9]$'
                     nodeLabel:
                       type: string
                       pattern: '^[a-zA-Z0-9][-A-Za-z0-9_.\/]{0,61}[a-zA-Z0-9]=[a-zA-Z0-9][-A-Za-z0-9_.]{0,61}[a-zA-Z0-9]$'
                   type: object
+                  required:
+                    - controllerIdentifier
                 networkConfig:
                   properties:
                     orchestrationCNI:

pkg/controller/constants.go

@@ -110,10 +110,12 @@ const (
 	OVN_K8S                    = "ovn-k8s"
 	OVNK8sNodeSubnetAnnotation = "k8s.ovn.org/node-subnets"
 	OVNK8sNodeIPAnnotation     = "k8s.ovn.org/node-primary-ifaddr"
-	OVNK8sNodeIPAnnotation2    = "k8s.ovn.org/host-addresses"
+	//k8s.ovn.org/host-addresses is changed to k8s.ovn.org/host-cidrs in openshift 4.14
+	OVNK8sNodeIPAnnotation2 = "k8s.ovn.org/host-addresses"
+	OvnK8sNodeIPAnnotation3 = "k8s.ovn.org/host-cidrs"
 
 	//Cilium CNI
-	CILIUM_K8S                      = "cilium-k8s"
+	CILIUM_Static                   = "cilium-static"
 	CiliumK8sNodeSubnetAnnotation12 = "io.cilium.network.ipv4-pod-cidr"
 	CiliumK8sNodeSubnetAnnotation13 = "network.cilium.io/ipv4-pod-cidr"
 

pkg/controller/controller.go

@@ -72,6 +72,12 @@ func NewController(params Params) *Controller {
 	}
 
 	log.Debug("Controller Created")
+	// fetch the CM token
+	err := ctlr.CMTokenManager.FetchToken()
+	if err != nil {
+		log.Errorf("Failed to Fetch Token: %v", err)
+		os.Exit(1)
+	}
 	// Sync CM token
 	go ctlr.CMTokenManager.SyncToken(make(chan struct{}))
 	ctlr.resourceQueue = workqueue.NewNamedRateLimitingQueue(
@@ -85,7 +91,7 @@ func NewController(params Params) *Controller {
 	}
 
 	// Initialize the controller with base resources in CIS config CR
-	ctlr.initInformers()
+	ctlr.initController()
 
 	// create the informers for namespaces and node
 	if err3 := ctlr.setupInformers(); err3 != nil {
@@ -96,13 +102,22 @@ func NewController(params Params) *Controller {
 	ctlr.NewRequestHandler(params.UserAgent, params.httpClientMetrics)
 	ctlr.RequestHandler.startRequestHandler()
 
+	// start response handler
 	go ctlr.responseHandler(ctlr.respChan)
 
+	// start the networkConfigHandler
+	if ctlr.networkManager != nil {
+		go ctlr.networkManager.NetworkConfigHandler()
+	}
 	// setup postmanager for bigip label
 	for bigip, _ := range ctlr.bigIpMap {
 		ctlr.RequestHandler.startPostManager(bigip)
 	}
+
+	// enable http endpoint
 	go ctlr.enableHttpEndpoint(params.HttpAddress)
+
+	// setup ipam
 	ctlr.setupIPAM(params)
 
 	go ctlr.Start()

pkg/controller/informerManager.go

@@ -3,6 +3,7 @@ package controller
 import (
 	"context"
 	cisapiv1 "github.com/F5Networks/k8s-bigip-ctlr/v3/config/apis/cis/v1"
+	"github.com/F5Networks/k8s-bigip-ctlr/v3/pkg/networkmanager"
 	log "github.com/F5Networks/k8s-bigip-ctlr/v3/pkg/vlogger"
 	v1 "k8s.io/api/core/v1"
 	metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -23,17 +24,26 @@ func (ctlr *Controller) setupInformers() error {
 	return nil
 }
 
-func (ctlr *Controller) initInformers() {
+func (ctlr *Controller) initController() {
 	// Initialize the controller with base resources in CIS config CR
 	key := strings.Split(ctlr.CISConfigCRKey, "/")
 	configCR, err := ctlr.clientsets.kubeCRClient.CisV1().DeployConfigs(key[0]).Get(context.TODO(), key[1], metaV1.GetOptions{})
 	if err != nil {
 		log.Errorf("%v", err)
 		os.Exit(1)
 	}
-	ctlr.processCNIConfig(configCR)
+
 	ctlr.updateResourceSelectorConfig(configCR.Spec.BaseConfig)
 	ctlr.updateBigIpConfigMap(configCR.Spec.BigIpConfig)
+
+	// process the CNI config
+	ctlr.processCNIConfig(configCR)
+	// create the network manager if required
+	if ctlr.StaticRoutingMode && ctlr.PoolMemberType != NodePort {
+		// create a new network manager
+		ctlr.networkManager = networkmanager.NewNetworkManager(ctlr.CMTokenManager, ctlr.ControllerIdentifier)
+	}
+
 	// update the agent params
 	ctlr.PostParams.AS3Config = configCR.Spec.AS3Config
 	ctlr.PostParams.tokenManager = ctlr.CMTokenManager
@@ -135,6 +145,7 @@ func (ctlr *Controller) updateResourceSelectorConfig(config cisapiv1.BaseConfig)
 		NamespaceLabel: config.NamespaceLabel,
 		RouteLabel:     config.RouteLabel,
 	}
+	ctlr.ControllerIdentifier = config.ControllerIdentifier
 	ctlr.resourceSelectorConfig.nativeResourceSelector, _ = createLabelSelector(DefaultNativeResourceLabel)
 	ctlr.resourceSelectorConfig.customResourceSelector, _ = createLabelSelector(DefaultCustomResourceLabel)
 }
@@ -178,7 +189,7 @@ func (ctlr *Controller) resetControllerForNamespaceLabel() {
 	// create new resource store
 	ctlr.resources = NewResourceStore()
 	// reinitialize the informers
-	ctlr.initInformers()
+	ctlr.initController()
 	ctlr.setupInformers()
 	ctlr.startInformers()
 	// process the resources
@@ -188,9 +199,6 @@ func (ctlr *Controller) resetControllerForNamespaceLabel() {
 	if ctlr.CISConfigCRKey != "" {
 		ctlr.processGlobalDeployConfigCR()
 	}
-	// process static routes after DeployConfig CR if present is processed, so as to support external cluster static routes during cis init
-	if ctlr.StaticRoutingMode {
-		clusterNodes := ctlr.getNodesFromAllClusters()
-		ctlr.processStaticRouteUpdate(clusterNodes)
-	}
+	// process static routes after DeployConfig CR if present is processed to support external cluster static routes
+	ctlr.processStaticRouteUpdate()
 }

pkg/controller/informerManager_test.go

@@ -62,7 +62,7 @@ var _ = Describe("Informers Tests", func() {
 		//
 		//})
 		It("Controller infromer setup for all namespaces", func() {
-			mockCtlr.initInformers()
+			mockCtlr.initController()
 			Expect(mockCtlr.resourceSelectorConfig.NodeLabel).To(Equal(""), "Failed to initialize informers")
 			Expect(mockCtlr.resourceSelectorConfig.RouteLabel).To(Equal(""), "Failed to initialize informers")
 			Expect(mockCtlr.resourceSelectorConfig.NamespaceLabel).To(Equal(""), "Failed to initialize informers")
@@ -92,7 +92,7 @@ var _ = Describe("Informers Tests", func() {
 			mockCtlr.stopInformers()
 		})
 		It("Controller reset with nodeLabel", func() {
-			mockCtlr.initInformers()
+			mockCtlr.initController()
 			mockCtlr.setupInformers()
 			newconfigCR := test.NewConfigCR(
 				configCRName,