Add static routing mode to network config (#3274)

config/apis/cis/v1/types.go

@@ -481,10 +481,11 @@ type NetworkConfig struct {
 }
 
 type CNIConfigMeta struct {
-	PoolMemberType string `json:"poolMemberType,omitempty"`
-	TunnelName     string `json:"tunnelName,omitempty"`
-	Shared         bool   `json:"shared,omitempty"`
-	NetworkCIDR    string `json:"networkCIDR,omitempty"`
+	PoolMemberType    string `json:"poolMemberType,omitempty"`
+	TunnelName        string `json:"tunnelName,omitempty"`
+	Shared            bool   `json:"shared,omitempty"`
+	NetworkCIDR       string `json:"networkCIDR,omitempty"`
+	StaticRoutingMode bool   `json:"staticRoutingMode,omitempty"`
 }
 
 type AS3Config struct {

docs/cis-3.x/README.md

@@ -34,7 +34,7 @@ kubectl create -f ./docs/config_examples/customResourceDefinitions/incubator/cus
 Step 4: Install CIS Deploy config CR
 
 ```shell
-kubectl create -f ./docs/cis-3.x/cis-deploy-config-cr.yaml
+kubectl create -f ./docs/cis-3.x/deploy-config/cis-deploy-config-cr.yaml
 ```
 
 Step 5: Create the kubernetes secret for Central Manager credentials

docs/cis-3.x/deploy-config/cis-deploy-config-cr.yaml

@@ -12,11 +12,12 @@ spec:
     nodeLabel: controller=cis
     controllerIdentifier: cluster-1
   networkConfig:
-    orchestrationCNI: ovn-static
+    orchestrationCNI: ovn-k8s
     metaData:
       poolMemberType: cluster
       sharedRouteMode: true
       networkCIDR: "10.1.0.0/16"
+      staticRoutingMode: true
   as3Config:
     debugAS3: true
     postDelayAS3: 10

docs/config_examples/customResourceDefinitions/incubator/customresourcedefinitions.yml

@@ -1118,58 +1118,78 @@ spec:
                   properties:
                     debugAS3:
                       type: boolean
+                      description: "Debug AS3 is used to enable or disable logging AS3 declaration being sent to BIG-IP"
                     postDelayAS3:
                       type: integer
+                      description: "time (in seconds) that CIS waits to post the available AS3 declaration to BIG-IP"
                     documentAPI:
                       type: boolean
+                      description: "documentAPI is used to enable or disable using centralmanager's two step deployment for AS3"
                   type: object
+                  description: AS3 Configuration for CIS
                 baseConfig:
                   properties:
                     controllerIdentifier:
                       type: string
+                      description: "Controller identifier is used to uniquely identify the CIS controller deployed in the cluster"
                       x-kubernetes-validations:
                         - message: "Controller identifier can not be changed. Please delete and recreate the CIS controller and deploy config CR."
                           rule: self == oldSelf
                     namespaceLabel:
                       type: string
                       pattern: '^[a-zA-Z0-9][-A-Za-z0-9_.\/]{0,61}[a-zA-Z0-9]=[a-zA-Z0-9][-A-Za-z0-9_.]{0,61}[a-zA-Z0-9]$'
+                      description: "CIS watches namespaces with this label"
                     nodeLabel:
                       type: string
                       pattern: '^[a-zA-Z0-9][-A-Za-z0-9_.\/]{0,61}[a-zA-Z0-9]=[a-zA-Z0-9][-A-Za-z0-9_.]{0,61}[a-zA-Z0-9]$'
+                      description: "CIS watches nodes only with this label, if not provided, CIS watches all nodes"
                   type: object
                   required:
                     - controllerIdentifier
                 networkConfig:
                   properties:
                     orchestrationCNI:
                       type: string
+                      enum: [ovn-k8s,cilium,flannel,antrea]
+                      description: "Orchestration CNI is used to specify the CNI plugin used in the cluster"
                     metaData:
                       type: object
                       properties:
                         poolMemberType:
                           type: string
                           enum: [cluster, nodeport, nodeportlocal]
+                          description: "Pool member type is used to specify the type of pool member used in the cluster"
+                        staticRoutingMode:
+                          type: boolean
+                          description: "Static routing mode is used to enable or disable configuration of static routes on bigip for pod network subnets"
                         tunnelName:
                           type: string
+                          description: "Tunnel name is used to specify the tunnel name configured on the BIG-IP for cluster mode routing"
                         sharedRouteMode:
                           type: boolean
+                          description: "Shared route mode is used to enable or disable creating static routes on the BIG-IP shared partition Common"
                         networkCIDR:
                           type: string
                           pattern: '(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){2}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\/(3[0-2]|[12]?[0-9])$'
+                          description: "flag to specify node network cidr to be used for static routing when node has multiple interfaces.This is supported only with CNI ovn-k8s"
                   type: object
                 bigIpConfig:
                   items:
                     properties:
                       bigIpAddress:
                         type: string
                         pattern: '^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])|(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$'
+                        description: "IP address of BIG-IP"
                       bigIpLabel:
                         type: string
+                        description: "To uniquely identify bigip ha pair"
                       haBigIpAddress:
                         type: string
                         pattern: '^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])|(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$'
+                        description: "IP address of bigip HA instance"
                       defaultPartition:
                         type: string
+                        description: "partition for the Big-IP kubernetes objects"
                     required:
                       - bigIpAddress
                       - bigIpLabel

pkg/controller/constants.go

@@ -115,10 +115,15 @@ const (
 	OvnK8sNodeIPAnnotation3 = "k8s.ovn.org/host-cidrs"
 
 	//Cilium CNI
-	CILIUM_Static                   = "cilium-static"
+	CILIUM                          = "cilium"
 	CiliumK8sNodeSubnetAnnotation12 = "io.cilium.network.ipv4-pod-cidr"
 	CiliumK8sNodeSubnetAnnotation13 = "network.cilium.io/ipv4-pod-cidr"
 
+	//CNI plugin
+	FLANNEL      = "flannel"
+	ANTREA       = "antrea"
+	OPENSHIFTSDN = "OpenShiftSDN"
+
 	F5VsWAFPolicy                      = "virtual-server.f5.com/waf"
 	F5VsAllowSourceRangeAnnotation     = "virtual-server.f5.com/allow-source-range"
 	MultiClusterServicesAnnotation     = "virtual-server.f5.com/multiClusterServices"

pkg/controller/node_poll_handler.go

@@ -275,7 +275,7 @@ func (ctlr *Controller) processStaticRouteUpdate() {
 						l3Forward.Name = fmt.Sprintf("%v/%v/%v", ctlr.ControllerIdentifier, node.Name, nodeIP)
 					}
 				}
-			} else if ctlr.OrchestrationCNI == CILIUM_Static {
+			} else if ctlr.OrchestrationCNI == CILIUM {
 				nodesubnet := ciliumPodCidr(node.ObjectMeta.Annotations)
 				if nodesubnet == "" {
 					log.Warningf("Cilium node podCIDR annotation not found on node %v, node has spec.podCIDR ?", node.Name)
@@ -301,6 +301,7 @@ func (ctlr *Controller) processStaticRouteUpdate() {
 					for _, addr := range nodeAddrs {
 						if addr.Type == addrType {
 							l3Forward.Config.Gateway = addr.Address
+							l3Forward.Config.L3ForwardType = networkmanager.L3RouteGateway
 							l3Forward.Name = fmt.Sprintf("%v/%v/%v", ctlr.ControllerIdentifier, node.Name, addr.Address)
 						}
 					}

pkg/controller/worker.go

@@ -3845,41 +3845,33 @@ func (ctlr *Controller) processCNIConfig(configCR *cisapiv1.DeployConfig) {
 
 	ctlr.OrchestrationCNI = configCR.Spec.NetworkConfig.OrchestrationCNI
 	ctlr.PoolMemberType = configCR.Spec.NetworkConfig.MetaData.PoolMemberType
+	ctlr.StaticRoutingMode = configCR.Spec.NetworkConfig.MetaData.StaticRoutingMode
 	if ctlr.PoolMemberType == "" {
 		ctlr.PoolMemberType = NodePort
 	}
 
 	if ctlr.PoolMemberType == NodePort || ctlr.PoolMemberType == NodePortLocal {
 		ctlr.shareNodes = true
-		if ctlr.isStaticRouteCNI() {
+		if ctlr.StaticRoutingMode {
 			log.Errorf("static route CNI: %v not supported with nodeport/nodeportlocal mode. Only supported with cluster mode", ctlr.OrchestrationCNI)
 			os.Exit(1)
 		}
 	} else if ctlr.PoolMemberType == Cluster {
-		if ctlr.OrchestrationCNI == "flannel" || ctlr.OrchestrationCNI == "cilium" ||
-			ctlr.OrchestrationCNI == "openshift-sdn" {
+		if ctlr.StaticRoutingMode {
+			ctlr.StaticRouteNodeCIDR = configCR.Spec.NetworkConfig.MetaData.NetworkCIDR
+		} else if ctlr.OrchestrationCNI == FLANNEL || ctlr.OrchestrationCNI == CILIUM ||
+			ctlr.OrchestrationCNI == OPENSHIFTSDN {
 			if configCR.Spec.NetworkConfig.MetaData.TunnelName == "" {
 				log.Errorf("tunnelName is not set in CIS Config CR")
 				os.Exit(1)
 			}
-		} else if ctlr.isStaticRouteCNI() {
-			ctlr.StaticRoutingMode = true
-			ctlr.StaticRouteNodeCIDR = configCR.Spec.NetworkConfig.MetaData.NetworkCIDR
 		} else {
 			log.Errorf("invalid CNI: %v configured in Config CR", ctlr.OrchestrationCNI)
 			os.Exit(1)
 		}
 	}
 }
 
-func (ctlr *Controller) isStaticRouteCNI() bool {
-	if ctlr.OrchestrationCNI == "flannel-static" || ctlr.OrchestrationCNI == CILIUM_Static ||
-		ctlr.OrchestrationCNI == "ovn-static" || ctlr.OrchestrationCNI == "antrea-static" {
-		return true
-	}
-	return false
-}
-
 func (ctlr *Controller) processConfigCR(configCR *cisapiv1.DeployConfig, isDelete bool) (error, bool) {
 	startTime := time.Now()
 	defer func() {