feat: support static routes

Support optional static routes in the CNI CmdAdd result.

Signed-off-by: Fred Rolland <frolland@nvidia.com>

README.md

@@ -350,6 +350,9 @@ spec:
     - matchExpressions:
         - key: node-role.kubernetes.io/worker
           operator: Exists
+  defautGateway: true # optional
+  routes: # optional
+  - dst: 5.5.0.0/24
 ```
 
 ##### IPv6 example
@@ -383,6 +386,10 @@ spec:
     * `endIP`: end IP of the exclude range (inclusive).
 
   * `nodeSelector` (optional): A list of node selector terms. The terms are ORed. Each term can have a list of matchExpressions that are ANDed. Only the nodes that match the provided labels will get assigned IP Blocks for the defined pool.
+  * `defautGateway` (optional): Add the pool gateway as default gateway in the pod static routes.
+  * `routes` (optional, list): contains CIDR to be added in the pod static routes via the pool gateway.
+
+    * `dst`: The destination of the static route, in CIDR notation.
 
 > __Notes:__
 >
@@ -422,6 +429,9 @@ spec:
       - matchExpressions:
           - key: node-role.kubernetes.io/worker
             operator: Exists
+  defautGateway: true # optional
+  routes: # optional
+  - dst: 5.5.0.0/24
 ```
 
 ##### IPv6 example
@@ -475,6 +485,10 @@ spec:
       * `prefix`: statically allocated prefix.
 
   * `nodeSelector`(optional): A list of node selector terms. The terms are ORed. Each term can have a list of matchExpressions that are ANDed. Only the nodes that match the provided labels will get assigned IP Blocks for the defined pool.
+  * `defautGateway` (optional): Add the pool gateway as default gateway in the pod static routes.
+  * `routes` (optional, list): contains CIDR to be added in the pod static routes via the pool gateway.
+
+    * `dst`: The destination of the static route, in CIDR notation.
 
 > __Notes:__
 >

api/grpc/proto/nvidia/ipam/node/v1/node.proto

@@ -125,6 +125,13 @@ message AllocationInfo {
   string gateway = 3;
   // type of the pool which is refered by the name in the pools field
   PoolType pool_type = 4;
+  // list of static routes
+  repeated Route routes = 5;
+}
+
+message Route {
+  // Static route destination in CIDR format
+  string dest = 1;
 }
 
 // IsAllocatedReply contains reply for IsAllocated rpc call

api/v1alpha1/cidrpool_test.go

@@ -94,6 +94,85 @@ var _ = Describe("CIDRPool", func() {
 		}
 		validatePoolAndCheckErr(&cidrPool, true)
 	})
+	It("Invalid - no gatewayIndex, defaultGateway true", func() {
+		cidrPool := v1alpha1.CIDRPool{
+			ObjectMeta: metav1.ObjectMeta{Name: "test"},
+			Spec: v1alpha1.CIDRPoolSpec{
+				CIDR:                 "fdf8:6aef:d1fe::/48",
+				PerNodeNetworkPrefix: 120,
+				DefaultGateway:       true,
+			},
+		}
+		validatePoolAndCheckErr(&cidrPool, false, ContainSubstring("spec.defaultGateway"))
+	})
+	It("Valid - routes", func() {
+		cidrPool := v1alpha1.CIDRPool{
+			ObjectMeta: metav1.ObjectMeta{Name: "test"},
+			Spec: v1alpha1.CIDRPoolSpec{
+				CIDR:                 "192.168.0.0/16",
+				PerNodeNetworkPrefix: 24,
+				GatewayIndex:         ptr.To[int32](100),
+				Routes: []v1alpha1.Route{
+					{
+						Dst: "5.5.0.0/16",
+					},
+					{
+						Dst: "10.7.1.0/24",
+					},
+				},
+			},
+		}
+		validatePoolAndCheckErr(&cidrPool, true)
+	})
+	It("Invalid - routes not CIDR", func() {
+		cidrPool := v1alpha1.CIDRPool{
+			ObjectMeta: metav1.ObjectMeta{Name: "test"},
+			Spec: v1alpha1.CIDRPoolSpec{
+				CIDR:                 "192.168.0.0/16",
+				PerNodeNetworkPrefix: 24,
+				GatewayIndex:         ptr.To[int32](100),
+				Routes: []v1alpha1.Route{
+					{
+						Dst: "5.5.0.0",
+					},
+				},
+			},
+		}
+		validatePoolAndCheckErr(&cidrPool, false, ContainSubstring("spec.routes"))
+	})
+	It("Invalid - routes without GatewayIndex", func() {
+		cidrPool := v1alpha1.CIDRPool{
+			ObjectMeta: metav1.ObjectMeta{Name: "test"},
+			Spec: v1alpha1.CIDRPoolSpec{
+				CIDR:                 "192.168.0.0/16",
+				PerNodeNetworkPrefix: 24,
+				Routes: []v1alpha1.Route{
+					{
+						Dst: "5.5.0.0",
+					},
+				},
+			},
+		}
+		validatePoolAndCheckErr(&cidrPool, false, ContainSubstring("spec.routes"))
+	})
+	It("Invalid - routes not same address family", func() {
+		ipPool := v1alpha1.CIDRPool{
+			ObjectMeta: metav1.ObjectMeta{Name: "test"},
+			Spec: v1alpha1.CIDRPoolSpec{
+				CIDR:                 "192.168.0.0/16",
+				PerNodeNetworkPrefix: 24,
+				Routes: []v1alpha1.Route{
+					{
+						Dst: "2001:db8:3333:4444::0/64",
+					},
+				},
+			},
+		}
+		Expect(ipPool.Validate().ToAggregate().Error()).
+			To(
+				ContainSubstring("spec.routes"),
+			)
+	})
 	DescribeTable("CIDR",
 		func(cidr string, prefix int32, isValid bool) {
 			cidrPool := v1alpha1.CIDRPool{

api/v1alpha1/cidrpool_type.go

@@ -48,6 +48,10 @@ type CIDRPoolSpec struct {
 	StaticAllocations []CIDRPoolStaticAllocation `json:"staticAllocations,omitempty"`
 	// selector for nodes, if empty match all nodes
 	NodeSelector *corev1.NodeSelector `json:"nodeSelector,omitempty"`
+	// if true, add gateway as default gateway in the routes list
+	DefaultGateway bool `json:"defautGateway,omitempty"`
+	// static routes list. The gateway used will according to the node allocation.
+	Routes []Route `json:"routes,omitempty"`
 }
 
 // CIDRPoolStatus contains the IP prefixes allocated to nodes

api/v1alpha1/cidrpool_validate.go

@@ -33,6 +33,20 @@ func (r *CIDRPool) Validate() field.ErrorList {
 	if r.Spec.NodeSelector != nil {
 		errList = append(errList, validateNodeSelector(r.Spec.NodeSelector, field.NewPath("spec"))...)
 	}
+	if r.Spec.GatewayIndex == nil {
+		if r.Spec.DefaultGateway {
+			errList = append(errList, field.Invalid(
+				field.NewPath("spec", "defaultGateway"), r.Spec.DefaultGateway,
+				"cannot be true if spec.gatewayIndex is not set"))
+		}
+		if len(r.Spec.Routes) > 0 {
+			errList = append(errList, field.Invalid(
+				field.NewPath("spec", "routes"), r.Spec.Routes,
+				"cannot be set if spec.gatewayIndex is not set"))
+		}
+	}
+	_, network, _ := net.ParseCIDR(r.Spec.CIDR)
+	errList = append(errList, validateRoutes(r.Spec.Routes, network, r.Spec.DefaultGateway, field.NewPath("spec"))...)
 	return errList
 }
 

api/v1alpha1/common_type.go

@@ -19,3 +19,9 @@ type ExcludeRange struct {
 	StartIP string `json:"startIP"`
 	EndIP   string `json:"endIP"`
 }
+
+// Route contains static route parameters
+type Route struct {
+	// The destination of the route, in CIDR notation
+	Dst string `json:"dst"`
+}

api/v1alpha1/ippool_test.go

@@ -156,4 +156,129 @@ var _ = Describe("Validate", func() {
 				ContainSubstring("spec.nodeSelector"),
 			)
 	})
+	It("Invalid - no Gateway, defaultGateway true", func() {
+		ipPool := v1alpha1.IPPool{
+			ObjectMeta: metav1.ObjectMeta{Name: "test"},
+			Spec: v1alpha1.IPPoolSpec{
+				Subnet:           "192.168.0.0/16",
+				PerNodeBlockSize: 128,
+				DefaultGateway:   true,
+			},
+		}
+		Expect(ipPool.Validate().ToAggregate().Error()).
+			To(
+				ContainSubstring("spec.defaultGateway"),
+			)
+	})
+	It("Valid - routes", func() {
+		ipPool := v1alpha1.IPPool{
+			ObjectMeta: metav1.ObjectMeta{Name: "test"},
+			Spec: v1alpha1.IPPoolSpec{
+				Subnet:           "192.168.0.0/16",
+				PerNodeBlockSize: 128,
+				Gateway:          "192.168.0.1",
+				Routes: []v1alpha1.Route{
+					{
+						Dst: "5.5.0.0/16",
+					},
+					{
+						Dst: "10.7.1.0/24",
+					},
+				},
+			},
+		}
+		Expect(ipPool.Validate()).To(BeEmpty())
+	})
+	It("Invalid - routes without Gateway", func() {
+		ipPool := v1alpha1.IPPool{
+			ObjectMeta: metav1.ObjectMeta{Name: "test"},
+			Spec: v1alpha1.IPPoolSpec{
+				Subnet:           "192.168.0.0/16",
+				PerNodeBlockSize: 128,
+				Routes: []v1alpha1.Route{
+					{
+						Dst: "5.5.0.0/16",
+					},
+				},
+			},
+		}
+		Expect(ipPool.Validate().ToAggregate().Error()).
+			To(
+				ContainSubstring("spec.routes"),
+			)
+	})
+	It("Invalid - routes not CIDR", func() {
+		ipPool := v1alpha1.IPPool{
+			ObjectMeta: metav1.ObjectMeta{Name: "test"},
+			Spec: v1alpha1.IPPoolSpec{
+				Subnet:           "192.168.0.0/16",
+				PerNodeBlockSize: 128,
+				Routes: []v1alpha1.Route{
+					{
+						Dst: "5.5.0.0",
+					},
+				},
+			},
+		}
+		Expect(ipPool.Validate().ToAggregate().Error()).
+			To(
+				ContainSubstring("spec.routes"),
+			)
+	})
+	It("Invalid - routes not same address family", func() {
+		ipPool := v1alpha1.IPPool{
+			ObjectMeta: metav1.ObjectMeta{Name: "test"},
+			Spec: v1alpha1.IPPoolSpec{
+				Subnet:           "192.168.0.0/16",
+				PerNodeBlockSize: 128,
+				Routes: []v1alpha1.Route{
+					{
+						Dst: "2001:db8:3333:4444::0/64",
+					},
+				},
+			},
+		}
+		Expect(ipPool.Validate().ToAggregate().Error()).
+			To(
+				ContainSubstring("spec.routes"),
+			)
+	})
+	It("Invalid - default routes with defaultGateway true - IPv6", func() {
+		ipPool := v1alpha1.IPPool{
+			ObjectMeta: metav1.ObjectMeta{Name: "test"},
+			Spec: v1alpha1.IPPoolSpec{
+				Subnet:           "2001:db8:3333:4444::0/64",
+				PerNodeBlockSize: 128,
+				DefaultGateway: true,
+				Routes: []v1alpha1.Route{
+					{
+						Dst: "::/0",
+					},
+				},
+			},
+		}
+		Expect(ipPool.Validate().ToAggregate().Error()).
+			To(
+				ContainSubstring("spec.routes"),
+			)
+	})
+	It("Invalid - default routes with defaultGateway true - IPv4", func() {
+		ipPool := v1alpha1.IPPool{
+			ObjectMeta: metav1.ObjectMeta{Name: "test"},
+			Spec: v1alpha1.IPPoolSpec{
+				Subnet:           "192.168.0.0/16",
+				PerNodeBlockSize: 128,
+				DefaultGateway: true,
+				Routes: []v1alpha1.Route{
+					{
+						Dst: "0.0.0.0/0",
+					},
+				},
+			},
+		}
+		Expect(ipPool.Validate().ToAggregate().Error()).
+			To(
+				ContainSubstring("spec.routes"),
+			)
+	})
 })

api/v1alpha1/ippool_type.go

@@ -45,6 +45,10 @@ type IPPoolSpec struct {
 	Gateway string `json:"gateway,omitempty"`
 	// selector for nodes, if empty match all nodes
 	NodeSelector *corev1.NodeSelector `json:"nodeSelector,omitempty"`
+	// if true, add gateway as default gateway in the routes list
+	DefaultGateway bool `json:"defautGateway,omitempty"`
+	// static routes list using the gateway specified in the spec.
+	Routes []Route `json:"routes,omitempty"`
 }
 
 // IPPoolStatus contains the IP ranges allocated to nodes

api/v1alpha1/ippool_validate.go

@@ -71,5 +71,21 @@ func (r *IPPool) Validate() field.ErrorList {
 	if r.Spec.NodeSelector != nil {
 		errList = append(errList, validateNodeSelector(r.Spec.NodeSelector, field.NewPath("spec"))...)
 	}
+
+	if r.Spec.Gateway == "" {
+		if r.Spec.DefaultGateway {
+			errList = append(errList, field.Invalid(
+				field.NewPath("spec", "defaultGateway"), r.Spec.DefaultGateway,
+				"cannot be true if spec.gateway is not set"))
+		}
+		if len(r.Spec.Routes) > 0 {
+			errList = append(errList, field.Invalid(
+				field.NewPath("spec", "routes"), r.Spec.Routes,
+				"cannot be set if spec.gateway is not set"))
+		}
+	}
+
+	errList = append(errList, validateRoutes(r.Spec.Routes, network, r.Spec.DefaultGateway, field.NewPath("spec"))...)
+
 	return errList
 }