Skip to content

Commit

Permalink
U2o support custom vpc (#2831)
Browse files Browse the repository at this point in the history 
* u2o support custom vpc

* fix golang lint

* merge failed

* support custome vpc u2o from release-1.11
  • Loading branch information
@changluyi
changluyi authored May 24, 2023
1 parent 2068d87 commit a1cf2b3
Show file tree
Hide file tree
Showing 9 changed files with 229 additions and 56 deletions.
2 changes: 2 additions & 0 deletions charts/templates/kube-ovn-crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1580,6 +1580,8 @@ spec:
type: string
u2oInterconnectionIP:
type: string
u2oInterconnectionVPC:
type: string
v4usingIPrange:
type: string
v4availableIPrange:
Expand Down
2 changes: 2 additions & 0 deletions dist/images/install.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2118,6 +2118,8 @@ spec:
type: string
u2oInterconnectionIP:
type: string
u2oInterconnectionVPC:
type: string
v4usingIPrange:
type: string
v4availableIPrange:
Expand Down
25 changes: 13 additions & 12 deletions pkg/apis/kubeovn/v1/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,18 +204,19 @@ type SubnetStatus struct {
// +patchStrategy=merge
Conditions []SubnetCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`

V4AvailableIPs float64 `json:"v4availableIPs"`
V4AvailableIPRange string `json:"v4availableIPrange"`
V4UsingIPs float64 `json:"v4usingIPs"`
V4UsingIPRange string `json:"v4usingIPrange"`
V6AvailableIPs float64 `json:"v6availableIPs"`
V6AvailableIPRange string `json:"v6availableIPrange"`
V6UsingIPs float64 `json:"v6usingIPs"`
V6UsingIPRange string `json:"v6usingIPrange"`
ActivateGateway string `json:"activateGateway"`
DHCPv4OptionsUUID string `json:"dhcpV4OptionsUUID"`
DHCPv6OptionsUUID string `json:"dhcpV6OptionsUUID"`
U2OInterconnectionIP string `json:"u2oInterconnectionIP"`
V4AvailableIPs float64 `json:"v4availableIPs"`
V4AvailableIPRange string `json:"v4availableIPrange"`
V4UsingIPs float64 `json:"v4usingIPs"`
V4UsingIPRange string `json:"v4usingIPrange"`
V6AvailableIPs float64 `json:"v6availableIPs"`
V6AvailableIPRange string `json:"v6availableIPrange"`
V6UsingIPs float64 `json:"v6usingIPs"`
V6UsingIPRange string `json:"v6usingIPrange"`
ActivateGateway string `json:"activateGateway"`
DHCPv4OptionsUUID string `json:"dhcpV4OptionsUUID"`
DHCPv6OptionsUUID string `json:"dhcpV6OptionsUUID"`
U2OInterconnectionIP string `json:"u2oInterconnectionIP"`
U2OInterconnectionVPC string `json:"u2oInterconnectionVPC"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
Expand Down
122 changes: 104 additions & 18 deletions pkg/controller/subnet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -587,9 +587,11 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error {
}
}

if err := c.reconcileU2OInterconnectionIP(subnet); err != nil {
klog.Errorf("failed to reconcile underlay subnet %s to overlay interconnection %v", subnet.Name, err)
return err
if subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway {
if err := c.reconcileU2OInterconnectionIP(subnet); err != nil {
klog.Errorf("failed to reconcile underlay subnet %s to overlay interconnection %v", subnet.Name, err)
return err
}
}

subnetList, err := c.subnetsLister.List(labels.Everything())
Expand Down Expand Up @@ -649,6 +651,11 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error {
gateway = subnet.Status.U2OInterconnectionIP
}

if err := c.clearOldU2OResource(subnet); err != nil {
klog.Errorf("clear subnet %s old u2o resource failed: %v", subnet.Name, err)
return err
}

// create or update logical switch
if err := c.ovnClient.CreateLogicalSwitch(subnet.Name, vpc.Status.Router, subnet.Spec.CIDRBlock, gateway, needRouter, randomAllocateGW); err != nil {
klog.Errorf("create logical switch %s: %v", subnet.Name, err)
Expand Down Expand Up @@ -714,6 +721,11 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error {
return err
}

subnet.Status.U2OInterconnectionVPC = ""
if subnet.Spec.U2OInterconnection {
subnet.Status.U2OInterconnectionVPC = vpc.Status.Router
}

if subnet.Spec.Private {
if err := c.ovnClient.SetLogicalSwitchPrivate(subnet.Name, subnet.Spec.CIDRBlock, subnet.Spec.AllowSubnets); err != nil {
c.patchSubnetStatus(subnet, "SetPrivateLogicalSwitchFailed", err.Error())
Expand Down Expand Up @@ -834,6 +846,13 @@ func (c *Controller) handleDeleteSubnet(subnet *kubeovnv1.Subnet) error {
}
}

if subnet.Spec.Vpc != c.config.ClusterRouter {
if err := c.deleteCustomVPCPolicyRoutesForSubnet(subnet); err != nil {
klog.Errorf("failed to delete custom vpc routes subnet %s, %v", subnet.Name, err)
return err
}
}

klog.Infof("delete policy route for %s subnet %s", subnet.Spec.GatewayType, subnet.Name)
if err := c.deletePolicyRouteByGatewayType(subnet, subnet.Spec.GatewayType, true); err != nil {
klog.Errorf("failed to delete policy route for overlay subnet %s, %v", subnet.Name, err)
Expand Down Expand Up @@ -1667,6 +1686,19 @@ func (c *Controller) reconcileOvnCustomVpcRoute(subnet *kubeovnv1.Subnet) error
return err
}
}

if subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway && subnet.Spec.U2OInterconnection && subnet.Status.U2OInterconnectionIP != "" {
if err := c.addPolicyRouteForU2OInterconn(subnet); err != nil {
klog.Errorf("failed to add policy route for underlay to overlay subnet interconnection %s %v", subnet.Name, err)
return err
}
}

if err := c.addCustomVPCPolicyRoutesForSubnet(subnet); err != nil {
klog.Error(err)
return err
}

return nil
}

Expand Down Expand Up @@ -2074,8 +2106,8 @@ func (c *Controller) addCommonRoutesForSubnet(subnet *kubeovnv1.Subnet) error {
match := fmt.Sprintf("ip%d.dst == %s", af, cidr)
action := ovnnb.LogicalRouterPolicyActionAllow
externalIDs := map[string]string{"vendor": util.CniTypeName, "subnet": subnet.Name}
klog.V(3).Infof("add policy route for router: %s, match %s, action %s, extrenalID %v", c.config.ClusterRouter, match, action, externalIDs)
if err := c.ovnClient.AddLogicalRouterPolicy(c.config.ClusterRouter, util.SubnetRouterPolicyPriority, match, action, nil, externalIDs); err != nil {
klog.V(3).Infof("add policy route for router: %s, match %s, action %s, externalID %v", subnet.Spec.Vpc, match, action, externalIDs)
if err := c.ovnClient.AddLogicalRouterPolicy(subnet.Spec.Vpc, util.SubnetRouterPolicyPriority, match, action, nil, externalIDs); err != nil {
klog.Errorf("failed to add logical router policy for CIDR %s of subnet %s: %v", cidr, subnet.Name, err)
return err
}
Expand Down Expand Up @@ -2364,7 +2396,7 @@ func (c *Controller) addPolicyRouteForU2OInterconn(subnet *kubeovnv1.Subnet) err
prio 31000 match: "ip4.dst == underlay subnet cidr && ip4.dst != node ips" action: allow
policy2:
prio 31000 match: "ip4.dst == node ips && ip4.src == underlay subnet cidr" action: reoute physical gw
prio 31000 match: "ip4.dst == node ips && ip4.src == underlay subnet cidr" action: reroute physical gw
policy3:
prio 29000 match: "ip4.src == underlay subnet cidr" action: reroute physical gw
Expand All @@ -2374,17 +2406,19 @@ func (c *Controller) addPolicyRouteForU2OInterconn(subnet *kubeovnv1.Subnet) err
policy3: underlay pod first access u2o interconnection lrp and then reoute to physical gw
*/
action := ovnnb.LogicalRouterPolicyActionAllow
klog.Infof("add u2o interconnection policy for router: %s, match %s, action %s", subnet.Spec.Vpc, match1, action)
if err := c.ovnClient.AddLogicalRouterPolicy(subnet.Spec.Vpc, util.SubnetRouterPolicyPriority, match1, action, nil, externalIDs); err != nil {
klog.Errorf("failed to add u2o interconnection policy1 for subnet %s %v", subnet.Name, err)
return err
}
if subnet.Spec.Vpc == c.config.ClusterRouter {
klog.Infof("add u2o interconnection policy for router: %s, match %s, action %s", subnet.Spec.Vpc, match1, action)
if err := c.ovnClient.AddLogicalRouterPolicy(subnet.Spec.Vpc, util.SubnetRouterPolicyPriority, match1, action, nil, externalIDs); err != nil {
klog.Errorf("failed to add u2o interconnection policy1 for subnet %s %v", subnet.Name, err)
return err
}

action = ovnnb.LogicalRouterPolicyActionReroute
klog.Infof("add u2o interconnection policy for router: %s, match %s, action %s", subnet.Spec.Vpc, match2, action)
if err := c.ovnClient.AddLogicalRouterPolicy(subnet.Spec.Vpc, util.SubnetRouterPolicyPriority, match2, action, []string{nextHop}, externalIDs); err != nil {
klog.Errorf("failed to add u2o interconnection policy2 for subnet %s %v", subnet.Name, err)
return err
action = ovnnb.LogicalRouterPolicyActionReroute
klog.Infof("add u2o interconnection policy for router: %s, match %s, action %s", subnet.Spec.Vpc, match2, action)
if err := c.ovnClient.AddLogicalRouterPolicy(subnet.Spec.Vpc, util.SubnetRouterPolicyPriority, match2, action, []string{nextHop}, externalIDs); err != nil {
klog.Errorf("failed to add u2o interconnection policy2 for subnet %s %v", subnet.Name, err)
return err
}
}

action = ovnnb.LogicalRouterPolicyActionReroute
Expand All @@ -2411,9 +2445,15 @@ func (c *Controller) deletePolicyRouteForU2OInterconn(subnet *kubeovnv1.Subnet)
return nil
}

lr := subnet.Status.U2OInterconnectionVPC
if lr == "" {
// old version field U2OInterconnectionVPC may be "" and then use subnet.Spec.Vpc
lr = subnet.Spec.Vpc
}

for _, policy := range policies {
klog.Infof("delete u2o interconnection policy for router %s with match %s priority %d", subnet.Spec.Vpc, policy.Match, policy.Priority)
if err = c.ovnClient.DeleteLogicalRouterPolicyByUUID(subnet.Spec.Vpc, policy.UUID); err != nil {
klog.Infof("delete u2o interconnection policy for router %s with match %s priority %d", lr, policy.Match, policy.Priority)
if err = c.ovnClient.DeleteLogicalRouterPolicyByUUID(lr, policy.UUID); err != nil {
klog.Errorf("failed to delete u2o interconnection policy for subnet %s: %v", subnet.Name, err)
return err
}
Expand Down Expand Up @@ -2464,3 +2504,49 @@ func (c *Controller) reconcileRouteTableForSubnet(subnet *kubeovnv1.Subnet) erro

return nil
}

func (c *Controller) addCustomVPCPolicyRoutesForSubnet(subnet *kubeovnv1.Subnet) error {
return c.addCommonRoutesForSubnet(subnet)
}

func (c *Controller) deleteCustomVPCPolicyRoutesForSubnet(subnet *kubeovnv1.Subnet) error {

for _, cidr := range strings.Split(subnet.Spec.CIDRBlock, ",") {
af := 4
if util.CheckProtocol(cidr) == kubeovnv1.ProtocolIPv6 {
af = 6
}
match := fmt.Sprintf("ip%d.dst == %s", af, cidr)
klog.Infof("delete policy route for router: %s, priority: %d, match %s", subnet.Spec.Vpc, util.SubnetRouterPolicyPriority, match)
if err := c.ovnClient.DeleteLogicalRouterPolicy(subnet.Spec.Vpc, util.SubnetRouterPolicyPriority, match); err != nil {
klog.Errorf("failed to delete logical router policy for CIDR %s of subnet %s: %v", cidr, subnet.Name, err)
return err
}
}
return nil
}

func (c *Controller) clearOldU2OResource(subnet *kubeovnv1.Subnet) error {
if subnet.Status.U2OInterconnectionVPC != "" &&
(!subnet.Spec.U2OInterconnection || (subnet.Spec.U2OInterconnection && subnet.Status.U2OInterconnectionVPC != subnet.Spec.Vpc)) {
// remove old u2o lsp and lrp first
lspName := fmt.Sprintf("%s-%s", subnet.Name, subnet.Status.U2OInterconnectionVPC)
lrpName := fmt.Sprintf("%s-%s", subnet.Status.U2OInterconnectionVPC, subnet.Name)
klog.Infof("clean subnet %s old u2o resource with lsp %s lrp %s ", subnet.Name, lspName, lrpName)
if err := c.ovnClient.DeleteLogicalSwitchPort(lspName); err != nil {
klog.Errorf("failed to delete u2o logical switch port %s: %v", lspName, err)
return err
}

if err := c.ovnClient.DeleteLogicalRouterPort(lrpName); err != nil {
klog.Errorf("failed to delete u2o logical router port %s: %v", lrpName, err)
return err
}

if err := c.deletePolicyRouteForU2OInterconn(subnet); err != nil {
klog.Errorf("failed to delete u2o policy route for u2o connection %s: %v", subnet.Name, err)
return err
}
}
return nil
}
3 changes: 2 additions & 1 deletion test/e2e/framework/vpc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ func (c *VpcClient) WaitToDisappear(name string, interval, timeout time.Duration
return nil
}

func MakeVpc(name, gatewayV4 string, enableExternal, enableBfd bool) *kubeovnv1.Vpc {
func MakeVpc(name, gatewayV4 string, enableExternal, enableBfd bool, namespaces []string) *kubeovnv1.Vpc {
routes := make([]*kubeovnv1.StaticRoute, 0, 1)
if gatewayV4 != "" {
routes = append(routes, &kubeovnv1.StaticRoute{
Expand All @@ -162,6 +162,7 @@ func MakeVpc(name, gatewayV4 string, enableExternal, enableBfd bool) *kubeovnv1.
StaticRoutes: routes,
EnableExternal: enableExternal,
EnableBfd: enableBfd,
Namespaces: namespaces,
},
}
return vpc
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/iptables-vpc-nat-gw/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ var _ = framework.Describe("[group:iptables-vpc-nat-gw]", func() {
overlaySubnetV4Cidr := "192.168.0.0/24"
overlaySubnetV4Gw := "192.168.0.1"
lanIp := "192.168.0.254"
vpc := framework.MakeVpc(vpcName, lanIp, false, false)
vpc := framework.MakeVpc(vpcName, lanIp, false, false, nil)
_ = vpcClient.CreateSync(vpc)

ginkgo.By("Creating custom overlay subnet")
Expand Down
Loading

0 comments on commit a1cf2b3

Please sign in to comment.