From e84ab121e5d6b0fb3797a8107a732b252fe0c0ff Mon Sep 17 00:00:00 2001 From: hzma Date: Wed, 12 Jun 2024 09:44:35 +0800 Subject: [PATCH] add ovn0 default route (#4127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 马洪贞 --- pkg/daemon/controller_linux.go | 3 +- pkg/daemon/controller_windows.go | 4 +-- pkg/daemon/init.go | 2 +- pkg/daemon/ovs_linux.go | 39 +++++++++++++++++++++++++- pkg/daemon/ovs_windows.go | 48 +++++++++++++++++++++++++++++++- 5 files changed, 90 insertions(+), 6 deletions(-) diff --git a/pkg/daemon/controller_linux.go b/pkg/daemon/controller_linux.go index b80a132adac..705ad59ec83 100644 --- a/pkg/daemon/controller_linux.go +++ b/pkg/daemon/controller_linux.go @@ -196,7 +196,8 @@ func (c *Controller) reconcileRouters(event *subnetEvent) error { joinCIDR := make([]string, 0, 2) cidrs := make([]string, 0, len(subnets)*2) for _, subnet := range subnets { - if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) || subnet.Spec.Vpc != c.config.ClusterRouter || !subnet.Status.IsReady() { + // The route for overlay subnet cidr via ovn0 should not be deleted even though subnet.Status has changed to not ready + if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) || subnet.Spec.Vpc != c.config.ClusterRouter { continue } diff --git a/pkg/daemon/controller_windows.go b/pkg/daemon/controller_windows.go index d348c5c5c95..f4c72fe1e73 100644 --- a/pkg/daemon/controller_windows.go +++ b/pkg/daemon/controller_windows.go @@ -53,9 +53,9 @@ func (c *Controller) reconcileRouters(_ *subnetEvent) error { gwIPv4, gwIPv6 := util.SplitStringIP(gateway) v4Cidrs, v6Cidrs := make([]string, 0, len(subnets)), make([]string, 0, len(subnets)) for _, subnet := range subnets { + // The route for overlay subnet cidr via ovn0 should not be deleted even though subnet.Status has changed to not ready if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) || - subnet.Spec.Vpc != c.config.ClusterRouter || - !subnet.Status.IsReady() { + subnet.Spec.Vpc != c.config.ClusterRouter { continue } diff --git a/pkg/daemon/init.go b/pkg/daemon/init.go index 86fe5aaad71..3c9e5520131 100644 --- a/pkg/daemon/init.go +++ b/pkg/daemon/init.go @@ -82,7 +82,7 @@ func InitNodeGateway(config *Configuration) error { } ipAddr = util.GetIPAddrWithMask(ip, cidr) - return configureNodeNic(portName, ipAddr, gw, mac, config.MTU) + return configureNodeNic(portName, ipAddr, gw, cidr, mac, config.MTU) } func InitMirror(config *Configuration) error { diff --git a/pkg/daemon/ovs_linux.go b/pkg/daemon/ovs_linux.go index 0a7f7eaaf37..0086855ed29 100644 --- a/pkg/daemon/ovs_linux.go +++ b/pkg/daemon/ovs_linux.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "net" "os" @@ -12,6 +13,7 @@ import ( "path/filepath" "regexp" "strings" + "syscall" "time" "github.com/containernetworking/plugins/pkg/ns" @@ -404,7 +406,7 @@ func waitNetworkReady(nic, ipAddr, gateway string, underlayGateway, verbose bool return nil } -func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int) error { +func configureNodeNic(portName, ip, gw, joinCIDR string, macAddr net.HardwareAddr, mtu int) error { ipStr := util.GetIPWithoutMask(ip) raw, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", util.NodeNic, "--", "set", "interface", util.NodeNic, "type=internal", "--", @@ -437,6 +439,41 @@ func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int return fmt.Errorf("can not set host nic %s qlen: %v", util.NodeNic, err) } + // check and add default route for ovn0 in case of can not add automatically + nodeNicRoutes, err := getNicExistRoutes(hostLink, gw) + if err != nil { + klog.Error(err) + return err + } + + var toAdd []netlink.Route + for _, c := range strings.Split(joinCIDR, ",") { + found := false + for _, r := range nodeNicRoutes { + if r.Dst.String() == c { + found = true + break + } + } + if !found { + _, cidr, _ := net.ParseCIDR(c) + toAdd = append(toAdd, netlink.Route{ + Dst: cidr, + Scope: netlink.SCOPE_UNIVERSE, + }) + } + } + if len(toAdd) > 0 { + klog.Infof("route to add for nic %s, %v", util.NodeNic, toAdd) + } + + for _, r := range toAdd { + r.LinkIndex = hostLink.Attrs().Index + if err = netlink.RouteReplace(&r); err != nil && !errors.Is(err, syscall.EEXIST) { + klog.Errorf("failed to replace route %v: %v", r, err) + } + } + // ping ovn0 gw to activate the flow klog.Infof("wait ovn0 gw ready") if err := waitNetworkReady(util.NodeNic, ip, gw, false, true, gatewayCheckMaxRetry); err != nil { diff --git a/pkg/daemon/ovs_windows.go b/pkg/daemon/ovs_windows.go index d27db196769..3b78eafc289 100644 --- a/pkg/daemon/ovs_windows.go +++ b/pkg/daemon/ovs_windows.go @@ -256,7 +256,7 @@ func waitNetworkReady(nic, ipAddr, gateway string, underlayGateway, verbose bool return nil } -func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int) error { +func configureNodeNic(portName, ip, gw, joinCIDR string, macAddr net.HardwareAddr, mtu int) error { ipStr := util.GetIPWithoutMask(ip) raw, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", util.NodeNic, "--", "set", "interface", util.NodeNic, "type=internal", "--", @@ -271,6 +271,52 @@ func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int return err } + // check and add default route for ovn0 in case of can not add automatically + // IPv4: 100.64.0.0/16 dev ovn0 proto kernel scope link src 100.64.0.2 + // IPv6: fd00:100:64::/112 dev ovn0 proto kernel metric 256 pref medium + adapter, err := util.GetNetAdapter(util.NodeNic, false) + if err != nil { + klog.Errorf("failed to get network adapter %s: %v", util.NodeNic, err) + return err + } + routes, err := util.GetNetRoute(adapter.InterfaceIndex) + if err != nil { + klog.Errorf("failed to get NetIPRoute with index %d: %v", adapter.InterfaceIndex, err) + return err + } + + var toAddV4, toAddV6 []string + for _, cidr := range strings.Split(joinCIDR, ",") { + found := false + for _, route := range routes { + if route.DestinationPrefix == cidr { + found = true + break + } + if !found { + if util.CheckProtocol(cidr) == kubeovnv1.ProtocolIPv4 { + toAddV4 = append(toAddV4, cidr) + } else { + toAddV6 = append(toAddV6, cidr) + } + } + } + } + if len(toAddV4) > 0 || len(toAddV6) > 0 { + klog.Infof("route to add for nic %s, ipv4 %v, ipv6 %v", util.NodeNic, toAddV4, toAddV6) + } + + for _, r := range toAddV4 { + if err = util.NewNetRoute(adapter.InterfaceIndex, r, "0.0.0.0"); err != nil { + klog.Errorf("failed to add ipv4 route %s: %v", r, err) + } + } + for _, r := range toAddV6 { + if err = util.NewNetRoute(adapter.InterfaceIndex, r, "::"); err != nil { + klog.Errorf("failed to add ipv6 route %s: %v", r, err) + } + } + // ping ovn0 gw to activate the flow klog.Infof("wait ovn0 gw ready") if err := waitNetworkReady(util.NodeNic, ip, gw, false, true, gatewayCheckMaxRetry); err != nil {