diff --git a/pkg/controller/external-gw.go b/pkg/controller/external-gw.go index e58b612ee0d..b3bcd8d0a7e 100644 --- a/pkg/controller/external-gw.go +++ b/pkg/controller/external-gw.go @@ -45,8 +45,9 @@ func (c *Controller) resyncExternalGateway() { if exGwEnabled == "true" && lastExGwCM != nil && reflect.DeepEqual(cm.Data, lastExGwCM) { return } + klog.Info("last external gw configmap: %v", lastExGwCM) if (lastExGwCM["type"] == "distributed" && cm.Data["type"] == "centralized") || - !reflect.DeepEqual(lastExGwCM["external-gw-nodes"], cm.Data["external-gw-nodes"]) { + lastExGwCM != nil && !reflect.DeepEqual(lastExGwCM["external-gw-nodes"], cm.Data["external-gw-nodes"]) { klog.Info("external gw nodes list changed, start to remove ovn external gw") if err := c.removeExternalGateway(); err != nil { klog.Errorf("failed to remove old ovn external gw, %v", err) @@ -94,6 +95,7 @@ func (c *Controller) removeExternalGateway() error { } } + klog.Infof("delete external gateway switch %s", c.config.ExternalGatewaySwitch) if err := c.ovnLegacyClient.DeleteGatewaySwitch(c.config.ExternalGatewaySwitch); err != nil { klog.Errorf("failed to delete external gateway switch, %v", err) return err @@ -117,6 +119,15 @@ func (c *Controller) establishExternalGateway(config map[string]string) error { lrpIp = config["nic-ip"] lrpMac = config["nic-mac"] } + lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch) + exist, err := c.ovnClient.LogicalRouterPortExists(lrpName) + if err != nil { + return err + } + if exist { + klog.Infof("lrp %s exist", lrpName) + return nil + } if err := c.ovnLegacyClient.CreateGatewaySwitch(c.config.ExternalGatewaySwitch, c.config.ExternalGatewayNet, c.config.ExternalGatewayVlanID, lrpIp, lrpMac, chassises); err != nil { klog.Errorf("failed to create external gateway switch, %v", err) return err diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go index d85ff561aed..22f54a0559f 100644 --- a/pkg/controller/gc.go +++ b/pkg/controller/gc.go @@ -65,6 +65,7 @@ func (c *Controller) gcLogicalRouterPort() error { } for _, lrp := range lrps { if !util.ContainsString(exceptPeerPorts, lrp) { + klog.Infof("gc logical router port %s", lrp) if err = c.ovnLegacyClient.DeleteLogicalRouterPort(lrp); err != nil { klog.Errorf("failed to delete logical router port %s, %v", lrp, err) return err diff --git a/pkg/controller/node.go b/pkg/controller/node.go index 55dd0dc8802..036a86d31f9 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -447,6 +447,7 @@ func (c *Controller) handleNodeAnnotationsForProviderNetworks(node *v1.Node) err func (c *Controller) handleDeleteNode(key string) error { portName := fmt.Sprintf("node-%s", key) + klog.Infof("delete logical switch port %s", portName) if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(portName); err != nil { klog.Errorf("failed to delete node switch port node-%s: %v", key, err) return err diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 94d1a43c939..69e946e3ff9 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -674,6 +674,7 @@ func (c *Controller) handleDeletePod(pod *v1.Pod) error { klog.Warningf("failed to get port '%s' sg, %v", port.Name, err) } // when lsp is deleted, the port of pod is deleted from any port-group automatically. + klog.Infof("gc logical switch port %s", port.Name) if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(port.Name); err != nil { klog.Errorf("failed to delete lsp %s, %v", port.Name, err) return err diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index c71a554bc3e..1774a1d7238 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "reflect" + "sort" "strings" "time" @@ -398,6 +399,7 @@ func checkAndUpdateExcludeIps(subnet *kubeovnv1.Subnet) bool { changed := false var excludeIps []string excludeIps = append(excludeIps, strings.Split(subnet.Spec.Gateway, ",")...) + sort.Strings(excludeIps) if len(subnet.Spec.ExcludeIps) == 0 { subnet.Spec.ExcludeIps = excludeIps changed = true @@ -413,6 +415,7 @@ func checkAndUpdateExcludeIps(subnet *kubeovnv1.Subnet) bool { } if !gwExists { subnet.Spec.ExcludeIps = append(subnet.Spec.ExcludeIps, gw) + sort.Strings(subnet.Spec.ExcludeIps) changed = true } } @@ -616,8 +619,15 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error { c.patchSubnetStatus(subnet, "ListLogicalSwitchFailed", err.Error()) return err } - - needRouter := subnet.Spec.Vlan == "" || subnet.Spec.LogicalGateway + var isUnderlayGateway, needRouter bool + if subnet.Spec.Vlan == "" { + // subnet is overlay, should add lrp between vpc and subnet, lrp ip is subnet gw + needRouter = true + } else { + // subnet is underlay, lrp is controlled by vpc spec enableExternal + needRouter = subnet.Spec.LogicalGateway + isUnderlayGateway = !subnet.Spec.LogicalGateway + } if !exist { subnet.Status.EnsureStandardConditions() // If multiple namespace use same ls name, only first one will success @@ -625,7 +635,6 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error { c.patchSubnetStatus(subnet, "CreateLogicalSwitchFailed", err.Error()) return err } - if needRouter { if err := c.reconcileRouterPortBySubnet(vpc, subnet); err != nil { klog.Errorf("failed to connect switch %s to router %s, %v", subnet.Name, vpc.Name, err) @@ -633,12 +642,19 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error { } } } else { + if !needRouter && isUnderlayGateway { + // do nothing if subnet is underlay vlan and use underlay gw + // TODO:// support update if spec changed + klog.Infof("skip reset external connection from vpc %s to switch %s", vpc.Status.Router, subnet.Name) + return nil + } // logical switch exists, only update other_config if err := c.ovnLegacyClient.SetLogicalSwitchConfig(subnet.Name, vpc.Status.Router, subnet.Spec.Protocol, subnet.Spec.CIDRBlock, subnet.Spec.Gateway, subnet.Spec.ExcludeIps, needRouter); err != nil { c.patchSubnetStatus(subnet, "SetLogicalSwitchConfigFailed", err.Error()) return err } if !needRouter { + klog.Infof("remove connection from router %s to switch %s", vpc.Status.Router, subnet.Name) if err := c.ovnLegacyClient.RemoveRouterPort(subnet.Name, vpc.Status.Router); err != nil { klog.Errorf("failed to remove router port from %s, %v", subnet.Name, err) return err @@ -803,12 +819,14 @@ func (c *Controller) handleDeleteSubnet(subnet *kubeovnv1.Subnet) error { } vpc, err := c.vpcsLister.Get(subnet.Spec.Vpc) if err == nil && vpc.Status.Router != "" { + klog.Infof("remove connection from router %s to switch %s", vpc.Status.Router, subnet.Name) if err = c.ovnLegacyClient.RemoveRouterPort(subnet.Name, vpc.Status.Router); err != nil { klog.Errorf("failed to delete router port %s %v", subnet.Name, err) return err } } else { if k8serrors.IsNotFound(err) { + klog.Infof("remove connection from router %s to switch %s", vpc.Status.Router, subnet.Name) if err = c.ovnLegacyClient.RemoveRouterPort(subnet.Name, util.DefaultVpc); err != nil { klog.Errorf("failed to delete router port %s %v", subnet.Name, err) return err @@ -894,6 +912,7 @@ func (c *Controller) reconcileVips(subnet *kubeovnv1.Subnet) error { continue } if !util.ContainsString(subnet.Spec.Vips, vip) { + klog.Infof("delete logical switch port %s", ret["name"][0]) if err = c.ovnLegacyClient.DeleteLogicalSwitchPort(ret["name"][0]); err != nil { klog.Errorf("failed to delete virtual port, %v", err) return err @@ -1025,12 +1044,16 @@ func (c *Controller) reconcileOvnRoute(subnet *kubeovnv1.Subnet) error { return err } - if !subnet.Spec.LogicalGateway { - if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(fmt.Sprintf("%s-%s", subnet.Name, c.config.ClusterRouter)); err != nil { + if subnet.Spec.LogicalGateway { + lspName := fmt.Sprintf("%s-%s", subnet.Name, c.config.ClusterRouter) + klog.Infof("delete logical switch port %s", lspName) + if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(lspName); err != nil { klog.Errorf("failed to delete lsp %s-%s, %v", subnet.Name, c.config.ClusterRouter, err) return err } - if err := c.ovnLegacyClient.DeleteLogicalRouterPort(fmt.Sprintf("%s-%s", c.config.ClusterRouter, subnet.Name)); err != nil { + lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, subnet.Name) + klog.Infof("delete logical router port %s", lrpName) + if err := c.ovnLegacyClient.DeleteLogicalRouterPort(lrpName); err != nil { klog.Errorf("failed to delete lrp %s-%s, %v", c.config.ClusterRouter, subnet.Name, err) return err } @@ -1514,6 +1537,7 @@ func checkAndFormatsExcludeIps(subnet *kubeovnv1.Subnet) bool { excludeIps = append(excludeIps, string(v.Start)+".."+string(v.End)) } } + sort.Strings(excludeIps) klog.V(3).Infof("excludeips before format is %v, after format is %v", subnet.Spec.ExcludeIps, excludeIps) if !reflect.DeepEqual(subnet.Spec.ExcludeIps, excludeIps) { subnet.Spec.ExcludeIps = excludeIps diff --git a/pkg/controller/vip.go b/pkg/controller/vip.go index 8802145456f..0473ec1b7df 100644 --- a/pkg/controller/vip.go +++ b/pkg/controller/vip.go @@ -300,7 +300,7 @@ func (c *Controller) acquireStaticIpAddress(subnetName, name, nicName, ip string func (c *Controller) acquireIpAddress(subnetName, name, nicName string) (string, string, string, error) { var skippedAddrs []string var v4ip, v6ip, mac string - checkConflict := false + checkConflict := true var err error for { v4ip, v6ip, mac, err = c.ipam.GetRandomAddress(name, nicName, mac, subnetName, skippedAddrs, checkConflict) diff --git a/pkg/controller/vlan.go b/pkg/controller/vlan.go index b4917c75a29..1921f10ac78 100644 --- a/pkg/controller/vlan.go +++ b/pkg/controller/vlan.go @@ -326,6 +326,7 @@ func (c *Controller) setLocalnetTag(subnet string, vlanID int) error { func (c *Controller) delLocalnet(subnet string) error { localnetPort := ovs.GetLocalnetName(subnet) + klog.Infof("delete logical switch port %s", localnetPort) if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(localnetPort); err != nil { klog.Errorf("failed to delete localnet port %s: %v", localnetPort, err) return err diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index bec679a9f51..f2a2d4536c8 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -192,9 +192,15 @@ func (c *Controller) reconcileRouterPorts(vpc *kubeovnv1.Vpc) error { return err } - klog.V(1).InfoS("router port not exists, trying to create", "vpc", vpc.Name, "subnet", subnetName) + if subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway { + // skip vlan subnet which use underlay gw + // vpc connect to external vlan subnet is controlled by vpc spec enableExternal + klog.Infof("no need to connect vpc '%s' to vlan subnet %s", router, subnet.Name) + return nil + } networks := util.GetIpAddrWithMask(subnet.Spec.Gateway, subnet.Spec.CIDRBlock) + klog.Infof("add vpc lrp %s, networks %s", routerPortName, networks) if err := c.ovnClient.AddLogicalRouterPort(router, routerPortName, "", networks); err != nil { klog.ErrorS(err, "unable to create router port", "vpc", vpc.Name, "subnet", subnetName) return err @@ -362,7 +368,9 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { } for _, oldPeer := range vpc.Status.VpcPeerings { if !util.ContainsString(newPeers, oldPeer) { - if err = c.ovnLegacyClient.DeleteLogicalRouterPort(fmt.Sprintf("%s-%s", vpc.Name, oldPeer)); err != nil { + lrpName := fmt.Sprintf("%s-%s", vpc.Name, oldPeer) + klog.Infof("delete logical router port %s", lrpName) + if err = c.ovnLegacyClient.DeleteLogicalRouterPort(lrpName); err != nil { klog.Errorf("failed to delete peer router port for vpc %s, %v", vpc.Name, err) return err } @@ -822,16 +830,36 @@ func (c *Controller) handleAddVpcExternal(key string) error { } func (c *Controller) handleDelVpcExternal(key string) error { - lrpEipName := fmt.Sprintf("%s-%s", key, c.config.ExternalGatewaySwitch) - cachedEip, err := c.ovnEipsLister.Get(lrpEipName) + cachedVpc, err := c.vpcsLister.Get(key) if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } return err } + lrpEipName := fmt.Sprintf("%s-%s", key, c.config.ExternalGatewaySwitch) klog.V(3).Infof("delete vpc lrp %s", lrpEipName) if err := c.ovnLegacyClient.DisconnectRouterToExternal(c.config.ExternalGatewaySwitch, key); err != nil { klog.Errorf("failed to disconnect router '%s' to external, %v", key, err) return err } + vpc := cachedVpc.DeepCopy() + vpc.Status.EnableExternal = cachedVpc.Spec.EnableExternal + bytes, err := vpc.Status.Bytes() + if err != nil { + return err + } + if _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Patch(context.Background(), + vpc.Name, types.MergePatchType, bytes, metav1.PatchOptions{}, "status"); err != nil { + return err + } + cachedEip, err := c.ovnEipsLister.Get(lrpEipName) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return err + } if err = c.handleDelOvnEipFinalizer(cachedEip); err != nil { klog.Errorf("failed to del finalizer for ovn eip, %v", err) return err diff --git a/pkg/ovs/ovn-nb-logical_router_port.go b/pkg/ovs/ovn-nb-logical_router_port.go index 3d73756f866..2f3cb5b91f6 100644 --- a/pkg/ovs/ovn-nb-logical_router_port.go +++ b/pkg/ovs/ovn-nb-logical_router_port.go @@ -12,6 +12,7 @@ import ( ovsclient "github.com/kubeovn/kube-ovn/pkg/ovsdb/client" "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" "github.com/kubeovn/kube-ovn/pkg/util" + "k8s.io/klog/v2" ) func (c OvnClient) GetLogicalRouterPort(name string, ignoreNotFound bool) (*ovnnb.LogicalRouterPort, error) { @@ -67,7 +68,7 @@ func (c OvnClient) AddLogicalRouterPort(lr, name, mac, networks string) error { return err } ops = append(ops, mutationOps...) - + klog.Infof("add vpc lrp %s, networks %s", name, networks) if err := Transact(c.ovnNbClient, "lrp-add", ops, c.ovnNbClient.Timeout); err != nil { return fmt.Errorf("failed to create logical router port %s: %v", name, err) } diff --git a/pkg/ovs/ovn-nbctl-legacy.go b/pkg/ovs/ovn-nbctl-legacy.go index ad83fa3ff13..fdf4cbf9838 100644 --- a/pkg/ovs/ovn-nbctl-legacy.go +++ b/pkg/ovs/ovn-nbctl-legacy.go @@ -108,6 +108,7 @@ func (c LegacyClient) SetICAutoRoute(enable bool, blackList []string) error { // DeleteLogicalSwitchPort delete logical switch port in ovn func (c LegacyClient) DeleteLogicalSwitchPort(port string) error { + klog.Infof("delete lsp %s", port) if _, err := c.ovnNbCommand(IfExists, "lsp-del", port); err != nil { return fmt.Errorf("failed to delete logical switch port %s, %v", port, err) } @@ -116,6 +117,7 @@ func (c LegacyClient) DeleteLogicalSwitchPort(port string) error { // DeleteLogicalRouterPort delete logical switch port in ovn func (c LegacyClient) DeleteLogicalRouterPort(port string) error { + klog.Infof("delete lrp %s", port) if _, err := c.ovnNbCommand(IfExists, "lrp-del", port); err != nil { return fmt.Errorf("failed to delete logical router port %s, %v", port, err) } @@ -123,7 +125,9 @@ func (c LegacyClient) DeleteLogicalRouterPort(port string) error { } func (c LegacyClient) CreateICLogicalRouterPort(az, mac, subnet string, chassises []string) error { - if _, err := c.ovnNbCommand(MayExist, "lrp-add", c.ClusterRouter, fmt.Sprintf("%s-ts", az), mac, subnet); err != nil { + lrpName := fmt.Sprintf("%s-ts", az) + klog.Infof("add vpc lrp %s", lrpName) + if _, err := c.ovnNbCommand(MayExist, "lrp-add", c.ClusterRouter, lrpName, mac, subnet); err != nil { return fmt.Errorf("failed to create ovn-ic lrp, %v", err) } if _, err := c.ovnNbCommand(MayExist, "lsp-add", util.InterconnectionSwitch, fmt.Sprintf("ts-%s", az), "--", @@ -405,6 +409,7 @@ func (c LegacyClient) ListPodLogicalSwitchPorts(pod, namespace string) ([]string } func (c LegacyClient) SetLogicalSwitchConfig(ls, lr, protocol, subnet, gateway string, excludeIps []string, needRouter bool) error { + klog.Infof("set logical switch: ls %s, lr %s, protocol %s, subnet %s, gw %s", ls, lr, protocol, subnet, gateway) var err error cidrBlocks := strings.Split(subnet, ",") temp := strings.Split(cidrBlocks[0], "/") @@ -551,6 +556,8 @@ func (c LegacyClient) ConnectRouterToExternal(externalNet, vpcRouter, lrpIpCidr, // add lrp and lsp between vpc router and external network lsTolr := fmt.Sprintf("%s-%s", externalNet, vpcRouter) lrTols := fmt.Sprintf("%s-%s", vpcRouter, externalNet) + klog.Infof("add vpc lrp %s, cidr %s", lrTols, lrpIpCidr) + klog.Infof("add lsp %s", lsTolr) _, err := c.ovnNbCommand( MayExist, "lrp-add", vpcRouter, lrTols, lrpMac, lrpIpCidr, "--", MayExist, "lsp-add", externalNet, lsTolr, "--", @@ -571,10 +578,12 @@ func (c LegacyClient) ConnectRouterToExternal(externalNet, vpcRouter, lrpIpCidr, func (c LegacyClient) DisconnectRouterToExternal(externalNet, vpcRouter string) error { lrTols := fmt.Sprintf("%s-%s", vpcRouter, externalNet) + klog.Infof("delete lrp %s", lrTols) if _, err := c.ovnNbCommand(IfExists, "lrp-del", lrTols); err != nil { return err } lsTolr := fmt.Sprintf("%s-%s", externalNet, vpcRouter) + klog.Infof("delete lsp %s", lsTolr) if _, err := c.ovnNbCommand(IfExists, "lsp-del", lsTolr); err != nil { return err } @@ -584,6 +593,8 @@ func (c LegacyClient) DisconnectRouterToExternal(externalNet, vpcRouter string) func (c LegacyClient) CreateGatewaySwitch(name, network string, vlan int, ip, mac string, chassises []string) error { lsTolr := fmt.Sprintf("%s-%s", name, c.ClusterRouter) lrTols := fmt.Sprintf("%s-%s", c.ClusterRouter, name) + klog.Infof("add vpc lrp %s, ip %s", lrTols, ip) + klog.Infof("add lsp %s", lsTolr) localnetPort := fmt.Sprintf("ln-%s", name) if name != util.ExternalGatewaySwitch { // compatiable with provider network and vlan and subnet @@ -625,6 +636,8 @@ func (c LegacyClient) CreateGatewaySwitch(name, network string, vlan int, ip, ma func (c LegacyClient) DeleteGatewaySwitch(name string) error { lrTols := fmt.Sprintf("%s-%s", c.ClusterRouter, name) + klog.Infof("delete gw switch %s", name) + klog.Infof("delete gw lrp %s", lrTols) _, err := c.ovnNbCommand( IfExists, "ls-del", name, "--", IfExists, "lrp-del", lrTols, @@ -819,6 +832,7 @@ func (c LegacyClient) DeleteLogicalRouter(lr string) error { func (c LegacyClient) RemoveRouterPort(ls, lr string) error { lsTolr := fmt.Sprintf("%s-%s", ls, lr) lrTols := fmt.Sprintf("%s-%s", lr, ls) + klog.Infof("remove router port %s, switch port %s", lrTols, lsTolr) _, err := c.ovnNbCommand(IfExists, "lsp-del", lsTolr, "--", IfExists, "lrp-del", lrTols) if err != nil { @@ -857,9 +871,11 @@ func (c LegacyClient) CreatePeerRouterPort(localRouter, remoteRouter, localRoute if len(results) == 0 { ipStr := strings.Split(localRouterPortIP, ",") if len(ipStr) == 2 { + klog.Infof("add vpc lrp %s", localRouterPort) _, err = c.ovnNbCommand(MayExist, "lrp-add", localRouter, localRouterPort, util.GenerateMac(), ipStr[0], ipStr[1], "--", "set", "logical_router_port", localRouterPort, fmt.Sprintf("peer=%s", remoteRouterPort)) } else { + klog.Infof("add vpc lrp %s", localRouterPort) _, err = c.ovnNbCommand(MayExist, "lrp-add", localRouter, localRouterPort, util.GenerateMac(), ipStr[0], "--", "set", "logical_router_port", localRouterPort, fmt.Sprintf("peer=%s", remoteRouterPort)) } @@ -869,6 +885,7 @@ func (c LegacyClient) CreatePeerRouterPort(localRouter, remoteRouter, localRoute } } + klog.Infof("set lrp %s, networks %s", localRouterPort, localRouterPortIP) _, err = c.ovnNbCommand("set", "logical_router_port", localRouterPort, fmt.Sprintf("networks=\"%s\"", strings.ReplaceAll(localRouterPortIP, ",", " "))) @@ -2765,6 +2782,7 @@ func (c *LegacyClient) UpdateRouterPortIPv6RA(ls, lr, cidrBlock, gateway, ipv6RA } ipv6RAConfigsStr = strings.ReplaceAll(ipv6RAConfigsStr, " ", "") + klog.Infof("set lrp %s, ipv6_prefix %s", lrTols, ipv6Prefix) _, err = c.ovnNbCommand("--", "set", "logical_router_port", lrTols, fmt.Sprintf("ipv6_prefix=%s", ipv6Prefix), fmt.Sprintf("ipv6_ra_configs=%s", ipv6RAConfigsStr)) if err != nil { @@ -2772,6 +2790,7 @@ func (c *LegacyClient) UpdateRouterPortIPv6RA(ls, lr, cidrBlock, gateway, ipv6RA return err } } else { + klog.Infof("set lrp %s", lrTols) _, err = c.ovnNbCommand("--", "set", "logical_router_port", lrTols, "ipv6_prefix=[]", "ipv6_ra_configs={}") if err != nil {