From 8611de8229214b732c858175bc2822a25bfcd02b Mon Sep 17 00:00:00 2001 From: hzma Date: Thu, 30 Dec 2021 15:47:15 +0800 Subject: [PATCH] modify pod's process of update for use multus cni as default cni --- pkg/controller/init.go | 15 +++++++------ pkg/controller/node.go | 16 ++++++++++++-- pkg/controller/pod.go | 48 ++++++++++++++++++++---------------------- pkg/daemon/handler.go | 4 ++-- pkg/util/const.go | 8 +++---- 5 files changed, 52 insertions(+), 39 deletions(-) diff --git a/pkg/controller/init.go b/pkg/controller/init.go index cfc1731320d..e9f144bd7c9 100644 --- a/pkg/controller/init.go +++ b/pkg/controller/init.go @@ -297,12 +297,15 @@ func (c *Controller) InitIPAM() error { return err } for _, pod := range pods { - if isPodAlive(pod) && pod.Annotations[util.AllocatedAnnotation] == "true" { - podNets, err := c.getPodKubeovnNets(pod) - if err != nil { - klog.Errorf("failed to get pod kubeovn nets %s.%s address %s: %v", pod.Name, pod.Namespace, pod.Annotations[util.IpAddressAnnotation], err) + podNets, err := c.getPodKubeovnNets(pod) + if err != nil { + klog.Errorf("failed to get pod kubeovn nets %s.%s address %s: %v", pod.Name, pod.Namespace, pod.Annotations[util.IpAddressAnnotation], err) + } + for _, podNet := range podNets { + if !isOvnSubnet(podNet.Subnet) { + continue } - for _, podNet := range podNets { + if isPodAlive(pod) && pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] == "true" { _, _, _, err := c.ipam.GetStaticAddress( fmt.Sprintf("%s/%s", pod.Namespace, pod.Name), ovs.PodNameToPortName(pod.Name, pod.Namespace, podNet.ProviderName), @@ -310,7 +313,7 @@ func (c *Controller) InitIPAM() error { pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podNet.ProviderName)], pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, podNet.ProviderName)], false) if err != nil { - klog.Errorf("failed to init pod %s.%s address %s: %v", pod.Name, pod.Namespace, pod.Annotations[util.IpAddressAnnotation], err) + klog.Errorf("failed to init pod %s.%s address %s: %v", pod.Name, pod.Namespace, pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)], err) } } if err = c.initAppendPodExternalIds(pod); err != nil { diff --git a/pkg/controller/node.go b/pkg/controller/node.go index f7402799bec..c79be747e7d 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -741,8 +741,20 @@ func (c *Controller) fetchPodsOnNode(nodeName string) ([]string, error) { continue } - if pod.Annotations[util.AllocatedAnnotation] == "true" { - ports = append(ports, fmt.Sprintf("%s.%s", pod.Name, pod.Namespace)) + podNets, err := c.getPodKubeovnNets(pod) + if err != nil { + klog.Errorf("failed to get pod nets %v", err) + return nil, err + } + + for _, podNet := range podNets { + if !isOvnSubnet(podNet.Subnet) { + continue + } + + if pod.Annotations != nil && pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] == "true" { + ports = append(ports, ovs.PodNameToPortName(pod.Name, pod.Namespace, podNet.ProviderName)) + } } } return ports, nil diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 53d008a8aab..2a489130f97 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -94,24 +94,23 @@ func (c *Controller) enqueueAddPod(obj interface{}) { return } // In case update event might lost during leader election - if p.Annotations != nil && - p.Annotations[util.AllocatedAnnotation] == "true" && - p.Status.HostIP != "" && p.Status.PodIP != "" { - for _, podNet := range podNets { - if !isOvnSubnet(podNet.Subnet) { - continue - } + for _, podNet := range podNets { + if !isOvnSubnet(podNet.Subnet) { + continue + } + if p.Annotations != nil && + p.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] == "true" && + p.Status.HostIP != "" && p.Status.PodIP != "" { if p.Annotations[fmt.Sprintf(util.RoutedAnnotationTemplate, podNet.ProviderName)] != "true" { c.updatePodQueue.Add(key) - break + return } } - return - } - if p.Annotations != nil && p.Annotations[util.AllocatedAnnotation] == "true" { - return + if p.Annotations != nil && p.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] == "true" { + return + } } klog.V(3).Infof("enqueue add pod %s", key) @@ -227,13 +226,12 @@ func (c *Controller) enqueueUpdatePod(oldObj, newObj interface{}) { } // pod assigned an ip - if newPod.Annotations[util.AllocatedAnnotation] == "true" && - newPod.Spec.NodeName != "" { - for _, podNet := range podNets { - if !isOvnSubnet(podNet.Subnet) { - continue - } + for _, podNet := range podNets { + if !isOvnSubnet(podNet.Subnet) { + continue + } + if newPod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] == "true" && newPod.Spec.NodeName != "" { if newPod.Annotations[fmt.Sprintf(util.RoutedAnnotationTemplate, podNet.ProviderName)] != "true" { klog.V(3).Infof("enqueue update pod %s", key) c.updatePodQueue.Add(key) @@ -487,8 +485,8 @@ func (c *Controller) handleAddPod(key string) error { pod.Annotations[fmt.Sprintf(util.GatewayAnnotationTemplate, podNet.ProviderName)] = subnet.Spec.Gateway pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, podNet.ProviderName)] = subnet.Name pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] = "true" - if pod.Annotations[util.PodNicAnnotation] == "" { - pod.Annotations[util.PodNicAnnotation] = c.config.PodNicType + if pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, podNet.ProviderName)] == "" { + pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, podNet.ProviderName)] = c.config.PodNicType } if err := util.ValidatePodCidr(podNet.Subnet.Spec.CIDRBlock, ipStr); err != nil { @@ -710,11 +708,6 @@ func (c *Controller) handleUpdatePod(key string) error { } pod := oripod.DeepCopy() - // in case update handler overlap the annotation when cache is not in sync - if pod.Annotations[util.AllocatedAnnotation] == "" { - return fmt.Errorf("no address has been allocated to %s/%s", namespace, name) - } - klog.Infof("update pod %s/%s", namespace, name) var podIP string @@ -729,6 +722,11 @@ func (c *Controller) handleUpdatePod(key string) error { if !isOvnSubnet(podNet.Subnet) { continue } + // in case update handler overlap the annotation when cache is not in sync + if pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] == "" { + return fmt.Errorf("no address has been allocated to %s/%s", namespace, name) + } + podIP = pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)] subnet = podNet.Subnet diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go index 63edd8dd92a..2b59b4af0d3 100644 --- a/pkg/daemon/handler.go +++ b/pkg/daemon/handler.go @@ -118,7 +118,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon if podRequest.DeviceID != "" { nicType = util.OffloadType } else { - nicType = pod.Annotations[util.PodNicAnnotation] + nicType = pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, podRequest.Provider)] } switch pod.Annotations[fmt.Sprintf(util.DefaultRouteAnnotationTemplate, podRequest.Provider)] { @@ -341,7 +341,7 @@ func (csh cniServerHandler) handleDel(req *restful.Request, resp *restful.Respon if podRequest.DeviceID != "" { nicType = util.OffloadType } else { - nicType = pod.Annotations[util.PodNicAnnotation] + nicType = pod.Annotations[fmt.Sprintf(util.PodNicAnnotationTemplate, podRequest.Provider)] } err = csh.deleteNic(podRequest.PodName, podRequest.PodNamespace, podRequest.ContainerID, podRequest.DeviceID, podRequest.IfName, nicType) if err != nil { diff --git a/pkg/util/const.go b/pkg/util/const.go index 5dc360a205a..4b7d97108ff 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -59,6 +59,7 @@ const ( ProviderNetworkInterfaceTemplate = "%s.provider-network.kubernetes.io/interface" ProviderNetworkMtuTemplate = "%s.provider-network.kubernetes.io/mtu" MirrorControlAnnotationTemplate = "%s.kubernetes.io/mirror" + PodNicAnnotationTemplate = "%s.kubernetes.io/pod_nic_type" ExcludeIpsAnnotation = "ovn.kubernetes.io/exclude_ips" @@ -123,10 +124,9 @@ const ( NodeRouterPolicyPriority = 30000 - PodNicAnnotation = "ovn.kubernetes.io/pod_nic_type" - VethType = "veth-pair" - OffloadType = "offload-port" - InternalType = "internal-port" + VethType = "veth-pair" + OffloadType = "offload-port" + InternalType = "internal-port" ChassisLoc = "/etc/openvswitch/system-id.conf" HostnameEnv = "KUBE_NODE_NAME"