Skip to content

Commit

Permalink
support alloc static ip from any subnet after ns supports multi subnets
Browse files Browse the repository at this point in the history
  • Loading branch information
@hongzhen-ma
hongzhen-ma committed Mar 31, 2022
1 parent d3f6431 commit 795037e
Showing 1 changed file with 97 additions and 19 deletions.
116 changes: 97 additions & 19 deletions pkg/controller/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -489,8 +489,8 @@ func (c *Controller) handleAddPod(key string) error {

// Avoid create lsp for already running pod in ovn-nb when controller restart
for _, podNet := range needAllocateSubnets(pod, podNets) {
subnet := podNet.Subnet
v4IP, v6IP, mac, err := c.acquireAddress(pod, podNet)
// the subnet may changed when alloc static ip from the latter subnet after ns supports multi subnets
v4IP, v6IP, mac, subnet, err := c.acquireAddress(pod, podNet)
if err != nil {
c.recorder.Eventf(pod, v1.EventTypeWarning, "AcquireAddressFailed", err.Error())
return err
Expand Down Expand Up @@ -1212,14 +1212,14 @@ func (c *Controller) validatePodIP(podName, subnetName, ipv4, ipv6 string) (bool
return true, true, nil
}

func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, string, string, error) {
func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, string, string, *kubeovnv1.Subnet, error) {
podName := c.getNameByPod(pod)
key := fmt.Sprintf("%s/%s", pod.Namespace, podName)

macStr := pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podNet.ProviderName)]
if macStr != "" {
if _, err := net.ParseMAC(macStr); err != nil {
return "", "", "", err
return "", "", "", podNet.Subnet, err
}
}

Expand All @@ -1232,14 +1232,14 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st

ipv4, ipv6, mac, err := c.ipam.GetRandomAddress(key, nicName, macStr, podNet.Subnet.Name, skippedAddrs, !podNet.AllowLiveMigration)
if err != nil {
return "", "", "", err
return "", "", "", podNet.Subnet, err
}
ipv4OK, ipv6OK, err := c.validatePodIP(pod.Name, podNet.Subnet.Name, ipv4, ipv6)
if err != nil {
return "", "", "", err
return "", "", "", podNet.Subnet, err
}
if ipv4OK && ipv6OK {
return ipv4, ipv6, mac, nil
return ipv4, ipv6, mac, podNet.Subnet, nil
}

if !ipv4OK {
Expand All @@ -1252,10 +1252,33 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
}

nicName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName)

// The static ip can be assigned from any subnet after ns supports multi subnets
nsNets, _ := c.getNsAvailableSubnets(pod)
found := false
for _, nsNet := range nsNets {
if nsNet.Subnet.Name == podNet.Subnet.Name {
found = true
break
}
}
if !found {
nsNets = append(nsNets, podNet)
}
var v4IP, v6IP, mac string
var err error

// Static allocate
if pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)] != "" {
ipStr := pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)]
return c.acquireStaticAddress(key, nicName, ipStr, macStr, podNet.Subnet.Name, podNet.AllowLiveMigration)

for _, net := range nsNets {
v4IP, v6IP, mac, err = c.acquireStaticAddress(key, nicName, ipStr, macStr, net.Subnet.Name, net.AllowLiveMigration)
if err == nil {
return v4IP, v6IP, mac, net.Subnet, nil
}
}
return v4IP, v6IP, mac, podNet.Subnet, err
}

// IPPool allocate
Expand All @@ -1265,27 +1288,36 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
}

if ok, _ := isStatefulSetPod(pod); !ok {
for _, staticIP := range ipPool {
if c.ipam.IsIPAssignedToPod(staticIP, podNet.Subnet.Name, key) {
klog.Errorf("static address %s for %s has been assigned", staticIP, key)
continue
}
if v4IP, v6IP, mac, err := c.acquireStaticAddress(key, nicName, staticIP, macStr, podNet.Subnet.Name, podNet.AllowLiveMigration); err == nil {
return v4IP, v6IP, mac, nil
} else {
klog.Errorf("acquire address %s for %s failed, %v", staticIP, key, err)
for _, net := range nsNets {
for _, staticIP := range ipPool {
if c.ipam.IsIPAssignedToPod(staticIP, net.Subnet.Name, key) {
klog.Errorf("static address %s for %s has been assigned", staticIP, key)
continue
}

v4IP, v6IP, mac, err = c.acquireStaticAddress(key, nicName, staticIP, macStr, net.Subnet.Name, net.AllowLiveMigration)
if err == nil {
return v4IP, v6IP, mac, net.Subnet, nil
}
}
}
klog.Errorf("acquire address %s for %s failed, %v", pod.Annotations[fmt.Sprintf(util.IpPoolAnnotationTemplate, podNet.ProviderName)], key, err)
} else {
tempStrs := strings.Split(pod.Name, "-")
numStr := tempStrs[len(tempStrs)-1]
index, _ := strconv.Atoi(numStr)
if index < len(ipPool) {
return c.acquireStaticAddress(key, nicName, ipPool[index], macStr, podNet.Subnet.Name, podNet.AllowLiveMigration)
for _, net := range nsNets {
v4IP, v6IP, mac, err = c.acquireStaticAddress(key, nicName, ipPool[index], macStr, net.Subnet.Name, net.AllowLiveMigration)
if err == nil {
return v4IP, v6IP, mac, net.Subnet, nil
}
}
klog.Errorf("acquire address %s for %s failed, %v", ipPool[index], key, err)
}
}
klog.Errorf("alloc address for %s failed, return NoAvailableAddress", key)
return "", "", "", ipam.ErrNoAvailable
return "", "", "", podNet.Subnet, ipam.ErrNoAvailable
}

func generatePatchPayload(annotations map[string]string, op string) []byte {
Expand Down Expand Up @@ -1460,3 +1492,49 @@ func (c *Controller) getNameByPod(pod *v1.Pod) string {
}
return podName
}

func (c *Controller) getNsAvailableSubnets(pod *v1.Pod) ([]*kubeovnNet, error) {
var result []*kubeovnNet

ns, err := c.namespacesLister.Get(pod.Namespace)
if err != nil {
klog.Errorf("failed to get namespace %s, %v", pod.Namespace, err)
return nil, err
}
if ns.Annotations == nil {
return nil, nil
}

subnetNames := ns.Annotations[util.LogicalSwitchAnnotation]
for _, subnetName := range strings.Split(subnetNames, ",") {
if subnetName == "" {
continue
}
subnet, err := c.subnetsLister.Get(subnetName)
if err != nil {
klog.Errorf("failed to get subnet %v", err)
return nil, err
}

switch subnet.Spec.Protocol {
case kubeovnv1.ProtocolIPv4:
fallthrough
case kubeovnv1.ProtocolDual:
if subnet.Status.V4AvailableIPs == 0 {
continue
}
case kubeovnv1.ProtocolIPv6:
if subnet.Status.V6AvailableIPs == 0 {
continue
}
}

result = append(result, &kubeovnNet{
Type: providerTypeOriginal,
ProviderName: subnet.Spec.Provider,
Subnet: subnet,
})
}

return result, nil
}

0 comments on commit 795037e

Please sign in to comment.