Merge pull request #3674 from spidernet-io/robot/cherrypick/pr3669/re…

…lease-v0.8

Fix: Statefulset pod should change IP when recreating with a changed pool in annotation

docs/usage/statefulset-zh_CN.md

@@ -26,7 +26,7 @@ StatefulSet 会在以下一些场景中会出现固定地址的使用：
 >
 > - 在 StatefulSet 副本经由`缩容`到`扩容`的变化过程中，Spiderpool 并不保证新扩容 Pod 能够获取到之前缩容 Pod 的 IP 地址。
 >
-> - 目前，当 StatefulSet 准备就绪并且其 Pod 正在运行时，即使修改 StatefulSet 注解指定了另一个 IP 池，并重启 Pod ，Pod IP 地址也不会生效到新的 IP 池范围内，而是继续使用旧的固定 IP 。
+> - 在 v0.9.4 及之前的的版本，当 StatefulSet 准备就绪并且其 Pod 正在运行时，即使修改 StatefulSet 注解指定了另一个 IP 池，并重启 Pod，Pod IP 地址也不会生效到新的 IP 池范围内，而是继续使用旧的固定 IP。当大于 0.9.4 版本之后更换 IP 池重启 Pod 会完成 IP 地址切换。
 
 ## 实施要求
 

docs/usage/statefulset.md

@@ -26,7 +26,7 @@ Many open-source CNI solutions provide limited support for fixing IP addresses f
 >
 > - During the transition from scaling down to scaling up StatefulSet replicas, Spiderpool does not guarantee that new Pods will inherit the IP addresses previously used by the scaled-down Pods.
 >
-> - Currently, when a StatefulSet Pod is running, modifying the StatefulSet annotation to specify a different IP pool and restarting the Pod will not cause the Pod IP addresses to be allocated from the new IP pool range. Instead, the Pod will continue using their existing fixed IP addresses.
+> - In version 0.9.4 and prior versions, when a StatefulSet is ready and its Pod is running, even if you modify the StatefulSet annotation to specify a different IP pool and restart the Pod, the Pod's IP address will not switch to the new IP pool range but will continue to use the old fixed IP. Starting from version 0.9.4 and above, changing the IP pool and restarting the Pod will complete the IP address switch.
 
 ## Prerequisites
 

pkg/ipam/allocate.go

@@ -68,7 +68,22 @@ func (i *ipam) Allocate(ctx context.Context, addArgs *models.IpamAddArgs) (*mode
 		logger.Debug("No Endpoint")
 	}
 
-	if (i.config.EnableStatefulSet && podTopController.APIVersion == appsv1.SchemeGroupVersion.String() && podTopController.Kind == constant.KindStatefulSet) ||
+	// Flag to indicate whether outdated IPs should be released.
+	// Check if StatefulSets are enabled in the configuration and
+	// if the pod's top controller is a StatefulSet.
+	// If an endpoint exists, attempt to release outdated IPs for
+	// the StatefulSet if necessary, and return an error if the
+	// operation fails.
+	releaseStsOutdatedIPFlag := false
+	if i.config.EnableStatefulSet && podTopController.APIVersion == appsv1.SchemeGroupVersion.String() && podTopController.Kind == constant.KindStatefulSet {
+		if endpoint != nil {
+			releaseStsOutdatedIPFlag, err = i.releaseStsOutdatedIPIfNeed(ctx, addArgs, pod, endpoint, podTopController)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	if (!releaseStsOutdatedIPFlag && i.config.EnableStatefulSet && podTopController.APIVersion == appsv1.SchemeGroupVersion.String() && podTopController.Kind == constant.KindStatefulSet) ||
 		(i.config.EnableKubevirtStaticIP && podTopController.APIVersion == kubevirtv1.SchemeGroupVersion.String() && podTopController.Kind == constant.KindKubevirtVMI) {
 		logger.Sugar().Infof("Try to retrieve the IP allocation of %s", podTopController.Kind)
 		addResp, err := i.retrieveStaticIPAllocation(ctx, *addArgs.IfName, pod, endpoint)
@@ -98,6 +113,60 @@ func (i *ipam) Allocate(ctx context.Context, addArgs *models.IpamAddArgs) (*mode
 	return addResp, nil
 }
 
+func (i *ipam) releaseStsOutdatedIPIfNeed(ctx context.Context, addArgs *models.IpamAddArgs,
+	pod *corev1.Pod, endpoint *spiderpoolv2beta1.SpiderEndpoint, podTopController types.PodTopController) (bool, error) {
+	logger := logutils.FromContext(ctx)
+
+	preliminary, err := i.getPoolCandidates(ctx, addArgs, pod, podTopController)
+	if err != nil {
+		return false, err
+	}
+	poolMap := make(map[string]map[string]struct{})
+	for _, candidates := range preliminary {
+		if _, ok := poolMap[candidates.NIC]; !ok {
+			poolMap[candidates.NIC] = make(map[string]struct{})
+		}
+		pools := candidates.Pools()
+		for _, pool := range pools {
+			poolMap[candidates.NIC][pool] = struct{}{}
+		}
+	}
+	endpointMap := make(map[string]map[string]struct{})
+	for _, ip := range endpoint.Status.Current.IPs {
+		if _, ok := endpointMap[ip.NIC]; !ok {
+			endpointMap[ip.NIC] = make(map[string]struct{})
+		}
+		if ip.IPv4Pool != nil && *ip.IPv4Pool != "" {
+			endpointMap[ip.NIC][*ip.IPv4Pool] = struct{}{}
+		}
+		if ip.IPv6Pool != nil && *ip.IPv6Pool != "" {
+			endpointMap[ip.NIC][*ip.IPv6Pool] = struct{}{}
+		}
+	}
+	if !checkNicPoolExistence(endpointMap, poolMap) {
+		logger.Sugar().Info("StatefulSet Pod need to release IP: owned pool %v, expected pools: %v", endpointMap, poolMap)
+		if endpoint.DeletionTimestamp == nil {
+			logger.Sugar().Infof("delete outdated endpoint of statefulset pod: %v/%v", endpoint.Namespace, endpoint.Name)
+			if err := i.endpointManager.DeleteEndpoint(ctx, endpoint); err != nil {
+				return false, err
+			}
+		}
+		err := i.release(ctx, endpoint.Status.Current.UID, endpoint.Status.Current.IPs)
+		if err != nil {
+			return false, err
+		}
+		logger.Sugar().Info("remove outdated of StatefulSet pod %s/%s: %v", endpoint.Namespace, endpoint.Name, endpoint.Status.Current.IPs)
+		if err := i.endpointManager.RemoveFinalizer(ctx, endpoint); err != nil {
+			return false, fmt.Errorf("failed to clean statefulset pod's Endpoint when expected ippool was changed: %v", err)
+		}
+		endpoint = nil
+		return true, nil
+	} else {
+		logger.Sugar().Debugf("StatefulSet Pod does not need to release IP: owned pool %v, expected pools: %v", endpointMap, poolMap)
+	}
+	return false, nil
+}
+
 func (i *ipam) retrieveStaticIPAllocation(ctx context.Context, nic string, pod *corev1.Pod, endpoint *spiderpoolv2beta1.SpiderEndpoint) (*models.IpamAddResponse, error) {
 	logger := logutils.FromContext(ctx)
 

pkg/ipam/utils.go

@@ -216,3 +216,19 @@ func validateAndMutateMultipleNICAnnotations(annoIPPoolsValue types.AnnoPodIPPoo
 
 	return nil
 }
+
+func checkNicPoolExistence(endpointMap, poolMap map[string]map[string]struct{}) bool {
+	for outerKey, innerMap := range endpointMap {
+		poolInnerMap, exists := poolMap[outerKey]
+		if !exists {
+			return false
+		}
+
+		for innerKey := range innerMap {
+			if _, exists := poolInnerMap[innerKey]; !exists {
+				return false
+			}
+		}
+	}
+	return true
+}

test/e2e/affinity/affinity_test.go

@@ -460,13 +460,13 @@ var _ = Describe("test Affinity", Label("affinity"), func() {
 			}
 			GinkgoWriter.Printf("StatefulSet %s/%s corresponding Pod IP allocations: %v \n", stsObject.Namespace, stsObject.Name, ipMap)
 
-			// A00009：Modify the annotated IPPool for a specified StatefulSet pod, the pod wouldn't change IP
+			// A00009：Modify the annotated IPPool for a specified StatefulSet pod, the pod will change IP
 			podIppoolAnnoStr = common.GeneratePodIPPoolAnnotations(frame, common.NIC1, []string{v4PoolName}, []string{v6PoolName})
 			stsObject, err = frame.GetStatefulSet(statefulSetName, namespace)
 			Expect(err).NotTo(HaveOccurred())
 			stsObject.Spec.Template.Annotations = map[string]string{constant.AnnoPodIPPool: podIppoolAnnoStr}
 			// Modify the ippool in annotation and update the statefulset
-			GinkgoWriter.Printf("try to update StatefulSet %s/%s template with new annotations: %v \n", stsObject.Namespace, stsObject.Name, stsObject.Spec.Template.Annotations)
+			GinkgoWriter.Printf("try to update StatefulSet %s/%s template from: %v, to new annotations: %v \n", stsObject.Namespace, stsObject.Name, stsObject.Spec.Template.Annotations, stsObject.Spec.Template.Annotations)
 			Expect(frame.UpdateResource(stsObject)).NotTo(HaveOccurred())
 
 			// Check that the container ID should be different
@@ -515,22 +515,22 @@ var _ = Describe("test Affinity", Label("affinity"), func() {
 					Expect(ok).NotTo(BeFalse(), "Failed to get IPv4 IP")
 					Expect(podIPv4).NotTo(BeEmpty(), "podIPv4 is a empty string")
 					d, ok := ipMap[podIPv4]
-					Expect(ok).To(BeTrue(), fmt.Sprintf("original StatefulSet Pod IP allcations: %v, new Pod %s/%s IPv4 %s", ipMap, pod.Namespace, pod.Name, podIPv4))
+					Expect(ok).To(BeFalse(), fmt.Sprintf("original StatefulSet Pod IP allcations: %v, new Pod %s/%s IPv4 %s", ipMap, pod.Namespace, pod.Name, podIPv4))
 					GinkgoWriter.Printf("Pod %v IP %v remains the same \n", d, podIPv4)
 				}
 				if frame.Info.IpV6Enabled {
 					podIPv6, ok := tools.CheckPodIpv6IPReady(&pod)
 					Expect(ok).NotTo(BeFalse(), "Failed to get IPv6 IP")
 					Expect(podIPv6).NotTo(BeEmpty(), "podIPv6 is a empty string")
 					d, ok := ipMap[podIPv6]
-					Expect(ok).To(BeTrue(), fmt.Sprintf("original StatefulSet Pod IP allcations: %v, new Pod %s/%s IPv6 %s", ipMap, pod.Namespace, pod.Name, podIPv6))
+					Expect(ok).To(BeFalse(), fmt.Sprintf("original StatefulSet Pod IP allcations: %v, new Pod %s/%s IPv6 %s", ipMap, pod.Namespace, pod.Name, podIPv6))
 					GinkgoWriter.Printf("Pod %v IP %v remains the same \n", d, podIPv6)
 				}
 				// WorkloadEndpoint UID remains the same
 				object, err := common.GetWorkloadByName(frame, pod.Namespace, pod.Name)
 				Expect(err).NotTo(HaveOccurred(), "Failed to get the same uid")
 				d, ok := uidMap[string(object.UID)]
-				Expect(ok).To(BeTrue(), "Failed to get the same uid")
+				Expect(ok).To(BeFalse(), "Unexpectedly got the same uid")
 				GinkgoWriter.Printf("Pod %v workloadendpoint UID %v remains the same \n", d, object.UID)
 			}