diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go
index f9c2f62bb62..d2b3310aefe 100644
--- a/pkg/controller/gc.go
+++ b/pkg/controller/gc.go
@@ -300,6 +300,8 @@ func (c *Controller) markAndCleanLSP() error {
 			if !lastNoPodLSP[lsp] {
 				noPodLSP[lsp] = true
 			} else {
+				nameNsMap := c.ovnClient.GetLspExternalIds(lsp)
+
 				klog.Infof("gc logical switch port %s", lsp)
 				if err := c.ovnClient.DeleteLogicalSwitchPort(lsp); err != nil {
 					klog.Errorf("failed to delete lsp %s, %v", lsp, err)
@@ -311,6 +313,11 @@ func (c *Controller) markAndCleanLSP() error {
 						return err
 					}
 				}
+
+				for podName, nsName := range nameNsMap {
+					key := fmt.Sprintf("%s/%s", nsName, podName)
+					c.ipam.ReleaseAddressByPod(key)
+				}
 			}
 		}
 	}
diff --git a/pkg/ovs/ovn-nbctl.go b/pkg/ovs/ovn-nbctl.go
index 32b33f12a94..4a24e718b5c 100644
--- a/pkg/ovs/ovn-nbctl.go
+++ b/pkg/ovs/ovn-nbctl.go
@@ -2543,3 +2543,38 @@ func (c Client) UpdateSubnetACL(ls string, acls []kubeovnv1.Acl) error {
 	}
 	return nil
 }
+
+func (c *Client) GetLspExternalIds(lsp string) map[string]string {
+	result, err := c.CustomFindEntity("Logical_Switch_Port", []string{"external_ids"}, fmt.Sprintf("name=%s", lsp))
+	if err != nil {
+		klog.Errorf("customFindEntity failed, %v", err)
+		return nil
+	}
+	if len(result) == 0 {
+		return nil
+	}
+
+	nameNsMap := make(map[string]string, 1)
+	for _, l := range result[0]["external_ids"] {
+		if len(strings.TrimSpace(l)) == 0 {
+			continue
+		}
+		parts := strings.Split(strings.TrimSpace(l), "=")
+		if len(parts) != 2 {
+			continue
+		}
+		if strings.TrimSpace(parts[0]) != "pod" {
+			continue
+		}
+
+		podInfo := strings.Split(strings.TrimSpace(parts[1]), "/")
+		if len(podInfo) != 2 {
+			continue
+		}
+		podNs := podInfo[0]
+		podName := podInfo[1]
+		nameNsMap[podName] = podNs
+	}
+
+	return nameNsMap
+}