diff --git a/pkg/controller/network_policy.go b/pkg/controller/network_policy.go
index 44b5afab5999..e30ecceecb7d 100644
--- a/pkg/controller/network_policy.go
+++ b/pkg/controller/network_policy.go
@@ -238,18 +238,6 @@ func (c *Controller) handleUpdateNp(key string) error {
 		}
 	}
 
-	ingressAsNames, err := c.ovnLegacyClient.ListNpAddressSet(np.Namespace, npName, "ingress")
-	if err != nil {
-		klog.Errorf("failed to list ingress address_set, %v", err)
-		return err
-	}
-	for _, ingressAsName := range ingressAsNames {
-		if err = c.ovnLegacyClient.DeleteAddressSet(ingressAsName); err != nil {
-			klog.Errorf("failed to delete np %s address set, %v", key, err)
-			return err
-		}
-	}
-
 	var ingressAclCmd []string
 	exist, err := c.ovnClient.PortGroupExists(pgName)
 	if err != nil {
@@ -381,18 +369,6 @@ func (c *Controller) handleUpdateNp(key string) error {
 		}
 	}
 
-	egressAsNames, err := c.ovnLegacyClient.ListNpAddressSet(np.Namespace, npName, "egress")
-	if err != nil {
-		klog.Errorf("failed to list egress address_set, %v", err)
-		return err
-	}
-	for _, egressAsName := range egressAsNames {
-		if err = c.ovnLegacyClient.DeleteAddressSet(egressAsName); err != nil {
-			klog.Errorf("failed to delete np %s address set, %v", key, err)
-			return err
-		}
-	}
-
 	var egressAclCmd []string
 	if exist, err = c.ovnClient.PortGroupExists(pgName); err != nil {
 		klog.Errorf("failed to query np %s port group, %v", key, err)