fix: gc rules

daemon/daemon.go

@@ -2,16 +2,20 @@
 
 import (
 	"context"
+	"encoding/binary"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net"
 	"net/netip"
 	"os"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/vishvananda/netlink"
+	"k8s.io/apimachinery/pkg/util/sets"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/AliyunContainerService/terway/deviceplugin"
@@ -79,6 +83,8 @@
 
 	wg sync.WaitGroup
 
+	gcRulesOnce sync.Once
+
 	rpc.UnimplementedTerwayBackendServer
 }
 
@@ -422,6 +428,21 @@
 		IPv6:    n.enableIPv6,
 	}
 
+	switch pod.PodNetworkType {
+	case daemon.PodNetworkTypeENIMultiIP:
+		reply.IPType = rpc.IPType_TypeENIMultiIP
+	case daemon.PodNetworkTypeVPCIP:
+		reply.IPType = rpc.IPType_TypeVPCIP
+	case daemon.PodNetworkTypeVPCENI:
+		reply.IPType = rpc.IPType_TypeVPCENI
+
+	default:
+		return nil, &types.Error{
+			Code: types.ErrInternalError,
+			Msg:  "Unknown pod network type",
+		}
+	}
+
 	// 2. Find old resource info
 	oldRes, err := n.getPodResource(pod)
 	if err != nil {
@@ -510,9 +531,14 @@
 	}
 	exist := make(map[string]bool)
 
+	existIPs := sets.Set[string]{}
+
 	for _, pod := range pods {
 		if !pod.SandboxExited {
 			exist[utils.PodInfoKey(pod.Namespace, pod.Name)] = true
+			if pod.PodIPs.IPv4 != nil {
+				existIPs.Insert(pod.PodIPs.IPv4.String())
+			}
 		}
 	}
 
@@ -523,6 +549,12 @@
 	podResources := getPodResources(objList)
 
 	for _, podRes := range podResources {
+		if podRes.PodInfo != nil {
+			if podRes.PodInfo.PodIPs.IPv4 != nil {
+				existIPs.Insert(podRes.PodInfo.PodIPs.IPv4.String())
+			}
+		}
+
 		podID := utils.PodInfoKey(podRes.PodInfo.Namespace, podRes.PodInfo.Name)
 		if _, ok := exist[podID]; ok {
 			continue
@@ -568,9 +600,38 @@
 		}
 		serviceLog.Info("removed pod", "pod", podID)
 	}
+
+	if os.Getenv("TERWAY_GC_RULES") == "true" {
+		n.gcRulesOnce.Do(func() {
+			gcLeakedRules(existIPs)
+		})
+	}
+
 	return nil
 }
 
+func gcLeakedRules(existIP sets.Set[string]) {
+	links, err := netlink.LinkList()
+	if err != nil {
+		serviceLog.Error(err, "error list links")
+		return
+	}
+
+	ipvlLinks := lo.Filter(links, func(item netlink.Link, index int) bool {
+		_, ok := item.(*netlink.IPVlan)
+		return ok
+	})
+
+	gcRoutes(ipvlLinks, existIP)
+
+	normalLinks := lo.Filter(links, func(item netlink.Link, index int) bool {
+		_, ok := item.(*netlink.Device)
+		return ok
+	})
+
+	gcTCFilters(normalLinks, existIP)
+}
+
 func (n *networkService) migrateEIP(ctx context.Context, objs []interface{}) error {
 	once := sync.Once{}
 
@@ -1341,3 +1402,84 @@
 	}
 	return podResources
 }
+
+func gcRoutes(links []netlink.Link, existIP sets.Set[string]) {
+	for _, link := range links {
+		routes, err := netlink.RouteList(link, netlink.FAMILY_V4)
+		if err != nil {
+			serviceLog.Error(err, "gc list route", "link", link)
+			return
+		}
+		for _, route := range routes {
+			if route.Dst == nil {
+				continue
+			}
+			// if not found
+			if existIP.Has(route.Dst.IP.String()) {
+				continue
+			}
+
+			serviceLog.Info("gc del route", "route", route)
+			err = netlink.RouteDel(&route)
+			if err != nil {
+				serviceLog.Error(err, "gc del route", "route", route)
+				return
+			}
+		}
+	}
+}
+
+func gcTCFilters(links []netlink.Link, existIP sets.Set[string]) {
+	// map ip to u32
+	toU32List := lo.Map(existIP.UnsortedList(), func(item string, index int) uint32 {
+		ip := net.ParseIP(item)
+		if ip == nil {
+			return 0
+		}
+		return binary.BigEndian.Uint32(ip.To4())
+	})
+	u32IPMap := lo.SliceToMap(toU32List, func(item uint32) (uint32, struct{}) {
+		return item, struct{}{}
+	})
+
+	for _, link := range links {
+		filters, err := netlink.FilterList(link, netlink.HANDLE_MIN_EGRESS)
+		if err != nil {
+			serviceLog.Error(err, "gc list filter", "link", link)
+			continue
+		}
+		for _, filter := range filters {
+			u32, ok := filter.(*netlink.U32)
+			if !ok {
+				continue
+			}
+			if u32.Priority != 50001 {
+				continue
+			}
+
+			if u32.Sel == nil || len(u32.Sel.Keys) != 1 {
+				continue
+			}
+			if len(u32.Actions) != 1 {
+				continue
+			}
+			_, ok = u32.Actions[0].(*netlink.VlanAction)
+			if !ok {
+				continue
+			}
+			if u32.Sel.Keys[0].Off != 12 {
+				continue
+			}
+			_, ok = u32IPMap[u32.Sel.Keys[0].Val]
+			if ok {
+				continue
+			}
+
+			serviceLog.Info("gc tc filter", "filter", filter)
+			err = netlink.FilterDel(filter)
+			if err != nil {
+				serviceLog.Error(err, "gc list filter", "link", link)
+			}
+		}
+	}
+}