From 8b5151f0ccc92eeef34bbf6950f658be9baa3b4f Mon Sep 17 00:00:00 2001 From: l1b0k Date: Sun, 28 Apr 2024 16:23:48 +0800 Subject: [PATCH 1/2] daemon: fix the ip type Signed-off-by: l1b0k --- daemon/daemon.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/daemon/daemon.go b/daemon/daemon.go index 4fb971be..d54f943d 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -422,6 +422,21 @@ func (n *networkService) GetIPInfo(ctx context.Context, r *rpc.GetInfoRequest) ( IPv6: n.enableIPv6, } + switch pod.PodNetworkType { + case daemon.PodNetworkTypeENIMultiIP: + reply.IPType = rpc.IPType_TypeENIMultiIP + case daemon.PodNetworkTypeVPCIP: + reply.IPType = rpc.IPType_TypeVPCIP + case daemon.PodNetworkTypeVPCENI: + reply.IPType = rpc.IPType_TypeVPCENI + + default: + return nil, &types.Error{ + Code: types.ErrInternalError, + Msg: "Unknown pod network type", + } + } + // 2. Find old resource info oldRes, err := n.getPodResource(pod) if err != nil { From a2193778287325942aee20026864216a5cc07dec Mon Sep 17 00:00:00 2001 From: l1b0k Date: Sun, 28 Apr 2024 17:51:31 +0800 Subject: [PATCH 2/2] daemon: add gc for route and tc filter Signed-off-by: l1b0k --- daemon/daemon.go | 127 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/daemon/daemon.go b/daemon/daemon.go index d54f943d..1aa84184 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -2,9 +2,11 @@ package daemon import ( "context" + "encoding/binary" "encoding/json" "errors" "fmt" + "net" "net/netip" "os" "strconv" @@ -12,6 +14,8 @@ import ( "sync" "time" + "github.com/vishvananda/netlink" + "k8s.io/apimachinery/pkg/util/sets" logf "sigs.k8s.io/controller-runtime/pkg/log" "github.com/AliyunContainerService/terway/deviceplugin" @@ -79,6 +83,8 @@ type networkService struct { wg sync.WaitGroup + gcRulesOnce sync.Once + rpc.UnimplementedTerwayBackendServer } @@ -525,9 +531,14 @@ func (n *networkService) gcPods(ctx context.Context) error { } exist := make(map[string]bool) + existIPs := sets.Set[string]{} + for _, pod := range pods { if !pod.SandboxExited { exist[utils.PodInfoKey(pod.Namespace, pod.Name)] = true + if pod.PodIPs.IPv4 != nil { + existIPs.Insert(pod.PodIPs.IPv4.String()) + } } } @@ -538,6 +549,12 @@ func (n *networkService) gcPods(ctx context.Context) error { podResources := getPodResources(objList) for _, podRes := range podResources { + if podRes.PodInfo != nil { + if podRes.PodInfo.PodIPs.IPv4 != nil { + existIPs.Insert(podRes.PodInfo.PodIPs.IPv4.String()) + } + } + podID := utils.PodInfoKey(podRes.PodInfo.Namespace, podRes.PodInfo.Name) if _, ok := exist[podID]; ok { continue @@ -583,9 +600,38 @@ func (n *networkService) gcPods(ctx context.Context) error { } serviceLog.Info("removed pod", "pod", podID) } + + if os.Getenv("TERWAY_GC_RULES") == "true" { + n.gcRulesOnce.Do(func() { + gcLeakedRules(existIPs) + }) + } + return nil } +func gcLeakedRules(existIP sets.Set[string]) { + links, err := netlink.LinkList() + if err != nil { + serviceLog.Error(err, "error list links") + return + } + + ipvlLinks := lo.Filter(links, func(item netlink.Link, index int) bool { + _, ok := item.(*netlink.IPVlan) + return ok + }) + + gcRoutes(ipvlLinks, existIP) + + normalLinks := lo.Filter(links, func(item netlink.Link, index int) bool { + _, ok := item.(*netlink.Device) + return ok + }) + + gcTCFilters(normalLinks, existIP) +} + func (n *networkService) migrateEIP(ctx context.Context, objs []interface{}) error { once := sync.Once{} @@ -1356,3 +1402,84 @@ func filterENINotFound(podResources []daemon.PodResources, attachedENIID map[str } return podResources } + +func gcRoutes(links []netlink.Link, existIP sets.Set[string]) { + for _, link := range links { + routes, err := netlink.RouteList(link, netlink.FAMILY_V4) + if err != nil { + serviceLog.Error(err, "gc list route", "link", link) + return + } + for _, route := range routes { + if route.Dst == nil { + continue + } + // if not found + if existIP.Has(route.Dst.IP.String()) { + continue + } + + serviceLog.Info("gc del route", "route", route) + err = netlink.RouteDel(&route) + if err != nil { + serviceLog.Error(err, "gc del route", "route", route) + return + } + } + } +} + +func gcTCFilters(links []netlink.Link, existIP sets.Set[string]) { + // map ip to u32 + toU32List := lo.Map(existIP.UnsortedList(), func(item string, index int) uint32 { + ip := net.ParseIP(item) + if ip == nil { + return 0 + } + return binary.BigEndian.Uint32(ip.To4()) + }) + u32IPMap := lo.SliceToMap(toU32List, func(item uint32) (uint32, struct{}) { + return item, struct{}{} + }) + + for _, link := range links { + filters, err := netlink.FilterList(link, netlink.HANDLE_MIN_EGRESS) + if err != nil { + serviceLog.Error(err, "gc list filter", "link", link) + continue + } + for _, filter := range filters { + u32, ok := filter.(*netlink.U32) + if !ok { + continue + } + if u32.Priority != 50001 { + continue + } + + if u32.Sel == nil || len(u32.Sel.Keys) != 1 { + continue + } + if len(u32.Actions) != 1 { + continue + } + _, ok = u32.Actions[0].(*netlink.VlanAction) + if !ok { + continue + } + if u32.Sel.Keys[0].Off != 12 { + continue + } + _, ok = u32IPMap[u32.Sel.Keys[0].Val] + if ok { + continue + } + + serviceLog.Info("gc tc filter", "filter", filter) + err = netlink.FilterDel(filter) + if err != nil { + serviceLog.Error(err, "gc list filter", "link", link) + } + } + } +}