Implement nftables masquerading for flannel

This PR allows flannel to use nftables natively instead of iptables.
This is used essentially to masquerade traffic coming from the pods.

The PR also fixes the clean-up mechanism in the iptables implementation.

Documentation/configuration.md

@@ -19,6 +19,9 @@ The value of the config is a JSON dictionary with the following keys:
 * `EnableIPv6` (bool): Enables ipv6 support
   Defaults to `false`
 
+* `EnableNFTables` (bool): (EXPERIMENTAL) If set to true, flannel uses nftables instead of iptables to masquerade the traffic.
+   Default to `false`
+
 * `SubnetLen` (integer): The size of the subnet allocated to each host.
    Defaults to 24 (i.e. /24) unless `Network` was configured to be smaller than a /22 in which case it is two less than the network.
 
@@ -128,3 +131,22 @@ FLANNEL_IPMASQ=true
 ## IPv6 only
 
 To use an IPv6-only environment use the same configuration of the Dual-stack section to enable IPv6 and add "EnableIPv4": false in the net-conf.json of the kube-flannel-cfg ConfigMap. In case of IPv6-only setup, please use the docker.io IPv6-only endpoint as described in the following link: https://www.docker.com/blog/beta-ipv6-support-on-docker-hub-registry/
+
+## nftables mode
+To enable `nftables` mode in flannel, set `EnableNFTables` to true in flannel configuration.
+
+Note: to test with kube-proxy, use kubeadm with the following configuration:
+```yaml
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: ClusterConfiguration
+kubernetesVersion: v1.29.0
+controllerManager:
+  extraArgs:
+    feature-gates: NFTablesProxyMode=true
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+mode: "nftables"
+featureGates:
+  NFTablesProxyMode: true
+```

Documentation/kube-flannel.yml

@@ -98,6 +98,7 @@ data:
   net-conf.json: |
     {
       "Network": "10.244.0.0/16",
+      "EnableNFTables": false,
       "Backend": {
         "Type": "vxlan"
       }

Makefile

@@ -6,7 +6,7 @@ QEMU_VERSION=v3.0.0
 BASH_UNIT_VERSION=v2.3.0
 
 # Default tag and architecture. Can be overridden
-TAG?=$(shell git describe --tags --always)
+TAG?=$(shell git describe --tags --dirty --always)
 ARCH?=amd64
 # Only enable CGO (and build the UDP backend) on AMD64
 ifeq ($(ARCH),amd64)

e2e/Dockerfile

@@ -11,6 +11,7 @@ RUN set -x \
     curl \
     tar gzip\
     iptables \
+    nftables \
     iproute2 \
     iputils \
  && if [ "${ARCH?required}" != "amd64" ]; then \

e2e/run-e2e-tests.sh

@@ -38,11 +38,12 @@ EOF
 
 write-flannel-conf(){
     local backend=$1
+    local enable_nftables=$2
     cp ../Documentation/kube-flannel.yml ./kube-flannel.yml
     yq -i 'select(.kind == "DaemonSet").spec.template.spec.containers[0].image |= strenv(FLANNEL_IMAGE)' ./kube-flannel.yml
     yq -i 'select(.kind == "DaemonSet").spec.template.spec.initContainers[1].image |= strenv(FLANNEL_IMAGE)' ./kube-flannel.yml
 
-    export flannel_conf="{ \"Network\": \"$FLANNEL_NET\", \"Backend\": { \"Type\": \"${backend}\" } }"
+    export flannel_conf="{ \"Network\": \"$FLANNEL_NET\", \"Backend\": { \"Type\": \"${backend}\" }, \"EnableNFTables\": ${enable_nftables} }"
 
     yq -i 'select(.metadata.name == "kube-flannel-cfg").data."net-conf.json" |= strenv(flannel_conf)' ./kube-flannel.yml
 
@@ -55,10 +56,11 @@ write-flannel-conf(){
 # This is not used at the moment since github runners don't support dual-stack networking
 write-flannel-conf-dual-stack(){
     local backend=$1
+    local enable_nftables=$2
     cp ../Documentation/kube-flannel.yml ./kube-flannel.yml
     yq -i 'select(.kind == "DaemonSet").spec.template.spec.containers[0].image |= strenv(FLANNEL_IMAGE)' ./kube-flannel.yml
 
-    export flannel_conf="{ \"EnableIPv6\": true, \"Network\": \"$FLANNEL_NET\", \"IPv6Network\":\"${FLANNEL_IP6NET}\", \"Backend\": { \"Type\": \"${backend}\" } }"
+    export flannel_conf="{ \"EnableIPv6\": true, \"Network\": \"$FLANNEL_NET\", \"IPv6Network\":\"${FLANNEL_IP6NET}\", \"Backend\": { \"Type\": \"${backend}\" }, \"EnableNFTables\": ${enable_nftables} }"
 
     yq -i 'select(.metadata.name == "kube-flannel-cfg").data."net-conf.json" |= strenv(flannel_conf)' ./kube-flannel.yml
 }
@@ -67,6 +69,10 @@ install-flannel() {
     kubectl --kubeconfig="${HOME}/.kube/config" apply -f ./kube-flannel.yml
 }
 
+delete-flannel() {
+    kubectl --kubeconfig="${HOME}/.kube/config" delete -f ./kube-flannel.yml
+}
+
 get_pod_ip() {
     local pod_name=$1
     kubectl --kubeconfig="${HOME}/.kube/config" get pod ${pod_name} --template '{{.status.podIP}}'
@@ -125,8 +131,9 @@ perf() {
 
 prepare_test() {
     local backend=$1
+    local enable_nftables=${2:-false}
     # install flannel version to test
-    write-flannel-conf ${backend} 
+    write-flannel-conf ${backend} ${enable_nftables}
 
     install-flannel
     # wait for nodes to be ready
@@ -150,32 +157,50 @@ test_vxlan() {
     prepare_test vxlan
     pings
     check_iptables
+    delete-flannel
+    check_iptables_removed
+}
+
+test_vxlan_nft() {
+    prepare_test vxlan true
+    pings
+    check_nftables
+    delete-flannel
+    check_nftables_removed
 }
 
 test_wireguard() {
     prepare_test wireguard
     pings
     check_iptables
+    delete-flannel
+    check_iptables_removed
 }
 
 test_host-gw() {
     prepare_test host-gw
     pings
     check_iptables
+    delete-flannel
+    check_iptables_removed
 }
 
 if [[ ${ARCH} == "amd64" ]]; then
 test_udp() {
     prepare_test udp
     pings
     check_iptables
+    delete-flannel
+    check_iptables_removed
 }
 fi
 
 test_ipip() {
     prepare_test ipip
     pings
     check_iptables
+    delete-flannel
+    check_iptables_removed
 }
 
 test_perf_vxlan() {
@@ -260,3 +285,95 @@ $(docker exec --privileged local-worker /usr/sbin/iptables -t filter -S FLANNEL-
                 "$(docker exec --privileged local-leader /usr/sbin/iptables -t filter -S FORWARD)
 $(docker exec --privileged local-leader /usr/sbin/iptables -t filter -S FLANNEL-FWD)" "Host 2 has not expected forward rules"
 }
+
+check_iptables_removed() {
+  local worker_podcidr=$(get_pod_cidr local-worker)
+  local leader_pod_cidr=$(get_pod_cidr local-leader)
+  read -r -d '' POSTROUTING_RULES_WORKER << EOM
+-N FLANNEL-POSTRTG
+EOM
+  read -r -d '' POSTROUTING_RULES_LEADER << EOM
+-N FLANNEL-POSTRTG
+EOM
+  read -r -d '' FORWARD_RULES << EOM
+-P FORWARD ACCEPT
+-A FORWARD -m conntrack --ctstate NEW -m comment --comment "kubernetes load balancer firewall" -j KUBE-PROXY-FIREWALL
+-A FORWARD -m comment --comment "kubernetes forwarding rules" -j KUBE-FORWARD
+-A FORWARD -m conntrack --ctstate NEW -m comment --comment "kubernetes service portals" -j KUBE-SERVICES
+-A FORWARD -m conntrack --ctstate NEW -m comment --comment "kubernetes externally-visible service portals" -j KUBE-EXTERNAL-SERVICES
+-N FLANNEL-FWD
+EOM
+# check that masquerade & forward rules have been removed
+  assert_equals "$POSTROUTING_RULES_WORKER" \
+                "$(docker exec --privileged local-worker /usr/sbin/iptables -t nat -S POSTROUTING | grep FLANNEL)$(docker exec --privileged local-worker /usr/sbin/iptables -t nat -S FLANNEL-POSTRTG)" "Host 1 has not expected postrouting rules"
+  assert_equals "$POSTROUTING_RULES_LEADER" \
+                "$(docker exec --privileged local-leader /usr/sbin/iptables -t nat -S POSTROUTING | grep FLANNEL)$(docker exec --privileged local-leader /usr/sbin/iptables -t nat -S FLANNEL-POSTRTG)" "Host 2 has not expected postrouting rules"
+  assert_equals "$FORWARD_RULES" \
+                "$(docker exec --privileged local-worker /usr/sbin/iptables -t filter -S FORWARD)
+$(docker exec --privileged local-worker /usr/sbin/iptables -t filter -S FLANNEL-FWD -w 5)" "Host 1 has not expected forward rules"
+  assert_equals "$FORWARD_RULES" \
+                "$(docker exec --privileged local-leader /usr/sbin/iptables -t filter -S FORWARD)
+$(docker exec --privileged local-leader /usr/sbin/iptables -t filter -S FLANNEL-FWD)" "Host 2 has not expected forward rules"
+}
+
+###nftables
+check_nftables() {
+  local worker_podcidr=$(get_pod_cidr local-worker)
+  local leader_podcidr=$(get_pod_cidr local-leader)
+  read -d '' POSTROUTING_RULES_WORKER << EOM
+table ip flannel-ipv4 {
+	chain postrtg {
+		type nat hook postrouting priority srcnat; policy accept;
+		meta mark 0x00004000 return
+		ip saddr ${worker_podcidr} ip daddr 10.42.0.0/16 return
+		ip saddr 10.42.0.0/16 ip daddr ${worker_podcidr} return
+		ip saddr != ${worker_podcidr} ip daddr 10.42.0.0/16 return
+		ip saddr 10.42.0.0/16 ip daddr != 224.0.0.0/4 masquerade fully-random
+		ip saddr != 10.42.0.0/16 ip daddr 10.42.0.0/16 masquerade fully-random
+	}
+}
+EOM
+  read -r -d '' POSTROUTING_RULES_LEADER << EOM
+table ip flannel-ipv4 {
+	chain postrtg {
+		type nat hook postrouting priority srcnat; policy accept;
+		meta mark 0x00004000 return
+		ip saddr ${leader_podcidr} ip daddr 10.42.0.0/16 return
+		ip saddr 10.42.0.0/16 ip daddr ${leader_podcidr} return
+		ip saddr != ${leader_podcidr} ip daddr 10.42.0.0/16 return
+		ip saddr 10.42.0.0/16 ip daddr != 224.0.0.0/4 masquerade fully-random
+		ip saddr != 10.42.0.0/16 ip daddr 10.42.0.0/16 masquerade fully-random
+	}
+}
+EOM
+  read -r -d '' FORWARD_RULES << EOM
+table ip flannel-ipv4 {
+	chain forward {
+		type filter hook input priority filter; policy accept;
+		ip saddr 10.42.0.0/16 accept
+		ip daddr 10.42.0.0/16 accept
+	}
+}
+EOM
+  # check masquerade & forward rules
+  assert_equals "$POSTROUTING_RULES_WORKER" \
+                "$(docker exec --privileged local-worker /usr/sbin/nft list chain flannel-ipv4 postrtg)" "Node worker does not have expected postrouting rules"
+  assert_equals "$POSTROUTING_RULES_LEADER" \
+                "$(docker exec --privileged local-leader /usr/sbin/nft list chain flannel-ipv4 postrtg)" "Node leader does not have expected postrouting rules"
+  assert_equals "$FORWARD_RULES" \
+                "$(docker exec --privileged local-worker /usr/sbin/nft list chain flannel-ipv4 forward)" "Node worker does not have expected forward rules"
+  assert_equals "$FORWARD_RULES" \
+                "$(docker exec --privileged local-leader /usr/sbin/nft list chain flannel-ipv4 forward)" "Node leader does not have expected forward rules"
+}
+
+check_nftables_removed() {
+  # check masquerade & forward rules
+  assert_equals "" \
+                "$(docker exec --privileged local-worker /usr/sbin/nft list chain flannel-ipv4 postrtg)" "Node worker has unexpected postrouting rules"
+  assert_equals "" \
+                "$(docker exec --privileged local-leader /usr/sbin/nft list chain flannel-ipv4 postrtg)" "Node leader has unexpected postrouting rules"
+  assert_equals "" \
+                "$(docker exec --privileged local-worker /usr/sbin/nft list chain flannel-ipv4 forward)" "Node worker has unexpected forward rules"
+  assert_equals "" \
+                "$(docker exec --privileged local-leader /usr/sbin/nft list chain flannel-ipv4 forward)" "Node leader has unexpected forward rules"
+}

go.mod

@@ -35,6 +35,7 @@ require (
 	github.com/avast/retry-go/v4 v4.5.1
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.872
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc v1.0.872
+	sigs.k8s.io/knftables v0.0.14
 )
 
 require (

go.sum

@@ -255,6 +255,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
+github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
@@ -781,6 +783,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=
 sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/knftables v0.0.14 h1:VzKQoDMCGBOH8c85sGrWSXSPCS0XrIpEfOlcCLBXiC0=
+sigs.k8s.io/knftables v0.0.14/go.mod h1:f/5ZLKYEUPUhVjUCg6l80ACdL7CIIyeL0DxfgojGRTk=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=

images/Dockerfile

@@ -27,7 +27,7 @@ RUN export GOOS=$(xx-info os) &&\
 
 FROM alpine:20231219
 RUN apk update && apk upgrade
-RUN apk add --no-cache iproute2 ca-certificates iptables strongswan iptables-legacy && update-ca-certificates
+RUN apk add --no-cache iproute2 ca-certificates nftables iptables strongswan iptables-legacy && update-ca-certificates
 RUN apk add wireguard-tools --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/community
 COPY --from=build /build/dist/flanneld /opt/bin/flanneld
 COPY dist/mk-docker-opts.sh /opt/bin/

main.go

@@ -36,6 +36,7 @@ import (
 	"github.com/flannel-io/flannel/pkg/subnet/kube"
 	"github.com/flannel-io/flannel/pkg/trafficmngr"
 	"github.com/flannel-io/flannel/pkg/trafficmngr/iptables"
+	"github.com/flannel-io/flannel/pkg/trafficmngr/nftables"
 	"github.com/flannel-io/flannel/pkg/version"
 	"golang.org/x/net/context"
 	log "k8s.io/klog/v2"
@@ -336,7 +337,15 @@ func main() {
 	}
 
 	//Create TrafficManager and instanciate it based on whether we use iptables or nftables
-	trafficMngr := newTrafficManager()
+	trafficMngr := newTrafficManager(config.EnableNFTables)
+	err = trafficMngr.Init(ctx, &wg)
+	if err != nil {
+		log.Error(err)
+		cancel()
+		wg.Wait()
+		os.Exit(1)
+	}
+
 	flannelIPv4Net := ip.IP4Net{}
 	flannelIpv6Net := ip.IP6Net{}
 	if config.EnableIPv4 {
@@ -365,7 +374,8 @@ func main() {
 		prevIPv6Networks := ReadIP6CIDRsFromSubnetFile(opts.subnetFile, "FLANNEL_IPV6_NETWORK")
 		prevIPv6Subnet := ReadIP6CIDRFromSubnetFile(opts.subnetFile, "FLANNEL_IPV6_SUBNET")
 
-		err = trafficMngr.SetupAndEnsureMasqRules(flannelIPv4Net, prevSubnet,
+		err = trafficMngr.SetupAndEnsureMasqRules(ctx,
+			flannelIPv4Net, prevSubnet,
 			prevNetworks,
 			flannelIpv6Net, prevIPv6Subnet,
 			prevIPv6Networks,
@@ -383,7 +393,7 @@ func main() {
 	// In Docker 1.12 and earlier, the default FORWARD chain policy was ACCEPT.
 	// In Docker 1.13 and later, Docker sets the default policy of the FORWARD chain to DROP.
 	if opts.iptablesForwardRules {
-		trafficMngr.SetupAndEnsureForwardRules(
+		trafficMngr.SetupAndEnsureForwardRules(ctx,
 			flannelIPv4Net,
 			flannelIpv6Net,
 			opts.iptablesResyncSeconds)
@@ -569,6 +579,13 @@ func ReadIP6CIDRsFromSubnetFile(path string, CIDRKey string) []ip.IP6Net {
 	return prevCIDRs
 }
 
-func newTrafficManager() trafficmngr.TrafficManager {
-	return iptables.IPTablesManager{}
+func newTrafficManager(useNftables bool) trafficmngr.TrafficManager {
+	if useNftables {
+		log.Info("Starting flannel in nftables mode")
+		return &nftables.NFTablesManager{}
+	} else {
+		log.Info("Starting flannel in iptables mode")
+		return &iptables.IPTablesManager{}
+
+	}
 }

pkg/subnet/config.go

@@ -27,20 +27,21 @@ import (
 )
 
 type Config struct {
-	EnableIPv4    bool
-	EnableIPv6    bool
-	Network       ip.IP4Net
-	IPv6Network   ip.IP6Net
-	Networks      []ip.IP4Net
-	IPv6Networks  []ip.IP6Net
-	SubnetMin     ip.IP4
-	SubnetMax     ip.IP4
-	IPv6SubnetMin *ip.IP6
-	IPv6SubnetMax *ip.IP6
-	SubnetLen     uint
-	IPv6SubnetLen uint
-	BackendType   string          `json:"-"`
-	Backend       json.RawMessage `json:",omitempty"`
+	EnableIPv4     bool
+	EnableIPv6     bool
+	EnableNFTables bool
+	Network        ip.IP4Net
+	IPv6Network    ip.IP6Net
+	Networks       []ip.IP4Net
+	IPv6Networks   []ip.IP6Net
+	SubnetMin      ip.IP4
+	SubnetMax      ip.IP4
+	IPv6SubnetMin  *ip.IP6
+	IPv6SubnetMax  *ip.IP6
+	SubnetLen      uint
+	IPv6SubnetLen  uint
+	BackendType    string          `json:"-"`
+	Backend        json.RawMessage `json:",omitempty"`
 }
 
 func parseBackendType(be json.RawMessage) (string, error) {