Auto discovery mtu

Discover mtu of primary interface if defaultMTU param is not
set in the antrea.yaml

cmd/antrea-agent/agent.go

@@ -94,7 +94,8 @@ func run(o *Options) error {
 	networkConfig := &config.NetworkConfig{
 		TunnelType:        ovsconfig.TunnelType(o.config.TunnelType),
 		TrafficEncapMode:  encapMode,
-		EnableIPSecTunnel: o.config.EnableIPSecTunnel}
+		EnableIPSecTunnel: o.config.EnableIPSecTunnel,
+		DefaultMTU:        o.config.DefaultMTU}
 
 	routeClient, err := route.NewClient(serviceCIDRNet, encapMode)
 	if err != nil {
@@ -113,7 +114,6 @@ func run(o *Options) error {
 		ifaceStore,
 		o.config.OVSBridge,
 		o.config.HostGateway,
-		o.config.DefaultMTU,
 		serviceCIDRNet,
 		networkConfig,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy))
@@ -162,7 +162,6 @@ func run(o *Options) error {
 	cniServer := cniserver.New(
 		o.config.CNISocket,
 		o.config.HostProcPathPrefix,
-		o.config.DefaultMTU,
 		nodeConfig,
 		k8sClient,
 		podUpdates,

cmd/antrea-agent/options.go

@@ -35,14 +35,6 @@ const (
 	defaultHostProcPathPrefix = "/host"
 	defaultServiceCIDR        = "10.96.0.0/12"
 	defaultTunnelType         = ovsconfig.GeneveTunnel
-	defaultMTUGeneve          = 1450
-	defaultMTUVXLAN           = 1450
-	defaultMTUGRE             = 1462
-	defaultMTUSTT             = 1500
-	defaultMTU                = 1500
-	// IPsec ESP can add a maximum of 38 bytes to the packet including the ESP
-	// header and trailer.
-	ipsecESPOverhead = 38
 )
 
 type Options struct {
@@ -148,26 +140,6 @@ func (o *Options) setDefaults() {
 	if o.config.TrafficEncapMode == "" {
 		o.config.TrafficEncapMode = config.TrafficEncapModeEncap.String()
 	}
-
-	if o.config.DefaultMTU == 0 {
-		ok, encapMode := config.GetTrafficEncapModeFromStr(o.config.TrafficEncapMode)
-		if ok && !encapMode.SupportsEncap() {
-			o.config.DefaultMTU = defaultMTU
-		} else if o.config.TunnelType == ovsconfig.VXLANTunnel {
-			o.config.DefaultMTU = defaultMTUVXLAN
-		} else if o.config.TunnelType == ovsconfig.GeneveTunnel {
-			o.config.DefaultMTU = defaultMTUGeneve
-		} else if o.config.TunnelType == ovsconfig.GRETunnel {
-			o.config.DefaultMTU = defaultMTUGRE
-		} else if o.config.TunnelType == ovsconfig.STTTunnel {
-			o.config.DefaultMTU = defaultMTUSTT
-		}
-
-		if o.config.EnableIPSecTunnel {
-			o.config.DefaultMTU -= ipsecESPOverhead
-		}
-	}
-
 	if o.config.APIPort == 0 {
 		o.config.APIPort = apis.AntreaAgentAPIPort
 	}

pkg/agent/agent.go

@@ -62,7 +62,6 @@ type Initializer struct {
 	ifaceStore      interfacestore.InterfaceStore
 	ovsBridge       string
 	hostGateway     string     // name of gateway port on the OVS bridge
-	mtu             int        // Pod network interface MTU
 	serviceCIDR     *net.IPNet // K8s Service ClusterIP CIDR
 	networkConfig   *config.NetworkConfig
 	nodeConfig      *config.NodeConfig
@@ -77,7 +76,6 @@ func NewInitializer(
 	ifaceStore interfacestore.InterfaceStore,
 	ovsBridge string,
 	hostGateway string,
-	mtu int,
 	serviceCIDR *net.IPNet,
 	networkConfig *config.NetworkConfig,
 	enableProxy bool) *Initializer {
@@ -89,7 +87,6 @@ func NewInitializer(
 		routeClient:     routeClient,
 		ovsBridge:       ovsBridge,
 		hostGateway:     hostGateway,
-		mtu:             mtu,
 		serviceCIDR:     serviceCIDR,
 		networkConfig:   networkConfig,
 		enableProxy:     enableProxy,
@@ -412,9 +409,9 @@ func (i *Initializer) setupGatewayInterface() error {
 	// Idempotent operation to set the gateway's MTU: we perform this operation regardless of
 	// whether or not the gateway interface already exists, as the desired MTU may change across
 	// restarts.
-	klog.V(4).Infof("Setting gateway interface %s MTU to %d", i.hostGateway, i.mtu)
+	klog.V(4).Infof("Setting gateway interface %s MTU to %d", i.hostGateway, i.nodeConfig.NodeMTU)
 
-	i.ovsBridgeClient.SetInterfaceMTU(i.hostGateway, i.mtu)
+	i.ovsBridgeClient.SetInterfaceMTU(i.hostGateway, i.nodeConfig.NodeMTU)
 	if err := i.configureGatewayInterface(gatewayIface); err != nil {
 		return err
 	}
@@ -543,16 +540,23 @@ func (i *Initializer) initNodeLocalConfig() error {
 	if err != nil {
 		return fmt.Errorf("failed to obtain local IP address from k8s: %w", err)
 	}
-	localAddr, _, err := util.GetIPNetDeviceFromIP(ipAddr)
+	localAddr, localIntf, err := util.GetIPNetDeviceFromIP(ipAddr)
 	if err != nil {
 		return fmt.Errorf("failed to get local IPNet:  %v", err)
 	}
 
+	mtu, err := i.getNodeMTU(localIntf)
+	if err != nil {
+		return err
+	}
+	klog.Infof("Setting node mtu=%d", mtu)
+
 	i.nodeConfig = &config.NodeConfig{
 		Name:            nodeName,
 		OVSBridge:       i.ovsBridge,
 		DefaultTunName:  defaultTunInterfaceName,
 		NodeIPAddr:      localAddr,
+		NodeMTU:         mtu,
 		UplinkNetConfig: new(config.AdapterNetConfig)}
 
 	if i.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() {
@@ -641,3 +645,26 @@ func getRoundInfo(bridgeClient ovsconfig.OVSBridgeClient) types.RoundInfo {
 
 	return roundInfo
 }
+
+func (i *Initializer) getNodeMTU(localIntf *net.Interface) (int, error) {
+	if i.networkConfig.DefaultMTU != 0 {
+		return i.networkConfig.DefaultMTU, nil
+	}
+	mtu := localIntf.MTU
+	if mtu <= 0 {
+		return 0, fmt.Errorf("Failed to fetch node mtu : %v", mtu)
+	}
+	if i.networkConfig.TrafficEncapMode.SupportsEncap() {
+		if i.networkConfig.TunnelType == ovsconfig.VXLANTunnel {
+			mtu -= config.VXLANOverhead
+		} else if i.networkConfig.TunnelType == ovsconfig.GeneveTunnel {
+			mtu -= config.GeneveOverhead
+		} else if i.networkConfig.TunnelType == ovsconfig.GRETunnel {
+			mtu -= config.GREOverhead
+		}
+	}
+	if i.networkConfig.EnableIPSecTunnel {
+		mtu -= config.IpsecESPOverhead
+	}
+	return mtu, nil
+}

pkg/agent/cniserver/server.go

@@ -93,7 +93,6 @@ type CNIServer struct {
 	serverVersion        string
 	nodeConfig           *config.NodeConfig
 	hostProcPathPrefix   string
-	defaultMTU           int
 	kubeClient           clientset.Interface
 	containerAccess      *containerAccessArbitrator
 	podConfigurator      *podConfigurator
@@ -182,7 +181,7 @@ func (s *CNIServer) loadNetworkConfig(request *cnipb.CniCmdRequest) (*CNIConfig,
 		s.updateLocalIPAMSubnet(cniConfig)
 	}
 	if cniConfig.MTU == 0 {
-		cniConfig.MTU = s.defaultMTU
+		cniConfig.MTU = s.nodeConfig.NodeMTU
 	}
 	klog.Infof("Load network configurations: %v", cniConfig)
 	return cniConfig, nil
@@ -496,7 +495,6 @@ func (s *CNIServer) CmdCheck(_ context.Context, request *cnipb.CniCmdRequest) (
 
 func New(
 	cniSocket, hostProcPathPrefix string,
-	defaultMTU int,
 	nodeConfig *config.NodeConfig,
 	kubeClient clientset.Interface,
 	podUpdates chan<- v1beta1.PodReference,
@@ -509,7 +507,6 @@ func New(
 		serverVersion:        cni.AntreaCNIVersion,
 		nodeConfig:           nodeConfig,
 		hostProcPathPrefix:   hostProcPathPrefix,
-		defaultMTU:           defaultMTU,
 		kubeClient:           kubeClient,
 		containerAccess:      newContainerAccessArbitrator(),
 		podUpdates:           podUpdates,

pkg/agent/config/node_config.go

@@ -33,6 +33,15 @@ const (
 	BridgeOFPort = 0xfffffffe
 )
 
+const (
+	VXLANOverhead  = 50
+	GeneveOverhead = 50
+	GREOverhead    = 38
+	// IPsec ESP can add a maximum of 38 bytes to the packet including the ESP
+	// header and trailer.
+	IpsecESPOverhead = 38
+)
+
 type GatewayConfig struct {
 	// Name is the name of host gateway, e.g. antrea-gw0.
 	Name string
@@ -69,6 +78,10 @@ type NodeConfig struct {
 	PodCIDR *net.IPNet
 	// The Node's IP used in Kubernetes. It has the network mask information.
 	NodeIPAddr *net.IPNet
+	// Set either via defaultMTU config in antrea.yaml or auto discovered.
+	// Auto discovery will use mtu value of the node's primary interface (eth0).
+	// For Encap mode, node mtu will be adjusted to account for encap header.
+	NodeMTU int
 	// The config of the gateway interface on the OVS bridge.
 	GatewayConfig *GatewayConfig
 	// The config of the OVS bridge uplink interface. Only for Windows Node.
@@ -86,4 +99,5 @@ type NetworkConfig struct {
 	TunnelType        ovsconfig.TunnelType
 	EnableIPSecTunnel bool
 	IPSecPSK          string
+	DefaultMTU        int
 }

pkg/agent/util/net.go

@@ -95,3 +95,26 @@ func listenUnix(address string) (net.Listener, error) {
 func dialUnix(address string) (net.Conn, error) {
 	return net.Dial("unix", address)
 }
+
+// GetIPNetDeviceFromIP returns a local IP/mask and associated device from IP.
+func GetIPNetDeviceFromIP(localIP net.IP) (*net.IPNet, *net.Interface, error) {
+	linkList, err := net.Interfaces()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	for _, link := range linkList {
+		addrList, err := link.Addrs()
+		if err != nil {
+			continue
+		}
+		for _, addr := range addrList {
+			if ipNet, ok := addr.(*net.IPNet); ok {
+				if ipNet.IP.Equal(localIP) {
+					return ipNet, &link, nil
+				}
+			}
+		}
+	}
+	return nil, nil, fmt.Errorf("unable to find local IP and device")
+}

pkg/agent/util/net_linux.go

@@ -25,32 +25,9 @@ import (
 	"github.com/containernetworking/plugins/pkg/ip"
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/vishvananda/netlink"
-	"golang.org/x/sys/unix"
 	"k8s.io/klog"
 )
 
-// GetIPNetDeviceFromIP returns a local IP/mask and associated device from IP.
-func GetIPNetDeviceFromIP(localIP net.IP) (*net.IPNet, netlink.Link, error) {
-	linkList, err := netlink.LinkList()
-	if err != nil {
-		return nil, nil, err
-	}
-
-	for _, link := range linkList {
-		addrList, err := netlink.AddrList(link, unix.AF_INET)
-		if err != nil {
-			klog.Errorf("Failed to get addr list for device %s", link)
-			continue
-		}
-		for _, addr := range addrList {
-			if addr.IP.Equal(localIP) {
-				return addr.IPNet, link, nil
-			}
-		}
-	}
-	return nil, nil, fmt.Errorf("unable to find local IP and device")
-}
-
 // GetNetLink returns dev link from name.
 func GetNetLink(dev string) netlink.Link {
 	link, err := netlink.LinkByName(dev)

pkg/agent/util/net_windows.go

@@ -202,29 +202,6 @@ func EnableHNSNetworkExtension(hnsNetID string, vSwitchExtension string) error {
 	return nil
 }
 
-// GetIPNetDeviceFromIP returns a local IP/mask and associated device from IP.
-func GetIPNetDeviceFromIP(localIP net.IP) (*net.IPNet, *net.Interface, error) {
-	linkList, err := net.Interfaces()
-	if err != nil {
-		return nil, nil, err
-	}
-
-	for _, link := range linkList {
-		addrList, err := link.Addrs()
-		if err != nil {
-			continue
-		}
-		for _, addr := range addrList {
-			if ipNet, ok := addr.(*net.IPNet); ok {
-				if ipNet.IP.Equal(localIP) {
-					return ipNet, &link, nil
-				}
-			}
-		}
-	}
-	return nil, nil, fmt.Errorf("unable to find local IP and device")
-}
-
 func SetLinkUp(name string) (net.HardwareAddr, int, error) {
 	// Set host gateway interface up.
 	if err := EnableHostInterface(name); err != nil {

test/integration/agent/cniserver_test.go

@@ -537,9 +537,9 @@ func (tester *cmdAddDelTester) cmdDelTest(tc testCase, dataDir string) {
 func newTester() *cmdAddDelTester {
 	tester := &cmdAddDelTester{}
 	ifaceStore := interfacestore.NewInterfaceStore()
+	testNodeConfig.NodeMTU = 1450
 	tester.server = cniserver.New(testSock,
 		"",
-		1450,
 		testNodeConfig,
 		k8sFake.NewSimpleClientset(),
 		make(chan v1beta1.PodReference, 100),
@@ -679,7 +679,6 @@ func setupChainTest(
 		routeMock = routetest.NewMockInterface(controller)
 		server = cniserver.New(testSock,
 			"",
-			1500,
 			testNodeConfig,
 			k8sFake.NewSimpleClientset(),
 			make(chan v1beta1.PodReference, 100),
@@ -803,6 +802,7 @@ func init() {
 	gwMAC, _ = net.ParseMAC("11:11:11:11:11:11")
 	nodeGateway := &config.GatewayConfig{IP: gwIP, MAC: gwMAC, Name: ""}
 	_, nodePodCIDR, _ := net.ParseCIDR("192.168.1.0/24")
+	nodeMTU := 1500
 
-	testNodeConfig = &config.NodeConfig{Name: nodeName, PodCIDR: nodePodCIDR, GatewayConfig: nodeGateway}
+	testNodeConfig = &config.NodeConfig{Name: nodeName, PodCIDR: nodePodCIDR, NodeMTU: nodeMTU, GatewayConfig: nodeGateway}
 }

test/integration/agent/route_test.go

@@ -46,11 +46,12 @@ func ExecOutputTrim(cmd string) (string, error) {
 
 var (
 	_, podCIDR, _       = net.ParseCIDR("10.10.10.0/24")
-	nodeIP, nodeLink, _ = util.GetIPNetDeviceFromIP(func() net.IP {
+	nodeIP, nodeIntf, _ = util.GetIPNetDeviceFromIP(func() net.IP {
 		conn, _ := net.Dial("udp", "8.8.8.8:80")
 		defer conn.Close()
 		return conn.LocalAddr().(*net.UDPAddr).IP
 	}())
+	nodeLink, _       = netlink.LinkByName(nodeIntf.Name)
 	localPeerIP       = ip.NextIP(nodeIP.IP)
 	remotePeerIP      = net.ParseIP("50.50.50.1")
 	_, serviceCIDR, _ = net.ParseCIDR("200.200.0.0/16")