From c76abd28958a2e2f772173fe76c32c2e48645cd8 Mon Sep 17 00:00:00 2001 From: Rahul Jain Date: Fri, 3 Jul 2020 08:44:18 +0530 Subject: [PATCH] Auto discovery mtu Discover mtu of primary interface if defaultMTU param is not set in the antrea.yaml --- cmd/antrea-agent/agent.go | 5 ++-- cmd/antrea-agent/options.go | 28 ------------------ pkg/agent/agent.go | 36 ++++++++++++++++++++---- pkg/agent/cniserver/server.go | 5 +--- pkg/agent/config/node_config.go | 13 +++++++++ pkg/agent/util/net.go | 23 +++++++++++++++ pkg/agent/util/net_linux.go | 23 --------------- pkg/agent/util/net_windows.go | 23 --------------- test/integration/agent/cniserver_test.go | 6 ++-- test/integration/agent/route_test.go | 3 +- 10 files changed, 74 insertions(+), 91 deletions(-) diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 4a9cbf86fab..48530bc7a6f 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -94,7 +94,8 @@ func run(o *Options) error { networkConfig := &config.NetworkConfig{ TunnelType: ovsconfig.TunnelType(o.config.TunnelType), TrafficEncapMode: encapMode, - EnableIPSecTunnel: o.config.EnableIPSecTunnel} + EnableIPSecTunnel: o.config.EnableIPSecTunnel, + DefaultMTU: o.config.DefaultMTU} routeClient, err := route.NewClient(serviceCIDRNet, encapMode) if err != nil { @@ -113,7 +114,6 @@ func run(o *Options) error { ifaceStore, o.config.OVSBridge, o.config.HostGateway, - o.config.DefaultMTU, serviceCIDRNet, networkConfig, features.DefaultFeatureGate.Enabled(features.AntreaProxy)) @@ -162,7 +162,6 @@ func run(o *Options) error { cniServer := cniserver.New( o.config.CNISocket, o.config.HostProcPathPrefix, - o.config.DefaultMTU, nodeConfig, k8sClient, podUpdates, diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index 82bbbf19c7f..9e95338e343 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -35,14 +35,6 @@ const ( defaultHostProcPathPrefix = "/host" defaultServiceCIDR = "10.96.0.0/12" defaultTunnelType = ovsconfig.GeneveTunnel - defaultMTUGeneve = 1450 - defaultMTUVXLAN = 1450 - defaultMTUGRE = 1462 - defaultMTUSTT = 1500 - defaultMTU = 1500 - // IPsec ESP can add a maximum of 38 bytes to the packet including the ESP - // header and trailer. - ipsecESPOverhead = 38 ) type Options struct { @@ -148,26 +140,6 @@ func (o *Options) setDefaults() { if o.config.TrafficEncapMode == "" { o.config.TrafficEncapMode = config.TrafficEncapModeEncap.String() } - - if o.config.DefaultMTU == 0 { - ok, encapMode := config.GetTrafficEncapModeFromStr(o.config.TrafficEncapMode) - if ok && !encapMode.SupportsEncap() { - o.config.DefaultMTU = defaultMTU - } else if o.config.TunnelType == ovsconfig.VXLANTunnel { - o.config.DefaultMTU = defaultMTUVXLAN - } else if o.config.TunnelType == ovsconfig.GeneveTunnel { - o.config.DefaultMTU = defaultMTUGeneve - } else if o.config.TunnelType == ovsconfig.GRETunnel { - o.config.DefaultMTU = defaultMTUGRE - } else if o.config.TunnelType == ovsconfig.STTTunnel { - o.config.DefaultMTU = defaultMTUSTT - } - - if o.config.EnableIPSecTunnel { - o.config.DefaultMTU -= ipsecESPOverhead - } - } - if o.config.APIPort == 0 { o.config.APIPort = apis.AntreaAgentAPIPort } diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 9d0ee46c8e1..b88f22613d4 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -62,7 +62,6 @@ type Initializer struct { ifaceStore interfacestore.InterfaceStore ovsBridge string hostGateway string // name of gateway port on the OVS bridge - mtu int // Pod network interface MTU serviceCIDR *net.IPNet // K8s Service ClusterIP CIDR networkConfig *config.NetworkConfig nodeConfig *config.NodeConfig @@ -77,7 +76,6 @@ func NewInitializer( ifaceStore interfacestore.InterfaceStore, ovsBridge string, hostGateway string, - mtu int, serviceCIDR *net.IPNet, networkConfig *config.NetworkConfig, enableProxy bool) *Initializer { @@ -89,7 +87,6 @@ func NewInitializer( routeClient: routeClient, ovsBridge: ovsBridge, hostGateway: hostGateway, - mtu: mtu, serviceCIDR: serviceCIDR, networkConfig: networkConfig, enableProxy: enableProxy, @@ -412,9 +409,9 @@ func (i *Initializer) setupGatewayInterface() error { // Idempotent operation to set the gateway's MTU: we perform this operation regardless of // whether or not the gateway interface already exists, as the desired MTU may change across // restarts. - klog.V(4).Infof("Setting gateway interface %s MTU to %d", i.hostGateway, i.mtu) + klog.V(4).Infof("Setting gateway interface %s MTU to %d", i.hostGateway, i.nodeConfig.NodeMTU) - i.ovsBridgeClient.SetInterfaceMTU(i.hostGateway, i.mtu) + i.ovsBridgeClient.SetInterfaceMTU(i.hostGateway, i.nodeConfig.NodeMTU) if err := i.configureGatewayInterface(gatewayIface); err != nil { return err } @@ -543,16 +540,20 @@ func (i *Initializer) initNodeLocalConfig() error { if err != nil { return fmt.Errorf("failed to obtain local IP address from k8s: %w", err) } - localAddr, _, err := util.GetIPNetDeviceFromIP(ipAddr) + localAddr, localIntf, err := util.GetIPNetDeviceFromIP(ipAddr) if err != nil { return fmt.Errorf("failed to get local IPNet: %v", err) } + mtu := i.getNodeMTU(localIntf) + klog.Infof("Setting node mtu=%d", mtu) + i.nodeConfig = &config.NodeConfig{ Name: nodeName, OVSBridge: i.ovsBridge, DefaultTunName: defaultTunInterfaceName, NodeIPAddr: localAddr, + NodeMTU: mtu, UplinkNetConfig: new(config.AdapterNetConfig)} if i.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { @@ -641,3 +642,26 @@ func getRoundInfo(bridgeClient ovsconfig.OVSBridgeClient) types.RoundInfo { return roundInfo } + +func (i *Initializer) getNodeMTU(localIntf *net.Interface) int { + if i.networkConfig.DefaultMTU != 0 { + return i.networkConfig.DefaultMTU + } + mtu := localIntf.MTU + if mtu <= 0 { + panic("Failed to fetch node mtu") + } + if i.networkConfig.TrafficEncapMode.SupportsEncap() { + if i.networkConfig.TunnelType == ovsconfig.VXLANTunnel { + mtu -= config.VXLANOverhead + } else if i.networkConfig.TunnelType == ovsconfig.GeneveTunnel { + mtu -= config.GeneveOverhead + } else if i.networkConfig.TunnelType == ovsconfig.GRETunnel { + mtu -= config.GREOverhead + } + } + if i.networkConfig.EnableIPSecTunnel { + mtu -= config.IpsecESPOverhead + } + return mtu +} diff --git a/pkg/agent/cniserver/server.go b/pkg/agent/cniserver/server.go index ed165719f07..96c63a43f25 100644 --- a/pkg/agent/cniserver/server.go +++ b/pkg/agent/cniserver/server.go @@ -93,7 +93,6 @@ type CNIServer struct { serverVersion string nodeConfig *config.NodeConfig hostProcPathPrefix string - defaultMTU int kubeClient clientset.Interface containerAccess *containerAccessArbitrator podConfigurator *podConfigurator @@ -182,7 +181,7 @@ func (s *CNIServer) loadNetworkConfig(request *cnipb.CniCmdRequest) (*CNIConfig, s.updateLocalIPAMSubnet(cniConfig) } if cniConfig.MTU == 0 { - cniConfig.MTU = s.defaultMTU + cniConfig.MTU = s.nodeConfig.NodeMTU } klog.Infof("Load network configurations: %v", cniConfig) return cniConfig, nil @@ -496,7 +495,6 @@ func (s *CNIServer) CmdCheck(_ context.Context, request *cnipb.CniCmdRequest) ( func New( cniSocket, hostProcPathPrefix string, - defaultMTU int, nodeConfig *config.NodeConfig, kubeClient clientset.Interface, podUpdates chan<- v1beta1.PodReference, @@ -509,7 +507,6 @@ func New( serverVersion: cni.AntreaCNIVersion, nodeConfig: nodeConfig, hostProcPathPrefix: hostProcPathPrefix, - defaultMTU: defaultMTU, kubeClient: kubeClient, containerAccess: newContainerAccessArbitrator(), podUpdates: podUpdates, diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go index def42a4935d..50db5679013 100644 --- a/pkg/agent/config/node_config.go +++ b/pkg/agent/config/node_config.go @@ -33,6 +33,15 @@ const ( BridgeOFPort = 0xfffffffe ) +const ( + VXLANOverhead = 50 + GeneveOverhead = 50 + GREOverhead = 38 + // IPsec ESP can add a maximum of 38 bytes to the packet including the ESP + // header and trailer. + IpsecESPOverhead = 38 +) + type GatewayConfig struct { // Name is the name of host gateway, e.g. antrea-gw0. Name string @@ -69,6 +78,9 @@ type NodeConfig struct { PodCIDR *net.IPNet // The Node's IP used in Kubernetes. It has the network mask information. NodeIPAddr *net.IPNet + // Set either via defaultMTU config in antrea.yaml or auto discovered. + // Auto discovery will use mtu value of the node's primary interface (eth0). + NodeMTU int // The config of the gateway interface on the OVS bridge. GatewayConfig *GatewayConfig // The config of the OVS bridge uplink interface. Only for Windows Node. @@ -86,4 +98,5 @@ type NetworkConfig struct { TunnelType ovsconfig.TunnelType EnableIPSecTunnel bool IPSecPSK string + DefaultMTU int } diff --git a/pkg/agent/util/net.go b/pkg/agent/util/net.go index a06cfe4d0a5..195ced6294c 100644 --- a/pkg/agent/util/net.go +++ b/pkg/agent/util/net.go @@ -95,3 +95,26 @@ func listenUnix(address string) (net.Listener, error) { func dialUnix(address string) (net.Conn, error) { return net.Dial("unix", address) } + +// GetIPNetDeviceFromIP returns a local IP/mask and associated device from IP. +func GetIPNetDeviceFromIP(localIP net.IP) (*net.IPNet, *net.Interface, error) { + linkList, err := net.Interfaces() + if err != nil { + return nil, nil, err + } + + for _, link := range linkList { + addrList, err := link.Addrs() + if err != nil { + continue + } + for _, addr := range addrList { + if ipNet, ok := addr.(*net.IPNet); ok { + if ipNet.IP.Equal(localIP) { + return ipNet, &link, nil + } + } + } + } + return nil, nil, fmt.Errorf("unable to find local IP and device") +} diff --git a/pkg/agent/util/net_linux.go b/pkg/agent/util/net_linux.go index 43d865b0037..66aacbe88e6 100644 --- a/pkg/agent/util/net_linux.go +++ b/pkg/agent/util/net_linux.go @@ -25,32 +25,9 @@ import ( "github.com/containernetworking/plugins/pkg/ip" "github.com/containernetworking/plugins/pkg/ns" "github.com/vishvananda/netlink" - "golang.org/x/sys/unix" "k8s.io/klog" ) -// GetIPNetDeviceFromIP returns a local IP/mask and associated device from IP. -func GetIPNetDeviceFromIP(localIP net.IP) (*net.IPNet, netlink.Link, error) { - linkList, err := netlink.LinkList() - if err != nil { - return nil, nil, err - } - - for _, link := range linkList { - addrList, err := netlink.AddrList(link, unix.AF_INET) - if err != nil { - klog.Errorf("Failed to get addr list for device %s", link) - continue - } - for _, addr := range addrList { - if addr.IP.Equal(localIP) { - return addr.IPNet, link, nil - } - } - } - return nil, nil, fmt.Errorf("unable to find local IP and device") -} - // GetNetLink returns dev link from name. func GetNetLink(dev string) netlink.Link { link, err := netlink.LinkByName(dev) diff --git a/pkg/agent/util/net_windows.go b/pkg/agent/util/net_windows.go index 4e51f9323c6..046f974f2ee 100644 --- a/pkg/agent/util/net_windows.go +++ b/pkg/agent/util/net_windows.go @@ -202,29 +202,6 @@ func EnableHNSNetworkExtension(hnsNetID string, vSwitchExtension string) error { return nil } -// GetIPNetDeviceFromIP returns a local IP/mask and associated device from IP. -func GetIPNetDeviceFromIP(localIP net.IP) (*net.IPNet, *net.Interface, error) { - linkList, err := net.Interfaces() - if err != nil { - return nil, nil, err - } - - for _, link := range linkList { - addrList, err := link.Addrs() - if err != nil { - continue - } - for _, addr := range addrList { - if ipNet, ok := addr.(*net.IPNet); ok { - if ipNet.IP.Equal(localIP) { - return ipNet, &link, nil - } - } - } - } - return nil, nil, fmt.Errorf("unable to find local IP and device") -} - func SetLinkUp(name string) (net.HardwareAddr, int, error) { // Set host gateway interface up. if err := EnableHostInterface(name); err != nil { diff --git a/test/integration/agent/cniserver_test.go b/test/integration/agent/cniserver_test.go index d3b9ed36424..0cfdb78bcb7 100644 --- a/test/integration/agent/cniserver_test.go +++ b/test/integration/agent/cniserver_test.go @@ -537,9 +537,9 @@ func (tester *cmdAddDelTester) cmdDelTest(tc testCase, dataDir string) { func newTester() *cmdAddDelTester { tester := &cmdAddDelTester{} ifaceStore := interfacestore.NewInterfaceStore() + testNodeConfig.NodeMTU = 1450 tester.server = cniserver.New(testSock, "", - 1450, testNodeConfig, k8sFake.NewSimpleClientset(), make(chan v1beta1.PodReference, 100), @@ -679,7 +679,6 @@ func setupChainTest( routeMock = routetest.NewMockInterface(controller) server = cniserver.New(testSock, "", - 1500, testNodeConfig, k8sFake.NewSimpleClientset(), make(chan v1beta1.PodReference, 100), @@ -803,6 +802,7 @@ func init() { gwMAC, _ = net.ParseMAC("11:11:11:11:11:11") nodeGateway := &config.GatewayConfig{IP: gwIP, MAC: gwMAC, Name: ""} _, nodePodCIDR, _ := net.ParseCIDR("192.168.1.0/24") + nodeMTU := 1500 - testNodeConfig = &config.NodeConfig{Name: nodeName, PodCIDR: nodePodCIDR, GatewayConfig: nodeGateway} + testNodeConfig = &config.NodeConfig{Name: nodeName, PodCIDR: nodePodCIDR, NodeMTU: nodeMTU, GatewayConfig: nodeGateway} } diff --git a/test/integration/agent/route_test.go b/test/integration/agent/route_test.go index e2613c7a577..b6c2c8b4f5f 100644 --- a/test/integration/agent/route_test.go +++ b/test/integration/agent/route_test.go @@ -46,11 +46,12 @@ func ExecOutputTrim(cmd string) (string, error) { var ( _, podCIDR, _ = net.ParseCIDR("10.10.10.0/24") - nodeIP, nodeLink, _ = util.GetIPNetDeviceFromIP(func() net.IP { + nodeIP, nodeIntf, _ = util.GetIPNetDeviceFromIP(func() net.IP { conn, _ := net.Dial("udp", "8.8.8.8:80") defer conn.Close() return conn.LocalAddr().(*net.UDPAddr).IP }()) + nodeLink, _ = netlink.LinkByName(nodeIntf.Name) localPeerIP = ip.NextIP(nodeIP.IP) remotePeerIP = net.ParseIP("50.50.50.1") _, serviceCIDR, _ = net.ParseCIDR("200.200.0.0/16")