From 3824430b13485067827d98c4d0070908bf67b61b Mon Sep 17 00:00:00 2001 From: Quan Tian Date: Mon, 10 May 2021 20:47:57 +0800 Subject: [PATCH] Update Node's MAC address to the Node's annotation for direct routing To bypass Windows host network when forwarding Pod egress traffic in noencap mode, antrea-agent needs to know peer Nodes' MAC addresses so that it can configure Openflow rules which route the packets to underlay network via uplink interface directly. To discover the MAC addresses, the PR makes each antrea-agent report its uplink's MAC address to the Node's annotation, then other agents can get it when the NodeRouteController configures route/flows to this Node. Signed-off-by: Quan Tian --- build/yamls/antrea-aks.yml | 1 + build/yamls/antrea-eks.yml | 1 + build/yamls/antrea-gke.yml | 1 + build/yamls/antrea-ipsec.yml | 1 + build/yamls/antrea.yml | 1 + build/yamls/base/agent-rbac.yml | 1 + pkg/agent/agent.go | 32 +++++++- pkg/agent/agent_test.go | 129 ++++++++++++++++++++++++++++++++ pkg/agent/agent_windows.go | 6 +- pkg/agent/types/annotations.go | 20 +++++ pkg/agent/util/net_windows.go | 2 +- pkg/util/env/env.go | 8 +- pkg/util/env/env_test.go | 4 +- 13 files changed, 194 insertions(+), 13 deletions(-) create mode 100644 pkg/agent/types/annotations.go diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index fa491dceb90..c45dae3bdeb 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -3052,6 +3052,7 @@ rules: - get - watch - list + - patch - apiGroups: - "" resources: diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index d1672cdb57c..c3970ecf15e 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -3052,6 +3052,7 @@ rules: - get - watch - list + - patch - apiGroups: - "" resources: diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 6712e154d6c..34cf1416f94 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -3052,6 +3052,7 @@ rules: - get - watch - list + - patch - apiGroups: - "" resources: diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 3bcf2d04073..4693730eb5c 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -3052,6 +3052,7 @@ rules: - get - watch - list + - patch - apiGroups: - "" resources: diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index ef6e58ddcb9..43a57865b8c 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -3052,6 +3052,7 @@ rules: - get - watch - list + - patch - apiGroups: - "" resources: diff --git a/build/yamls/base/agent-rbac.yml b/build/yamls/base/agent-rbac.yml index d1b900d573e..13c94107750 100644 --- a/build/yamls/base/agent-rbac.yml +++ b/build/yamls/base/agent-rbac.yml @@ -18,6 +18,7 @@ rules: - get - watch - list + - patch - apiGroups: - "" resources: diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index a29fa056d86..b99a0c79040 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -16,6 +16,7 @@ package agent import ( "context" + "encoding/json" "fmt" "net" "os" @@ -25,8 +26,10 @@ import ( "github.com/containernetworking/plugins/pkg/ip" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apitypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" "k8s.io/klog/v2" "antrea.io/antrea/pkg/agent/cniserver" @@ -53,6 +56,9 @@ const ( maxRetryForRoundNumSave = 5 ) +// getIPNetDeviceFromIP is meant to be overridden for testing. +var getIPNetDeviceFromIP = util.GetIPNetDeviceFromIP + // Initializer knows how to setup host networking, OpenVSwitch, and Openflow. type Initializer struct { client clientset.Interface @@ -637,11 +643,31 @@ func (i *Initializer) initNodeLocalConfig() error { ipAddr, err := noderoute.GetNodeAddr(node) if err != nil { - return fmt.Errorf("failed to obtain local IP address from k8s: %w", err) + return fmt.Errorf("failed to obtain local IP address from K8s: %w", err) } - localAddr, localIntf, err := util.GetIPNetDeviceFromIP(ipAddr) + localAddr, localIntf, err := getIPNetDeviceFromIP(ipAddr) if err != nil { - return fmt.Errorf("failed to get local IPNet: %v", err) + return fmt.Errorf("failed to get local IPNet device with IP %v: %v", ipAddr, err) + } + + // Update the Node's MAC address in the annotations of the Node. The MAC address will be used for direct routing by + // OVS in noencap case on Windows Nodes. As a mixture of Linux and Windows nodes is possible, Linux Nodes' MAC + // addresses should be reported too to make them discoverable for Windows Nodes. + if i.networkConfig.TrafficEncapMode.SupportsNoEncap() { + klog.Infof("Updating Node MAC annotation") + patch, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": map[string]string{ + types.NodeMACAddressAnnotationKey: localIntf.HardwareAddr.String(), + }, + }, + }) + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + _, err := i.client.CoreV1().Nodes().Patch(context.TODO(), nodeName, apitypes.MergePatchType, patch, metav1.PatchOptions{}) + return err + }); err != nil { + return err + } } i.nodeConfig = &config.NodeConfig{ diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index 8eff9ae08ef..2f9df21718e 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -15,19 +15,27 @@ package agent import ( + "context" "fmt" "net" + "os" "testing" mock "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" "antrea.io/antrea/pkg/agent/cniserver" "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/interfacestore" + "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/ovs/ovsconfig" ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing" + "antrea.io/antrea/pkg/util/env" ) func newAgentInitializer(ovsBridgeClient ovsconfig.OVSBridgeClient, ifaceStore interfacestore.InterfaceStore) *Initializer { @@ -134,3 +142,124 @@ func TestGetRoundInfo(t *testing.T) { roundInfo = getRoundInfo(mockOVSBridgeClient) assert.Equal(t, uint64(initialRoundNum), roundInfo.RoundNum, "Unexpected round number") } + +func TestInitNodeLocalConfig(t *testing.T) { + nodeName := "node1" + ovsBridge := "br-int" + nodeIPStr := "192.168.10.10" + _, nodeIPNet, _ := net.ParseCIDR("192.168.10.10/24") + macAddr, _ := net.ParseMAC("00:00:5e:00:53:01") + ipDevice := &net.Interface{ + Index: 10, + MTU: 1500, + Name: "ens160", + HardwareAddr: macAddr, + } + podCIDRStr := "172.16.10.0/24" + _, podCIDR, _ := net.ParseCIDR(podCIDRStr) + tests := []struct { + name string + trafficEncapMode config.TrafficEncapModeType + tunnelType ovsconfig.TunnelType + mtu int + expectedMTU int + expectedNodeAnnotation map[string]string + }{ + { + name: "noencap mode", + trafficEncapMode: config.TrafficEncapModeNoEncap, + mtu: 0, + expectedMTU: 1500, + expectedNodeAnnotation: map[string]string{types.NodeMACAddressAnnotationKey: macAddr.String()}, + }, + { + name: "hybrid mode", + trafficEncapMode: config.TrafficEncapModeHybrid, + expectedMTU: 1500, + expectedNodeAnnotation: map[string]string{types.NodeMACAddressAnnotationKey: macAddr.String()}, + }, + { + name: "encap mode", + trafficEncapMode: config.TrafficEncapModeEncap, + expectedMTU: 1500, + expectedNodeAnnotation: nil, + }, + { + name: "encap mode, geneve tunnel", + trafficEncapMode: config.TrafficEncapModeEncap, + tunnelType: ovsconfig.GeneveTunnel, + expectedMTU: 1450, + expectedNodeAnnotation: nil, + }, + { + name: "encap mode, mtu specified", + trafficEncapMode: config.TrafficEncapModeEncap, + tunnelType: ovsconfig.GeneveTunnel, + mtu: 1400, + expectedMTU: 1400, + expectedNodeAnnotation: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + Spec: corev1.NodeSpec{ + PodCIDR: podCIDRStr, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: nodeIPStr, + }, + }, + }, + } + client := fake.NewSimpleClientset(node) + ifaceStore := interfacestore.NewInterfaceStore() + defer mockGetIPNetDeviceFromIP(nodeIPNet, ipDevice)() + defer mockNodeNameEnv(nodeName)() + + initializer := &Initializer{ + client: client, + ifaceStore: ifaceStore, + mtu: tt.mtu, + ovsBridge: ovsBridge, + networkConfig: &config.NetworkConfig{ + TrafficEncapMode: tt.trafficEncapMode, + TunnelType: tt.tunnelType, + }, + } + require.NoError(t, initializer.initNodeLocalConfig()) + expectedNodeConfig := config.NodeConfig{ + Name: nodeName, + OVSBridge: ovsBridge, + DefaultTunName: defaultTunInterfaceName, + PodIPv4CIDR: podCIDR, + NodeIPAddr: nodeIPNet, + NodeMTU: tt.expectedMTU, + UplinkNetConfig: new(config.AdapterNetConfig), + } + assert.Equal(t, expectedNodeConfig, *initializer.nodeConfig) + node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, tt.expectedNodeAnnotation, node.Annotations) + }) + } +} + +func mockGetIPNetDeviceFromIP(ipNet *net.IPNet, ipDevice *net.Interface) func() { + prevGetIPNetDeviceFromIP := getIPNetDeviceFromIP + getIPNetDeviceFromIP = func(localIP net.IP) (*net.IPNet, *net.Interface, error) { + return ipNet, ipDevice, nil + } + return func() { getIPNetDeviceFromIP = prevGetIPNetDeviceFromIP } +} + +func mockNodeNameEnv(name string) func() { + _ = os.Setenv(env.NodeNameEnvKey, name) + return func() { os.Unsetenv(env.NodeNameEnvKey) } +} diff --git a/pkg/agent/agent_windows.go b/pkg/agent/agent_windows.go index 9f3b3d4052a..2b93af516e5 100644 --- a/pkg/agent/agent_windows.go +++ b/pkg/agent/agent_windows.go @@ -71,7 +71,7 @@ func (i *Initializer) prepareHostNetwork() error { // Create HNS network. subnetCIDR := i.nodeConfig.PodIPv4CIDR if subnetCIDR == nil { - return fmt.Errorf("Failed to find valid IPv4 PodCIDR") + return fmt.Errorf("failed to find valid IPv4 PodCIDR") } return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeIPAddr, adapter) } @@ -82,7 +82,7 @@ func (i *Initializer) prepareOVSBridge() error { hnsNetwork, err := hcsshim.GetHNSNetworkByName(util.LocalHNSNetwork) defer func() { // prepareOVSBridge only works on windows platform. The operation has a chance to fail on the first time agent - // starts up when OVS bridge uplink and local inteface have not been configured. If the operation fails, the + // starts up when OVS bridge uplink and local interface have not been configured. If the operation fails, the // host can not communicate with external network. To make sure the agent can connect to API server in // next retry, this step deletes OVS bridge and HNS network created previously which will restore the // host network. @@ -144,7 +144,7 @@ func (i *Initializer) prepareOVSBridge() error { // Move network configuration of uplink interface to OVS bridge local interface. // - The net configuration of uplink will be restored by OS if the attached HNS network is deleted. - // - When ovs-switchd is down, antrea-agent will disable OVS Extension. The OVS bridge local interface will work + // - When ovs-vswitchd is down, antrea-agent will disable OVS Extension. The OVS bridge local interface will work // like a normal interface on host and is responsible for forwarding host traffic. if err = util.EnableHostInterface(brName); err != nil { return err diff --git a/pkg/agent/types/annotations.go b/pkg/agent/types/annotations.go new file mode 100644 index 00000000000..c835cb34e10 --- /dev/null +++ b/pkg/agent/types/annotations.go @@ -0,0 +1,20 @@ +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +const ( + // NodeMACAddressAnnotationKey represents the key of the Node's MAC address in the Annotations of the Node. + NodeMACAddressAnnotationKey string = "node.antrea.io/mac-address" +) diff --git a/pkg/agent/util/net_windows.go b/pkg/agent/util/net_windows.go index d8223f66d29..c95cd1a0d34 100644 --- a/pkg/agent/util/net_windows.go +++ b/pkg/agent/util/net_windows.go @@ -400,7 +400,7 @@ func GetLocalBroadcastIP(ipNet *net.IPNet) net.IP { return lastAddr } -// GetDefaultGatewayByInterfaceIndex returns the default gateway configured on the speicified interface. +// GetDefaultGatewayByInterfaceIndex returns the default gateway configured on the specified interface. func GetDefaultGatewayByInterfaceIndex(ifIndex int) (string, error) { cmd := fmt.Sprintf("$(Get-NetRoute -InterfaceIndex %d -DestinationPrefix 0.0.0.0/0 ).NextHop", ifIndex) defaultGW, err := CallPSCommand(cmd) diff --git a/pkg/util/env/env.go b/pkg/util/env/env.go index 253768d8754..e37606b6abe 100644 --- a/pkg/util/env/env.go +++ b/pkg/util/env/env.go @@ -21,9 +21,9 @@ import ( "k8s.io/klog/v2" ) -// nodeNameEnvKey is environment variable. +// NodeNameEnvKey is environment variable. const ( - nodeNameEnvKey = "NODE_NAME" + NodeNameEnvKey = "NODE_NAME" podNameEnvKey = "POD_NAME" podNamespaceEnvKey = "POD_NAMESPACE" svcAcctNameEnvKey = "SERVICEACCOUNT_NAME" @@ -37,11 +37,11 @@ const ( // - Environment variable NODE_NAME, which should be set by Downward API // - OS's hostname func GetNodeName() (string, error) { - nodeName := os.Getenv(nodeNameEnvKey) + nodeName := os.Getenv(NodeNameEnvKey) if nodeName != "" { return nodeName, nil } - klog.Infof("Environment variable %s not found, using hostname instead", nodeNameEnvKey) + klog.Infof("Environment variable %s not found, using hostname instead", NodeNameEnvKey) var err error nodeName, err = os.Hostname() if err != nil { diff --git a/pkg/util/env/env_test.go b/pkg/util/env/env_test.go index 8e76eab83ff..960ec62ad49 100644 --- a/pkg/util/env/env_test.go +++ b/pkg/util/env/env_test.go @@ -39,8 +39,8 @@ func TestGetNodeName(t *testing.T) { func compareNodeName(k, v string, t *testing.T) { if k != "" { - _ = os.Setenv(nodeNameEnvKey, k) - defer os.Unsetenv(nodeNameEnvKey) + _ = os.Setenv(NodeNameEnvKey, k) + defer os.Unsetenv(NodeNameEnvKey) } nodeName, err := GetNodeName() if err != nil {