Update Node's MAC address to the Node's annotation for direct routing

To bypass Windows host network when forwarding Pod egress traffic in
noencap mode, antrea-agent needs to know peer Nodes' MAC addresses so
that it can configure Openflow rules which route the packets to underlay
network via uplink interface directly.

To discover the MAC addresses, the PR makes each antrea-agent report its
uplink's MAC address to the Node's annotation, then other agents can get
it when the NodeRouteController configures route/flows to this Node.

Signed-off-by: Quan Tian <qtian@vmware.com>

build/yamls/antrea-aks.yml

@@ -3052,6 +3052,7 @@ rules:
   - get
   - watch
   - list
+  - patch
 - apiGroups:
   - ""
   resources:

build/yamls/antrea-eks.yml

@@ -3052,6 +3052,7 @@ rules:
   - get
   - watch
   - list
+  - patch
 - apiGroups:
   - ""
   resources:

build/yamls/antrea-gke.yml

@@ -3052,6 +3052,7 @@ rules:
   - get
   - watch
   - list
+  - patch
 - apiGroups:
   - ""
   resources:

build/yamls/antrea-ipsec.yml

@@ -3052,6 +3052,7 @@ rules:
   - get
   - watch
   - list
+  - patch
 - apiGroups:
   - ""
   resources:

build/yamls/antrea.yml

@@ -3052,6 +3052,7 @@ rules:
   - get
   - watch
   - list
+  - patch
 - apiGroups:
   - ""
   resources:

build/yamls/base/agent-rbac.yml

@@ -18,6 +18,7 @@ rules:
       - get
       - watch
       - list
+      - patch
   - apiGroups:
       - ""
     resources:

pkg/agent/agent.go

@@ -16,6 +16,7 @@ package agent
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"os"
@@ -25,8 +26,10 @@ import (
 
 	"github.com/containernetworking/plugins/pkg/ip"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apitypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/retry"
 	"k8s.io/klog/v2"
 
 	"antrea.io/antrea/pkg/agent/cniserver"
@@ -53,6 +56,9 @@ const (
 	maxRetryForRoundNumSave = 5
 )
 
+// getIPNetDeviceFromIP is meant to be overridden for testing.
+var getIPNetDeviceFromIP = util.GetIPNetDeviceFromIP
+
 // Initializer knows how to setup host networking, OpenVSwitch, and Openflow.
 type Initializer struct {
 	client          clientset.Interface
@@ -637,11 +643,31 @@ func (i *Initializer) initNodeLocalConfig() error {
 
 	ipAddr, err := noderoute.GetNodeAddr(node)
 	if err != nil {
-		return fmt.Errorf("failed to obtain local IP address from k8s: %w", err)
+		return fmt.Errorf("failed to obtain local IP address from K8s: %w", err)
 	}
-	localAddr, localIntf, err := util.GetIPNetDeviceFromIP(ipAddr)
+	localAddr, localIntf, err := getIPNetDeviceFromIP(ipAddr)
 	if err != nil {
-		return fmt.Errorf("failed to get local IPNet:  %v", err)
+		return fmt.Errorf("failed to get local IPNet device with IP %v: %v", ipAddr, err)
+	}
+
+	// Update the Node's MAC address in the annotations of the Node. The MAC address will be used for direct routing by
+	// OVS in noencap case on Windows Nodes. As a mixture of Linux and Windows nodes is possible, Linux Nodes' MAC
+	// addresses should be reported too to make them discoverable for Windows Nodes.
+	if i.networkConfig.TrafficEncapMode.SupportsNoEncap() {
+		klog.Infof("Updating Node MAC annotation")
+		patch, _ := json.Marshal(map[string]interface{}{
+			"metadata": map[string]interface{}{
+				"annotations": map[string]string{
+					types.NodeMACAddressAnnotationKey: localIntf.HardwareAddr.String(),
+				},
+			},
+		})
+		if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			_, err := i.client.CoreV1().Nodes().Patch(context.TODO(), nodeName, apitypes.MergePatchType, patch, metav1.PatchOptions{})
+			return err
+		}); err != nil {
+			return err
+		}
 	}
 
 	i.nodeConfig = &config.NodeConfig{

pkg/agent/agent_test.go

@@ -15,19 +15,26 @@
 package agent
 
 import (
+	"context"
 	"fmt"
 	"net"
+	"os"
 	"testing"
 
 	mock "github.com/golang/mock/gomock"
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
 
 	"antrea.io/antrea/pkg/agent/cniserver"
 	"antrea.io/antrea/pkg/agent/config"
 	"antrea.io/antrea/pkg/agent/interfacestore"
+	"antrea.io/antrea/pkg/agent/types"
 	"antrea.io/antrea/pkg/ovs/ovsconfig"
 	ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing"
+	"antrea.io/antrea/pkg/util/env"
 )
 
 func newAgentInitializer(ovsBridgeClient ovsconfig.OVSBridgeClient, ifaceStore interfacestore.InterfaceStore) *Initializer {
@@ -134,3 +141,155 @@ func TestGetRoundInfo(t *testing.T) {
 	roundInfo = getRoundInfo(mockOVSBridgeClient)
 	assert.Equal(t, uint64(initialRoundNum), roundInfo.RoundNum, "Unexpected round number")
 }
+
+func TestInitNodeLocalConfig(t *testing.T) {
+	nodeName := "node1"
+	ovsBridge := "br-int"
+	nodeIPStr := "192.168.10.10"
+	_, nodeIPNet, _ := net.ParseCIDR("192.168.10.10/24")
+	macAddr, _ := net.ParseMAC("00:00:5e:00:53:01")
+	ipDevice := &net.Interface{
+		Index:        10,
+		MTU:          1500,
+		Name:         "ens160",
+		HardwareAddr: macAddr,
+	}
+	podCIDRStr := "172.16.10.0/24"
+	_, podCIDR, _ := net.ParseCIDR(podCIDRStr)
+	tests := []struct {
+		name                   string
+		trafficEncapMode       config.TrafficEncapModeType
+		tunnelType             ovsconfig.TunnelType
+		mtu                    int
+		expectedNodeConfig     config.NodeConfig
+		expectedNodeAnnotation map[string]string
+	}{
+		{
+			name:             "noencap mode",
+			trafficEncapMode: config.TrafficEncapModeNoEncap,
+			mtu:              0,
+			expectedNodeConfig: config.NodeConfig{
+				Name:            nodeName,
+				OVSBridge:       ovsBridge,
+				DefaultTunName:  defaultTunInterfaceName,
+				PodIPv4CIDR:     podCIDR,
+				NodeIPAddr:      nodeIPNet,
+				NodeMTU:         1500,
+				UplinkNetConfig: new(config.AdapterNetConfig),
+			},
+			expectedNodeAnnotation: map[string]string{types.NodeMACAddressAnnotationKey: macAddr.String()},
+		},
+		{
+			name:             "hybrid mode",
+			trafficEncapMode: config.TrafficEncapModeHybrid,
+			expectedNodeConfig: config.NodeConfig{
+				Name:            nodeName,
+				OVSBridge:       ovsBridge,
+				DefaultTunName:  defaultTunInterfaceName,
+				PodIPv4CIDR:     podCIDR,
+				NodeIPAddr:      nodeIPNet,
+				NodeMTU:         1500,
+				UplinkNetConfig: new(config.AdapterNetConfig),
+			},
+			expectedNodeAnnotation: map[string]string{types.NodeMACAddressAnnotationKey: macAddr.String()},
+		},
+		{
+			name:             "encap mode",
+			trafficEncapMode: config.TrafficEncapModeEncap,
+			expectedNodeConfig: config.NodeConfig{
+				Name:            nodeName,
+				OVSBridge:       ovsBridge,
+				DefaultTunName:  defaultTunInterfaceName,
+				PodIPv4CIDR:     podCIDR,
+				NodeIPAddr:      nodeIPNet,
+				NodeMTU:         1500,
+				UplinkNetConfig: new(config.AdapterNetConfig),
+			},
+			expectedNodeAnnotation: nil,
+		},
+		{
+			name:             "encap mode, geneve tunnel",
+			trafficEncapMode: config.TrafficEncapModeEncap,
+			tunnelType:       ovsconfig.GeneveTunnel,
+			expectedNodeConfig: config.NodeConfig{
+				Name:            nodeName,
+				OVSBridge:       ovsBridge,
+				DefaultTunName:  defaultTunInterfaceName,
+				PodIPv4CIDR:     podCIDR,
+				NodeIPAddr:      nodeIPNet,
+				NodeMTU:         1450,
+				UplinkNetConfig: new(config.AdapterNetConfig),
+			},
+			expectedNodeAnnotation: nil,
+		},
+		{
+			name:             "encap mode, mtu specified",
+			trafficEncapMode: config.TrafficEncapModeEncap,
+			tunnelType:       ovsconfig.GeneveTunnel,
+			mtu:              1400,
+			expectedNodeConfig: config.NodeConfig{
+				Name:            nodeName,
+				OVSBridge:       ovsBridge,
+				DefaultTunName:  defaultTunInterfaceName,
+				PodIPv4CIDR:     podCIDR,
+				NodeIPAddr:      nodeIPNet,
+				NodeMTU:         1400,
+				UplinkNetConfig: new(config.AdapterNetConfig),
+			},
+			expectedNodeAnnotation: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			node := &corev1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: nodeName,
+				},
+				Spec: corev1.NodeSpec{
+					PodCIDR: podCIDRStr,
+				},
+				Status: corev1.NodeStatus{
+					Addresses: []corev1.NodeAddress{
+						{
+							Type:    corev1.NodeInternalIP,
+							Address: nodeIPStr,
+						},
+					},
+				},
+			}
+			client := fake.NewSimpleClientset(node)
+			ifaceStore := interfacestore.NewInterfaceStore()
+			defer mockGetIPNetDeviceFromIP(nodeIPNet, ipDevice)()
+			defer mockNodeNameEnv(nodeName)()
+
+			initializer := &Initializer{
+				client:     client,
+				ifaceStore: ifaceStore,
+				mtu:        tt.mtu,
+				ovsBridge:  ovsBridge,
+				networkConfig: &config.NetworkConfig{
+					TrafficEncapMode: tt.trafficEncapMode,
+					TunnelType:       tt.tunnelType,
+				},
+			}
+			assert.NoError(t, initializer.initNodeLocalConfig())
+			assert.Equal(t, tt.expectedNodeConfig, *initializer.nodeConfig)
+			node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
+			assert.NoError(t, err)
+			assert.Equal(t, tt.expectedNodeAnnotation, node.Annotations)
+		})
+	}
+}
+
+func mockGetIPNetDeviceFromIP(ipNet *net.IPNet, ipDevice *net.Interface) func() {
+	prevGetIPNetDeviceFromIP := getIPNetDeviceFromIP
+	getIPNetDeviceFromIP = func(localIP net.IP) (*net.IPNet, *net.Interface, error) {
+		return ipNet, ipDevice, nil
+	}
+	return func() { getIPNetDeviceFromIP = prevGetIPNetDeviceFromIP }
+}
+
+func mockNodeNameEnv(name string) func() {
+	_ = os.Setenv(env.NodeNameEnvKey, name)
+	return func() { os.Unsetenv(env.NodeNameEnvKey) }
+}

pkg/agent/agent_windows.go

@@ -71,7 +71,7 @@ func (i *Initializer) prepareHostNetwork() error {
 	// Create HNS network.
 	subnetCIDR := i.nodeConfig.PodIPv4CIDR
 	if subnetCIDR == nil {
-		return fmt.Errorf("Failed to find valid IPv4 PodCIDR")
+		return fmt.Errorf("failed to find valid IPv4 PodCIDR")
 	}
 	return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeIPAddr, adapter)
 }
@@ -82,7 +82,7 @@ func (i *Initializer) prepareOVSBridge() error {
 	hnsNetwork, err := hcsshim.GetHNSNetworkByName(util.LocalHNSNetwork)
 	defer func() {
 		// prepareOVSBridge only works on windows platform. The operation has a chance to fail on the first time agent
-		// starts up when OVS bridge uplink and local inteface have not been configured. If the operation fails, the
+		// starts up when OVS bridge uplink and local interface have not been configured. If the operation fails, the
 		// host can not communicate with external network. To make sure the agent can connect to API server in
 		// next retry, this step deletes OVS bridge and HNS network created previously which will restore the
 		// host network.
@@ -144,7 +144,7 @@ func (i *Initializer) prepareOVSBridge() error {
 
 	// Move network configuration of uplink interface to OVS bridge local interface.
 	// - The net configuration of uplink will be restored by OS if the attached HNS network is deleted.
-	// - When ovs-switchd is down, antrea-agent will disable OVS Extension. The OVS bridge local interface will work
+	// - When ovs-vswitchd is down, antrea-agent will disable OVS Extension. The OVS bridge local interface will work
 	//   like a normal interface on host and is responsible for forwarding host traffic.
 	if err = util.EnableHostInterface(brName); err != nil {
 		return err

pkg/agent/types/annotations.go

@@ -0,0 +1,20 @@
+// Copyright 2021 Antrea Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+const (
+	// NodeMACAddressAnnotationKey represents the key of the Node's MAC address in the Annotations of the Node.
+	NodeMACAddressAnnotationKey string = "node.antrea.io/mac-address"
+)

pkg/agent/util/net_windows.go

@@ -400,7 +400,7 @@ func GetLocalBroadcastIP(ipNet *net.IPNet) net.IP {
 	return lastAddr
 }
 
-// GetDefaultGatewayByInterfaceIndex returns the default gateway configured on the speicified interface.
+// GetDefaultGatewayByInterfaceIndex returns the default gateway configured on the specified interface.
 func GetDefaultGatewayByInterfaceIndex(ifIndex int) (string, error) {
 	cmd := fmt.Sprintf("$(Get-NetRoute -InterfaceIndex %d -DestinationPrefix 0.0.0.0/0 ).NextHop", ifIndex)
 	defaultGW, err := CallPSCommand(cmd)

pkg/util/env/env.go

@@ -21,9 +21,9 @@ import (
 	"k8s.io/klog/v2"
 )
 
-// nodeNameEnvKey is environment variable.
+// NodeNameEnvKey is environment variable.
 const (
-	nodeNameEnvKey     = "NODE_NAME"
+	NodeNameEnvKey     = "NODE_NAME"
 	podNameEnvKey      = "POD_NAME"
 	podNamespaceEnvKey = "POD_NAMESPACE"
 	svcAcctNameEnvKey  = "SERVICEACCOUNT_NAME"
@@ -37,11 +37,11 @@ const (
 // - Environment variable NODE_NAME, which should be set by Downward API
 // - OS's hostname
 func GetNodeName() (string, error) {
-	nodeName := os.Getenv(nodeNameEnvKey)
+	nodeName := os.Getenv(NodeNameEnvKey)
 	if nodeName != "" {
 		return nodeName, nil
 	}
-	klog.Infof("Environment variable %s not found, using hostname instead", nodeNameEnvKey)
+	klog.Infof("Environment variable %s not found, using hostname instead", NodeNameEnvKey)
 	var err error
 	nodeName, err = os.Hostname()
 	if err != nil {

pkg/util/env/env_test.go

@@ -39,8 +39,8 @@ func TestGetNodeName(t *testing.T) {
 
 func compareNodeName(k, v string, t *testing.T) {
 	if k != "" {
-		_ = os.Setenv(nodeNameEnvKey, k)
-		defer os.Unsetenv(nodeNameEnvKey)
+		_ = os.Setenv(NodeNameEnvKey, k)
+		defer os.Unsetenv(NodeNameEnvKey)
 	}
 	nodeName, err := GetNodeName()
 	if err != nil {