allow primary nic for secondary bridge

If primary nic is configured with secondary bridge antrea-agent will
move primary nic configuration(IPs and Routes) to the secondary bridge,
and will revert the change on shutdown.

Signed-off-by: Daman Arora <aroradaman@gmail.com>

cmd/antrea-agent/agent.go

@@ -728,6 +728,7 @@ func run(o *Options) error {
 	}
 
 	if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) {
+		defer secondarynetwork.RestoreOVSBridge(&o.config.SecondaryNetwork)
 		if err := secondarynetwork.Initialize(
 			o.config.ClientConnection, o.config.KubeAPIServerOverride,
 			k8sClient, localPodInformer.Get(), nodeConfig.Name,

pkg/agent/agent_linux.go

@@ -166,30 +166,6 @@ func (i *Initializer) saveHostRoutes() error {
 	return nil
 }
 
-// restoreHostRoutes restores the host routes which are lost when moving the IP
-// configuration of uplink interface to the OVS bridge interface during
-// the Antrea bridge initialization stage.
-// The backup routes are restored after the IP configuration changes.
-func (i *Initializer) restoreHostRoutes() error {
-	return i.restoreHostRoutesToInterface(i.nodeConfig.UplinkNetConfig.Name)
-}
-
-func (i *Initializer) restoreHostRoutesToInterface(ifaceName string) error {
-	iface, err := net.InterfaceByName(ifaceName)
-	if err != nil {
-		return nil
-	}
-	for _, routeInterface := range i.nodeConfig.UplinkNetConfig.Routes {
-		route := routeInterface.(netlink.Route)
-		newRoute := route
-		newRoute.LinkIndex = iface.Index
-		if err := netlink.RouteReplace(&newRoute); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func (i *Initializer) ConnectUplinkToOVSBridge() error {
 	// Return immediately on Linux if connectUplinkToBridge is false.
 	if !i.connectUplinkToBridge {
@@ -198,18 +174,24 @@ func (i *Initializer) ConnectUplinkToOVSBridge() error {
 	klog.InfoS("Bridging uplink to OVS bridge")
 	var err error
 	uplinkNetConfig := i.nodeConfig.UplinkNetConfig
-	uplinkName := uplinkNetConfig.Name
-	bridgedUplinkName := util.GenerateUplinkInterfaceName(uplinkNetConfig.Name)
-	uplinkIPs := uplinkNetConfig.IPs
 
-	// If the uplink port already exists, just return.
-	if uplinkOFPort, err := i.ovsBridgeClient.GetOFPort(bridgedUplinkName, false); err == nil {
-		klog.InfoS("Uplink already exists, skip the configuration", "uplink", bridgedUplinkName, "port", uplinkOFPort)
-		return nil
+	externalIDs := map[string]interface{}{
+		interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaHost,
 	}
-
-	if err := util.RenameInterface(uplinkName, bridgedUplinkName); err != nil {
-		return fmt.Errorf("failed to change uplink interface name: err=%w", err)
+	bridgedUplinkName, exists, err := util.PrepareHostInterfaceConnection(
+		i.ovsBridgeClient,
+		uplinkNetConfig.Name,
+		uplinkNetConfig.Index,
+		int32(i.nodeConfig.HostInterfaceOFPort),
+		uplinkNetConfig.MAC,
+		uplinkNetConfig.IPs,
+		uplinkNetConfig.Routes,
+		externalIDs)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return nil
 	}
 
 	// Create uplink port.
@@ -226,64 +208,6 @@ func (i *Initializer) ConnectUplinkToOVSBridge() error {
 	klog.InfoS("Allocated OpenFlow port for uplink interface", "port", bridgedUplinkName, "ofPort", uplinkOFPort)
 	uplinkInterface.OVSPortConfig = &interfacestore.OVSPortConfig{uplinkPortUUID, uplinkOFPort} //nolint: govet
 	i.ifaceStore.AddInterface(uplinkInterface)
-
-	// Create local port.
-	externalIDs := map[string]interface{}{
-		interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaHost,
-	}
-	if _, err = i.ovsBridgeClient.CreateInternalPort(uplinkName, int32(i.nodeConfig.HostInterfaceOFPort), uplinkNetConfig.MAC.String(), externalIDs); err != nil {
-		return fmt.Errorf("cannot create host interface port %s: err=%w", uplinkName, err)
-	}
-
-	// Move network configuration of uplink interface to OVS bridge local interface.
-	// The net configuration of uplink will be restored by RestoreOVSBridge when shutting down.
-	wait.PollUntilContextTimeout(context.TODO(), 100*time.Millisecond, 10000*time.Millisecond, true,
-		func(ctx context.Context) (bool, error) {
-			// Wait a few seconds for OVS bridge local port.
-			link, err := netlink.LinkByName(uplinkName)
-			if err != nil {
-				klog.V(4).InfoS("OVS bridge local port is not ready", "port", uplinkName, "err", err)
-				return false, nil
-			}
-			klog.InfoS("OVS bridge local port is ready", "type", link.Type(), "attrs", link.Attrs())
-			return true, nil
-		})
-	localLink, err := netlink.LinkByName(uplinkName)
-	if err != nil {
-		return err
-	}
-	if _, _, err = util.SetLinkUp(uplinkName); err != nil {
-		return err
-	}
-
-	// Check if uplink is configured with an IPv6 address: if it is, we need to ensure that IPv6
-	// is enabled on the OVS internal port as we need to move all IP addresses over.
-	uplinkHasIPv6Address := false
-	for _, ip := range uplinkIPs {
-		if ip.IP.To4() == nil {
-			uplinkHasIPv6Address = true
-			break
-		}
-	}
-	if uplinkHasIPv6Address {
-		klog.InfoS("Uplink has IPv6 address, ensuring that IPv6 is enabled on bridge local port", "port", uplinkName)
-		if err := util.EnsureIPv6EnabledOnInterface(uplinkName); err != nil {
-			klog.ErrorS(err, "Failed to ensure that IPv6 is enabled on bridge local port, moving uplink IPs to bridge is likely to fail", "port", uplinkName)
-		}
-	}
-
-	if err = util.ConfigureLinkAddresses(localLink.Attrs().Index, uplinkIPs); err != nil {
-		return err
-	}
-	if err = util.ConfigureLinkAddresses(uplinkNetConfig.Index, nil); err != nil {
-		return err
-	}
-	// Restore the host routes which are lost when moving the network configuration of the
-	// uplink interface to OVS bridge interface.
-	if err = i.restoreHostRoutes(); err != nil {
-		return err
-	}
-
 	return nil
 }
 
@@ -294,33 +218,15 @@ func (i *Initializer) RestoreOVSBridge() {
 		return
 	}
 	klog.InfoS("Restoring bridge config to uplink...")
-	uplinkNetConfig := i.nodeConfig.UplinkNetConfig
-	uplinkName := ""
-	bridgedUplinkName := ""
-	if uplinkNetConfig != nil {
-		uplinkName = uplinkNetConfig.Name
-		bridgedUplinkName = util.GenerateUplinkInterfaceName(uplinkName)
-	}
-	brName := i.ovsBridge
 
-	if uplinkName != "" {
-		uplinkIPs := uplinkNetConfig.IPs
-		if err := util.DeleteOVSPort(brName, uplinkName); err != nil {
-			klog.ErrorS(err, "Delete OVS port failed", "port", uplinkName)
-		}
-		if err := util.DeleteOVSPort(brName, bridgedUplinkName); err != nil {
-			klog.ErrorS(err, "Delete OVS port failed", "port", bridgedUplinkName)
-		}
-		if err := util.RenameInterface(bridgedUplinkName, uplinkName); err != nil {
-			klog.ErrorS(err, "Restore uplink name failed", "uplink", bridgedUplinkName)
-		}
-		if err := util.ConfigureLinkAddresses(uplinkNetConfig.Index, uplinkIPs); err != nil {
-			klog.ErrorS(err, "Configure IP to uplink failed", "uplink", uplinkName)
-		}
-		if err := i.restoreHostRoutesToInterface(uplinkName); err != nil {
-			klog.ErrorS(err, "Configure route to uplink interface failed", "uplink", uplinkName)
-		}
+	_, ifaceIPs, ifaceRoutes, err := util.GetInterfaceConfig(i.nodeConfig.UplinkNetConfig.Name)
+	if err != nil {
+		klog.ErrorS(err, "Failed to get interface config", "interface", i.nodeConfig.UplinkNetConfig.Name)
+		ifaceIPs = i.nodeConfig.UplinkNetConfig.IPs
+		ifaceRoutes = i.nodeConfig.UplinkNetConfig.Routes
 	}
+
+	util.RestoreHostInterfaceConfiguration(i.ovsBridge, i.nodeConfig.UplinkNetConfig.Name, ifaceIPs, ifaceRoutes)
 	klog.InfoS("Finished to restore bridge config to uplink...")
 }
 

pkg/agent/secondarynetwork/init.go

@@ -27,6 +27,7 @@ import (
 
 	"antrea.io/antrea/pkg/agent/interfacestore"
 	"antrea.io/antrea/pkg/agent/secondarynetwork/podwatch"
+	"antrea.io/antrea/pkg/agent/util"
 	agentconfig "antrea.io/antrea/pkg/config/agent"
 	"antrea.io/antrea/pkg/ovs/ovsconfig"
 	"antrea.io/antrea/pkg/util/channel"
@@ -48,13 +49,34 @@ func Initialize(
 	nodeName string,
 	podUpdateSubscriber channel.Subscriber,
 	stopCh <-chan struct{},
-	config *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB) error {
+	secNetConfig *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB) error {
 
-	ovsBridgeClient, err := createOVSBridge(config.OVSBridges, ovsdb)
+	ovsBridgeClient, err := createOVSBridge(secNetConfig.OVSBridges, ovsdb)
 	if err != nil {
 		return err
 	}
 
+	// We only support moving and restoring of interface configuration to OVS Bridge for the single physical interface case.
+	phyInterfaces := make([]string, len(secNetConfig.OVSBridges[0].PhysicalInterfaces))
+	copy(phyInterfaces, secNetConfig.OVSBridges[0].PhysicalInterfaces)
+	if len(phyInterfaces) == 1 {
+		externalIDs := map[string]interface{}{
+			interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaHost,
+		}
+		ifaceName := phyInterfaces[0]
+		iface, ifaceIPs, ifaceRoutes, err := util.GetInterfaceConfig(ifaceName)
+		bridgedName, _, err := util.PrepareHostInterfaceConnection(ovsBridgeClient, iface.Name, iface.Index,
+			0, iface.HardwareAddr, ifaceIPs, ifaceRoutes, externalIDs)
+		if err != nil {
+			return err
+		}
+		phyInterfaces[0] = bridgedName
+	}
+
+	if err = connectPhyInterfacesToOVSBridge(ovsBridgeClient, phyInterfaces); err != nil {
+		return err
+	}
+
 	// Create the NetworkAttachmentDefinition client, which handles access to secondary network object
 	// definition from the API Server.
 	netAttachDefClient, err := createNetworkAttachDefClient(clientConnectionConfig, kubeAPIServerOverride)
@@ -74,38 +96,54 @@ func Initialize(
 	return nil
 }
 
-// TODO: check and update bridge configuration.
+// RestoreOVSBridge restores interface configuration from secondary-bridge back to host-interface.
+func RestoreOVSBridge(secNetConfig *agentconfig.SecondaryNetworkConfig) {
+	if len(secNetConfig.OVSBridges[0].PhysicalInterfaces) == 1 {
+		iface, ifaceIPs, ifaceRoutes, err := util.GetInterfaceConfig(secNetConfig.OVSBridges[0].PhysicalInterfaces[0])
+		if err != nil {
+			klog.ErrorS(err, "Failed to get interface config", "interface", secNetConfig.OVSBridges[0].PhysicalInterfaces[0])
+			return
+		}
+		util.RestoreHostInterfaceConfiguration(secNetConfig.OVSBridges[0].BridgeName, iface.Name, ifaceIPs, ifaceRoutes)
+	}
+}
+
 func createOVSBridge(bridges []agentconfig.OVSBridgeConfig, ovsdb *ovsdb.OVSDB) (ovsconfig.OVSBridgeClient, error) {
 	if len(bridges) == 0 {
 		return nil, nil
 	}
 	// Only one OVS bridge is supported.
 	bridgeConfig := bridges[0]
-
-	for _, phyInterface := range bridgeConfig.PhysicalInterfaces {
-		if _, err := interfaceByNameFn(phyInterface); err != nil {
-			return nil, fmt.Errorf("failed to get interface %s: %v", phyInterface, err)
-		}
-	}
-
 	ovsBridgeClient := newOVSBridgeFn(bridgeConfig.BridgeName, ovsconfig.OVSDatapathSystem, ovsdb)
 	if err := ovsBridgeClient.Create(); err != nil {
 		return nil, fmt.Errorf("failed to create OVS bridge %s: %v", bridgeConfig.BridgeName, err)
 	}
 	klog.InfoS("OVS bridge created", "bridge", bridgeConfig.BridgeName)
+	return ovsBridgeClient, nil
+}
+
+func connectPhyInterfacesToOVSBridge(ovsBridgeClient ovsconfig.OVSBridgeClient, phyInterfaces []string) error {
+	for _, phyInterface := range phyInterfaces {
+		if _, err := interfaceByNameFn(phyInterface); err != nil {
+			return fmt.Errorf("failed to get interface %s: %v", phyInterface, err)
+		}
+	}
 
-	for i, phyInterface := range bridgeConfig.PhysicalInterfaces {
+	externalIDs := map[string]interface{}{
+		interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaUplink,
+	}
+	for i, phyInterface := range phyInterfaces {
 		if _, err := ovsBridgeClient.GetOFPort(phyInterface, false); err == nil {
-			klog.V(2).InfoS("Physical interface already connected to OVS bridge, skip the configuration", "device", phyInterface, "bridge", bridgeConfig.BridgeName)
+			klog.V(2).InfoS("Physical interface already connected to secondary OVS bridge, skip the configuration", "device", phyInterface)
 			continue
 		}
 
-		if _, err := ovsBridgeClient.CreateUplinkPort(phyInterface, int32(i), map[string]interface{}{interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaUplink}); err != nil {
-			return nil, fmt.Errorf("failed to create OVS uplink port %s: %v", phyInterface, err)
+		if _, err := ovsBridgeClient.CreateUplinkPort(phyInterface, int32(i), externalIDs); err != nil {
+			return fmt.Errorf("failed to create OVS uplink port %s: %v", phyInterface, err)
 		}
-		klog.InfoS("Physical interface added to OVS bridge", "device", phyInterface, "bridge", bridgeConfig.BridgeName)
+		klog.InfoS("Physical interface added to secondary OVS bridge", "device", phyInterface)
 	}
-	return ovsBridgeClient, nil
+	return nil
 }
 
 // CreateNetworkAttachDefClient creates net-attach-def client handle from the given config.