Skip to content

Commit

Permalink
Support Pod secondary interfaces on VLAN network
Browse files Browse the repository at this point in the history
This commit adds support for connecting Pod secondary interfaces to a
VLAN network. A Pod secondary interface configured with the VLAN
networkType will be connected to the secondary network OVS bridge, and
the specified VLAN tag will be set on the OVS port.
This commit mostly follows the existing SR-IOV secondary network
implementation, in which a Pod controller configures Pod secondary
interfaces based on the local Pod events. It thus inherits the existing
limitations, like secondary interface configuration is not persisted on
the Node and may get lost after antrea-agent restarts.

Signed-off-by: Jianjun Shen <shenj@vmware.com>
  • Loading branch information
jianjuns committed Aug 12, 2023
1 parent 2302cfe commit 5db5adf
Show file tree
Hide file tree
Showing 19 changed files with 896 additions and 374 deletions.
2 changes: 0 additions & 2 deletions cmd/antrea-agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -694,8 +694,6 @@ func run(o *Options) error {
if err := secondarynetwork.Initialize(
o.config.ClientConnection, o.config.KubeAPIServerOverride,
k8sClient, localPodInformer, nodeConfig.Name, cniPodInfoStore,
// safe to call given that cniServer.Initialize has been called already.
cniServer.GetPodConfigurator(),
stopCh,
&o.config.SecondaryNetwork, ovsdbConnection); err != nil {
return fmt.Errorf("failed to initialize secondary network: %v", err)
Expand Down
5 changes: 4 additions & 1 deletion pkg/agent/cniserver/interface_configuration_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,10 @@ func (ic *ifConfigurator) configureContainerLinkVeth(
mtu int,
result *current.Result,
) error {
hostIfaceName := util.GenerateContainerInterfaceName(podName, podNamespace, containerID)
// Include the container veth interface name in the name generation, as one Pod can have more
// than one interfaces inc. secondary interfaces, while the host interface name must be
// be unique.
hostIfaceName := util.GenerateContainerHostVethName(podName, podNamespace, containerID, containerIfaceName)

hostIface := &current.Interface{Name: hostIfaceName}
containerIface := &current.Interface{Name: containerIfaceName, Sandbox: containerNetNS}
Expand Down
5 changes: 3 additions & 2 deletions pkg/agent/cniserver/pod_configuration.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,9 @@ func newPodConfigurator(
gatewayMAC net.HardwareAddr,
ovsDatapathType ovsconfig.OVSDatapathType,
isOvsHardwareOffloadEnabled bool,
disableTXChecksumOffload bool,
podUpdateNotifier channel.Notifier,
podInfoStore cnipodcache.CNIPodInfoStore,
disableTXChecksumOffload bool,
) (*podConfigurator, error) {
ifConfigurator, err := newInterfaceConfigurator(ovsDatapathType, isOvsHardwareOffloadEnabled, disableTXChecksumOffload)
if err != nil {
Expand Down Expand Up @@ -259,7 +259,8 @@ func (pc *podConfigurator) configureInterfaces(
// Note that the IP address should be advertised after Pod OpenFlow entries are installed, otherwise the packet might
// be dropped by OVS.
if err = pc.ifConfigurator.advertiseContainerAddr(containerNetNS, containerIface.Name, &result.Result); err != nil {
klog.Errorf("Failed to advertise IP address for container %s: %v", containerID, err)
// Do not return an error and fail the interface creation.
klog.ErrorS(err, "Failed to advertise IP address for container", "container ID", containerID)
}
// Mark the manipulation as success to cancel deferred operations.
success = true
Expand Down
6 changes: 4 additions & 2 deletions pkg/agent/cniserver/pod_configuration_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,9 @@ func TestConfigureSriovSecondaryInterface(t *testing.T) {
name: "advertise-failure",
podSriovVFDeviceID: "vf2",
advertiseErr: fmt.Errorf("unable to advertise on the sriov link"),
expectedErr: fmt.Errorf("failed to advertise IP address for container %s: unable to advertise on the sriov link", containerID),
// When advertiseContainerAddr returns an error, it is logged, but does not
// cause ConfigureSriovSecondaryInterface to also return an error.

}, {
name: "success",
podSriovVFDeviceID: "vf3",
Expand All @@ -499,7 +501,7 @@ func createPodConfigurator(controller *gomock.Controller, testIfaceConfigurator
mockOFClient = openflowtest.NewMockClient(controller)
ifaceStore = interfacestore.NewInterfaceStore()
mockRoute = routetest.NewMockInterface(controller)
configurator, _ := newPodConfigurator(mockOVSBridgeClient, mockOFClient, mockRoute, ifaceStore, gwMAC, "system", false, channel.NewSubscribableChannel("PodUpdate", 100), nil, false)
configurator, _ := newPodConfigurator(mockOVSBridgeClient, mockOFClient, mockRoute, ifaceStore, gwMAC, "system", false, false, channel.NewSubscribableChannel("PodUpdate", 100), nil)
configurator.ifConfigurator = testIfaceConfigurator
return configurator
}
Expand Down
103 changes: 103 additions & 0 deletions pkg/agent/cniserver/secondary.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
// Copyright 2021 Antrea Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cniserver

import (
"fmt"

current "github.com/containernetworking/cni/pkg/types/100"
"k8s.io/klog/v2"

"antrea.io/antrea/pkg/ovs/ovsconfig"
)

func NewSecondaryInterfaceConfigurator(ovsBridgeClient ovsconfig.OVSBridgeClient) (*podConfigurator, error) {
return newPodConfigurator(ovsBridgeClient, nil, nil, nil, nil, ovsconfig.OVSDatapathSystem, false, false, nil, nil)
}

// ConfigureSriovSecondaryInterface configures a SR-IOV secondary interface for a Pod.
func (pc *podConfigurator) ConfigureSriovSecondaryInterface(
podName, podNamespace string,
containerID, containerNetNS, containerInterfaceName string,
mtu int,
podSriovVFDeviceID string,
result *current.Result) error {
if podSriovVFDeviceID == "" {
return fmt.Errorf("error getting the Pod SR-IOV VF device ID")
}

err := pc.ifConfigurator.configureContainerLink(podName, podNamespace, containerID, containerNetNS, containerInterfaceName, mtu, "", podSriovVFDeviceID, result, nil)
if err != nil {
return err
}
hostIface := result.Interfaces[0]
containerIface := result.Interfaces[1]
klog.InfoS("Configured SR-IOV interface", "Pod", klog.KRef(podNamespace, podName), "interface", containerInterfaceName, "hostInterface", hostIface)

if err = pc.ifConfigurator.advertiseContainerAddr(containerNetNS, containerIface.Name, result); err != nil {
klog.ErrorS(err, "Failed to advertise IP address for SR-IOV interface", "container ID", containerID, "interface", containerInterfaceName)
}
return nil
}

// ConfigureVLANSecondaryInterface configures a VLAN secondary interface on the secondary network
// OVS bridge, and returns the OVS port UUID.
func (pc *podConfigurator) ConfigureVLANSecondaryInterface(
podName, podNamespace string,
containerID, containerNetNS, containerInterfaceName string,
mtu int, vlanID uint16,
result *current.Result) (string, error) {
// TODO: revisit the possibility of reusing configureInterfaces(), connectInterfaceToOVS()
// removeInterfaces() code, and using InterfaceStore to store secondary interface info.
if err := pc.ifConfigurator.configureContainerLink(podName, podNamespace, containerID, containerNetNS, containerInterfaceName, mtu, "", "", result, nil); err != nil {
return "", err
}
hostIface := result.Interfaces[0]
containerIface := result.Interfaces[1]

success := false
defer func() {
if !success {
if err := pc.ifConfigurator.removeContainerLink(containerID, hostIface.Name); err != nil {
klog.ErrorS(err, "failed to roll back veth creation", "container ID", containerID, "interface", containerInterfaceName)
}
}
}()

// Use the outer veth interface name as the OVS port name.
ovsPortName := hostIface.Name
ovsPortUUID, err := pc.createOVSPort(ovsPortName, nil, vlanID)
if err != nil {
return "", fmt.Errorf("failed to add OVS port for container %s: %v", containerID, err)
}
klog.InfoS("Configured VLAN interface", "Pod", klog.KRef(podNamespace, podName), "interface", containerInterfaceName, "hostInterface", hostIface)

if err := pc.ifConfigurator.advertiseContainerAddr(containerNetNS, containerIface.Name, result); err != nil {
klog.ErrorS(err, "Failed to advertise IP address for VLAN interface", "container ID", containerID, "interface", containerInterfaceName)
}
success = true
return ovsPortUUID, nil
}

// DeleteVLANSecondaryInterface deletes a VLAN secondary interface.
func (pc *podConfigurator) DeleteVLANSecondaryInterface(containerID, hostInterfaceName, ovsPortUUID string) error {
if err := pc.ovsBridgeClient.DeletePort(ovsPortUUID); err != nil {
return fmt.Errorf("failed to delete OVS port for container %s: %v", containerID, err)
}
if err := pc.ifConfigurator.removeContainerLink(containerID, hostInterfaceName); err != nil {
return err
}
return nil
}
13 changes: 4 additions & 9 deletions pkg/agent/cniserver/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -387,10 +387,6 @@ func (s *CNIServer) validatePrevResult(cfgArgs *cnipb.CniCmdArgs, prevResult *cu
return nil
}

func (s *CNIServer) GetPodConfigurator() *podConfigurator {
return s.podConfigurator
}

// Declared variables for testing
var (
ipamSecondaryNetworkAdd = ipam.SecondaryNetworkAdd
Expand Down Expand Up @@ -531,8 +527,7 @@ func (s *CNIServer) CmdAdd(ctx context.Context, request *cnipb.CniCmdRequest) (*
if s.secondaryNetworkEnabled {
// Go cache the CNI server info at CNIConfigInfo cache, for podWatch usage
cniInfo := &cnipodcache.CNIConfigInfo{CNIVersion: cniVersion, PodName: podName, PodNamespace: podNamespace,
ContainerID: cniConfig.ContainerId, ContainerNetNS: netNS, PodCNIDeleted: false,
MTU: cniConfig.MTU}
ContainerID: cniConfig.ContainerId, ContainerNetNS: netNS, PodCNIDeleted: false}
s.podConfigurator.podInfoStore.AddCNIConfigInfo(cniInfo)
}

Expand Down Expand Up @@ -668,9 +663,9 @@ func (s *CNIServer) Initialize(

s.podConfigurator, err = newPodConfigurator(
ovsBridgeClient, ofClient, s.routeClient, ifaceStore, s.nodeConfig.GatewayConfig.MAC,
ovsBridgeClient.GetOVSDatapathType(), ovsBridgeClient.IsHardwareOffloadEnabled(), podUpdateNotifier,
podInfoStore, s.disableTXChecksumOffload,
)
ovsBridgeClient.GetOVSDatapathType(), ovsBridgeClient.IsHardwareOffloadEnabled(),
s.disableTXChecksumOffload,
podUpdateNotifier, podInfoStore)
if err != nil {
return fmt.Errorf("error during initialize podConfigurator: %v", err)
}
Expand Down
13 changes: 6 additions & 7 deletions pkg/agent/cniserver/server_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ func TestValidatePrevResult(t *testing.T) {
cniConfig.Ifname = ifname
cniConfig.Netns = "invalid_netns"
sriovVFDeviceID := ""
cniServer.podConfigurator, _ = newPodConfigurator(nil, nil, nil, nil, nil, "", false, channel.NewSubscribableChannel("PodUpdate", 100), nil, false)
cniServer.podConfigurator, _ = newPodConfigurator(nil, nil, nil, nil, nil, "", false, false, channel.NewSubscribableChannel("PodUpdate", 100), nil)
response := cniServer.validatePrevResult(cniConfig.CniCmdArgs, prevResult, sriovVFDeviceID)
checkErrorResponse(t, response, cnipb.ErrorCode_CHECK_INTERFACE_FAILURE, "")
})
Expand All @@ -109,7 +109,7 @@ func TestValidatePrevResult(t *testing.T) {
cniConfig.Netns = "invalid_netns"
sriovVFDeviceID := "0000:03:00.6"
prevResult.Interfaces = []*current.Interface{hostIface, containerIface}
cniServer.podConfigurator, _ = newPodConfigurator(nil, nil, nil, nil, nil, "", true, channel.NewSubscribableChannel("PodUpdate", 100), nil, false)
cniServer.podConfigurator, _ = newPodConfigurator(nil, nil, nil, nil, nil, "", true, false, channel.NewSubscribableChannel("PodUpdate", 100), nil)
response := cniServer.validatePrevResult(cniConfig.CniCmdArgs, prevResult, sriovVFDeviceID)
checkErrorResponse(t, response, cnipb.ErrorCode_CHECK_INTERFACE_FAILURE, "")
})
Expand All @@ -122,7 +122,7 @@ func TestRemoveInterface(t *testing.T) {
ifaceStore = interfacestore.NewInterfaceStore()
mockRoute = routetest.NewMockInterface(controller)
gwMAC, _ := net.ParseMAC("00:00:11:11:11:11")
podConfigurator, err := newPodConfigurator(mockOVSBridgeClient, mockOFClient, mockRoute, ifaceStore, gwMAC, "system", false, channel.NewSubscribableChannel("PodUpdate", 100), nil, false)
podConfigurator, err := newPodConfigurator(mockOVSBridgeClient, mockOFClient, mockRoute, ifaceStore, gwMAC, "system", false, false, channel.NewSubscribableChannel("PodUpdate", 100), nil)
require.Nil(t, err, "No error expected in podConfigurator constructor")

containerMAC, _ := net.ParseMAC("aa:bb:cc:dd:ee:ff")
Expand Down Expand Up @@ -203,7 +203,7 @@ func newMockCNIServer(t *testing.T, controller *gomock.Controller, ipamDriver ip
gwMAC, _ := net.ParseMAC("00:00:11:11:11:11")
gateway := &config.GatewayConfig{Name: "", IPv4: gwIPv4, MAC: gwMAC}
cniServer.nodeConfig = &config.NodeConfig{Name: "node1", PodIPv4CIDR: nodePodCIDRv4, GatewayConfig: gateway}
cniServer.podConfigurator, _ = newPodConfigurator(mockOVSBridgeClient, mockOFClient, mockRoute, ifaceStore, gwMAC, "system", false, channel.NewSubscribableChannel("PodUpdate", 100), nil, false)
cniServer.podConfigurator, _ = newPodConfigurator(mockOVSBridgeClient, mockOFClient, mockRoute, ifaceStore, gwMAC, "system", false, false, channel.NewSubscribableChannel("PodUpdate", 100), nil)
cniServer.enableSecondaryNetworkIPAM = enableSecondaryNetworkIPAM
cniServer.isChaining = isChaining
cniServer.secondaryNetworkEnabled = secondaryNetworkEnabled
Expand Down Expand Up @@ -494,8 +494,7 @@ func TestCmdDel(t *testing.T) {
cniserver.podConfigurator.ifConfigurator = testIfaceConfigurator
if tc.secondaryNetworkEnabled {
cniInfo := &cnipodcache.CNIConfigInfo{CNIVersion: supportedCNIVersion, PodName: tc.podName, PodNamespace: testPodNamespace,
ContainerID: containerID, ContainerNetNS: netns, PodCNIDeleted: false,
MTU: 1450}
ContainerID: containerID, ContainerNetNS: netns, PodCNIDeleted: false}
cniserver.podConfigurator.podInfoStore.AddCNIConfigInfo(cniInfo)
}
if tc.ipamDel {
Expand Down Expand Up @@ -639,7 +638,7 @@ func TestReconcile(t *testing.T) {
cniServer := newCNIServer(t)
cniServer.routeClient = mockRoute
gwMAC, _ := net.ParseMAC("00:00:11:11:11:11")
cniServer.podConfigurator, _ = newPodConfigurator(mockOVSBridgeClient, mockOFClient, mockRoute, ifaceStore, gwMAC, "system", false, channel.NewSubscribableChannel("PodUpdate", 100), nil, false)
cniServer.podConfigurator, _ = newPodConfigurator(mockOVSBridgeClient, mockOFClient, mockRoute, ifaceStore, gwMAC, "system", false, false, channel.NewSubscribableChannel("PodUpdate", 100), nil)
cniServer.podConfigurator.ifConfigurator = newTestInterfaceConfigurator()
cniServer.nodeConfig = &config.NodeConfig{
Name: nodeName,
Expand Down
117 changes: 0 additions & 117 deletions pkg/agent/cniserver/sriov.go

This file was deleted.

12 changes: 6 additions & 6 deletions pkg/agent/secondarynetwork/cnipodcache/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,19 +35,19 @@ type CNIPodInfoCache struct {
}

// Add CNIPodInfo to local cache store.
func (c *CNIPodInfoCache) AddCNIConfigInfo(CNIConfig *CNIConfigInfo) {
c.cache.Add(CNIConfig)
func (c *CNIPodInfoCache) AddCNIConfigInfo(cniConfig *CNIConfigInfo) {
c.cache.Add(cniConfig)
}

// Delete CNIPodInfo from local cache store.
func (c *CNIPodInfoCache) DeleteCNIConfigInfo(CNIConfig *CNIConfigInfo) {
c.cache.Delete(CNIConfig)
func (c *CNIPodInfoCache) DeleteCNIConfigInfo(cniConfig *CNIConfigInfo) {
c.cache.Delete(cniConfig)
}

func (c *CNIPodInfoCache) SetPodCNIDeleted(CNIConfig *CNIConfigInfo) {
func (c *CNIPodInfoCache) SetPodCNIDeleted(cniConfig *CNIConfigInfo) {
c.Lock()
defer c.Unlock()
CNIConfig.PodCNIDeleted = true
cniConfig.PodCNIDeleted = true
}

// Retrieve a valid CNI cache (PodCNIDeleted is not true) entry for the given Pod name and namespace.
Expand Down
Loading

0 comments on commit 5db5adf

Please sign in to comment.