Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix for working with network namespaces #576

Merged
merged 1 commit into from
Mar 17, 2023
Merged

Fix for working with network namespaces #576

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 34 additions & 38 deletions pkg/kernel/link.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
package kernel

import (
"io/fs"
"io/ioutil"
"os"
"path/filepath"
Expand Down Expand Up @@ -172,18 +173,6 @@ func (l *link) GetLink() netlink.Link {
// address and/or target interface name.
func FindHostDevice(pciAddress, name string, namespaces ...netns.NsHandle) (Link, error) {
// TODO: add support for shared l interfaces (like Mellanox NICs)

current, err := nshandle.Current()
if err != nil {
return nil, err
}
defer func() {
if err := netns.Set(current); err != nil {
panic(errors.Wrapf(err, "failed to switch back to the current net NS: %v", current).Error())
}
_ = current.Close()
}()

attempts := []func(netns.NsHandle, string, string) (netlink.Link, error){
searchByPCIAddress,
searchByName,
Expand All @@ -210,37 +199,44 @@ func FindHostDevice(pciAddress, name string, namespaces ...netns.NsHandle) (Link

func searchByPCIAddress(ns netns.NsHandle, name, pciAddress string) (netlink.Link, error) {
// execute in context of the pod's namespace
err := netns.Set(ns)
currentNs, err := nshandle.Current()
if err != nil {
return nil, errors.Errorf("failed to enter namespace: %s", err)
return nil, err
}
defer func() { _ = currentNs.Close() }()

pciDevicePath := filepath.Join("/sys/bus/pci/devices", pciAddress)
netDir, err := findNetDir(pciDevicePath)
if err != nil {
return nil, errors.Errorf("no net directory under pci device %s: %q", pciAddress, err)
}
var link netlink.Link
err = nshandle.RunIn(currentNs, ns, func() error {
var netDir string
pciDevicePath := filepath.Join("/sys/bus/pci/devices", pciAddress)
netDir, err = findNetDir(pciDevicePath)
if err != nil {
return errors.Errorf("no net directory under pci device %s: %q", pciAddress, err)
}

fInfos, err := ioutil.ReadDir(netDir)
if err != nil {
return nil, errors.Errorf("failed to read net directory %s: %q", netDir, err)
}
var fInfos []fs.FileInfo
fInfos, err = ioutil.ReadDir(netDir)
if err != nil {
return errors.Errorf("failed to read net directory %s: %q", netDir, err)
}

names := make([]string, 0)
for _, f := range fInfos {
names = append(names, f.Name())
}
names := make([]string, 0)
for _, f := range fInfos {
names = append(names, f.Name())
}

if len(names) == 0 {
return nil, errors.Errorf("no links with PCI address %s found", pciAddress)
}
if len(names) == 0 {
return errors.Errorf("no links with PCI address %s found", pciAddress)
}

link, err := netlink.LinkByName(names[0])
if err != nil {
return nil, errors.Errorf("error getting host device with PCI address %s", pciAddress)
}
link, err = netlink.LinkByName(names[0])
if err != nil {
return errors.Errorf("error getting host device with PCI address %s", pciAddress)
}
return nil
})

return link, nil
return link, err
}

func findNetDir(basePath string) (string, error) {
Expand Down Expand Up @@ -272,13 +268,13 @@ func findNetDir(basePath string) (string, error) {

func searchByName(ns netns.NsHandle, name, pciAddress string) (netlink.Link, error) {
// execute in context of the pod's namespace
err := netns.Set(ns)
handle, err := netlink.NewHandleAt(ns)
if err != nil {
return nil, errors.Errorf("failed to switch to namespace: %s", err)
return nil, errors.Errorf("failed to create netlink handler: %s", err)
}

// get link
link, err := netlink.LinkByName(name)
link, err := handle.LinkByName(name)
if err != nil {
return nil, errors.Errorf("failed to get link with name %s", name)
}
Expand Down
95 changes: 60 additions & 35 deletions pkg/kernel/networkservice/inject/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,42 +37,59 @@ import (
"github.com/networkservicemesh/sdk-kernel/pkg/kernel/tools/nshandle"
)

func moveInterfaceToAnotherNamespace(ifName string, curNetNS, fromNetNS, toNetNS netns.NsHandle) error {
return nshandle.RunIn(curNetNS, fromNetNS, func() error {
link, err := netlink.LinkByName(ifName)
if err != nil {
return errors.Wrapf(err, "failed to get net interface: %v", ifName)
}
func moveInterfaceToAnotherNamespace(ifName string, fromNetNS, toNetNS netns.NsHandle) error {
handle, err := netlink.NewHandleAt(fromNetNS)
if err != nil {
return errors.Wrap(err, "failed to create netlink fromNetNS handle")
}

if err := netlink.LinkSetNsFd(link, int(toNetNS)); err != nil {
return errors.Wrapf(err, "failed to move net interface to net NS: %v %v", ifName, toNetNS)
}
link, err := handle.LinkByName(ifName)
if err != nil {
return errors.Wrapf(err, "failed to get net interface: %v", ifName)
}

return nil
})
if err := handle.LinkSetNsFd(link, int(toNetNS)); err != nil {
return errors.Wrapf(err, "failed to move net interface to net NS: %v %v", ifName, toNetNS)
}
return nil
}

func renameInterface(origIfName, desiredIfName string, curNetNS, targetNetNS netns.NsHandle) error {
return nshandle.RunIn(curNetNS, targetNetNS, func() error {
link, err := netlink.LinkByName(origIfName)
if err != nil {
return errors.Wrapf(err, "failed to get net interface: %v", origIfName)
}
func renameInterface(origIfName, desiredIfName string, targetNetNS netns.NsHandle) error {
handle, err := netlink.NewHandleAt(targetNetNS)
if err != nil {
return errors.Wrap(err, "failed to create netlink targetNetNS handle")
}

if err = netlink.LinkSetDown(link); err != nil {
return errors.Wrapf(err, "failed to rename net interface: %v -> %v", origIfName, desiredIfName)
}
link, err := handle.LinkByName(origIfName)
if err != nil {
return errors.Wrapf(err, "failed to get net interface: %v", origIfName)
}

if err = netlink.LinkSetName(link, desiredIfName); err != nil {
return errors.Wrapf(err, "failed to rename net interface: %v -> %v", origIfName, desiredIfName)
}
if err = handle.LinkSetDown(link); err != nil {
return errors.Wrapf(err, "failed to down net interface: %v -> %v", origIfName, desiredIfName)
}

if err = netlink.LinkSetUp(link); err != nil {
return errors.Wrapf(err, "failed to rename net interface: %v -> %v", origIfName, desiredIfName)
}
if err = handle.LinkSetName(link, desiredIfName); err != nil {
return errors.Wrapf(err, "failed to rename net interface: %v -> %v", origIfName, desiredIfName)
}
return nil
}

return nil
})
func upInterface(ifName string, targetNetNS netns.NsHandle) error {
handle, err := netlink.NewHandleAt(targetNetNS)
if err != nil {
return errors.Wrap(err, "failed to create netlink NS handle")
}

link, err := handle.LinkByName(ifName)
if err != nil {
return errors.Wrapf(err, "failed to get net interface: %v", ifName)
}

if err = handle.LinkSetUp(link); err != nil {
return errors.Wrapf(err, "failed to up net interface: %v", ifName)
}
return nil
}

func move(ctx context.Context, conn *networkservice.Connection, vfRefCountMap map[string]int, vfRefCountMutex sync.Locker, isClient, isMoveBack bool) error {
Expand Down Expand Up @@ -119,7 +136,12 @@ func move(ctx context.Context, conn *networkservice.Connection, vfRefCountMap ma
ifName := mech.GetInterfaceName()
if !isMoveBack {
err = moveToContNetNS(vfConfig, vfRefCountMap, vfRefKey, ifName, hostNetNS, contNetNS)
vfConfig.ContNetNS = contNetNS
if err != nil {
// If we got an error, try to move back the vf to the host namespace
_ = moveToHostNetNS(vfConfig, vfRefCountMap, vfRefKey, ifName, hostNetNS, contNetNS)
} else {
vfConfig.ContNetNS = contNetNS
}
} else {
err = moveToHostNetNS(vfConfig, vfRefCountMap, vfRefKey, ifName, hostNetNS, contNetNS)
}
Expand All @@ -146,12 +168,15 @@ func moveToContNetNS(vfConfig *vfconfig.VFConfig, vfRefCountMap map[string]int,
return
}
if vfConfig != nil && vfConfig.VFInterfaceName != ifName {
err = moveInterfaceToAnotherNamespace(vfConfig.VFInterfaceName, hostNetNS, hostNetNS, contNetNS)
err = moveInterfaceToAnotherNamespace(vfConfig.VFInterfaceName, hostNetNS, contNetNS)
if err == nil {
err = renameInterface(vfConfig.VFInterfaceName, ifName, hostNetNS, contNetNS)
err = renameInterface(vfConfig.VFInterfaceName, ifName, contNetNS)
if err == nil {
err = upInterface(ifName, contNetNS)
}
}
} else {
err = moveInterfaceToAnotherNamespace(ifName, hostNetNS, hostNetNS, contNetNS)
err = moveInterfaceToAnotherNamespace(ifName, hostNetNS, contNetNS)
}
return err
}
Expand All @@ -178,17 +203,17 @@ func moveToHostNetNS(vfConfig *vfconfig.VFConfig, vfRefCountMap map[string]int,
}
return nil
}
err := renameInterface(ifName, vfConfig.VFInterfaceName, hostNetNS, contNetNS)
err := renameInterface(ifName, vfConfig.VFInterfaceName, contNetNS)
if err == nil {
err = moveInterfaceToAnotherNamespace(vfConfig.VFInterfaceName, hostNetNS, contNetNS, hostNetNS)
err = moveInterfaceToAnotherNamespace(vfConfig.VFInterfaceName, contNetNS, hostNetNS)
}
return err
}
link, _ := kernellink.FindHostDevice("", ifName, hostNetNS)
if link != nil {
return nil
}
return moveInterfaceToAnotherNamespace(ifName, hostNetNS, contNetNS, hostNetNS)
return moveInterfaceToAnotherNamespace(ifName, contNetNS, hostNetNS)
}
return nil
}