Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ex-lock utils setupVF finished #17

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Add ex-lock utils setupVF finished #17

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e321f4e
Add ex-lock utils setupVF finished
SchoIsles Sep 21, 2017
ca134da
remove code intrusion, be more pretty
SchoIsles Sep 25, 2017
1057675
Some changes
hustcat Oct 18, 2017
d7a965f
create lock file if not exist
SchoIsles Nov 7, 2017
7f04a52
Merge branch 'master' into master
SchoIsles Dec 1, 2017
2611088
Merge branch 'master' of github.com:hustcat/sriov-cni
SchoIsles Dec 8, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions sriov/lock.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package main

import (
"fmt"
"os"
"syscall"
)

var defaultDeviceDir = "/var/lib/cni/devices"

type FileLocker struct {
f *os.File
}

func NewFileLocker(vf int) (*FileLocker, error) {
if err := os.MkdirAll(defaultDeviceDir, 0644); err != nil {
return nil, err
}

path := fmt.Sprintf("%s/%d.lock", defaultDeviceDir, vf)

file, err := os.Open(path)
if err == nil {
return &FileLocker{file}, nil
}

newfile, err := os.Create(path)
if err != nil {
return nil, err
}
return &FileLocker{newfile}, nil

}

// Close closes underlying file
func (l *FileLocker) Close() error {
return l.f.Close()
}

// Lock acquires an exclusive lock
func (l *FileLocker) Lock() error {
return syscall.Flock(int(l.f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
}

// Unlock releases the lock
func (l *FileLocker) Unlock() error {
return syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN)
}
46 changes: 30 additions & 16 deletions sriov/sriov.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ func setupVF(conf *SriovConf, ifName string, netns ns.NetNS) error {
var (
err error
vfDevName string
locker *FileLocker
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about make locker as global variable and initialize it in init function?

)

vfIdx := 0
Expand All @@ -37,17 +38,18 @@ func setupVF(conf *SriovConf, ifName string, netns ns.NetNS) error {

if args.VF != 0 {
vfIdx = int(args.VF)
vfDevName, err = getVFDeviceName(masterName, vfIdx)
vfDevName, locker, err = getVFDeviceName(masterName, vfIdx)
if err != nil {
return err
}
} else {
// alloc a free virtual function
if vfIdx, vfDevName, err = allocFreeVF(masterName); err != nil {
if vfIdx, vfDevName, locker, err = allocFreeVF(masterName); err != nil {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe have no need to change allocFreeVF function, We can change it as follows:

locker.Lock()
allocFreeVF()
locker.UnLock()

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, It's more better implementation. locker is global, Lock() maybe need an angument vfId and make it saved, Unlock() free the vfId and opened file. I will try

return err
}
}

defer locker.Unlock()
m, err := netlink.LinkByName(masterName)
if err != nil {
return fmt.Errorf("failed to lookup master %q: %v", masterName, err)
Expand Down Expand Up @@ -201,36 +203,37 @@ func renameLink(curName, newName string) error {
return netlink.LinkSetName(link, newName)
}

func allocFreeVF(master string) (int, string, error) {
func allocFreeVF(master string) (int, string, *FileLocker, error) {
vfIdx := -1
devName := ""

sriovFile := fmt.Sprintf("/sys/class/net/%s/device/sriov_numvfs", master)
if _, err := os.Lstat(sriovFile); err != nil {
return -1, "", fmt.Errorf("failed to open the sriov_numfs of device %q: %v", master, err)
return -1, "", nil, fmt.Errorf("failed to open the sriov_numfs of device %q: %v", master, err)
}

data, err := ioutil.ReadFile(sriovFile)
if err != nil {
return -1, "", fmt.Errorf("failed to read the sriov_numfs of device %q: %v", master, err)
return -1, "", nil, fmt.Errorf("failed to read the sriov_numfs of device %q: %v", master, err)
}

if len(data) == 0 {
return -1, "", fmt.Errorf("no data in the file %q", sriovFile)
return -1, "", nil, fmt.Errorf("no data in the file %q", sriovFile)
}

sriovNumfs := strings.TrimSpace(string(data))
vfTotal, err := strconv.Atoi(sriovNumfs)
if err != nil {
return -1, "", fmt.Errorf("failed to convert sriov_numfs(byte value) to int of device %q: %v", master, err)
return -1, "", nil, fmt.Errorf("failed to convert sriov_numfs(byte value) to int of device %q: %v", master, err)
}

if vfTotal <= 0 {
return -1, "", fmt.Errorf("no virtual function in the device %q: %v", master)
return -1, "", nil, fmt.Errorf("no virtual function in the device %q: %v", master)
}

var locker *FileLocker
for vf := 0; vf < vfTotal; vf++ {
devName, err = getVFDeviceName(master, vf)
devName, locker, err = getVFDeviceName(master, vf)

// got a free vf
if err == nil {
Expand All @@ -240,26 +243,37 @@ func allocFreeVF(master string) (int, string, error) {
}

if vfIdx == -1 {
return -1, "", fmt.Errorf("can not get a free virtual function in directory %s", master)
return -1, "", nil, fmt.Errorf("can not get a free virtual function in directory %s", master)
}
return vfIdx, devName, nil
return vfIdx, devName, locker, nil
}

func getVFDeviceName(master string, vf int) (string, error) {
func getVFDeviceName(master string, vf int) (string, *FileLocker, error) {
vfDir := fmt.Sprintf("/sys/class/net/%s/device/virtfn%d/net", master, vf)
if _, err := os.Lstat(vfDir); err != nil {
return "", fmt.Errorf("failed to open the virtfn%d dir of the device %q: %v", vf, master, err)
return "", nil, fmt.Errorf("failed to open the virtfn%d dir of the device %q: %v", vf, master, err)
}

infos, err := ioutil.ReadDir(vfDir)
if err != nil {
return "", fmt.Errorf("failed to read the virtfn%d dir of the device %q: %v", vf, master, err)
return "", nil, fmt.Errorf("failed to read the virtfn%d dir of the device %q: %v", vf, master, err)
}

if len(infos) != 1 {
return "", fmt.Errorf("no network device in directory %s", vfDir)
return "", nil, fmt.Errorf("no network device in directory %s", vfDir)
}
return infos[0].Name(), nil

locker, err := NewFileLocker(vf)
if err != nil {
return "", nil, fmt.Errorf("failed to create file locker for %d: %v", vf, err)
}

err = locker.Lock()
if err != nil {
return "", nil, fmt.Errorf("failed to add exclude lock to %d: %v", vf, err)
}

return infos[0].Name(), locker, nil
}

func main() {
Expand Down