From e321f4e6a41eedae1d059146f1bdc7bbc5b1fd25 Mon Sep 17 00:00:00 2001 From: okletswin Date: Thu, 21 Sep 2017 15:25:50 +0800 Subject: [PATCH 1/4] Add ex-lock utils setupVF finished --- sriov/lock.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ sriov/sriov.go | 46 ++++++++++++++++++++++++++++++---------------- 2 files changed, 78 insertions(+), 16 deletions(-) create mode 100644 sriov/lock.go diff --git a/sriov/lock.go b/sriov/lock.go new file mode 100644 index 000000000..e71a65d2d --- /dev/null +++ b/sriov/lock.go @@ -0,0 +1,48 @@ +package main + +import ( + "fmt" + "os" + "syscall" +) + +var defaultDeviceDir = "/var/lib/cni/devices" + +type FileLocker struct { + f *os.File +} + +func NewFileLocker(vf int) (*FileLocker, error) { + if err := os.MkdirAll(defaultDeviceDir, 0644); err != nil { + return nil, err + } + + path := fmt.Sprintf("%s/%d.lock", defaultDeviceDir, vf) + + file, err := os.Open(path) + if err == nil { + return &FileLocker{file}, nil + } + + newfile, err := os.Create(path) + if err != nil { + return nil, err + } + return &FileLocker{newfile}, nil + +} + +// Close closes underlying file +func (l *FileLocker) Close() error { + return l.f.Close() +} + +// Lock acquires an exclusive lock +func (l *FileLocker) Lock() error { + return syscall.Flock(int(l.f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) +} + +// Unlock releases the lock +func (l *FileLocker) Unlock() error { + return syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN) +} \ No newline at end of file diff --git a/sriov/sriov.go b/sriov/sriov.go index dcd629aa6..3d15ec111 100644 --- a/sriov/sriov.go +++ b/sriov/sriov.go @@ -29,6 +29,7 @@ func setupVF(conf *SriovConf, ifName string, netns ns.NetNS) error { var ( err error vfDevName string + locker *FileLocker ) vfIdx := 0 @@ -37,17 +38,18 @@ func setupVF(conf *SriovConf, ifName string, netns ns.NetNS) error { if args.VF != 0 { vfIdx = int(args.VF) - vfDevName, err = getVFDeviceName(masterName, vfIdx) + vfDevName, locker, err = getVFDeviceName(masterName, vfIdx) if err != nil { return err } } else { // alloc a free virtual function - if vfIdx, vfDevName, err = allocFreeVF(masterName); err != nil { + if vfIdx, vfDevName, locker, err = allocFreeVF(masterName); err != nil { return err } } + defer locker.Unlock() m, err := netlink.LinkByName(masterName) if err != nil { return fmt.Errorf("failed to lookup master %q: %v", masterName, err) @@ -201,36 +203,37 @@ func renameLink(curName, newName string) error { return netlink.LinkSetName(link, newName) } -func allocFreeVF(master string) (int, string, error) { +func allocFreeVF(master string) (int, string, *FileLocker, error) { vfIdx := -1 devName := "" sriovFile := fmt.Sprintf("/sys/class/net/%s/device/sriov_numvfs", master) if _, err := os.Lstat(sriovFile); err != nil { - return -1, "", fmt.Errorf("failed to open the sriov_numfs of device %q: %v", master, err) + return -1, "", nil, fmt.Errorf("failed to open the sriov_numfs of device %q: %v", master, err) } data, err := ioutil.ReadFile(sriovFile) if err != nil { - return -1, "", fmt.Errorf("failed to read the sriov_numfs of device %q: %v", master, err) + return -1, "", nil, fmt.Errorf("failed to read the sriov_numfs of device %q: %v", master, err) } if len(data) == 0 { - return -1, "", fmt.Errorf("no data in the file %q", sriovFile) + return -1, "", nil, fmt.Errorf("no data in the file %q", sriovFile) } sriovNumfs := strings.TrimSpace(string(data)) vfTotal, err := strconv.Atoi(sriovNumfs) if err != nil { - return -1, "", fmt.Errorf("failed to convert sriov_numfs(byte value) to int of device %q: %v", master, err) + return -1, "", nil, fmt.Errorf("failed to convert sriov_numfs(byte value) to int of device %q: %v", master, err) } if vfTotal <= 0 { - return -1, "", fmt.Errorf("no virtual function in the device %q: %v", master) + return -1, "", nil, fmt.Errorf("no virtual function in the device %q: %v", master) } + var locker *FileLocker for vf := 0; vf < vfTotal; vf++ { - devName, err = getVFDeviceName(master, vf) + devName, locker, err = getVFDeviceName(master, vf) // got a free vf if err == nil { @@ -240,26 +243,37 @@ func allocFreeVF(master string) (int, string, error) { } if vfIdx == -1 { - return -1, "", fmt.Errorf("can not get a free virtual function in directory %s", master) + return -1, "", nil, fmt.Errorf("can not get a free virtual function in directory %s", master) } - return vfIdx, devName, nil + return vfIdx, devName, locker, nil } -func getVFDeviceName(master string, vf int) (string, error) { +func getVFDeviceName(master string, vf int) (string, *FileLocker, error) { vfDir := fmt.Sprintf("/sys/class/net/%s/device/virtfn%d/net", master, vf) if _, err := os.Lstat(vfDir); err != nil { - return "", fmt.Errorf("failed to open the virtfn%d dir of the device %q: %v", vf, master, err) + return "", nil, fmt.Errorf("failed to open the virtfn%d dir of the device %q: %v", vf, master, err) } infos, err := ioutil.ReadDir(vfDir) if err != nil { - return "", fmt.Errorf("failed to read the virtfn%d dir of the device %q: %v", vf, master, err) + return "", nil, fmt.Errorf("failed to read the virtfn%d dir of the device %q: %v", vf, master, err) } if len(infos) != 1 { - return "", fmt.Errorf("no network device in directory %s", vfDir) + return "", nil, fmt.Errorf("no network device in directory %s", vfDir) } - return infos[0].Name(), nil + + locker, err := NewFileLocker(vf) + if err != nil { + return "", nil, fmt.Errorf("failed to create file locker for %d: %v", vf, err) + } + + err = locker.Lock() + if err != nil { + return "", nil, fmt.Errorf("failed to add exclude lock to %d: %v", vf, err) + } + + return infos[0].Name(), locker, nil } func main() { From ca134dacf52f3af9bee8c12a77ad9cf426d47ea9 Mon Sep 17 00:00:00 2001 From: okletswin Date: Mon, 25 Sep 2017 16:17:13 +0800 Subject: [PATCH 2/4] remove code intrusion, be more pretty still make lock in getVFDeviceName --- sriov/lock.go | 41 ++++++++++++++++++----------------------- sriov/sriov.go | 43 ++++++++++++++++++------------------------- 2 files changed, 36 insertions(+), 48 deletions(-) diff --git a/sriov/lock.go b/sriov/lock.go index e71a65d2d..d8be360e2 100644 --- a/sriov/lock.go +++ b/sriov/lock.go @@ -8,41 +8,36 @@ import ( var defaultDeviceDir = "/var/lib/cni/devices" +var locker = &FileLocker{} + type FileLocker struct { - f *os.File + file *os.File } -func NewFileLocker(vf int) (*FileLocker, error) { - if err := os.MkdirAll(defaultDeviceDir, 0644); err != nil { - return nil, err - } - +// Lock acquires an exclusive lock +func (l *FileLocker) Lock(vf int) error { path := fmt.Sprintf("%s/%d.lock", defaultDeviceDir, vf) - file, err := os.Open(path) - if err == nil { - return &FileLocker{file}, nil + if err := os.MkdirAll(defaultDeviceDir, 0644); err != nil { + return err } - newfile, err := os.Create(path) + file, err := os.Open(path) if err != nil { - return nil, err + file, err := os.Create(path) + if err != nil { + return err + } + l.file = file + } else { + l.file = file } - return &FileLocker{newfile}, nil - -} -// Close closes underlying file -func (l *FileLocker) Close() error { - return l.f.Close() -} - -// Lock acquires an exclusive lock -func (l *FileLocker) Lock() error { - return syscall.Flock(int(l.f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + return syscall.Flock(int(l.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) } // Unlock releases the lock func (l *FileLocker) Unlock() error { - return syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN) + defer l.file.Close() + return syscall.Flock(int(l.file.Fd()), syscall.LOCK_UN) } \ No newline at end of file diff --git a/sriov/sriov.go b/sriov/sriov.go index 3d15ec111..2cd38fa99 100644 --- a/sriov/sriov.go +++ b/sriov/sriov.go @@ -29,7 +29,6 @@ func setupVF(conf *SriovConf, ifName string, netns ns.NetNS) error { var ( err error vfDevName string - locker *FileLocker ) vfIdx := 0 @@ -38,13 +37,13 @@ func setupVF(conf *SriovConf, ifName string, netns ns.NetNS) error { if args.VF != 0 { vfIdx = int(args.VF) - vfDevName, locker, err = getVFDeviceName(masterName, vfIdx) + vfDevName, err = getVFDeviceName(masterName, vfIdx) if err != nil { return err } } else { // alloc a free virtual function - if vfIdx, vfDevName, locker, err = allocFreeVF(masterName); err != nil { + if vfIdx, vfDevName, err = allocFreeVF(masterName); err != nil { return err } } @@ -203,37 +202,36 @@ func renameLink(curName, newName string) error { return netlink.LinkSetName(link, newName) } -func allocFreeVF(master string) (int, string, *FileLocker, error) { +func allocFreeVF(master string) (int, string, error) { vfIdx := -1 devName := "" sriovFile := fmt.Sprintf("/sys/class/net/%s/device/sriov_numvfs", master) if _, err := os.Lstat(sriovFile); err != nil { - return -1, "", nil, fmt.Errorf("failed to open the sriov_numfs of device %q: %v", master, err) + return -1, "", fmt.Errorf("failed to open the sriov_numfs of device %q: %v", master, err) } data, err := ioutil.ReadFile(sriovFile) if err != nil { - return -1, "", nil, fmt.Errorf("failed to read the sriov_numfs of device %q: %v", master, err) + return -1, "", fmt.Errorf("failed to read the sriov_numfs of device %q: %v", master, err) } if len(data) == 0 { - return -1, "", nil, fmt.Errorf("no data in the file %q", sriovFile) + return -1, "", fmt.Errorf("no data in the file %q", sriovFile) } sriovNumfs := strings.TrimSpace(string(data)) vfTotal, err := strconv.Atoi(sriovNumfs) if err != nil { - return -1, "", nil, fmt.Errorf("failed to convert sriov_numfs(byte value) to int of device %q: %v", master, err) + return -1, "", fmt.Errorf("failed to convert sriov_numfs(byte value) to int of device %q: %v", master, err) } if vfTotal <= 0 { - return -1, "", nil, fmt.Errorf("no virtual function in the device %q: %v", master) + return -1, "", fmt.Errorf("no virtual function in the device %q: %v", master) } - var locker *FileLocker for vf := 0; vf < vfTotal; vf++ { - devName, locker, err = getVFDeviceName(master, vf) + devName, err = getVFDeviceName(master, vf) // got a free vf if err == nil { @@ -243,37 +241,32 @@ func allocFreeVF(master string) (int, string, *FileLocker, error) { } if vfIdx == -1 { - return -1, "", nil, fmt.Errorf("can not get a free virtual function in directory %s", master) + return -1, "", fmt.Errorf("can not get a free virtual function in directory %s", master) } - return vfIdx, devName, locker, nil + return vfIdx, devName, nil } -func getVFDeviceName(master string, vf int) (string, *FileLocker, error) { +func getVFDeviceName(master string, vf int) (string, error) { vfDir := fmt.Sprintf("/sys/class/net/%s/device/virtfn%d/net", master, vf) if _, err := os.Lstat(vfDir); err != nil { - return "", nil, fmt.Errorf("failed to open the virtfn%d dir of the device %q: %v", vf, master, err) + return "", fmt.Errorf("failed to open the virtfn%d dir of the device %q: %v", vf, master, err) } infos, err := ioutil.ReadDir(vfDir) if err != nil { - return "", nil, fmt.Errorf("failed to read the virtfn%d dir of the device %q: %v", vf, master, err) + return "", fmt.Errorf("failed to read the virtfn%d dir of the device %q: %v", vf, master, err) } if len(infos) != 1 { - return "", nil, fmt.Errorf("no network device in directory %s", vfDir) + return "", fmt.Errorf("no network device in directory %s", vfDir) } - locker, err := NewFileLocker(vf) + err = locker.Lock(vf) if err != nil { - return "", nil, fmt.Errorf("failed to create file locker for %d: %v", vf, err) + return "", fmt.Errorf("failed to add exclude lock to %d: %v", vf, err) } - err = locker.Lock() - if err != nil { - return "", nil, fmt.Errorf("failed to add exclude lock to %d: %v", vf, err) - } - - return infos[0].Name(), locker, nil + return infos[0].Name(), nil } func main() { From 10576758a50c681a35c0f74d5c44818a7665d60d Mon Sep 17 00:00:00 2001 From: Ye Yin Date: Wed, 18 Oct 2017 16:44:20 +0800 Subject: [PATCH 3/4] Some changes --- sriov/lock.go | 48 ++++++++++++++++++++---------------------------- sriov/sriov.go | 33 +++++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/sriov/lock.go b/sriov/lock.go index d8be360e2..773b4746c 100644 --- a/sriov/lock.go +++ b/sriov/lock.go @@ -1,43 +1,35 @@ package main import ( - "fmt" "os" "syscall" ) -var defaultDeviceDir = "/var/lib/cni/devices" - -var locker = &FileLocker{} - -type FileLocker struct { - file *os.File +type FileLock struct { + f *os.File } -// Lock acquires an exclusive lock -func (l *FileLocker) Lock(vf int) error { - path := fmt.Sprintf("%s/%d.lock", defaultDeviceDir, vf) - - if err := os.MkdirAll(defaultDeviceDir, 0644); err != nil { - return err - } - - file, err := os.Open(path) +// NewFileLock opens file/dir at path and returns unlocked FileLock object +func NewFileLock(path string) (*FileLock, error) { + f, err := os.Open(path) if err != nil { - file, err := os.Create(path) - if err != nil { - return err - } - l.file = file - } else { - l.file = file + return nil, err } - return syscall.Flock(int(l.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + return &FileLock{f}, nil +} + +// Close closes underlying file +func (l *FileLock) Close() error { + return l.f.Close() +} + +// Lock acquires an exclusive lock +func (l *FileLock) Lock() error { + return syscall.Flock(int(l.f.Fd()), syscall.LOCK_EX) } // Unlock releases the lock -func (l *FileLocker) Unlock() error { - defer l.file.Close() - return syscall.Flock(int(l.file.Fd()), syscall.LOCK_UN) -} \ No newline at end of file +func (l *FileLock) Unlock() error { + return syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN) +} diff --git a/sriov/sriov.go b/sriov/sriov.go index 2cd38fa99..310c64efa 100644 --- a/sriov/sriov.go +++ b/sriov/sriov.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "net" "os" + "path/filepath" "runtime" "strconv" "strings" @@ -18,6 +19,10 @@ import ( . "github.com/hustcat/sriov-cni/config" ) +var defaultDataDir = "/var/lib/cni/sriov" + +var locker *FileLock + func init() { // this ensures that main runs only on main thread (thread group leader). // since namespace ops (unshare, setns) are done for a single thread, we @@ -25,6 +30,18 @@ func init() { runtime.LockOSThread() } +func initLocker() error { + var err error + + if err = os.MkdirAll(defaultDataDir, 0644); err != nil { + return err + } + + path := filepath.Join(defaultDataDir, "sriov.lock") + locker, err = NewFileLock(path) + return err +} + func setupVF(conf *SriovConf, ifName string, netns ns.NetNS) error { var ( err error @@ -48,7 +65,6 @@ func setupVF(conf *SriovConf, ifName string, netns ns.NetNS) error { } } - defer locker.Unlock() m, err := netlink.LinkByName(masterName) if err != nil { return fmt.Errorf("failed to lookup master %q: %v", masterName, err) @@ -134,6 +150,11 @@ func releaseVF(conf *SriovConf, ifName string, netns ns.NetNS) error { } func cmdAdd(args *skel.CmdArgs) error { + if err := initLocker(); err != nil { + return err + } + defer locker.Close() + n, err := LoadConf(args.StdinData, args.Args) if err != nil { return err @@ -230,6 +251,11 @@ func allocFreeVF(master string) (int, string, error) { return -1, "", fmt.Errorf("no virtual function in the device %q: %v", master) } + if err = locker.Lock(); err != nil { + return -1, "", fmt.Errorf("failed to get lock: %v", err) + } + defer locker.Unlock() + for vf := 0; vf < vfTotal; vf++ { devName, err = getVFDeviceName(master, vf) @@ -261,11 +287,6 @@ func getVFDeviceName(master string, vf int) (string, error) { return "", fmt.Errorf("no network device in directory %s", vfDir) } - err = locker.Lock(vf) - if err != nil { - return "", fmt.Errorf("failed to add exclude lock to %d: %v", vf, err) - } - return infos[0].Name(), nil } From d7a965faf5bd3ca8a45aeb7d25144d094cf31dd5 Mon Sep 17 00:00:00 2001 From: okletswin Date: Tue, 7 Nov 2017 21:56:38 +0800 Subject: [PATCH 4/4] create lock file if not exist --- sriov/lock.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sriov/lock.go b/sriov/lock.go index 773b4746c..fb272b2c7 100644 --- a/sriov/lock.go +++ b/sriov/lock.go @@ -11,7 +11,7 @@ type FileLock struct { // NewFileLock opens file/dir at path and returns unlocked FileLock object func NewFileLock(path string) (*FileLock, error) { - f, err := os.Open(path) + f, err := os.OpenFile(path, os.O_RDONLY|os.O_CREATE, 0644) if err != nil { return nil, err }