From 8246a5171372b5043958851bf1ff7516f345faeb Mon Sep 17 00:00:00 2001 From: Abhinandan Prativadi Date: Sun, 20 Aug 2017 11:35:11 -0700 Subject: [PATCH] Making IP allocation method configurable This commit allows IP allocation method to be changed based on the requirement. For eg if swarm manager wants IP allocation to be serialized it can set the serializable option. If docker daemon wants IP allocation to be first available then that can be done by set the serial option to false/ignoring setting an option Signed-off-by: Abhinandan Prativadi --- bitseq/sequence.go | 24 +++-- bitseq/sequence_test.go | 176 ++++++------------------------------ idm/idm.go | 4 +- ipam/allocator.go | 16 +++- ipam/allocator_test.go | 2 +- ipamapi/labels.go | 10 ++ libnetwork_internal_test.go | 2 +- 7 files changed, 68 insertions(+), 166 deletions(-) create mode 100644 ipamapi/labels.go diff --git a/bitseq/sequence.go b/bitseq/sequence.go index c6c1fc9788..a209d1ec7c 100644 --- a/bitseq/sequence.go +++ b/bitseq/sequence.go @@ -199,22 +199,22 @@ func (h *Handle) getCopy() *Handle { } // SetAnyInRange atomically sets the first unset bit in the specified range in the sequence and returns the corresponding ordinal -func (h *Handle) SetAnyInRange(start, end uint64) (uint64, error) { +func (h *Handle) SetAnyInRange(start, end uint64, serial bool) (uint64, error) { if end < start || end >= h.bits { return invalidPos, fmt.Errorf("invalid bit range [%d, %d]", start, end) } if h.Unselected() == 0 { return invalidPos, ErrNoBitAvailable } - return h.set(0, start, end, true, false) + return h.set(0, start, end, true, false, serial) } // SetAny atomically sets the first unset bit in the sequence and returns the corresponding ordinal -func (h *Handle) SetAny() (uint64, error) { +func (h *Handle) SetAny(serial bool) (uint64, error) { if h.Unselected() == 0 { return invalidPos, ErrNoBitAvailable } - return h.set(0, 0, h.bits-1, true, false) + return h.set(0, 0, h.bits-1, true, false, serial) } // Set atomically sets the corresponding bit in the sequence @@ -222,7 +222,7 @@ func (h *Handle) Set(ordinal uint64) error { if err := h.validateOrdinal(ordinal); err != nil { return err } - _, err := h.set(ordinal, 0, 0, false, false) + _, err := h.set(ordinal, 0, 0, false, false, false) return err } @@ -231,7 +231,7 @@ func (h *Handle) Unset(ordinal uint64) error { if err := h.validateOrdinal(ordinal); err != nil { return err } - _, err := h.set(ordinal, 0, 0, false, true) + _, err := h.set(ordinal, 0, 0, false, true, false) return err } @@ -300,7 +300,7 @@ func (h *Handle) CheckConsistency() error { } // set/reset the bit -func (h *Handle) set(ordinal, start, end uint64, any bool, release bool) (uint64, error) { +func (h *Handle) set(ordinal, start, end uint64, any bool, release bool, serial bool) (uint64, error) { var ( bitPos uint64 bytePos uint64 @@ -310,6 +310,7 @@ func (h *Handle) set(ordinal, start, end uint64, any bool, release bool) (uint64 for { var store datastore.DataStore + curr := uint64(0) h.Lock() store = h.store h.Unlock() @@ -320,12 +321,15 @@ func (h *Handle) set(ordinal, start, end uint64, any bool, release bool) (uint64 } h.Lock() + if serial { + curr = h.curr + } // Get position if available if release { bytePos, bitPos = ordinalToPos(ordinal) } else { if any { - bytePos, bitPos, err = getAvailableFromCurrent(h.head, start, h.curr, end) + bytePos, bitPos, err = getAvailableFromCurrent(h.head, start, curr, end) ret = posToOrdinal(bytePos, bitPos) if err == nil { h.curr = ret + 1 @@ -517,9 +521,9 @@ func getFirstAvailable(head *sequence, start uint64) (uint64, uint64, error) { return invalidPos, invalidPos, ErrNoBitAvailable } -//getAvailableFromCurrent will look for available ordinal from the current ordinal. +// getAvailableFromCurrent will look for available ordinal from the current ordinal. // If none found then it will loop back to the start to check of the available bit. -//This can be further optimized to check from start till curr in case of a rollover +// This can be further optimized to check from start till curr in case of a rollover func getAvailableFromCurrent(head *sequence, start, curr, end uint64) (uint64, uint64, error) { var bytePos, bitPos uint64 if curr != 0 && curr > start { diff --git a/bitseq/sequence_test.go b/bitseq/sequence_test.go index e716eba48c..dc63706b39 100644 --- a/bitseq/sequence_test.go +++ b/bitseq/sequence_test.go @@ -1,4 +1,4 @@ -SetAny(false)package bitseq +package bitseq import ( "fmt" @@ -613,7 +613,7 @@ func TestSetUnset(t *testing.T) { if _, err := hnd.SetAny(false); err != ErrNoBitAvailable { t.Fatal("Expected error. Got success") } - if _, err := hnd.SetAnyInRange(10, 20); err != ErrNoBitAvailable { + if _, err := hnd.SetAnyInRange(10, 20, false); err != ErrNoBitAvailable { t.Fatal("Expected error. Got success") } if err := hnd.Set(50); err != ErrBitAllocated { @@ -647,7 +647,7 @@ func TestOffsetSetUnset(t *testing.T) { t.Fatal("Expected error. Got success") } - if _, err := hnd.SetAnyInRange(10, 20); err != ErrNoBitAvailable { + if _, err := hnd.SetAnyInRange(10, 20, false); err != ErrNoBitAvailable { t.Fatal("Expected error. Got success") } @@ -656,7 +656,7 @@ func TestOffsetSetUnset(t *testing.T) { } //At this point sequence is (0xffffffff, 9)->(0x7fffffff, 1)->(0xffffffff, 22)->end - if o, err = hnd.SetAnyInRange(32, 500); err != nil { + if o, err = hnd.SetAnyInRange(32, 500, false); err != nil { t.Fatal(err) } @@ -675,15 +675,15 @@ func TestSetInRange(t *testing.T) { firstAv := uint64(100*blockLen + blockLen - 1) - if o, err := hnd.SetAnyInRange(4, 3); err == nil { + if o, err := hnd.SetAnyInRange(4, 3, false); err == nil { t.Fatalf("Expected failure. Got success with ordinal:%d", o) } - if o, err := hnd.SetAnyInRange(0, numBits); err == nil { + if o, err := hnd.SetAnyInRange(0, numBits, false); err == nil { t.Fatalf("Expected failure. Got success with ordinal:%d", o) } - o, err := hnd.SetAnyInRange(100*uint64(blockLen), 101*uint64(blockLen)) + o, err := hnd.SetAnyInRange(100*uint64(blockLen), 101*uint64(blockLen), false) if err != nil { t.Fatalf("Unexpected failure: (%d, %v)", o, err) } @@ -691,19 +691,19 @@ func TestSetInRange(t *testing.T) { t.Fatalf("Unexpected ordinal: %d", o) } - if o, err := hnd.SetAnyInRange(0, uint64(blockLen)); err == nil { + if o, err := hnd.SetAnyInRange(0, uint64(blockLen), false); err == nil { t.Fatalf("Expected failure. Got success with ordinal:%d", o) } - if o, err := hnd.SetAnyInRange(0, firstAv-1); err == nil { + if o, err := hnd.SetAnyInRange(0, firstAv-1, false); err == nil { t.Fatalf("Expected failure. Got success with ordinal:%d", o) } - if o, err := hnd.SetAnyInRange(111*uint64(blockLen), 161*uint64(blockLen)); err == nil { + if o, err := hnd.SetAnyInRange(111*uint64(blockLen), 161*uint64(blockLen), false); err == nil { t.Fatalf("Expected failure. Got success with ordinal:%d", o) } - o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen)) + o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen), false) if err != nil { t.Fatal(err) } @@ -711,7 +711,7 @@ func TestSetInRange(t *testing.T) { t.Fatalf("Unexpected ordinal: %d", o) } - o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen)) + o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen), false) if err != nil { t.Fatal(err) } @@ -719,17 +719,17 @@ func TestSetInRange(t *testing.T) { t.Fatalf("Unexpected ordinal: %d", o) } - o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen)) + o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen), false) if err == nil { t.Fatalf("Expected failure. Got success with ordinal:%d", o) } - if _, err := hnd.SetAnyInRange(0, numBits-1); err != nil { + if _, err := hnd.SetAnyInRange(0, numBits-1, false); err != nil { t.Fatalf("Unexpected failure: %v", err) } // set one bit using the set range with 1 bit size range - if _, err := hnd.SetAnyInRange(uint64(163*blockLen-1), uint64(163*blockLen-1)); err != nil { + if _, err := hnd.SetAnyInRange(uint64(163*blockLen-1), uint64(163*blockLen-1), false); err != nil { t.Fatal(err) } @@ -741,12 +741,12 @@ func TestSetInRange(t *testing.T) { // set all bit in the first range for hnd.Unselected() > 22 { - if o, err := hnd.SetAnyInRange(0, 7); err != nil { + if o, err := hnd.SetAnyInRange(0, 7, false); err != nil { t.Fatalf("Unexpected failure: (%d, %v)", o, err) } } // try one more set, which should fail - o, err = hnd.SetAnyInRange(0, 7) + o, err = hnd.SetAnyInRange(0, 7, false) if err == nil { t.Fatalf("Expected failure. Got success with ordinal:%d", o) } @@ -756,13 +756,13 @@ func TestSetInRange(t *testing.T) { // set all bit in a second range for hnd.Unselected() > 14 { - if o, err := hnd.SetAnyInRange(8, 15); err != nil { + if o, err := hnd.SetAnyInRange(8, 15, false); err != nil { t.Fatalf("Unexpected failure: (%d, %v)", o, err) } } // try one more set, which should fail - o, err = hnd.SetAnyInRange(0, 15) + o, err = hnd.SetAnyInRange(0, 15, false) if err == nil { t.Fatalf("Expected failure. Got success with ordinal:%d", o) } @@ -772,131 +772,11 @@ func TestSetInRange(t *testing.T) { // set all bit in a range which includes the last bit for hnd.Unselected() > 12 { - if o, err := hnd.SetAnyInRange(28, 29); err != nil { + if o, err := hnd.SetAnyInRange(28, 29, false); err != nil { t.Fatalf("Unexpected failure: (%d, %v)", o, err) } } - o, err = hnd.SetAnyInRange(28, 29) - if err == nil { - t.Fatalf("Expected failure. Got success with ordinal:%d", o) - } - if err != ErrNoBitAvailable { - t.Fatalf("Unexpected error: %v", err) - } -} - -func TestSetInRangeSerial(t *testing.T) { - numBits := uint64(1024 * blockLen) - hnd, err := NewHandle("", nil, "", numBits) - if err != nil { - t.Fatal(err) - } - hnd.head = getTestSequence() - - firstAv := uint64(100*blockLen + blockLen - 1) - - if o, err := hnd.SetAnyInRange(4, 3); err == nil { - t.Fatalf("Expected failure. Got success with ordinal:%d", o) - } - - if o, err := hnd.SetAnyInRange(0, numBits); err == nil { - t.Fatalf("Expected failure. Got success with ordinal:%d", o) - } - - o, err := hnd.SetAnyInRange(100*uint64(blockLen), 101*uint64(blockLen)) - if err != nil { - t.Fatalf("Unexpected failure: (%d, %v)", o, err) - } - if o != firstAv { - t.Fatalf("Unexpected ordinal: %d", o) - } - - if o, err := hnd.SetAnyInRange(0, uint64(blockLen)); err == nil { - t.Fatalf("Expected failure. Got success with ordinal:%d", o) - } - - if o, err := hnd.SetAnyInRange(0, firstAv-1); err == nil { - t.Fatalf("Expected failure. Got success with ordinal:%d", o) - } - - if o, err := hnd.SetAnyInRange(111*uint64(blockLen), 161*uint64(blockLen)); err == nil { - t.Fatalf("Expected failure. Got success with ordinal:%d", o) - } - - o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen)) - if err != nil { - t.Fatal(err) - } - if o != 161*uint64(blockLen)+30 { - t.Fatalf("Unexpected ordinal: %d", o) - } - - o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen)) - if err != nil { - t.Fatal(err) - } - if o != 161*uint64(blockLen)+31 { - t.Fatalf("Unexpected ordinal: %d", o) - } - - o, err = hnd.SetAnyInRange(161*uint64(blockLen), 162*uint64(blockLen)) - if err == nil { - t.Fatalf("Expected failure. Got success with ordinal:%d", o) - } - - if _, err := hnd.SetAnyInRange(0, numBits-1); err != nil { - t.Fatalf("Unexpected failure: %v", err) - } - - // set one bit using the set range with 1 bit size range - if _, err := hnd.SetAnyInRange(uint64(163*blockLen-1), uint64(163*blockLen-1)); err != nil { - t.Fatal(err) - } - - // create a non multiple of 32 mask - hnd, err = NewHandle("", nil, "", 30) - if err != nil { - t.Fatal(err) - } - - // set all bit in the first range - for hnd.Unselected() > 22 { - if o, err := hnd.SetAnyInRange(0, 7); err != nil { - t.Fatalf("Unexpected failure: (%d, %v)", o, err) - } - } - // try one more set, which should fail - o, err = hnd.SetAnyInRange(0, 7) - if err == nil { - t.Fatalf("Expected failure. Got success with ordinal:%d", o) - } - if err != ErrNoBitAvailable { - t.Fatalf("Unexpected error: %v", err) - } - - // set all bit in a second range - for hnd.Unselected() > 14 { - if o, err := hnd.SetAnyInRange(8, 15); err != nil { - t.Fatalf("Unexpected failure: (%d, %v)", o, err) - } - } - - // try one more set, which should fail - o, err = hnd.SetAnyInRange(0, 15) - if err == nil { - t.Fatalf("Expected failure. Got success with ordinal:%d", o) - } - if err != ErrNoBitAvailable { - t.Fatalf("Unexpected error: %v", err) - } - - // set all bit in a range which includes the last bit - for hnd.Unselected() > 12 { - if o, err := hnd.SetAnyInRange(28, 29); err != nil { - t.Fatalf("Unexpected failure: (%d, %v)", o, err) - } - } - o, err = hnd.SetAnyInRange(28, 29) + o, err = hnd.SetAnyInRange(28, 29, false) if err == nil { t.Fatalf("Expected failure. Got success with ordinal:%d", o) } @@ -926,7 +806,7 @@ func TestSetAnyInRange(t *testing.T) { t.Fatal(err) } - o, err := hnd.SetAnyInRange(128, 255) + o, err := hnd.SetAnyInRange(128, 255, false) if err != nil { t.Fatal(err) } @@ -934,7 +814,7 @@ func TestSetAnyInRange(t *testing.T) { t.Fatalf("Unexpected ordinal: %d", o) } - o, err = hnd.SetAnyInRange(128, 255) + o, err = hnd.SetAnyInRange(128, 255, false) if err != nil { t.Fatal(err) } @@ -943,7 +823,7 @@ func TestSetAnyInRange(t *testing.T) { t.Fatalf("Unexpected ordinal: %d", o) } - o, err = hnd.SetAnyInRange(246, 255) + o, err = hnd.SetAnyInRange(246, 255, false) if err != nil { t.Fatal(err) } @@ -951,7 +831,7 @@ func TestSetAnyInRange(t *testing.T) { t.Fatalf("Unexpected ordinal: %d", o) } - o, err = hnd.SetAnyInRange(246, 255) + o, err = hnd.SetAnyInRange(246, 255, false) if err != nil { t.Fatal(err) } @@ -1373,7 +1253,7 @@ func TestSetRollover(t *testing.T) { // Allocate first half of the bits for i := 0; i < numBits/2; i++ { - _, err := hnd.SetAny() + _, err := hnd.SetAny(true) if err != nil { t.Fatalf("Unexpected failure on allocation %d: %v\n%s", i, err, hnd) } @@ -1401,7 +1281,7 @@ func TestSetRollover(t *testing.T) { //request to allocate for remaining half of the bits for i := 0; i < numBits/2; i++ { - _, err := hnd.SetAny() + _, err := hnd.SetAny(true) if err != nil { t.Fatalf("Unexpected failure on allocation %d: %v\nSeed: %d\n%s", i, err, seed, hnd) } @@ -1414,7 +1294,7 @@ func TestSetRollover(t *testing.T) { } for i := 0; i < numBits/4; i++ { - _, err := hnd.SetAny() + _, err := hnd.SetAny(true) if err != nil { t.Fatalf("Unexpected failure on allocation %d: %v\nSeed: %d\n%s", i, err, seed, hnd) } diff --git a/idm/idm.go b/idm/idm.go index 7e449a0dc8..e784d2179d 100644 --- a/idm/idm.go +++ b/idm/idm.go @@ -38,7 +38,7 @@ func (i *Idm) GetID() (uint64, error) { if i.handle == nil { return 0, errors.New("ID set is not initialized") } - ordinal, err := i.handle.SetAny() + ordinal, err := i.handle.SetAny(false) return i.start + ordinal, err } @@ -65,7 +65,7 @@ func (i *Idm) GetIDInRange(start, end uint64) (uint64, error) { return 0, errors.New("Requested range does not belong to the set") } - ordinal, err := i.handle.SetAnyInRange(start-i.start, end-i.start) + ordinal, err := i.handle.SetAnyInRange(start-i.start, end-i.start, false) return i.start + ordinal, err } diff --git a/ipam/allocator.go b/ipam/allocator.go index b3876ffded..7b18744a75 100644 --- a/ipam/allocator.go +++ b/ipam/allocator.go @@ -457,7 +457,15 @@ func (a *Allocator) RequestAddress(poolID string, prefAddress net.IP, opts map[s return nil, nil, types.InternalErrorf("could not find bitmask in datastore for %s on address %v request from pool %s: %v", k.String(), prefAddress, poolID, err) } - ip, err := a.getAddress(p.Pool, bm, prefAddress, p.Range) + // In order to request for a serial ip address allocation, callers can pass in the option to request + // IP allocation serially or first available IP in the subnet + var serial bool + if opts != nil { + if val, ok := opts[ipamapi.AllocSerialPrefix]; ok { + serial = (val == "true") + } + } + ip, err := a.getAddress(p.Pool, bm, prefAddress, p.Range, serial) if err != nil { return nil, nil, err } @@ -522,7 +530,7 @@ func (a *Allocator) ReleaseAddress(poolID string, address net.IP) error { return bm.Unset(ipToUint64(h)) } -func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddress net.IP, ipr *AddressRange) (net.IP, error) { +func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddress net.IP, ipr *AddressRange, serial bool) (net.IP, error) { var ( ordinal uint64 err error @@ -535,7 +543,7 @@ func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddres return nil, ipamapi.ErrNoAvailableIPs } if ipr == nil && prefAddress == nil { - ordinal, err = bitmask.SetAny() + ordinal, err = bitmask.SetAny(serial) } else if prefAddress != nil { hostPart, e := types.GetHostPartIP(prefAddress, base.Mask) if e != nil { @@ -544,7 +552,7 @@ func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddres ordinal = ipToUint64(types.GetMinimalIP(hostPart)) err = bitmask.Set(ordinal) } else { - ordinal, err = bitmask.SetAnyInRange(ipr.Start, ipr.End) + ordinal, err = bitmask.SetAnyInRange(ipr.Start, ipr.End, serial) } switch err { diff --git a/ipam/allocator_test.go b/ipam/allocator_test.go index 4454860e11..7108eaa644 100644 --- a/ipam/allocator_test.go +++ b/ipam/allocator_test.go @@ -1033,7 +1033,7 @@ func assertGetAddress(t *testing.T, subnet string) { start := time.Now() run := 0 for err != ipamapi.ErrNoAvailableIPs { - _, err = a.getAddress(sub, bm, nil, nil) + _, err = a.getAddress(sub, bm, nil, nil, false) run++ } if printTime { diff --git a/ipamapi/labels.go b/ipamapi/labels.go new file mode 100644 index 0000000000..065e983914 --- /dev/null +++ b/ipamapi/labels.go @@ -0,0 +1,10 @@ +package ipamapi + +const ( + // Prefix constant marks the reserved label space for libnetwork + Prefix = "com.docker.network" + + // AllocSerialPrefix constant marks the reserved label space for libnetwork ipam + // allocation ordering.(serial/first available) + AllocSerialPrefix = Prefix + "ipam.serial" +) diff --git a/libnetwork_internal_test.go b/libnetwork_internal_test.go index 805f3aaa8f..58742cf5e1 100644 --- a/libnetwork_internal_test.go +++ b/libnetwork_internal_test.go @@ -614,7 +614,7 @@ func TestIpamReleaseOnNetDriverFailures(t *testing.T) { } defer ep.Delete(false) - expectedIP, _ := types.ParseCIDR("10.34.0.2/16") + expectedIP, _ := types.ParseCIDR("10.34.0.1/16") if !types.CompareIPNet(ep.Info().Iface().Address(), expectedIP) { t.Fatalf("Ipam release must have failed, endpoint has unexpected address: %v", ep.Info().Iface().Address()) }