Merge pull request #752 from mgleung/prev-lease

flannel reads from created subnet.env file on startup

Documentation/reservations.md

@@ -25,6 +25,16 @@ This shows that there is a single lease (`10.5.34.0/24`) which will expire in 85
 The `"PublicIP"` value is how flannel knows to reuse this lease when restarted. 
 This means that if the public IP changes, then the flannel subnet will change too.
 
+In case a host is unable to renew its lease before the lease expires (e.g. a host takes a long time to restart and the timing lines up with when the lease would normally be renewed), flannel will then attempt to renew the last lease that it has saved in its subnet config file (which, unless specified, is located at `/var/run/flannel/subnet.env`)
+```bash
+cat /var/run/flannel/subnet.env
+FLANNEL_NETWORK=10.5.0.0/16
+FLANNEL_SUBNET=10.5.34.1/24
+FLANNEL_MTU=1450
+FLANNEL_IPMASQ=false
+```
+In this case, if flannel fails to retrieve an existing lease from etcd, it will attempt to renew lease specified in `FLANNEL_SUBNET` (`10.5.34.1/24`).  It will only renew this lease if the subnet specified is valid for the current etcd network configuration otherwise it will allocate a new lease.
+
 ## Reservations
 
 flannel also supports reservations for the subnet assigned to a host. Reservations

Documentation/running.md

@@ -46,6 +46,23 @@ FLANNEL_SUBNET=10.5.72.1/24
 FLANNEL_MTU=1450
 FLANNEL_IPMASQ=false
 ```
+Each time flannel is restarted, it will attempt to access the `FLANNEL_SUBNET` value written in this subnet config file. This prevents each host from needing to update its network information in case a host is unable to renew its lease before it expires (e.g. a host was restarting during the time flannel would normally renew its lease).
+
+The `FLANNEL_SUBNET` value is also only used if it is valid for the etcd network config. For instance, a `FLANNEL_SUBNET` value of `10.5.72.1/24` will not be used if the etcd network value is set to `10.6.0.0/16` since it is not within that network range.
+
+Subnet config value is `10.5.72.1/24`
+```bash
+cat /var/run/flannel/subnet.env
+FLANNEL_NETWORK=10.5.0.0/16
+FLANNEL_SUBNET=10.5.72.1/24
+FLANNEL_MTU=1450
+FLANNEL_IPMASQ=false
+```
+etcd network value is `10.6.0.0/16`. Since `10.5.72.1/24` is outside of this network, a new lease will be allocated.
+```bash
+etcdctl get /coreos.com/network/config
+{ "Network": "10.6.0.0/16", "Backend": {"Type": "vxlan"}}
+```
 
 ## Interface selection
 

glide.yaml

@@ -58,3 +58,5 @@ import:
   - pkg/runtime
   - pkg/util/wait
   - pkg/watch
+- package: github.com/joho/godotenv
+  version: v1.1

main.go

@@ -41,6 +41,8 @@ import (
 
 	"time"
 
+	"github.com/joho/godotenv"
+
 	// Backends need to be imported for their init() to get executed and them to register
 	"github.com/coreos/flannel/backend"
 	_ "github.com/coreos/flannel/backend/alivpc"
@@ -118,7 +120,10 @@ func newSubnetManager() (subnet.Manager, error) {
 		Password:  opts.etcdPassword,
 	}
 
-	return etcdv2.NewLocalManager(cfg)
+	// Attempt to renew the lease for the subnet specified in the subnetFile
+	prevSubnet := ReadSubnetFromSubnetFile(opts.subnetFile)
+
+	return etcdv2.NewLocalManager(cfg, prevSubnet)
 }
 
 func main() {
@@ -456,3 +461,19 @@ func mustRunHealthz() {
 		panic(err)
 	}
 }
+
+func ReadSubnetFromSubnetFile(path string) ip.IP4Net {
+	var prevSubnet ip.IP4Net
+	if _, err := os.Stat(path); !os.IsNotExist(err) {
+		prevSubnetVals, err := godotenv.Read(path)
+		if err != nil {
+			log.Errorf("Couldn't fetch previous subnet from subnet file at %s: %s", path, err)
+		} else if prevSubnetString, ok := prevSubnetVals["FLANNEL_SUBNET"]; ok {
+			err = prevSubnet.UnmarshalJSON([]byte(prevSubnetString))
+			if err != nil {
+				log.Errorf("Couldn't parse previous subnet from subnet file at %s: %s", path, err)
+			}
+		}
+	}
+	return prevSubnet
+}

pkg/ip/ipnet.go

@@ -159,6 +159,10 @@ func (n IP4Net) Contains(ip IP4) bool {
 	return (uint32(n.IP) & n.Mask()) == (uint32(ip) & n.Mask())
 }
 
+func (n IP4Net) Empty() bool {
+	return n.IP == IP4(0) && n.PrefixLen == uint(0)
+}
+
 // json.Marshaler impl
 func (n IP4Net) MarshalJSON() ([]byte, error) {
 	return []byte(fmt.Sprintf(`"%s"`, n)), nil

subnet/etcdv2/local_manager.go

@@ -33,7 +33,8 @@ const (
 )
 
 type LocalManager struct {
-	registry Registry
+	registry       Registry
+	previousSubnet ip.IP4Net
 }
 
 type watchCursor struct {
@@ -68,17 +69,18 @@ func (c watchCursor) String() string {
 	return strconv.FormatUint(c.index, 10)
 }
 
-func NewLocalManager(config *EtcdConfig) (Manager, error) {
+func NewLocalManager(config *EtcdConfig, prevSubnet ip.IP4Net) (Manager, error) {
 	r, err := newEtcdSubnetRegistry(config, nil)
 	if err != nil {
 		return nil, err
 	}
-	return newLocalManager(r), nil
+	return newLocalManager(r, prevSubnet), nil
 }
 
-func newLocalManager(r Registry) Manager {
+func newLocalManager(r Registry, prevSubnet ip.IP4Net) Manager {
 	return &LocalManager{
-		registry: r,
+		registry:       r,
+		previousSubnet: prevSubnet,
 	}
 }
 
@@ -155,10 +157,24 @@ func (m *LocalManager) tryAcquireLease(ctx context.Context, config *Config, extI
 		}
 	}
 
-	// no existing match, grab a new one
-	sn, err := m.allocateSubnet(config, leases)
-	if err != nil {
-		return nil, err
+	// no existing match, check if there was a previous subnet to use
+	var sn ip.IP4Net
+	// Check if the previous subnet is a part of the network and of the right subnet length
+	if !m.previousSubnet.Empty() && isSubnetConfigCompat(config, m.previousSubnet) {
+		// use previous subnet
+		log.Infof("Found previously leased subnet (%v), reusing", m.previousSubnet)
+		sn = m.previousSubnet
+	} else {
+		// Create error message for info
+		if !m.previousSubnet.Empty() {
+			log.Errorf("Found previously leased subnet (%v) that is not compatible with the Etcd network config, ignoring", m.previousSubnet)
+		}
+
+		// no existing match, grab a new one
+		sn, err = m.allocateSubnet(config, leases)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	exp, err := m.registry.createSubnet(ctx, sn, attrs, subnetTTL)

subnet/etcdv2/mock_subnet.go

@@ -15,9 +15,14 @@
 package etcdv2
 
 import (
+	"github.com/coreos/flannel/pkg/ip"
 	"github.com/coreos/flannel/subnet"
 )
 
 func NewMockManager(registry *MockSubnetRegistry) subnet.Manager {
-	return newLocalManager(registry)
+	return newLocalManager(registry, ip.IP4Net{})
+}
+
+func NewMockManagerWithSubnet(registry *MockSubnetRegistry, sn ip.IP4Net) subnet.Manager {
+	return newLocalManager(registry, sn)
 }

subnet/etcdv2/subnet_test.go

@@ -76,6 +76,30 @@ func TestAcquireLease(t *testing.T) {
 	if !l.Subnet.Equal(l2.Subnet) {
 		t.Fatalf("AcquireLease did not reuse subnet; expected %v, got %v", l.Subnet, l2.Subnet)
 	}
+
+	// Test if a previous subnet will be used
+	msr2 := newDummyRegistry()
+	prevSubnet := ip.IP4Net{ip.MustParseIP4("10.3.6.0"), 24}
+	sm2 := NewMockManagerWithSubnet(msr2, prevSubnet)
+	prev, err := sm2.AcquireLease(context.Background(), &attrs)
+	if err != nil {
+		t.Fatal("AcquireLease failed: ", err)
+	}
+	if !prev.Subnet.Equal(prevSubnet) {
+		t.Fatalf("AcquireLease did not reuse subnet from previous run; expected %v, got %v", prevSubnet, prev.Subnet)
+	}
+
+	// Test that a previous subnet will not be used if it does not match the registry config
+	msr3 := newDummyRegistry()
+	invalidSubnet := ip.IP4Net{ip.MustParseIP4("10.4.1.0"), 24}
+	sm3 := NewMockManagerWithSubnet(msr3, invalidSubnet)
+	l3, err := sm3.AcquireLease(context.Background(), &attrs)
+	if err != nil {
+		t.Fatal("AcquireLease failed: ", err)
+	}
+	if l3.Subnet.Equal(invalidSubnet) {
+		t.Fatalf("AcquireLease reused invalid subnet from previous run; reused %v", l3.Subnet)
+	}
 }
 
 func TestConfigChanged(t *testing.T) {