flannel reads from created subnet.env file on startup

Added feature to allow flannel to restart in case of etcd failures and
still keep the same subnet address for the hosts.

Fixes flannel-io#610 flannel-io#29

Documentation/reservations.md

@@ -25,6 +25,16 @@ This shows that there is a single lease (`10.5.34.0/24`) which will expire in 85
 The `"PublicIP"` value is how flannel knows to reuse this lease when restarted. 
 This means that if the public IP changes, then the flannel subnet will change too.
 
+In case something happens to etcd and it is either unavailable or loses the record of its leases, flannel will then attempt to renew the last lease that it has saved in its subnet config file (which, unless specified, is located at `/var/run/flannel/subnet.env`)
+```bash
+cat /var/run/flannel/subnet.env
+FLANNEL_NETWORK=10.5.0.0/16
+FLANNEL_SUBNET=10.5.34.1/24
+FLANNEL_MTU=1450
+FLANNEL_IPMASQ=false
+```
+In this case, if flannel fails to retrieve a lease from etcd, it will attempt to renew lease specified in `FLANNEL_SUBNET` (`10.5.34.1/24`).  It will only renew this lease if the subnet specified is valid for the current etcd network configuration otherwise it will allocate a new lease.
+
 ## Reservations
 
 flannel also supports reservations for the subnet assigned to a host. Reservations

Documentation/running.md

@@ -46,6 +46,7 @@ FLANNEL_SUBNET=10.5.72.1/24
 FLANNEL_MTU=1450
 FLANNEL_IPMASQ=false
 ```
+The `FLANNEL_SUBNET` value written in this subnet config file will be checked each time flannel starts and will be reused if the subnet still matches the values stored in the etcd config.
 
 ## Interface selection
 

glide.yaml

@@ -58,3 +58,5 @@ import:
   - pkg/runtime
   - pkg/util/wait
   - pkg/watch
+- package: github.com/joho/godotenv
+  version: v1.1

main.go

@@ -23,9 +23,9 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"syscall"
-	"strconv"
 
 	"github.com/coreos/pkg/flagutil"
 	log "github.com/golang/glog"
@@ -40,6 +40,8 @@ import (
 
 	"time"
 
+	"github.com/joho/godotenv"
+
 	// Backends need to be imported for their init() to get executed and them to register
 	"github.com/coreos/flannel/backend"
 	_ "github.com/coreos/flannel/backend/alivpc"
@@ -115,6 +117,8 @@ func newSubnetManager() (subnet.Manager, error) {
 		Password:  opts.etcdPassword,
 	}
 
+	ReadSubnetFromSubnetFile(opts.subnetFile, cfg)
+
 	return etcdv2.NewLocalManager(cfg)
 }
 
@@ -410,3 +414,17 @@ func mustRunHealthz() {
 		panic(err)
 	}
 }
+
+func ReadSubnetFromSubnetFile(path string, config *etcdv2.EtcdConfig) {
+	if _, err := os.Stat(path); !os.IsNotExist(err) {
+		prevSubnetVals, err := godotenv.Read(path)
+		if err != nil {
+			log.Errorf("Couldn't fetch previous subnet from subnet file at %s: %s", path, err)
+		} else if prevSubnetString, ok := prevSubnetVals["FLANNEL_SUBNET"]; ok {
+			err = config.PreviousSubnet.UnmarshalJSON([]byte(prevSubnetString))
+			if err != nil {
+				log.Errorf("Couldn't parse previous subnet from subnet file at %s: %s", path, err)
+			}
+		}
+	}
+}

pkg/ip/ipnet.go

@@ -159,6 +159,10 @@ func (n IP4Net) Contains(ip IP4) bool {
 	return (uint32(n.IP) & n.Mask()) == (uint32(ip) & n.Mask())
 }
 
+func (n IP4Net) Empty() bool {
+	return n.IP == IP4(0) && n.PrefixLen == uint(0)
+}
+
 // json.Marshaler impl
 func (n IP4Net) MarshalJSON() ([]byte, error) {
 	return []byte(fmt.Sprintf(`"%s"`, n)), nil

subnet/etcdv2/local_manager.go

@@ -33,7 +33,8 @@ const (
 )
 
 type LocalManager struct {
-	registry Registry
+	registry       Registry
+	previousSubnet ip.IP4Net
 }
 
 type watchCursor struct {
@@ -73,12 +74,13 @@ func NewLocalManager(config *EtcdConfig) (Manager, error) {
 	if err != nil {
 		return nil, err
 	}
-	return newLocalManager(r), nil
+	return newLocalManager(r, config.PreviousSubnet), nil
 }
 
-func newLocalManager(r Registry) Manager {
+func newLocalManager(r Registry, prevSubnet ip.IP4Net) Manager {
 	return &LocalManager{
-		registry: r,
+		registry:       r,
+		previousSubnet: prevSubnet,
 	}
 }
 
@@ -155,10 +157,24 @@ func (m *LocalManager) tryAcquireLease(ctx context.Context, config *Config, extI
 		}
 	}
 
-	// no existing match, grab a new one
-	sn, err := m.allocateSubnet(config, leases)
-	if err != nil {
-		return nil, err
+	// no existing match, check if there was a previous subnet to use
+	var sn ip.IP4Net
+	// Check if the previous subnet is a part of the network and of the right subnet length
+	if !m.previousSubnet.Empty() && isSubnetConfigCompat(config, m.previousSubnet) {
+		// use previous subnet
+		log.Infof("Found previously leased subnet (%v), reusing", m.previousSubnet)
+		sn = m.previousSubnet
+	} else {
+		// Create error message for info
+		if !m.previousSubnet.Empty() {
+			log.Errorf("Found previously leased subnet (%v) that is not compatible with the Etcd network config, ignoring", m.previousSubnet)
+		}
+
+		// no existing match, grab a new one
+		sn, err = m.allocateSubnet(config, leases)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	exp, err := m.registry.createSubnet(ctx, sn, attrs, subnetTTL)

subnet/etcdv2/mock_subnet.go

@@ -15,9 +15,14 @@
 package etcdv2
 
 import (
+	"github.com/coreos/flannel/pkg/ip"
 	"github.com/coreos/flannel/subnet"
 )
 
 func NewMockManager(registry *MockSubnetRegistry) subnet.Manager {
-	return newLocalManager(registry)
+	return newLocalManager(registry, ip.IP4Net{})
+}
+
+func NewMockManagerWithSubnet(registry *MockSubnetRegistry, sn ip.IP4Net) subnet.Manager {
+	return newLocalManager(registry, sn)
 }

subnet/etcdv2/registry.go

@@ -48,13 +48,14 @@ type Registry interface {
 }
 
 type EtcdConfig struct {
-	Endpoints []string
-	Keyfile   string
-	Certfile  string
-	CAFile    string
-	Prefix    string
-	Username  string
-	Password  string
+	Endpoints      []string
+	Keyfile        string
+	Certfile       string
+	CAFile         string
+	Prefix         string
+	Username       string
+	Password       string
+	PreviousSubnet ip.IP4Net
 }
 
 type etcdNewFunc func(c *EtcdConfig) (etcd.KeysAPI, error)

subnet/etcdv2/subnet_test.go

@@ -76,6 +76,30 @@ func TestAcquireLease(t *testing.T) {
 	if !l.Subnet.Equal(l2.Subnet) {
 		t.Fatalf("AcquireLease did not reuse subnet; expected %v, got %v", l.Subnet, l2.Subnet)
 	}
+
+	// Test if a previous subnet will be used
+	msr2 := newDummyRegistry()
+	prevSubnet := ip.IP4Net{ip.MustParseIP4("10.3.6.0"), 24}
+	sm2 := NewMockManagerWithSubnet(msr2, prevSubnet)
+	prev, err := sm2.AcquireLease(context.Background(), &attrs)
+	if err != nil {
+		t.Fatal("AcquireLease failed: ", err)
+	}
+	if !prev.Subnet.Equal(prevSubnet) {
+		t.Fatalf("AcquireLease did not reuse subnet from previous run; expected %v, got %v", prevSubnet, prev.Subnet)
+	}
+
+	// Test that a previous subnet will not be used if it does not match the registry config
+	msr3 := newDummyRegistry()
+	invalidSubnet := ip.IP4Net{ip.MustParseIP4("10.4.1.0"), 24}
+	sm3 := NewMockManagerWithSubnet(msr3, invalidSubnet)
+	l3, err := sm3.AcquireLease(context.Background(), &attrs)
+	if err != nil {
+		t.Fatal("AcquireLease failed: ", err)
+	}
+	if l3.Subnet.Equal(invalidSubnet) {
+		t.Fatalf("AcquireLease reused invalid subnet from previous run; reused %v", l3.Subnet)
+	}
 }
 
 func TestConfigChanged(t *testing.T) {