From 4045bfc39326c97f2a3b4d022d394cba39a7b77f Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Fri, 27 Jul 2018 11:18:03 -0400 Subject: [PATCH 1/4] Change ReallocateEgressIPs() to return the full allocation, not just changes --- pkg/network/common/egressip.go | 16 ++-------------- pkg/network/common/egressip_test.go | 3 --- pkg/network/master/egressip.go | 9 +++++++-- 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/pkg/network/common/egressip.go b/pkg/network/common/egressip.go index 258204c2851b..cb2477c3faae 100644 --- a/pkg/network/common/egressip.go +++ b/pkg/network/common/egressip.go @@ -506,13 +506,12 @@ func (eit *EgressIPTracker) findEgressIPAllocation(ip net.IP, allocation map[str return bestNode, otherNodes } -// ReallocateEgressIPs returns a map from Node name to array-of-Egress-IP. Unchanged nodes are not included. +// ReallocateEgressIPs returns a map from Node name to array-of-Egress-IP for all auto-allocated egress IPs func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string { eit.Lock() defer eit.Unlock() allocation := make(map[string][]string) - changed := make(map[string]bool) alreadyAllocated := make(map[string]bool) for _, node := range eit.nodes { if len(node.parsedCIDRs) > 0 { @@ -520,8 +519,7 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string { } } // For each active egress IP, if it still fits within some egress CIDR on its node, - // add it to that node's allocation. (Otherwise add the node to the "changed" map, - // since we'll be removing this egress IP from it.) + // add it to that node's allocation. for egressIP, eip := range eit.egressIPs { if eip.assignedNodeIP == "" { continue @@ -536,8 +534,6 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string { } if found { allocation[node.nodeName] = append(allocation[node.nodeName], egressIP) - } else { - changed[node.nodeName] = true } // (We set alreadyAllocated even if the egressIP will be removed from // its current node; we can't assign it to a new node until the next @@ -553,7 +549,6 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string { nodeName, otherNodes := eit.findEgressIPAllocation(eip.parsed, allocation) if nodeName != "" && !otherNodes { allocation[nodeName] = append(allocation[nodeName], egressIP) - changed[nodeName] = true alreadyAllocated[egressIP] = true } } @@ -565,15 +560,8 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string { nodeName, _ := eit.findEgressIPAllocation(eip.parsed, allocation) if nodeName != "" { allocation[nodeName] = append(allocation[nodeName], egressIP) - changed[nodeName] = true } } - // Remove unchanged nodes from the return value - for _, node := range eit.nodes { - if !changed[node.nodeName] { - delete(allocation, node.nodeName) - } - } return allocation } diff --git a/pkg/network/common/egressip_test.go b/pkg/network/common/egressip_test.go index 91143892ab88..8e4146a407a9 100644 --- a/pkg/network/common/egressip_test.go +++ b/pkg/network/common/egressip_test.go @@ -864,9 +864,6 @@ func TestEgressCIDRAllocation(t *testing.T) { t.Fatalf("%v", err) } allocation = eit.ReallocateEgressIPs() - if len(allocation) != 0 { - t.Fatalf("Unexpected allocation: %#v", allocation) - } updateAllocations(eit, allocation) err = w.assertNoChanges() if err != nil { diff --git a/pkg/network/master/egressip.go b/pkg/network/master/egressip.go index 38b149e48c8d..303c30c52e3f 100644 --- a/pkg/network/master/egressip.go +++ b/pkg/network/master/egressip.go @@ -6,6 +6,7 @@ import ( "time" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" utilwait "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/retry" @@ -82,8 +83,12 @@ func (eim *egressIPManager) maybeDoUpdateEgressCIDRs() (bool, error) { return err } - hs.EgressIPs = egressIPs - _, err = eim.networkClient.Network().HostSubnets().Update(hs) + oldIPs := sets.NewString(hs.EgressIPs...) + newIPs := sets.NewString(egressIPs...) + if !oldIPs.Equal(newIPs) { + hs.EgressIPs = egressIPs + _, err = eim.networkClient.Network().HostSubnets().Update(hs) + } return err }) if resultErr != nil { From f13ae4e82f65cc250b7cdc7474dc8c5a2770032c Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Sun, 22 Jul 2018 10:24:59 -0400 Subject: [PATCH 2/4] Track node online/offline state in master egress IP allocator and don't allocate egress IPs to offline nodes --- pkg/network/common/egressip.go | 27 +++++-- pkg/network/common/egressip_test.go | 109 ++++++++++++++++++++++++++++ pkg/network/master/egressip.go | 91 +++++++++++++++++++++++ 3 files changed, 220 insertions(+), 7 deletions(-) diff --git a/pkg/network/common/egressip.go b/pkg/network/common/egressip.go index cb2477c3faae..2293ff713de9 100644 --- a/pkg/network/common/egressip.go +++ b/pkg/network/common/egressip.go @@ -450,22 +450,32 @@ func (eit *EgressIPTracker) SetNodeOffline(nodeIP string, offline bool) { eit.egressIPChanged(eg) } } + + if node.requestedCIDRs.Len() != 0 { + eit.updateEgressCIDRs = true + } + eit.syncEgressIPs() } +func (eit *EgressIPTracker) lookupNodeIP(ip string) string { + eit.Lock() + defer eit.Unlock() + + if node := eit.nodesByNodeIP[ip]; node != nil { + return node.sdnIP + } + return ip +} + // Ping a node and return whether or not it is online. We do this by trying to open a TCP // connection to the "discard" service (port 9); if the node is offline, the attempt will // time out with no response (and we will return false). If the node is online then we // presumably will get a "connection refused" error; the code below assumes that anything // other than timing out indicates that the node is online. func (eit *EgressIPTracker) Ping(ip string, timeout time.Duration) bool { - eit.Lock() - defer eit.Unlock() - // If the caller used a public node IP, replace it with the SDN IP - if node := eit.nodesByNodeIP[ip]; node != nil { - ip = node.sdnIP - } + ip = eit.lookupNodeIP(ip) conn, err := net.DialTimeout("tcp", ip+":9", timeout) if conn != nil { @@ -485,6 +495,9 @@ func (eit *EgressIPTracker) findEgressIPAllocation(ip net.IP, allocation map[str otherNodes := false for _, node := range eit.nodes { + if node.offline { + continue + } egressIPs, exists := allocation[node.nodeName] if !exists { continue @@ -532,7 +545,7 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string { break } } - if found { + if found && !node.offline { allocation[node.nodeName] = append(allocation[node.nodeName], egressIP) } // (We set alreadyAllocated even if the egressIP will be removed from diff --git a/pkg/network/common/egressip_test.go b/pkg/network/common/egressip_test.go index 8e4146a407a9..8db9e93ab2ff 100644 --- a/pkg/network/common/egressip_test.go +++ b/pkg/network/common/egressip_test.go @@ -73,6 +73,20 @@ func (w *testEIPWatcher) assertNoChanges() error { return w.assertChanges() } +func (w *testEIPWatcher) flushChanges() { + w.changes = []string{} +} + +func (w *testEIPWatcher) assertUpdateEgressCIDRsNotification() error { + for _, change := range w.changes { + if change == "update egress CIDRs" { + w.flushChanges() + return nil + } + } + return fmt.Errorf("expected change \"update egress CIDRs\", got %#v", w.changes) +} + func setupEgressIPTracker(t *testing.T) (*EgressIPTracker, *testEIPWatcher) { watcher := &testEIPWatcher{} return NewEgressIPTracker(watcher), watcher @@ -1028,3 +1042,98 @@ func TestEgressNodeRenumbering(t *testing.T) { t.Fatalf("%v", err) } } + +func TestEgressCIDRAllocationOffline(t *testing.T) { + eit, w := setupEgressIPTracker(t) + + // Create nodes... + updateHostSubnetEgress(eit, &networkapi.HostSubnet{ + HostIP: "172.17.0.3", + EgressIPs: []string{}, + EgressCIDRs: []string{"172.17.0.0/24", "172.17.1.0/24"}, + }) + updateHostSubnetEgress(eit, &networkapi.HostSubnet{ + HostIP: "172.17.0.4", + EgressIPs: []string{}, + EgressCIDRs: []string{"172.17.0.0/24"}, + }) + updateHostSubnetEgress(eit, &networkapi.HostSubnet{ + HostIP: "172.17.0.5", + EgressIPs: []string{}, + EgressCIDRs: []string{"172.17.1.0/24"}, + }) + + // Create namespaces + updateNetNamespaceEgress(eit, &networkapi.NetNamespace{ + NetID: 100, + EgressIPs: []string{"172.17.0.100"}, + }) + updateNetNamespaceEgress(eit, &networkapi.NetNamespace{ + NetID: 101, + EgressIPs: []string{"172.17.0.101"}, + }) + updateNetNamespaceEgress(eit, &networkapi.NetNamespace{ + NetID: 102, + EgressIPs: []string{"172.17.0.102"}, + }) + updateNetNamespaceEgress(eit, &networkapi.NetNamespace{ + NetID: 200, + EgressIPs: []string{"172.17.1.200"}, + }) + updateNetNamespaceEgress(eit, &networkapi.NetNamespace{ + NetID: 201, + EgressIPs: []string{"172.17.1.201"}, + }) + updateNetNamespaceEgress(eit, &networkapi.NetNamespace{ + NetID: 202, + EgressIPs: []string{"172.17.1.202"}, + }) + + // In a perfect world, we'd get 2 IPs on each node, but depending on processing + // order, this isn't guaranteed. Eg, if the three 172.17.0.x IPs get processed + // first, we could get two of them on node-3 and one on node-4. Then the first two + // 172.17.1.x IPs get assigned to node-5, and the last one could go to either + // node-3 or node-5. Regardless of order, node-3 is guaranteed to get at least + // two IPs since there's no way either node-4 or node-5 could be assigned a + // third IP if node-3 still only had one. + allocation := eit.ReallocateEgressIPs() + node3ips := allocation["node-3"] + node4ips := allocation["node-4"] + node5ips := allocation["node-5"] + if len(node3ips) < 2 || len(node4ips) == 0 || len(node5ips) == 0 || + len(node3ips)+len(node4ips)+len(node5ips) != 6 { + t.Fatalf("Bad IP allocation: %#v", allocation) + } + updateAllocations(eit, allocation) + + w.flushChanges() + + // Now take node-3 offline + eit.SetNodeOffline("172.17.0.3", true) + err := w.assertUpdateEgressCIDRsNotification() + if err != nil { + t.Fatalf("%v", err) + } + + // First reallocation should empty out node-3 + allocation = eit.ReallocateEgressIPs() + if node3ips, ok := allocation["node-3"]; !ok || len(node3ips) != 0 { + t.Fatalf("Bad IP allocation: %#v", allocation) + } + updateAllocations(eit, allocation) + + err = w.assertUpdateEgressCIDRsNotification() + if err != nil { + t.Fatalf("%v", err) + } + + // Next reallocation should reassign egress IPs to node-4 and node-5 + allocation = eit.ReallocateEgressIPs() + node3ips = allocation["node-3"] + node4ips = allocation["node-4"] + node5ips = allocation["node-5"] + if len(node3ips) != 0 || len(node4ips) != 3 || len(node5ips) != 3 { + t.Fatalf("Bad IP allocation: %#v", allocation) + } + updateAllocations(eit, allocation) +} diff --git a/pkg/network/master/egressip.go b/pkg/network/master/egressip.go index 303c30c52e3f..528d69d2a2c6 100644 --- a/pkg/network/master/egressip.go +++ b/pkg/network/master/egressip.go @@ -5,6 +5,8 @@ import ( "sync" "time" + "github.com/golang/glog" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" utilwait "k8s.io/apimachinery/pkg/util/wait" @@ -24,6 +26,15 @@ type egressIPManager struct { updatePending bool updatedAgain bool + + monitorNodes map[string]*egressNode + stop chan struct{} +} + +type egressNode struct { + ip string + offline bool + retries int } func newEgressIPManager() *egressIPManager { @@ -76,6 +87,7 @@ func (eim *egressIPManager) maybeDoUpdateEgressCIDRs() (bool, error) { // we won't process that until this reallocation is complete. allocation := eim.tracker.ReallocateEgressIPs() + monitorNodes := make(map[string]*egressNode, len(allocation)) for nodeName, egressIPs := range allocation { resultErr := retry.RetryOnConflict(retry.DefaultBackoff, func() error { hs, err := eim.hostSubnetInformer.Lister().Get(nodeName) @@ -83,6 +95,12 @@ func (eim *egressIPManager) maybeDoUpdateEgressCIDRs() (bool, error) { return err } + if node := eim.monitorNodes[hs.HostIP]; node != nil { + monitorNodes[hs.HostIP] = node + } else { + monitorNodes[hs.HostIP] = &egressNode{ip: hs.HostIP} + } + oldIPs := sets.NewString(hs.EgressIPs...) newIPs := sets.NewString(egressIPs...) if !oldIPs.Equal(newIPs) { @@ -96,9 +114,82 @@ func (eim *egressIPManager) maybeDoUpdateEgressCIDRs() (bool, error) { } } + eim.monitorNodes = monitorNodes + if len(monitorNodes) > 0 { + if eim.stop == nil { + eim.stop = make(chan struct{}) + go eim.poll(eim.stop) + } + } else { + if eim.stop != nil { + close(eim.stop) + eim.stop = nil + } + } + return true, nil } +const ( + pollInterval = 5 * time.Second + repollInterval = time.Second + maxRetries = 2 +) + +func (eim *egressIPManager) poll(stop chan struct{}) { + retry := false + for { + select { + case <-stop: + return + default: + } + + start := time.Now() + retry := eim.check(retry) + if !retry { + // If less than pollInterval has passed since start, then sleep until it has + time.Sleep(start.Add(pollInterval).Sub(time.Now())) + } + } +} + +func (eim *egressIPManager) check(retrying bool) bool { + var timeout time.Duration + if retrying { + timeout = repollInterval + } else { + timeout = pollInterval + } + + needRetry := false + for _, node := range eim.monitorNodes { + if retrying && node.retries == 0 { + continue + } + + online := eim.tracker.Ping(node.ip, timeout) + if node.offline && online { + glog.Infof("Node %s is back online", node.ip) + node.offline = false + eim.tracker.SetNodeOffline(node.ip, false) + } else if !node.offline && !online { + node.retries++ + if node.retries > maxRetries { + glog.Warningf("Node %s is offline", node.ip) + node.retries = 0 + node.offline = true + eim.tracker.SetNodeOffline(node.ip, true) + } else { + glog.V(2).Infof("Node %s may be offline... retrying", node.ip) + needRetry = true + } + } + } + + return needRetry +} + func (eim *egressIPManager) ClaimEgressIP(vnid uint32, egressIP, nodeIP string) { } From 6bb345b656609aa4cea0d6af28a723b0c9341023 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Tue, 31 Jul 2018 07:20:03 -0400 Subject: [PATCH 3/4] Rebalance auto-assigned egress IPs when necessary --- pkg/network/common/egressip.go | 71 ++++++++++++++++++++++++----- pkg/network/common/egressip_test.go | 34 ++++++++++++++ 2 files changed, 94 insertions(+), 11 deletions(-) diff --git a/pkg/network/common/egressip.go b/pkg/network/common/egressip.go index 2293ff713de9..554307627b29 100644 --- a/pkg/network/common/egressip.go +++ b/pkg/network/common/egressip.go @@ -498,10 +498,7 @@ func (eit *EgressIPTracker) findEgressIPAllocation(ip net.IP, allocation map[str if node.offline { continue } - egressIPs, exists := allocation[node.nodeName] - if !exists { - continue - } + egressIPs := allocation[node.nodeName] for _, parsed := range node.parsedCIDRs { if parsed.Contains(ip) { if bestNode != "" { @@ -519,13 +516,13 @@ func (eit *EgressIPTracker) findEgressIPAllocation(ip net.IP, allocation map[str return bestNode, otherNodes } -// ReallocateEgressIPs returns a map from Node name to array-of-Egress-IP for all auto-allocated egress IPs -func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string { - eit.Lock() - defer eit.Unlock() +func (eit *EgressIPTracker) makeEmptyAllocation() (map[string][]string, map[string]bool) { + return make(map[string][]string), make(map[string]bool) +} + +func (eit *EgressIPTracker) allocateExistingEgressIPs(allocation map[string][]string, alreadyAllocated map[string]bool) bool { + removedEgressIPs := false - allocation := make(map[string][]string) - alreadyAllocated := make(map[string]bool) for _, node := range eit.nodes { if len(node.parsedCIDRs) > 0 { allocation[node.nodeName] = make([]string, 0, node.requestedIPs.Len()) @@ -534,7 +531,7 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string { // For each active egress IP, if it still fits within some egress CIDR on its node, // add it to that node's allocation. for egressIP, eip := range eit.egressIPs { - if eip.assignedNodeIP == "" { + if eip.assignedNodeIP == "" || alreadyAllocated[egressIP] { continue } node := eip.nodes[0] @@ -547,6 +544,8 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string { } if found && !node.offline { allocation[node.nodeName] = append(allocation[node.nodeName], egressIP) + } else { + removedEgressIPs = true } // (We set alreadyAllocated even if the egressIP will be removed from // its current node; we can't assign it to a new node until the next @@ -554,6 +553,10 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string { alreadyAllocated[egressIP] = true } + return removedEgressIPs +} + +func (eit *EgressIPTracker) allocateNewEgressIPs(allocation map[string][]string, alreadyAllocated map[string]bool) { // Allocate pending egress IPs that can only go to a single node for egressIP, eip := range eit.egressIPs { if alreadyAllocated[egressIP] || len(eip.namespaces) == 0 { @@ -575,6 +578,52 @@ func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string { allocation[nodeName] = append(allocation[nodeName], egressIP) } } +} + +// ReallocateEgressIPs returns a map from Node name to array-of-Egress-IP for all auto-allocated egress IPs +func (eit *EgressIPTracker) ReallocateEgressIPs() map[string][]string { + eit.Lock() + defer eit.Unlock() + + allocation, alreadyAllocated := eit.makeEmptyAllocation() + removedEgressIPs := eit.allocateExistingEgressIPs(allocation, alreadyAllocated) + eit.allocateNewEgressIPs(allocation, alreadyAllocated) + if removedEgressIPs { + // Process the removals now; we'll get called again afterward and can + // check for balance then. + return allocation + } + + // Compare the allocation to what we would have gotten if we started from scratch, + // to see if things have gotten too unbalanced. (In particular, if a node goes + // offline, gets emptied, and then comes back online, we want to move a bunch of + // egress IPs back onto that node.) + fullReallocation, alreadyAllocated := eit.makeEmptyAllocation() + eit.allocateNewEgressIPs(fullReallocation, alreadyAllocated) + + emptyNodes := []string{} + for nodeName, fullEgressIPs := range fullReallocation { + incrementalEgressIPs := allocation[nodeName] + if len(incrementalEgressIPs) < len(fullEgressIPs)/2 { + emptyNodes = append(emptyNodes, nodeName) + } + } + + if len(emptyNodes) > 0 { + // Make a new incremental allocation, but skipping all of the egress IPs + // that got assigned to the "empty" nodes in the full reallocation; this + // will cause them to be dropped from their current nodes and then later + // reassigned (to one of the "empty" nodes, for balance). + allocation, alreadyAllocated = eit.makeEmptyAllocation() + for _, nodeName := range emptyNodes { + for _, egressIP := range fullReallocation[nodeName] { + alreadyAllocated[egressIP] = true + } + } + eit.allocateExistingEgressIPs(allocation, alreadyAllocated) + eit.allocateNewEgressIPs(allocation, alreadyAllocated) + eit.updateEgressCIDRs = true + } return allocation } diff --git a/pkg/network/common/egressip_test.go b/pkg/network/common/egressip_test.go index 8db9e93ab2ff..c3b4ec02b0e6 100644 --- a/pkg/network/common/egressip_test.go +++ b/pkg/network/common/egressip_test.go @@ -1136,4 +1136,38 @@ func TestEgressCIDRAllocationOffline(t *testing.T) { t.Fatalf("Bad IP allocation: %#v", allocation) } updateAllocations(eit, allocation) + + // Bring node-3 back + eit.SetNodeOffline("172.17.0.3", false) + err = w.assertUpdateEgressCIDRsNotification() + if err != nil { + t.Fatalf("%v", err) + } + + // First reallocation should remove some IPs from node-4 and node-5 but not add + // them to node-3. As above, the "balanced" allocation we're aiming for may not + // be perfect, but it has to be planning to assign at least 2 IPs to node-3. + allocation = eit.ReallocateEgressIPs() + node3ips = allocation["node-3"] + node4ips = allocation["node-4"] + node5ips = allocation["node-5"] + if len(node3ips) != 0 || len(node4ips)+len(node5ips) > 4 { + t.Fatalf("Bad IP allocation: %#v", allocation) + } + updateAllocations(eit, allocation) + + err = w.assertUpdateEgressCIDRsNotification() + if err != nil { + t.Fatalf("%v", err) + } + + // Next reallocation should reassign egress IPs to node-3 + allocation = eit.ReallocateEgressIPs() + node3ips = allocation["node-3"] + node4ips = allocation["node-4"] + node5ips = allocation["node-5"] + if len(node3ips) < 2 || len(node4ips) == 0 || len(node5ips) == 0 { + t.Fatalf("Bad IP allocation: %#v", allocation) + } + updateAllocations(eit, allocation) } From 31c32182e240cb6e26ba41ac40d8b9143f79e1c7 Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Tue, 21 Aug 2018 13:01:41 -0400 Subject: [PATCH 4/4] Test deleting all EgressIPs from auto-assigned NetNamespace --- pkg/network/common/egressip_test.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pkg/network/common/egressip_test.go b/pkg/network/common/egressip_test.go index c3b4ec02b0e6..393f9a92a9f7 100644 --- a/pkg/network/common/egressip_test.go +++ b/pkg/network/common/egressip_test.go @@ -958,24 +958,33 @@ func TestEgressCIDRAllocation(t *testing.T) { t.Fatalf("%v", err) } - // Changing the EgressIPs of a namespace should drop the old allocation and create a new one + // Changing/Removing the EgressIPs of a namespace should drop the old allocation and create a new one updateNetNamespaceEgress(eit, &networkapi.NetNamespace{ NetID: 46, EgressIPs: []string{"172.17.0.202"}, // was 172.17.0.200 }) + updateNetNamespaceEgress(eit, &networkapi.NetNamespace{ + NetID: 44, + EgressIPs: []string{}, // was 172.17.1.1 + }) err = w.assertChanges( "release 172.17.0.200 on 172.17.0.4", "namespace 46 dropped", "update egress CIDRs", + "release 172.17.1.1 on 172.17.0.3", + "namespace 44 normal", + "update egress CIDRs", ) if err != nil { t.Fatalf("%v", err) } allocation = eit.ReallocateEgressIPs() - for _, ip := range allocation["node-4"] { - if ip == "172.17.0.200" { - t.Fatalf("reallocation failed to drop unused egress IP 172.17.0.200: %#v", allocation) + for _, nodeAllocation := range allocation { + for _, ip := range nodeAllocation { + if ip == "172.17.1.1" || ip == "172.17.0.200" { + t.Fatalf("reallocation failed to drop unused egress IP %s: %#v", ip, allocation) + } } } updateAllocations(eit, allocation) @@ -983,6 +992,7 @@ func TestEgressCIDRAllocation(t *testing.T) { "claim 172.17.0.202 on 172.17.0.4 for namespace 46", "namespace 46 via 172.17.0.202 on 172.17.0.4", "update egress CIDRs", + "update egress CIDRs", ) if err != nil { t.Fatalf("%v", err)