Skip to content

Commit

Permalink
Use utilwait.ExponentialBackoff instead of looping
Browse files Browse the repository at this point in the history
  • Loading branch information
danwinship committed Oct 7, 2016
1 parent cfcd39e commit 54b856a
Show file tree
Hide file tree
Showing 2 changed files with 44 additions and 34 deletions.
53 changes: 29 additions & 24 deletions pkg/sdn/plugin/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@ import (
"github.com/openshift/origin/pkg/util/netutils"

kapi "k8s.io/kubernetes/pkg/api"
kapierrors "k8s.io/kubernetes/pkg/api/errors"
kexec "k8s.io/kubernetes/pkg/util/exec"
"k8s.io/kubernetes/pkg/util/sysctl"
utilwait "k8s.io/kubernetes/pkg/util/wait"
)

const (
Expand Down Expand Up @@ -47,21 +49,24 @@ func getPluginVersion(multitenant bool) []string {
}

func (plugin *OsdnNode) getLocalSubnet() (string, error) {
// timeout: 30 secs
retries := 60
retryInterval := 500 * time.Millisecond

var err error
var subnet *osapi.HostSubnet
// Try every retryInterval and bail-out if it exceeds max retries
for i := 0; i < retries; i++ {
backoff := utilwait.Backoff{
Duration: 100 * time.Millisecond,
Factor: 2,
Steps: 8,
}
err := utilwait.ExponentialBackoff(backoff, func() (bool, error) {
var err error
subnet, err = plugin.osClient.HostSubnets().Get(plugin.hostName)
if err == nil {
break
return true, nil
} else if kapierrors.IsNotFound(err) {
glog.Warningf("Could not find an allocated subnet for node: %s, Waiting...", plugin.hostName)
return false, nil
} else {
return false, err
}
glog.Warningf("Could not find an allocated subnet for node: %s, Waiting...", plugin.hostName)
time.Sleep(retryInterval)
}
})
if err != nil {
return "", fmt.Errorf("Failed to get subnet for this host: %s, error: %v", plugin.hostName, err)
}
Expand Down Expand Up @@ -142,33 +147,33 @@ func (plugin *OsdnNode) alreadySetUp(localSubnetGatewayCIDR, clusterNetworkCIDR
}

func deleteLocalSubnetRoute(device, localSubnetCIDR string) {
const (
timeInterval = 100 * time.Millisecond
maxIntervals = 20
)

for i := 0; i < maxIntervals; i++ {
backoff := utilwait.Backoff{
Duration: 100 * time.Millisecond,
Factor: 1.25,
Steps: 6,
}
err := utilwait.ExponentialBackoff(backoff, func() (bool, error) {
itx := ipcmd.NewTransaction(kexec.New(), device)
routes, err := itx.GetRoutes()
if err != nil {
glog.Errorf("Could not get routes for dev %s: %v", device, err)
return
return false, fmt.Errorf("could not get routes: %v", err)
}
for _, route := range routes {
if strings.Contains(route, localSubnetCIDR) {
itx.DeleteRoute(localSubnetCIDR)
err = itx.EndTransaction()
if err != nil {
glog.Errorf("Could not delete subnet route %s from dev %s: %v", localSubnetCIDR, device, err)
return false, fmt.Errorf("could not delete route: %v", err)
}
return
return true, nil
}
}
return false, nil
})

time.Sleep(timeInterval)
if err != nil {
glog.Errorf("Error removing %s route from dev %s: %v; if the route appears later it will not be deleted.", localSubnetCIDR, device, err)
}

glog.Errorf("Timed out looking for %s route for dev %s; if it appears later it will not be deleted.", localSubnetCIDR, device)
}

func (plugin *OsdnNode) SetupSDN() (bool, error) {
Expand Down
25 changes: 15 additions & 10 deletions pkg/sdn/plugin/vnids_node.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,17 +78,22 @@ func (vmap *nodeVNIDMap) GetVNID(name string) (uint32, error) {
// So, use this method to alleviate this problem. This method will
// retry vnid lookup before giving up.
func (vmap *nodeVNIDMap) WaitAndGetVNID(name string) (uint32, error) {
// Try few times up to 2 seconds
retries := 20
retryInterval := 100 * time.Millisecond
for i := 0; i < retries; i++ {
if id, err := vmap.GetVNID(name); err == nil {
return id, nil
}
time.Sleep(retryInterval)
var id uint32
backoff := utilwait.Backoff{
Duration: 100 * time.Millisecond,
Factor: 1.5,
Steps: 5,
}
err := utilwait.ExponentialBackoff(backoff, func() (bool, error) {
var err error
id, err = vmap.GetVNID(name)
return err == nil, nil
})
if err == nil {
return id, nil
} else {
return 0, fmt.Errorf("Failed to find netid for namespace: %s in vnid map", name)
}

return 0, fmt.Errorf("Failed to find netid for namespace: %s in vnid map", name)
}

func (vmap *nodeVNIDMap) setVNID(name string, id uint32) {
Expand Down

0 comments on commit 54b856a

Please sign in to comment.