Skip to content

Commit

Permalink
Avoid using force detach of ENIs
Browse files Browse the repository at this point in the history
* Not using force
* Added retries
* Related to #359
  • Loading branch information
Claes Mogren committed May 9, 2019
1 parent b68da39 commit 4334376
Showing 1 changed file with 19 additions and 10 deletions.
29 changes: 19 additions & 10 deletions pkg/awsutils/awsutils.go
Original file line number Diff line number Diff line change
Expand Up @@ -731,22 +731,31 @@ func (cache *EC2InstanceMetadataCache) FreeENI(eniName string) error {
log.Errorf("Failed to retrieve ENI %s attachment id: %v", eniName, err)
return errors.Wrap(err, "FreeENI: failed to retrieve ENI's attachment id")
}

log.Debugf("Found ENI %s attachment id: %s ", eniName, aws.StringValue(attachID))

// Detach it first
detachInput := &ec2.DetachNetworkInterfaceInput{
AttachmentId: attachID,
Force: aws.Bool(true),
}

start := time.Now()
_, err = cache.ec2SVC.DetachNetworkInterface(detachInput)
awsAPILatency.WithLabelValues("DetachNetworkInterface", fmt.Sprint(err != nil)).Observe(msSince(start))
if err != nil {
awsAPIErrInc("DetachNetworkInterface", err)
log.Errorf("Failed to detach ENI %s %v", eniName, err)
return errors.Wrap(err, "FreeENI: failed to detach ENI from instance")
// Retry detaching the ENI from the instance
var retry int
for retry = 0; retry <= maxENIDeleteRetries; retry++ {
start := time.Now()
_, err = cache.ec2SVC.DetachNetworkInterface(detachInput)
awsAPILatency.WithLabelValues("DetachNetworkInterface", fmt.Sprint(err != nil)).Observe(msSince(start))
if err != nil {
awsAPIErrInc("DetachNetworkInterface", err)
log.Errorf("Failed to detach ENI %s %v", eniName, err)
if retry == maxENIDeleteRetries {
return errors.New("unable to detach ENI from EC2 instance, giving up")
}
} else {
log.Infof("Successfully detached ENI: %s", eniName)
break
}

log.Debugf("Not able to detach ENI yet (attempt %d/%d): %v ", retry, maxENIDeleteRetries, err)
time.Sleep(retryDeleteENIInternal)
}

// It may take awhile for EC2-VPC to detach ENI from instance
Expand Down

0 comments on commit 4334376

Please sign in to comment.