Skip to content

Commit

Permalink
Final retries after timeouts creating, draining, and deleting ASGs an…
Browse files Browse the repository at this point in the history 
…d autoscaling helpers
  • Loading branch information
@ryndaniels
ryndaniels committed Aug 6, 2019
1 parent aa531ce commit 563413f
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 60 deletions.
54 changes: 43 additions & 11 deletions aws/resource_aws_autoscaling_group.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -596,6 +596,9 @@ func resourceAwsAutoscalingGroupCreate(d *schema.ResourceData, meta interface{})

return nil
})
if isResourceTimeoutError(err) {
_, err = conn.CreateAutoScalingGroup(&createOpts)
}
if err != nil {
return fmt.Errorf("Error creating AutoScaling Group: %s", err)
}
Expand Down Expand Up @@ -1005,23 +1008,38 @@ func resourceAwsAutoscalingGroupDelete(d *schema.ResourceData, meta interface{})
// Successful delete
return nil
})
if isResourceTimeoutError(err) {
_, err = conn.DeleteAutoScalingGroup(&deleteopts)
if isAWSErr(err, "InvalidGroup.NotFound", "") {
return nil
}
}
if err != nil {
return err
return fmt.Errorf("Error deleting autoscaling group: %s", err)
}

return resource.Retry(d.Timeout(schema.TimeoutDelete), func() *resource.RetryError {
if g, _ = getAwsAutoscalingGroup(d.Id(), conn); g != nil {
return resource.RetryableError(
fmt.Errorf("Auto Scaling Group still exists"))
var group *autoscaling.Group
err = resource.Retry(d.Timeout(schema.TimeoutDelete), func() *resource.RetryError {
group, err = getAwsAutoscalingGroup(d.Id(), conn)

if group != nil {
return resource.RetryableError(fmt.Errorf("Auto Scaling Group still exists"))
}
return nil
})
if isResourceTimeoutError(err) {
group, err = getAwsAutoscalingGroup(d.Id(), conn)
if group != nil {
return fmt.Errorf("Auto Scaling Group still exists")
}
}
if err != nil {
return fmt.Errorf("Error deleting autoscaling group: %s", err)
}
return nil
}

func getAwsAutoscalingGroup(
asgName string,
conn *autoscaling.AutoScaling) (*autoscaling.Group, error) {

func getAwsAutoscalingGroup(asgName string, conn *autoscaling.AutoScaling) (*autoscaling.Group, error) {
describeOpts := autoscaling.DescribeAutoScalingGroupsInput{
AutoScalingGroupNames: []*string{aws.String(asgName)},
}
Expand Down Expand Up @@ -1069,7 +1087,8 @@ func resourceAwsAutoscalingGroupDrain(d *schema.ResourceData, meta interface{})

// Next, wait for the autoscale group to drain
log.Printf("[DEBUG] Waiting for group to have zero instances")
return resource.Retry(d.Timeout(schema.TimeoutDelete), func() *resource.RetryError {
var g *autoscaling.Group
err := resource.Retry(d.Timeout(schema.TimeoutDelete), func() *resource.RetryError {
g, err := getAwsAutoscalingGroup(d.Id(), conn)
if err != nil {
return resource.NonRetryableError(err)
Expand All @@ -1085,8 +1104,21 @@ func resourceAwsAutoscalingGroupDrain(d *schema.ResourceData, meta interface{})
}

return resource.RetryableError(
fmt.Errorf("group still has %d instances", len(g.Instances)))
fmt.Errorf("Group still has %d instances", len(g.Instances)))
})
if isResourceTimeoutError(err) {
g, err = getAwsAutoscalingGroup(d.Id(), conn)
if err != nil {
return fmt.Errorf("Error getting autoscaling group info when draining: %s", err)
}
if g != nil && len(g.Instances) > 0 {
return fmt.Errorf("Group still has %d instances", len(g.Instances))
}
}
if err != nil {
return fmt.Errorf("Error draining autoscaling group: %s", err)
}
return nil
}

func enableASGSuspendedProcesses(d *schema.ResourceData, conn *autoscaling.AutoScaling) error {
Expand Down
119 changes: 71 additions & 48 deletions aws/resource_aws_autoscaling_group_waiting.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,63 +44,32 @@ func waitForASGCapacity(
d.SetId("")
return nil
}
elbis, err := getELBInstanceStates(g, meta)
if err != nil {
return resource.NonRetryableError(err)
}
albis, err := getTargetGroupInstanceStates(g, meta)
if err != nil {
return resource.NonRetryableError(err)
}

haveASG := 0
haveELB := 0

for _, i := range g.Instances {
if i.HealthStatus == nil || i.InstanceId == nil || i.LifecycleState == nil {
continue
}

if !strings.EqualFold(*i.HealthStatus, "Healthy") {
continue
}

if !strings.EqualFold(*i.LifecycleState, "InService") {
continue
}
satisfied, reason := isELBCapacitySatisfied(d, meta, g, satisfiedFunc)
if satisfied {
return nil
}

haveASG++
return resource.RetryableError(fmt.Errorf("%q: Waiting up to %s: %s", d.Id(), wait, reason))
})
if isResourceTimeoutError(err) {
g, err := getAwsAutoscalingGroup(d.Id(), meta.(*AWSClient).autoscalingconn)

inAllLbs := true
for _, states := range elbis {
state, ok := states[*i.InstanceId]
if !ok || !strings.EqualFold(state, "InService") {
inAllLbs = false
}
}
for _, states := range albis {
state, ok := states[*i.InstanceId]
if !ok || !strings.EqualFold(state, "healthy") {
inAllLbs = false
}
}
if inAllLbs {
haveELB++
}
if err != nil {
return fmt.Errorf("Error getting autoscaling group info: %s", err)
}

satisfied, reason := satisfiedFunc(d, haveASG, haveELB)

log.Printf("[DEBUG] %q Capacity: %d ASG, %d ELB/ALB, satisfied: %t, reason: %q",
d.Id(), haveASG, haveELB, satisfied, reason)
if g == nil {
log.Printf("[WARN] Autoscaling Group (%s) not found, removing from state", d.Id())
d.SetId("")
return nil
}

satisfied, _ := isELBCapacitySatisfied(d, meta, g, satisfiedFunc)
if satisfied {
return nil
}

return resource.RetryableError(
fmt.Errorf("%q: Waiting up to %s: %s", d.Id(), wait, reason))
})
}

if err == nil {
return nil
Expand All @@ -126,6 +95,60 @@ func waitForASGCapacity(
return fmt.Errorf("%s. Most recent activity: %s", err, recentStatus)
}

func isELBCapacitySatisfied(d *schema.ResourceData, meta interface{}, g *autoscaling.Group, satisfiedFunc capacitySatisfiedFunc) (bool, string) {
elbis, err := getELBInstanceStates(g, meta)
if err != nil {
return false, fmt.Sprintf("Error getting ELB instance states: %s", err)
}
albis, err := getTargetGroupInstanceStates(g, meta)
if err != nil {
return false, fmt.Sprintf("Error getting target group instance states: %s", err)
}

haveASG := 0
haveELB := 0

for _, i := range g.Instances {
if i.HealthStatus == nil || i.InstanceId == nil || i.LifecycleState == nil {
continue
}

if !strings.EqualFold(*i.HealthStatus, "Healthy") {
continue
}

if !strings.EqualFold(*i.LifecycleState, "InService") {
continue
}

haveASG++

inAllLbs := true
for _, states := range elbis {
state, ok := states[*i.InstanceId]
if !ok || !strings.EqualFold(state, "InService") {
inAllLbs = false
}
}
for _, states := range albis {
state, ok := states[*i.InstanceId]
if !ok || !strings.EqualFold(state, "healthy") {
inAllLbs = false
}
}
if inAllLbs {
haveELB++
}
}

satisfied, reason := satisfiedFunc(d, haveASG, haveELB)

log.Printf("[DEBUG] %q Capacity: %d ASG, %d ELB/ALB, satisfied: %t, reason: %q",
d.Id(), haveASG, haveELB, satisfied, reason)

return satisfied, reason
}

type capacitySatisfiedFunc func(*schema.ResourceData, int, int) (bool, string)

// capacitySatisfiedCreate treats all targets as minimums
Expand Down
9 changes: 8 additions & 1 deletion aws/resource_aws_autoscaling_lifecycle_hook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func resourceAwsAutoscalingLifecycleHook() *schema.Resource {

func resourceAwsAutoscalingLifecycleHookPutOp(conn *autoscaling.AutoScaling, params *autoscaling.PutLifecycleHookInput) error {
log.Printf("[DEBUG] AutoScaling PutLifecyleHook: %s", params)
return resource.Retry(5*time.Minute, func() *resource.RetryError {
err := resource.Retry(5*time.Minute, func() *resource.RetryError {
_, err := conn.PutLifecycleHook(params)

if err != nil {
Expand All @@ -78,6 +78,13 @@ func resourceAwsAutoscalingLifecycleHookPutOp(conn *autoscaling.AutoScaling, par
}
return nil
})
if isResourceTimeoutError(err) {
_, err = conn.PutLifecycleHook(params)
}
if err != nil {
return fmt.Errorf("Error putting autoscaling lifecycle hook: %s", err)
}
return nil
}

func resourceAwsAutoscalingLifecycleHookPut(d *schema.ResourceData, meta interface{}) error {
Expand Down

0 comments on commit 563413f

Please sign in to comment.