From f0e077d7a995390121b32f7b3a5c6008b4d4d808 Mon Sep 17 00:00:00 2001 From: Darren Haken Date: Fri, 5 Jan 2018 16:34:27 +0000 Subject: [PATCH 1/4] Add autoscaling policy on EMR instance groups #713 --- aws/resource_aws_emr_cluster.go | 97 +++++++++++++++++++--------- aws/resource_aws_emr_cluster_test.go | 33 ++++++++++ 2 files changed, 101 insertions(+), 29 deletions(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index bc8c9108f92..d2559c92f5d 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -173,6 +173,12 @@ func resourceAwsEMRCluster() *schema.Resource { Optional: true, Default: 0, }, + "autoscaling_policy": { + Type: schema.TypeString, + Optional: true, + DiffSuppressFunc: suppressEquivalentAwsPolicyDiffs, + ValidateFunc: validateJsonString, + }, "instance_role": { Type: schema.TypeString, Required: true, @@ -724,6 +730,13 @@ func flattenInstanceGroups(igs []*emr.InstanceGroup) []map[string]interface{} { attrs["instance_count"] = *ig.RequestedInstanceCount attrs["instance_role"] = *ig.InstanceGroupType attrs["instance_type"] = *ig.InstanceType + + if ig.AutoScalingPolicy != nil { + attrs["autoscaling_policy"] = *ig.AutoScalingPolicy + } else { + attrs["autoscaling_policy"] = "" + } + attrs["name"] = *ig.Name result = append(result, attrs) } @@ -871,7 +884,7 @@ func expandBootstrapActions(bootstrapActions []interface{}) []*emr.BootstrapActi } func expandInstanceGroupConfigs(instanceGroupConfigs []interface{}) []*emr.InstanceGroupConfig { - configsOut := []*emr.InstanceGroupConfig{} + instanceGroupConfig := []*emr.InstanceGroupConfig{} for _, raw := range instanceGroupConfigs { configAttributes := raw.(map[string]interface{}) @@ -886,42 +899,68 @@ func expandInstanceGroupConfigs(instanceGroupConfigs []interface{}) []*emr.Insta InstanceCount: aws.Int64(int64(configInstanceCount)), } - if bidPrice, ok := configAttributes["bid_price"]; ok { - if bidPrice != "" { - config.BidPrice = aws.String(bidPrice.(string)) - config.Market = aws.String("SPOT") - } else { - config.Market = aws.String("ON_DEMAND") + applyBidPrice(config, configAttributes) + applyEbsConfig(configAttributes, config) + applyAutoScalingPolicy(configAttributes, config) + + instanceGroupConfig = append(instanceGroupConfig, config) + } + + return instanceGroupConfig +} + +func applyBidPrice(config *emr.InstanceGroupConfig, configAttributes map[string]interface{}) { + if bidPrice, ok := configAttributes["bid_price"]; ok { + if bidPrice != "" { + config.BidPrice = aws.String(bidPrice.(string)) + config.Market = aws.String("SPOT") + } else { + config.Market = aws.String("ON_DEMAND") + } + } +} + +func applyEbsConfig(configAttributes map[string]interface{}, config *emr.InstanceGroupConfig) { + if rawEbsConfigs, ok := configAttributes["ebs_config"]; ok { + ebsConfig := &emr.EbsConfiguration{} + + ebsBlockDeviceConfigs := make([]*emr.EbsBlockDeviceConfig, 0) + for _, rawEbsConfig := range rawEbsConfigs.(*schema.Set).List() { + rawEbsConfig := rawEbsConfig.(map[string]interface{}) + ebsBlockDeviceConfig := &emr.EbsBlockDeviceConfig{ + VolumesPerInstance: aws.Int64(int64(rawEbsConfig["volumes_per_instance"].(int))), + VolumeSpecification: &emr.VolumeSpecification{ + SizeInGB: aws.Int64(int64(rawEbsConfig["size"].(int))), + VolumeType: aws.String(rawEbsConfig["type"].(string)), + }, } + if v, ok := rawEbsConfig["iops"].(int); ok && v != 0 { + ebsBlockDeviceConfig.VolumeSpecification.Iops = aws.Int64(int64(v)) + } + ebsBlockDeviceConfigs = append(ebsBlockDeviceConfigs, ebsBlockDeviceConfig) } + ebsConfig.EbsBlockDeviceConfigs = ebsBlockDeviceConfigs - if rawEbsConfigs, ok := configAttributes["ebs_config"]; ok { - ebsConfig := &emr.EbsConfiguration{} + config.EbsConfiguration = ebsConfig + } +} - ebsBlockDeviceConfigs := make([]*emr.EbsBlockDeviceConfig, 0) - for _, rawEbsConfig := range rawEbsConfigs.(*schema.Set).List() { - rawEbsConfig := rawEbsConfig.(map[string]interface{}) - ebsBlockDeviceConfig := &emr.EbsBlockDeviceConfig{ - VolumesPerInstance: aws.Int64(int64(rawEbsConfig["volumes_per_instance"].(int))), - VolumeSpecification: &emr.VolumeSpecification{ - SizeInGB: aws.Int64(int64(rawEbsConfig["size"].(int))), - VolumeType: aws.String(rawEbsConfig["type"].(string)), - }, - } - if v, ok := rawEbsConfig["iops"].(int); ok && v != 0 { - ebsBlockDeviceConfig.VolumeSpecification.Iops = aws.Int64(int64(v)) - } - ebsBlockDeviceConfigs = append(ebsBlockDeviceConfigs, ebsBlockDeviceConfig) - } - ebsConfig.EbsBlockDeviceConfigs = ebsBlockDeviceConfigs +func applyAutoScalingPolicy(configAttributes map[string]interface{}, config *emr.InstanceGroupConfig) { + if rawAutoScalingPolicy, ok := configAttributes["autoscaling_policy"]; ok { + autoScalingConfig, _ := expandAutoScalingPolicy(rawAutoScalingPolicy.(string)) + config.AutoScalingPolicy = autoScalingConfig + } +} - config.EbsConfiguration = ebsConfig - } +func expandAutoScalingPolicy(rawDefinitions string) (*emr.AutoScalingPolicy, error) { + var policy *emr.AutoScalingPolicy - configsOut = append(configsOut, config) + err := json.Unmarshal([]byte(rawDefinitions), &policy) + if err != nil { + return nil, fmt.Errorf("Error decoding JSON: %s", err) } - return configsOut + return policy, nil } func expandConfigures(input string) []*emr.Configuration { diff --git a/aws/resource_aws_emr_cluster_test.go b/aws/resource_aws_emr_cluster_test.go index afae05c76ff..f103b16c690 100644 --- a/aws/resource_aws_emr_cluster_test.go +++ b/aws/resource_aws_emr_cluster_test.go @@ -1346,6 +1346,39 @@ resource "aws_emr_cluster" "tf-test-cluster" { volumes_per_instance = 1 } bid_price = "0.30" + autoscaling_policy = < Date: Fri, 5 Jan 2018 18:14:49 +0000 Subject: [PATCH 2/4] Add EMR docs for autoscaling instace group #713 --- website/docs/r/emr_cluster.html.md | 46 +++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/website/docs/r/emr_cluster.html.md b/website/docs/r/emr_cluster.html.md index 12fd848b357..a01eb77fe93 100644 --- a/website/docs/r/emr_cluster.html.md +++ b/website/docs/r/emr_cluster.html.md @@ -30,6 +30,50 @@ resource "aws_emr_cluster" "emr-test-cluster" { instance_profile = "${aws_iam_instance_profile.emr_profile.arn}" } + instance_group { + instance_role = "CORE" + instance_type = "c4.large" + instance_count = "1" + ebs_config { + size = "40" + type = "gp2" + volumes_per_instance = 1 + } + bid_price = "0.30" + autoscaling_policy = < Date: Tue, 13 Feb 2018 09:00:21 +0000 Subject: [PATCH 3/4] Change emr autoscaling to use validate json func - As per the review process for the PR with @bflad - https://github.com/terraform-providers/terraform-provider-aws/pull/2877/files/7307a17ae67702f28be85d052e7e922e00bbd9e5#diff-4bc1c34afa1c12afe92754aeb99c560c --- aws/resource_aws_emr_cluster.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index d2559c92f5d..9d4f7bf5203 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -15,6 +15,7 @@ import ( "github.com/aws/aws-sdk-go/service/emr" "github.com/hashicorp/terraform/helper/resource" "github.com/hashicorp/terraform/helper/schema" + "github.com/hashicorp/terraform/helper/structure" ) func resourceAwsEMRCluster() *schema.Resource { @@ -176,8 +177,12 @@ func resourceAwsEMRCluster() *schema.Resource { "autoscaling_policy": { Type: schema.TypeString, Optional: true, - DiffSuppressFunc: suppressEquivalentAwsPolicyDiffs, + DiffSuppressFunc: suppressEquivalentJsonDiffs, ValidateFunc: validateJsonString, + StateFunc: func(v interface{}) string { + jsonString, _ := normalizeJsonString(v) + return jsonString + }, }, "instance_role": { Type: schema.TypeString, From 4c48218591d085f546339fece4ac44b87d02affa Mon Sep 17 00:00:00 2001 From: Darren Haken Date: Tue, 13 Feb 2018 14:22:42 +0000 Subject: [PATCH 4/4] Fix build error --- aws/resource_aws_emr_cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index ba46e378540..2cec704bee2 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -180,7 +180,7 @@ func resourceAwsEMRCluster() *schema.Resource { DiffSuppressFunc: suppressEquivalentJsonDiffs, ValidateFunc: validateJsonString, StateFunc: func(v interface{}) string { - jsonString, _ := normalizeJsonString(v) + jsonString, _ := structure.NormalizeJsonString(v) return jsonString }, },