From 34f4d854ebdaedac5e62d80eb4dca6bf6d2c59a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Bra=C5=84ski?= Date: Wed, 15 Apr 2020 17:25:16 +0200 Subject: [PATCH 1/3] Add dummy bootstrap action that use md5 of configuration_json as an argument to prevent force recreation --- main.tf | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 4b3f363..d5c2c88 100644 --- a/main.tf +++ b/main.tf @@ -349,6 +349,19 @@ resource "aws_iam_role_policy_attachment" "ec2_autoscaling" { policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforAutoScalingRole" } +# This dummy bootstrap action is needed because of terraform bug https://github.com/terraform-providers/terraform-provider-aws/issues/12683 +# When javax.jdo.option.ConnectionPassword is used in configuration_json then every plan will result in force recreation. +locals { + bootstrap_action = concat( + [{ + path = "file:/bin/echo", + name = "Dummy bootstrap action to prevent EMR cluster recration when configuration_json has parameter javax.jdo.option.ConnectionPassword.", + args = [md5(jsonencode(var.configurations_json))] + }], + var.bootstrap_action + ) +} + resource "aws_emr_cluster" "default" { count = var.enabled ? 1 : 0 name = module.label.id @@ -409,7 +422,7 @@ resource "aws_emr_cluster" "default" { security_configuration = var.security_configuration dynamic "bootstrap_action" { - for_each = var.bootstrap_action + for_each = local.bootstrap_action content { path = bootstrap_action.value.path name = bootstrap_action.value.name @@ -427,7 +440,7 @@ resource "aws_emr_cluster" "default" { tags = module.label.tags lifecycle { - ignore_changes = ["kerberos_attributes", "step"] + ignore_changes = ["kerberos_attributes", "step", "configurations_json"] } } From 31e8be35de23d82845234ebb64ec360cd5f633c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Bra=C5=84ski?= Date: Mon, 20 Apr 2020 16:35:48 +0200 Subject: [PATCH 2/3] Terratest fixed, empty string as new default for configurations_json --- README.md | 7 ++++++- docs/terraform.md | 2 +- examples/complete/variables.tf | 2 +- main.tf | 12 ++++++------ variables.tf | 2 +- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index dc267b4..f53ba9e 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ Available targets: | applications | A list of applications for the cluster. Valid values are: Flink, Ganglia, Hadoop, HBase, HCatalog, Hive, Hue, JupyterHub, Livy, Mahout, MXNet, Oozie, Phoenix, Pig, Presto, Spark, Sqoop, TensorFlow, Tez, Zeppelin, and ZooKeeper (as of EMR 5.25.0). Case insensitive | list(string) | - | yes | | attributes | Additional attributes (_e.g._ "1") | list(string) | `` | no | | bootstrap_action | List of bootstrap actions that will be run before Hadoop is started on the cluster nodes | object | `` | no | -| configurations_json | A JSON string for supplying list of configurations for the EMR cluster. See https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-configure-apps.html for more details | string | `null` | no | +| configurations_json | A JSON string for supplying list of configurations for the EMR cluster. See https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-configure-apps.html for more details | string | `` | no | | core_instance_group_autoscaling_policy | String containing the EMR Auto Scaling Policy JSON for the Core instance group | string | `null` | no | | core_instance_group_bid_price | Bid price for each EC2 instance in the Core instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances | string | `null` | no | | core_instance_group_ebs_iops | The number of I/O operations per second (IOPS) that the Core volume supports | number | `null` | no | @@ -310,6 +310,10 @@ We deliver 10x the value for a fraction of the cost of a full-time engineer. Our Join our [Open Source Community][slack] on Slack. It's **FREE** for everyone! Our "SweetOps" community is where you get to talk with others who share a similar vision for how to rollout and manage infrastructure. This is the best place to talk shop, ask questions, solicit feedback, and work together as a community to build totally *sweet* infrastructure. +## Discourse Forums + +Participate in our [Discourse Forums][discourse]. Here you'll find answers to commonly asked questions. Most questions will be related to the enormous number of projects we support on our GitHub. Come here to collaborate on answers, find solutions, and get ideas about the products and services we value. It only takes a minute to get started! Just sign in with SSO using your GitHub account. + ## Newsletter Sign up for [our newsletter][newsletter] that covers everything on our technology radar. Receive updates on what we're up to on GitHub as well as awesome new projects we discover. @@ -423,6 +427,7 @@ Check out [our other projects][github], [follow us on twitter][twitter], [apply [testimonial]: https://cpco.io/leave-testimonial?utm_source=github&utm_medium=readme&utm_campaign=cloudposse/terraform-aws-emr-cluster&utm_content=testimonial [office_hours]: https://cloudposse.com/office-hours?utm_source=github&utm_medium=readme&utm_campaign=cloudposse/terraform-aws-emr-cluster&utm_content=office_hours [newsletter]: https://cpco.io/newsletter?utm_source=github&utm_medium=readme&utm_campaign=cloudposse/terraform-aws-emr-cluster&utm_content=newsletter + [discourse]: https://ask.sweetops.com/?utm_source=github&utm_medium=readme&utm_campaign=cloudposse/terraform-aws-emr-cluster&utm_content=discourse [email]: https://cpco.io/email?utm_source=github&utm_medium=readme&utm_campaign=cloudposse/terraform-aws-emr-cluster&utm_content=email [commercial_support]: https://cpco.io/commercial-support?utm_source=github&utm_medium=readme&utm_campaign=cloudposse/terraform-aws-emr-cluster&utm_content=commercial_support [we_love_open_source]: https://cpco.io/we-love-open-source?utm_source=github&utm_medium=readme&utm_campaign=cloudposse/terraform-aws-emr-cluster&utm_content=we_love_open_source diff --git a/docs/terraform.md b/docs/terraform.md index d2bc2f3..6f8a3e5 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -6,7 +6,7 @@ | applications | A list of applications for the cluster. Valid values are: Flink, Ganglia, Hadoop, HBase, HCatalog, Hive, Hue, JupyterHub, Livy, Mahout, MXNet, Oozie, Phoenix, Pig, Presto, Spark, Sqoop, TensorFlow, Tez, Zeppelin, and ZooKeeper (as of EMR 5.25.0). Case insensitive | list(string) | - | yes | | attributes | Additional attributes (_e.g._ "1") | list(string) | `` | no | | bootstrap_action | List of bootstrap actions that will be run before Hadoop is started on the cluster nodes | object | `` | no | -| configurations_json | A JSON string for supplying list of configurations for the EMR cluster. See https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-configure-apps.html for more details | string | `null` | no | +| configurations_json | A JSON string for supplying list of configurations for the EMR cluster. See https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-configure-apps.html for more details | string | `` | no | | core_instance_group_autoscaling_policy | String containing the EMR Auto Scaling Policy JSON for the Core instance group | string | `null` | no | | core_instance_group_bid_price | Bid price for each EC2 instance in the Core instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances | string | `null` | no | | core_instance_group_ebs_iops | The number of I/O operations per second (IOPS) that the Core volume supports | number | `null` | no | diff --git a/examples/complete/variables.tf b/examples/complete/variables.tf index 73fc58c..2e1dc52 100644 --- a/examples/complete/variables.tf +++ b/examples/complete/variables.tf @@ -46,7 +46,7 @@ variable "applications" { variable "configurations_json" { type = string description = "A JSON string for supplying list of configurations for the EMR cluster" - default = null + default = "" } variable "core_instance_group_instance_type" { diff --git a/main.tf b/main.tf index d5c2c88..694efd3 100644 --- a/main.tf +++ b/main.tf @@ -353,12 +353,12 @@ resource "aws_iam_role_policy_attachment" "ec2_autoscaling" { # When javax.jdo.option.ConnectionPassword is used in configuration_json then every plan will result in force recreation. locals { bootstrap_action = concat( - [{ - path = "file:/bin/echo", - name = "Dummy bootstrap action to prevent EMR cluster recration when configuration_json has parameter javax.jdo.option.ConnectionPassword.", - args = [md5(jsonencode(var.configurations_json))] - }], - var.bootstrap_action + [{ + path = "file:/bin/echo", + name = "Dummy bootstrap action to prevent EMR cluster recration when configuration_json has parameter javax.jdo.option.ConnectionPassword.", + args = [md5(jsonencode(var.configurations_json))] + }], + var.bootstrap_action ) } diff --git a/variables.tf b/variables.tf index 33e4abd..5f3fbf7 100644 --- a/variables.tf +++ b/variables.tf @@ -125,7 +125,7 @@ variable "applications" { variable "configurations_json" { type = string description = "A JSON string for supplying list of configurations for the EMR cluster. See https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-configure-apps.html for more details" - default = null + default = "" } variable "key_name" { From 6989a4fe693cea92ba18b5bb9145fce9701bdc60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Bra=C5=84ski?= Date: Mon, 20 Apr 2020 16:53:39 +0200 Subject: [PATCH 3/3] More descriptive comment for changes around bootstrap_action --- main.tf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 694efd3..6cc20eb 100644 --- a/main.tf +++ b/main.tf @@ -350,7 +350,9 @@ resource "aws_iam_role_policy_attachment" "ec2_autoscaling" { } # This dummy bootstrap action is needed because of terraform bug https://github.com/terraform-providers/terraform-provider-aws/issues/12683 -# When javax.jdo.option.ConnectionPassword is used in configuration_json then every plan will result in force recreation. +# When javax.jdo.option.ConnectionPassword is used in configuration_json then every plan will result in force recreation of EMR cluster. +# To mitigate this issue dummy bootstrap action `echo` was introduced. It is executed with an argument of a hash generated from configuration. +# This in tandem with lifecycle ignore_changes for `configurations_json` will only trigger EMR recreation when hash of configuration will change. locals { bootstrap_action = concat( [{ @@ -439,6 +441,7 @@ resource "aws_emr_cluster" "default" { tags = module.label.tags + # configurations_json changes are ignored because of terraform bug. Configuration changes are applied via local.bootstrap_action. lifecycle { ignore_changes = ["kerberos_attributes", "step", "configurations_json"] }