From 76bebd55480aac482adcd45c7201e6053f72b2e4 Mon Sep 17 00:00:00 2001 From: Mayur Hadole Date: Wed, 7 Jun 2023 22:01:30 -0400 Subject: [PATCH 1/7] add CREATING_SNAPSHOT status In a successful MWAA version upgrade scenario, the status will show UPDATING, then CREATING_SNAPSHOT as Amazon MWAA captures a backup of metadata. Finally, the status will return first to UPDATING, then to AVAILABLE when the procedure is done. --- internal/service/mwaa/environment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/service/mwaa/environment.go b/internal/service/mwaa/environment.go index 5be70ef013f..4f2a9bf2b8e 100644 --- a/internal/service/mwaa/environment.go +++ b/internal/service/mwaa/environment.go @@ -635,7 +635,7 @@ func waitEnvironmentCreated(ctx context.Context, conn *mwaa.MWAA, name string, t func waitEnvironmentUpdated(ctx context.Context, conn *mwaa.MWAA, name string, timeout time.Duration) (*mwaa.Environment, error) { stateConf := &retry.StateChangeConf{ - Pending: []string{mwaa.EnvironmentStatusUpdating}, + Pending: []string{mwaa.EnvironmentStatusUpdating, mwaa.EnvironmentStatusCreatingSnapshot}, Target: []string{mwaa.EnvironmentStatusAvailable}, Refresh: statusEnvironment(ctx, conn, name), Timeout: timeout, From 10336589088f3dde93e7297f346539641a22a22d Mon Sep 17 00:00:00 2001 From: Mayur Hadole Date: Wed, 7 Jun 2023 22:34:29 -0400 Subject: [PATCH 2/7] add release notes --- .changelog/31833.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/31833.txt diff --git a/.changelog/31833.txt b/.changelog/31833.txt new file mode 100644 index 00000000000..9ce914771fc --- /dev/null +++ b/.changelog/31833.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +resource/aws_mwaa_environment: Add CREATING_SNAPSHOT status while waiting for MWAA version upgrade +``` \ No newline at end of file From 1e9b87f911b6a9f28f5346ff6dc16a1697bf25bc Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Thu, 8 Jun 2023 10:27:31 -0400 Subject: [PATCH 3/7] Tweak CHANGELOG entry. --- .changelog/31833.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changelog/31833.txt b/.changelog/31833.txt index 9ce914771fc..5d27208871e 100644 --- a/.changelog/31833.txt +++ b/.changelog/31833.txt @@ -1,3 +1,3 @@ ```release-note:enhancement -resource/aws_mwaa_environment: Add CREATING_SNAPSHOT status while waiting for MWAA version upgrade +resource/aws_mwaa_environment: Consider `CREATING_SNAPSHOT` a valid pending state for resource update ``` \ No newline at end of file From d6428698eaae855e5fd7c94919888f24e7909c96 Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Thu, 8 Jun 2023 11:15:53 -0400 Subject: [PATCH 4/7] r/aws_mwaa_environment: Upgrading your environment to a new major version of Apache Airflow forces replacement of the resource. --- .changelog/31833.txt | 6 +- internal/service/mwaa/environment.go | 24 +++++++- internal/service/mwaa/environment_test.go | 67 +++++++++++++++++++++++ 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/.changelog/31833.txt b/.changelog/31833.txt index 5d27208871e..11874055cb5 100644 --- a/.changelog/31833.txt +++ b/.changelog/31833.txt @@ -1,3 +1,7 @@ ```release-note:enhancement resource/aws_mwaa_environment: Consider `CREATING_SNAPSHOT` a valid pending state for resource update -``` \ No newline at end of file +``` + +```release-note:note +resource/aws_mwaa_environment: Upgrading your environment to a new major version of Apache Airflow forces replacement of the resource +``` diff --git a/internal/service/mwaa/environment.go b/internal/service/mwaa/environment.go index 4f2a9bf2b8e..3d009346b70 100644 --- a/internal/service/mwaa/environment.go +++ b/internal/service/mwaa/environment.go @@ -9,7 +9,9 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/mwaa" "github.com/hashicorp/aws-sdk-go-base/v2/awsv1shim/v2/tfawserr" + gversion "github.com/hashicorp/go-version" "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/customdiff" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/retry" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" @@ -264,7 +266,25 @@ func ResourceEnvironment() *schema.Resource { }, }, - CustomizeDiff: verify.SetTagsDiff, + CustomizeDiff: customdiff.Sequence( + customdiff.ForceNewIf("airflow_version", func(ctx context.Context, d *schema.ResourceDiff, meta interface{}) bool { + o, n := d.GetChange("airflow_version") + + if oldVersion, err := gversion.NewVersion(o.(string)); err != nil { + if newVersion, err := gversion.NewVersion(n.(string)); err != nil { + // https://docs.aws.amazon.com/mwaa/latest/userguide/airflow-versions.html#airflow-versions-upgrade: + // Amazon MWAA supports minor version upgrades. + // This means you can upgrade your environment from version x.4.z to x.5.z. + // However, you cannot upgrade your environment to a new major version of Apache Airflow. + // For example, upgrading from version 1.y.z to 2.y.z is not supported. + return oldVersion.Segments()[0] < newVersion.Segments()[0] + } + } + + return false + }), + verify.SetTagsDiff, + ), } } @@ -346,7 +366,6 @@ func resourceEnvironmentCreate(ctx context.Context, d *schema.ResourceData, meta input.WeeklyMaintenanceWindowStart = aws.String(v.(string)) } - log.Printf("[INFO] Creating MWAA Environment: %s", input) /* Execution roles created just before the MWAA Environment may result in ValidationExceptions due to IAM permission propagation delays. @@ -511,7 +530,6 @@ func resourceEnvironmentUpdate(ctx context.Context, d *schema.ResourceData, meta input.WeeklyMaintenanceWindowStart = aws.String(d.Get("weekly_maintenance_window_start").(string)) } - log.Printf("[INFO] Updating MWAA Environment: %s", input) _, err := conn.UpdateEnvironmentWithContext(ctx, input) if err != nil { diff --git a/internal/service/mwaa/environment_test.go b/internal/service/mwaa/environment_test.go index c03f4d1d001..fe9e91ca8fb 100644 --- a/internal/service/mwaa/environment_test.go +++ b/internal/service/mwaa/environment_test.go @@ -2,9 +2,11 @@ package mwaa_test import ( "context" + "errors" "fmt" "testing" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/mwaa" sdkacctest "github.com/hashicorp/terraform-plugin-testing/helper/acctest" "github.com/hashicorp/terraform-plugin-testing/helper/resource" @@ -350,6 +352,42 @@ func TestAccMWAAEnvironment_pluginsS3ObjectVersion(t *testing.T) { }) } +func TestAccMWAAEnvironment_updateAirflowVersionMinor(t *testing.T) { + ctx := acctest.Context(t) + var environment1, environment2 mwaa.Environment + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + resourceName := "aws_mwaa_environment.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t) }, + ErrorCheck: acctest.ErrorCheck(t, mwaa.EndpointsID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckEnvironmentDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccEnvironmentConfig_airflowVersion(rName, "2.4.3"), + Check: resource.ComposeTestCheckFunc( + testAccCheckEnvironmentExists(ctx, resourceName, &environment1), + resource.TestCheckResourceAttr(resourceName, "airflow_version", "2.4.3"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccEnvironmentConfig_airflowVersion(rName, "2.5.1"), + Check: resource.ComposeTestCheckFunc( + testAccCheckEnvironmentExists(ctx, resourceName, &environment2), + testAccCheckEnvironmentNotRecreated(&environment2, &environment1), + resource.TestCheckResourceAttr(resourceName, "airflow_version", "2.5.1"), + ), + }, + }, + }) +} + func testAccCheckEnvironmentExists(ctx context.Context, n string, v *mwaa.Environment) resource.TestCheckFunc { return func(s *terraform.State) error { rs, ok := s.RootModule().Resources[n] @@ -401,6 +439,16 @@ func testAccCheckEnvironmentDestroy(ctx context.Context) resource.TestCheckFunc } } +func testAccCheckEnvironmentNotRecreated(i, j *mwaa.Environment) resource.TestCheckFunc { + return func(s *terraform.State) error { + if !i.CreatedAt.Equal(aws.TimeValue(j.CreatedAt)) { + return errors.New("MWAA Environment was recreated") + } + + return nil + } +} + func testAccEnvironmentConfig_base(rName string) string { return acctest.ConfigCompose(acctest.ConfigAvailableAZsNoOptIn(), fmt.Sprintf(` data "aws_partition" "current" {} @@ -833,3 +881,22 @@ resource "aws_s3_object" "plugins" { } `, rName, content)) } + +func testAccEnvironmentConfig_airflowVersion(rName, airflowVersion string) string { + return acctest.ConfigCompose(testAccEnvironmentConfig_base(rName), fmt.Sprintf(` +resource "aws_mwaa_environment" "test" { + dag_s3_path = aws_s3_object.dags.key + execution_role_arn = aws_iam_role.test.arn + name = %[1]q + + network_configuration { + security_group_ids = [aws_security_group.test.id] + subnet_ids = aws_subnet.private[*].id + } + + source_bucket_arn = aws_s3_bucket.test.arn + + airflow_version = %[2]q +} +`, rName, airflowVersion)) +} From 1c3bac85a309cf08eac9f712f0177c1de23a12ef Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Thu, 8 Jun 2023 13:58:02 -0400 Subject: [PATCH 5/7] Additional calls to 'testAccCheckEnvironmentNotRecreated'. --- internal/service/mwaa/environment_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/service/mwaa/environment_test.go b/internal/service/mwaa/environment_test.go index fe9e91ca8fb..7e555d41b3a 100644 --- a/internal/service/mwaa/environment_test.go +++ b/internal/service/mwaa/environment_test.go @@ -152,7 +152,7 @@ func TestAccMWAAEnvironment_airflowOptions(t *testing.T) { func TestAccMWAAEnvironment_log(t *testing.T) { ctx := acctest.Context(t) - var environment mwaa.Environment + var environment1, environment2 mwaa.Environment rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_mwaa_environment.test" @@ -165,7 +165,7 @@ func TestAccMWAAEnvironment_log(t *testing.T) { { Config: testAccEnvironmentConfig_logging(rName, "true", mwaa.LoggingLevelCritical), Check: resource.ComposeTestCheckFunc( - testAccCheckEnvironmentExists(ctx, resourceName, &environment), + testAccCheckEnvironmentExists(ctx, resourceName, &environment1), resource.TestCheckResourceAttr(resourceName, "logging_configuration.#", "1"), resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.#", "1"), @@ -202,7 +202,8 @@ func TestAccMWAAEnvironment_log(t *testing.T) { { Config: testAccEnvironmentConfig_logging(rName, "false", mwaa.LoggingLevelInfo), Check: resource.ComposeTestCheckFunc( - testAccCheckEnvironmentExists(ctx, resourceName, &environment), + testAccCheckEnvironmentExists(ctx, resourceName, &environment2), + testAccCheckEnvironmentNotRecreated(&environment2, &environment1), resource.TestCheckResourceAttr(resourceName, "logging_configuration.#", "1"), resource.TestCheckResourceAttr(resourceName, "logging_configuration.0.dag_processing_logs.#", "1"), @@ -313,7 +314,7 @@ func TestAccMWAAEnvironment_full(t *testing.T) { func TestAccMWAAEnvironment_pluginsS3ObjectVersion(t *testing.T) { ctx := acctest.Context(t) - var environment mwaa.Environment + var environment1, environment2 mwaa.Environment rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) resourceName := "aws_mwaa_environment.test" s3ObjectResourceName := "aws_s3_object.plugins" @@ -327,7 +328,7 @@ func TestAccMWAAEnvironment_pluginsS3ObjectVersion(t *testing.T) { { Config: testAccEnvironmentConfig_pluginsS3ObjectVersion(rName, "test"), Check: resource.ComposeTestCheckFunc( - testAccCheckEnvironmentExists(ctx, resourceName, &environment), + testAccCheckEnvironmentExists(ctx, resourceName, &environment1), resource.TestCheckResourceAttrPair(resourceName, "plugins_s3_object_version", s3ObjectResourceName, "version_id"), ), }, @@ -339,7 +340,8 @@ func TestAccMWAAEnvironment_pluginsS3ObjectVersion(t *testing.T) { { Config: testAccEnvironmentConfig_pluginsS3ObjectVersion(rName, "test-updated"), Check: resource.ComposeTestCheckFunc( - testAccCheckEnvironmentExists(ctx, resourceName, &environment), + testAccCheckEnvironmentExists(ctx, resourceName, &environment2), + testAccCheckEnvironmentNotRecreated(&environment2, &environment1), resource.TestCheckResourceAttrPair(resourceName, "plugins_s3_object_version", s3ObjectResourceName, "version_id"), ), }, From c944e5b86e4fb4276c9c165dd8193888e486d125 Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Thu, 8 Jun 2023 14:02:25 -0400 Subject: [PATCH 6/7] Prevent crash. --- internal/service/mwaa/environment.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/service/mwaa/environment.go b/internal/service/mwaa/environment.go index 3d009346b70..3e1f80016d7 100644 --- a/internal/service/mwaa/environment.go +++ b/internal/service/mwaa/environment.go @@ -272,12 +272,14 @@ func ResourceEnvironment() *schema.Resource { if oldVersion, err := gversion.NewVersion(o.(string)); err != nil { if newVersion, err := gversion.NewVersion(n.(string)); err != nil { - // https://docs.aws.amazon.com/mwaa/latest/userguide/airflow-versions.html#airflow-versions-upgrade: - // Amazon MWAA supports minor version upgrades. - // This means you can upgrade your environment from version x.4.z to x.5.z. - // However, you cannot upgrade your environment to a new major version of Apache Airflow. - // For example, upgrading from version 1.y.z to 2.y.z is not supported. - return oldVersion.Segments()[0] < newVersion.Segments()[0] + if len(oldVersion.Segments()) > 0 && len(newVersion.Segments()) > 0 { + // https://docs.aws.amazon.com/mwaa/latest/userguide/airflow-versions.html#airflow-versions-upgrade: + // Amazon MWAA supports minor version upgrades. + // This means you can upgrade your environment from version x.4.z to x.5.z. + // However, you cannot upgrade your environment to a new major version of Apache Airflow. + // For example, upgrading from version 1.y.z to 2.y.z is not supported. + return oldVersion.Segments()[0] < newVersion.Segments()[0] + } } } From a7e8bfa5ed85a166d9738b22df37c0343f4e41bd Mon Sep 17 00:00:00 2001 From: Kit Ewbank Date: Thu, 8 Jun 2023 14:04:53 -0400 Subject: [PATCH 7/7] Prevent crash. --- internal/service/mwaa/environment.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/internal/service/mwaa/environment.go b/internal/service/mwaa/environment.go index 3e1f80016d7..f7a75c185d7 100644 --- a/internal/service/mwaa/environment.go +++ b/internal/service/mwaa/environment.go @@ -270,16 +270,14 @@ func ResourceEnvironment() *schema.Resource { customdiff.ForceNewIf("airflow_version", func(ctx context.Context, d *schema.ResourceDiff, meta interface{}) bool { o, n := d.GetChange("airflow_version") - if oldVersion, err := gversion.NewVersion(o.(string)); err != nil { - if newVersion, err := gversion.NewVersion(n.(string)); err != nil { - if len(oldVersion.Segments()) > 0 && len(newVersion.Segments()) > 0 { - // https://docs.aws.amazon.com/mwaa/latest/userguide/airflow-versions.html#airflow-versions-upgrade: - // Amazon MWAA supports minor version upgrades. - // This means you can upgrade your environment from version x.4.z to x.5.z. - // However, you cannot upgrade your environment to a new major version of Apache Airflow. - // For example, upgrading from version 1.y.z to 2.y.z is not supported. - return oldVersion.Segments()[0] < newVersion.Segments()[0] - } + if oldVersion, err := gversion.NewVersion(o.(string)); err == nil { + if newVersion, err := gversion.NewVersion(n.(string)); err == nil { + // https://docs.aws.amazon.com/mwaa/latest/userguide/airflow-versions.html#airflow-versions-upgrade: + // Amazon MWAA supports minor version upgrades. + // This means you can upgrade your environment from version x.4.z to x.5.z. + // However, you cannot upgrade your environment to a new major version of Apache Airflow. + // For example, upgrading from version 1.y.z to 2.y.z is not supported. + return oldVersion.Segments()[0] < newVersion.Segments()[0] } }