From 227953f6575acfe2c4249f33b273abca8aa20899 Mon Sep 17 00:00:00 2001 From: Nathan McKinley Date: Thu, 15 Nov 2018 00:20:04 +0000 Subject: [PATCH] Container cluster support for cluster autoprovisioning. --- google-beta/resource_container_cluster.go | 132 ++++++++++++++++++ .../resource_container_cluster_test.go | 79 +++++++++++ .../docs/r/container_cluster.html.markdown | 20 +++ 3 files changed, 231 insertions(+) diff --git a/google-beta/resource_container_cluster.go b/google-beta/resource_container_cluster.go index 4ffb60358bd..15789931091 100644 --- a/google-beta/resource_container_cluster.go +++ b/google-beta/resource_container_cluster.go @@ -186,6 +186,41 @@ func resourceContainerCluster() *schema.Resource { }, }, + "cluster_autoscaling": { + Type: schema.TypeList, + Computed: true, + MaxItems: 1, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "enabled": { + Type: schema.TypeBool, + Required: true, + }, + "resource_limits": { + Type: schema.TypeList, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "resource_type": { + Type: schema.TypeString, + Required: true, + }, + "minimum": { + Type: schema.TypeInt, + Optional: true, + }, + "maximum": { + Type: schema.TypeInt, + Optional: true, + }, + }, + }, + }, + }, + }, + }, + "cluster_ipv4_cidr": { Type: schema.TypeString, Optional: true, @@ -604,6 +639,7 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er Enabled: d.Get("enable_binary_authorization").(bool), ForceSendFields: []string{"Enabled"}, }, + Autoscaling: expandClusterAutoscaling(d.Get("cluster_autoscaling"), d), MasterAuth: expandMasterAuth(d.Get("master_auth")), ResourceLabels: expandStringMap(d, "resource_labels"), } @@ -781,6 +817,9 @@ func resourceContainerClusterRead(d *schema.ResourceData, meta interface{}) erro d.Set("subnetwork", cluster.NetworkConfig.Subnetwork) d.Set("enable_binary_authorization", cluster.BinaryAuthorization != nil && cluster.BinaryAuthorization.Enabled) d.Set("enable_tpu", cluster.EnableTpu) + if err := d.Set("cluster_autoscaling", flattenClusterAutoscaling(cluster.Autoscaling)); err != nil { + return err + } if err := d.Set("node_config", flattenNodeConfig(cluster.NodeConfig)); err != nil { return err } @@ -963,6 +1002,23 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er d.SetPartial("enable_binary_authorization") } + if d.HasChange("cluster_autoscaling") { + req := &containerBeta.UpdateClusterRequest{ + Update: &containerBeta.ClusterUpdate{ + DesiredClusterAutoscaling: expandClusterAutoscaling(d.Get("cluster_autoscaling"), d), + }} + + updateF := updateFunc(req, "updating GKE cluster autoscaling") + // Call update serially. + if err := lockedCall(lockKey, updateF); err != nil { + return err + } + + log.Printf("[INFO] GKE cluster %s's cluster-wide autoscaling has been updated", d.Id()) + + d.SetPartial("cluster_autoscaling") + } + if d.HasChange("maintenance_policy") { var req *containerBeta.SetMaintenancePolicyRequest if mp, ok := d.GetOk("maintenance_policy"); ok { @@ -1472,6 +1528,63 @@ func expandMaintenancePolicy(configured interface{}) *containerBeta.MaintenanceP } } +func expandClusterAutoscaling(configured interface{}, d *schema.ResourceData) *containerBeta.ClusterAutoscaling { + l, ok := configured.([]interface{}) + if !ok || l == nil || len(l) == 0 || l[0] == nil { + // Before master version 1.11.2, we must send 'nil' values if autoscaling isn't + // turned on - the cluster will return an error even if we're setting + // EnableNodeAutoprovisioning to false. + cmv, err := version.NewVersion(d.Get("master_version").(string)) + if err != nil { + log.Printf("[DEBUG] Could not parse master_version into version (%q), trying min_master_version.", d.Get("master_version").(string)) + cmv, err = version.NewVersion(d.Get("min_master_version").(string)) + if err != nil { + log.Printf("[DEBUG] Could not parse min_master_version into version (%q), assuming we are not already using cluster autoscaling.", d.Get("min_master_version").(string)) + // This deserves a little explanation. The only reason we would ever want to send + // `EnableNodeAutoprovisioning: false` is because we think we might need to + // disable it (e.g. it is already enabled). Otherwise, there is no difference + // between sending `nil` and sending `EnableNodeAutoprovisioning: false`. + // The only circumstance in which neither master_version nor min_master_version + // can be parsed into version objects would be if the user has not set either one, + // and we have not yet had a `read` call. e.g. first-time creates, and possibly + // some circumstance related to import. It is probably safe to assume that + // we are not going to be changing cluster autoscaling from on to off in those + // circumstances. Therefore, if we don't know what version we're running, and + // the user has not requested cluster autoscaling, we'll fail "safe" and not touch + // it. + cmv, _ = version.NewVersion("0.0.0") + } + } + dmv, _ := version.NewVersion("1.11.2") + if cmv.LessThan(dmv) { + return nil + } else { + return &containerBeta.ClusterAutoscaling{ + EnableNodeAutoprovisioning: false, + ForceSendFields: []string{"EnableNodeAutoprovisioning"}, + } + } + } + r := &containerBeta.ClusterAutoscaling{} + if config, ok := l[0].(map[string]interface{}); ok { + r.EnableNodeAutoprovisioning = config["enabled"].(bool) + if limits, ok := config["resource_limits"]; ok { + if lmts, ok := limits.([]interface{}); ok { + for _, v := range lmts { + limit := v.(map[string]interface{}) + r.ResourceLimits = append(r.ResourceLimits, &containerBeta.ResourceLimit{ + ResourceType: limit["resource_type"].(string), + // Here we're relying on *not* setting ForceSendFields for 0-values. + Minimum: int64(limit["minimum"].(int)), + Maximum: int64(limit["maximum"].(int)), + }) + } + } + } + } + return r +} + func expandMasterAuth(configured interface{}) *containerBeta.MasterAuth { l := configured.([]interface{}) if len(l) == 0 || l[0] == nil { @@ -1701,6 +1814,25 @@ func flattenMasterAuth(ma *containerBeta.MasterAuth) []map[string]interface{} { return masterAuth } +func flattenClusterAutoscaling(a *containerBeta.ClusterAutoscaling) []map[string]interface{} { + r := make(map[string]interface{}) + if a == nil || !a.EnableNodeAutoprovisioning { + r["enabled"] = false + } else { + resourceLimits := make([]interface{}, 0, len(a.ResourceLimits)) + for _, rl := range a.ResourceLimits { + resourceLimits = append(resourceLimits, map[string]interface{}{ + "resource_type": rl.ResourceType, + "minimum": rl.Minimum, + "maximum": rl.Maximum, + }) + } + r["resource_limits"] = resourceLimits + r["enabled"] = true + } + return []map[string]interface{}{r} +} + func flattenMasterAuthorizedNetworksConfig(c *containerBeta.MasterAuthorizedNetworksConfig) []map[string]interface{} { if c == nil { return nil diff --git a/google-beta/resource_container_cluster_test.go b/google-beta/resource_container_cluster_test.go index 4453da5aca9..6630d1ce020 100644 --- a/google-beta/resource_container_cluster_test.go +++ b/google-beta/resource_container_cluster_test.go @@ -1203,6 +1203,48 @@ func TestAccContainerCluster_withPodSecurityPolicy(t *testing.T) { }) } +func TestAccContainerCluster_autoprovisioning(t *testing.T) { + t.Parallel() + + clusterName := fmt.Sprintf("cluster-test-%s", acctest.RandString(10)) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckContainerClusterDestroy, + Steps: []resource.TestStep{ + { + Config: testAccContainerCluster_autoprovisioning(clusterName, true), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("google_container_cluster.with_autoprovisioning", + "cluster_autoscaling.0.enabled", "true"), + ), + }, + { + ResourceName: "google_container_cluster.with_autoprovisioning", + ImportStateIdPrefix: "us-central1-a/", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version"}, + }, + { + Config: testAccContainerCluster_autoprovisioning(clusterName, false), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("google_container_cluster.with_autoprovisioning", + "cluster_autoscaling.0.enabled", "false"), + ), + }, + { + ResourceName: "google_container_cluster.with_autoprovisioning", + ImportStateIdPrefix: "us-central1-a/", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version"}, + }, + }, + }) +} + func TestAccContainerCluster_sharedVpc(t *testing.T) { t.Parallel() @@ -2141,6 +2183,43 @@ resource "google_container_cluster" "with_node_pool" { }`, cluster, nodePool) } +func testAccContainerCluster_autoprovisioning(cluster string, autoprovisioning bool) string { + config := fmt.Sprintf(` +data "google_container_engine_versions" "central1a" { + zone = "us-central1-a" +} + +resource "google_container_cluster" "with_autoprovisioning" { + name = "%s" + zone = "us-central1-a" + min_master_version = "${data.google_container_engine_versions.central1a.latest_master_version}" + node_version = "${data.google_container_engine_versions.central1a.latest_node_version}" + initial_node_count = 3 +`, cluster) + if autoprovisioning { + config += ` + cluster_autoscaling { + enabled = true + resource_limits { + resource_type = "cpu" + maximum = 2 + } + resource_limits { + resource_type = "memory" + maximum = 2048 + } + }` + } else { + config += ` + cluster_autoscaling { + enabled = false + }` + } + config += ` +}` + return config +} + func testAccContainerCluster_withNodePoolAutoscaling(cluster, np string) string { return fmt.Sprintf(` resource "google_container_cluster" "with_node_pool" { diff --git a/website/docs/r/container_cluster.html.markdown b/website/docs/r/container_cluster.html.markdown index b7edbbc25ba..d2ecbe0ab33 100644 --- a/website/docs/r/container_cluster.html.markdown +++ b/website/docs/r/container_cluster.html.markdown @@ -92,6 +92,11 @@ output "cluster_ca_certificate" { * `cluster_ipv4_cidr` - (Optional) The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. +* `cluster_autoscaling` - (Optional, [Beta](https://terraform.io/docs/providers/google/provider_versions.html)) + Configuration for cluster autoscaling (also called autoprovisioning), as described in + [the docs](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-provisioning). + Structure is documented below. + * `description` - (Optional) Description of the cluster. * `enable_binary_authorization` - (Optional, [Beta](https://terraform.io/docs/providers/google/provider_versions.html)) Enable Binary Authorization for this cluster. @@ -219,6 +224,21 @@ addons_config { } ``` +The `cluster_autoscaling` block supports: +* `enabled` - (Required) Whether cluster autoscaling (also called autoprovisioning) is + enabled. To set this to true, make sure your config meets the rest of the + requirements. Notably, you'll need `min_master_version` of at least `1.11.2`. +* `resource_limits` - (Optional) A list of limits on the autoprovisioning. + See [the docs](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-provisioning) + for an explanation of what options are available. If enabling autoprovisioning, make + sure to set at least `cpu` and `memory`. Structure is documented below. + +The `resource_limits` block supports: +* `resource_type` - (Required) See [the docs](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-provisioning) + for a list of permitted types - `cpu`, `memory`, and others. +* `minimum` - (Optional) The minimum value for the resource type specified. +* `maximum` - (Optional) The maximum value for the resource type specified. + The `maintenance_policy` block supports: * `daily_maintenance_window` - (Required) Time window specified for daily maintenance operations.