Skip to content

Commit

Permalink
Container cluster support for cluster autoprovisioning.
Browse files Browse the repository at this point in the history
  • Loading branch information
@nat-henderson @modular-magician
nat-henderson authored and modular-magician committed Nov 15, 2018
1 parent 489f4be commit 227953f
Show file tree
Hide file tree
Showing 3 changed files with 231 additions and 0 deletions.
132 changes: 132 additions & 0 deletions google-beta/resource_container_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,41 @@ func resourceContainerCluster() *schema.Resource {
},
},

"cluster_autoscaling": {
Type: schema.TypeList,
Computed: true,
MaxItems: 1,
Optional: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"enabled": {
Type: schema.TypeBool,
Required: true,
},
"resource_limits": {
Type: schema.TypeList,
Optional: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"resource_type": {
Type: schema.TypeString,
Required: true,
},
"minimum": {
Type: schema.TypeInt,
Optional: true,
},
"maximum": {
Type: schema.TypeInt,
Optional: true,
},
},
},
},
},
},
},

"cluster_ipv4_cidr": {
Type: schema.TypeString,
Optional: true,
Expand Down Expand Up @@ -604,6 +639,7 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er
Enabled: d.Get("enable_binary_authorization").(bool),
ForceSendFields: []string{"Enabled"},
},
Autoscaling: expandClusterAutoscaling(d.Get("cluster_autoscaling"), d),
MasterAuth: expandMasterAuth(d.Get("master_auth")),
ResourceLabels: expandStringMap(d, "resource_labels"),
}
Expand Down Expand Up @@ -781,6 +817,9 @@ func resourceContainerClusterRead(d *schema.ResourceData, meta interface{}) erro
d.Set("subnetwork", cluster.NetworkConfig.Subnetwork)
d.Set("enable_binary_authorization", cluster.BinaryAuthorization != nil && cluster.BinaryAuthorization.Enabled)
d.Set("enable_tpu", cluster.EnableTpu)
if err := d.Set("cluster_autoscaling", flattenClusterAutoscaling(cluster.Autoscaling)); err != nil {
return err
}
if err := d.Set("node_config", flattenNodeConfig(cluster.NodeConfig)); err != nil {
return err
}
Expand Down Expand Up @@ -963,6 +1002,23 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er
d.SetPartial("enable_binary_authorization")
}

if d.HasChange("cluster_autoscaling") {
req := &containerBeta.UpdateClusterRequest{
Update: &containerBeta.ClusterUpdate{
DesiredClusterAutoscaling: expandClusterAutoscaling(d.Get("cluster_autoscaling"), d),
}}

updateF := updateFunc(req, "updating GKE cluster autoscaling")
// Call update serially.
if err := lockedCall(lockKey, updateF); err != nil {
return err
}

log.Printf("[INFO] GKE cluster %s's cluster-wide autoscaling has been updated", d.Id())

d.SetPartial("cluster_autoscaling")
}

if d.HasChange("maintenance_policy") {
var req *containerBeta.SetMaintenancePolicyRequest
if mp, ok := d.GetOk("maintenance_policy"); ok {
Expand Down Expand Up @@ -1472,6 +1528,63 @@ func expandMaintenancePolicy(configured interface{}) *containerBeta.MaintenanceP
}
}

func expandClusterAutoscaling(configured interface{}, d *schema.ResourceData) *containerBeta.ClusterAutoscaling {
l, ok := configured.([]interface{})
if !ok || l == nil || len(l) == 0 || l[0] == nil {
// Before master version 1.11.2, we must send 'nil' values if autoscaling isn't
// turned on - the cluster will return an error even if we're setting
// EnableNodeAutoprovisioning to false.
cmv, err := version.NewVersion(d.Get("master_version").(string))
if err != nil {
log.Printf("[DEBUG] Could not parse master_version into version (%q), trying min_master_version.", d.Get("master_version").(string))
cmv, err = version.NewVersion(d.Get("min_master_version").(string))
if err != nil {
log.Printf("[DEBUG] Could not parse min_master_version into version (%q), assuming we are not already using cluster autoscaling.", d.Get("min_master_version").(string))
// This deserves a little explanation. The only reason we would ever want to send
// `EnableNodeAutoprovisioning: false` is because we think we might need to
// disable it (e.g. it is already enabled). Otherwise, there is no difference
// between sending `nil` and sending `EnableNodeAutoprovisioning: false`.
// The only circumstance in which neither master_version nor min_master_version
// can be parsed into version objects would be if the user has not set either one,
// and we have not yet had a `read` call. e.g. first-time creates, and possibly
// some circumstance related to import. It is probably safe to assume that
// we are not going to be changing cluster autoscaling from on to off in those
// circumstances. Therefore, if we don't know what version we're running, and
// the user has not requested cluster autoscaling, we'll fail "safe" and not touch
// it.
cmv, _ = version.NewVersion("0.0.0")
}
}
dmv, _ := version.NewVersion("1.11.2")
if cmv.LessThan(dmv) {
return nil
} else {
return &containerBeta.ClusterAutoscaling{
EnableNodeAutoprovisioning: false,
ForceSendFields: []string{"EnableNodeAutoprovisioning"},
}
}
}
r := &containerBeta.ClusterAutoscaling{}
if config, ok := l[0].(map[string]interface{}); ok {
r.EnableNodeAutoprovisioning = config["enabled"].(bool)
if limits, ok := config["resource_limits"]; ok {
if lmts, ok := limits.([]interface{}); ok {
for _, v := range lmts {
limit := v.(map[string]interface{})
r.ResourceLimits = append(r.ResourceLimits, &containerBeta.ResourceLimit{
ResourceType: limit["resource_type"].(string),
// Here we're relying on *not* setting ForceSendFields for 0-values.
Minimum: int64(limit["minimum"].(int)),
Maximum: int64(limit["maximum"].(int)),
})
}
}
}
}
return r
}

func expandMasterAuth(configured interface{}) *containerBeta.MasterAuth {
l := configured.([]interface{})
if len(l) == 0 || l[0] == nil {
Expand Down Expand Up @@ -1701,6 +1814,25 @@ func flattenMasterAuth(ma *containerBeta.MasterAuth) []map[string]interface{} {
return masterAuth
}

func flattenClusterAutoscaling(a *containerBeta.ClusterAutoscaling) []map[string]interface{} {
r := make(map[string]interface{})
if a == nil || !a.EnableNodeAutoprovisioning {
r["enabled"] = false
} else {
resourceLimits := make([]interface{}, 0, len(a.ResourceLimits))
for _, rl := range a.ResourceLimits {
resourceLimits = append(resourceLimits, map[string]interface{}{
"resource_type": rl.ResourceType,
"minimum": rl.Minimum,
"maximum": rl.Maximum,
})
}
r["resource_limits"] = resourceLimits
r["enabled"] = true
}
return []map[string]interface{}{r}
}

func flattenMasterAuthorizedNetworksConfig(c *containerBeta.MasterAuthorizedNetworksConfig) []map[string]interface{} {
if c == nil {
return nil
Expand Down
79 changes: 79 additions & 0 deletions google-beta/resource_container_cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1203,6 +1203,48 @@ func TestAccContainerCluster_withPodSecurityPolicy(t *testing.T) {
})
}

func TestAccContainerCluster_autoprovisioning(t *testing.T) {
t.Parallel()

clusterName := fmt.Sprintf("cluster-test-%s", acctest.RandString(10))

resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckContainerClusterDestroy,
Steps: []resource.TestStep{
{
Config: testAccContainerCluster_autoprovisioning(clusterName, true),
Check: resource.ComposeTestCheckFunc(
resource.TestCheckResourceAttr("google_container_cluster.with_autoprovisioning",
"cluster_autoscaling.0.enabled", "true"),
),
},
{
ResourceName: "google_container_cluster.with_autoprovisioning",
ImportStateIdPrefix: "us-central1-a/",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"min_master_version"},
},
{
Config: testAccContainerCluster_autoprovisioning(clusterName, false),
Check: resource.ComposeTestCheckFunc(
resource.TestCheckResourceAttr("google_container_cluster.with_autoprovisioning",
"cluster_autoscaling.0.enabled", "false"),
),
},
{
ResourceName: "google_container_cluster.with_autoprovisioning",
ImportStateIdPrefix: "us-central1-a/",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"min_master_version"},
},
},
})
}

func TestAccContainerCluster_sharedVpc(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -2141,6 +2183,43 @@ resource "google_container_cluster" "with_node_pool" {
}`, cluster, nodePool)
}

func testAccContainerCluster_autoprovisioning(cluster string, autoprovisioning bool) string {
config := fmt.Sprintf(`
data "google_container_engine_versions" "central1a" {
zone = "us-central1-a"
}
resource "google_container_cluster" "with_autoprovisioning" {
name = "%s"
zone = "us-central1-a"
min_master_version = "${data.google_container_engine_versions.central1a.latest_master_version}"
node_version = "${data.google_container_engine_versions.central1a.latest_node_version}"
initial_node_count = 3
`, cluster)
if autoprovisioning {
config += `
cluster_autoscaling {
enabled = true
resource_limits {
resource_type = "cpu"
maximum = 2
}
resource_limits {
resource_type = "memory"
maximum = 2048
}
}`
} else {
config += `
cluster_autoscaling {
enabled = false
}`
}
config += `
}`
return config
}

func testAccContainerCluster_withNodePoolAutoscaling(cluster, np string) string {
return fmt.Sprintf(`
resource "google_container_cluster" "with_node_pool" {
Expand Down
20 changes: 20 additions & 0 deletions website/docs/r/container_cluster.html.markdown
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,11 @@ output "cluster_ca_certificate" {
* `cluster_ipv4_cidr` - (Optional) The IP address range of the kubernetes pods in
this cluster. Default is an automatically assigned CIDR.

* `cluster_autoscaling` - (Optional, [Beta](https://terraform.io/docs/providers/google/provider_versions.html))
Configuration for cluster autoscaling (also called autoprovisioning), as described in
[the docs](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-provisioning).
Structure is documented below.

* `description` - (Optional) Description of the cluster.

* `enable_binary_authorization` - (Optional, [Beta](https://terraform.io/docs/providers/google/provider_versions.html)) Enable Binary Authorization for this cluster.
Expand Down Expand Up @@ -219,6 +224,21 @@ addons_config {
}
```

The `cluster_autoscaling` block supports:
* `enabled` - (Required) Whether cluster autoscaling (also called autoprovisioning) is
enabled. To set this to true, make sure your config meets the rest of the
requirements. Notably, you'll need `min_master_version` of at least `1.11.2`.
* `resource_limits` - (Optional) A list of limits on the autoprovisioning.
See [the docs](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-provisioning)
for an explanation of what options are available. If enabling autoprovisioning, make
sure to set at least `cpu` and `memory`. Structure is documented below.

The `resource_limits` block supports:
* `resource_type` - (Required) See [the docs](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-provisioning)
for a list of permitted types - `cpu`, `memory`, and others.
* `minimum` - (Optional) The minimum value for the resource type specified.
* `maximum` - (Optional) The maximum value for the resource type specified.

The `maintenance_policy` block supports:

* `daily_maintenance_window` - (Required) Time window specified for daily maintenance operations.
Expand Down

0 comments on commit 227953f

Please sign in to comment.