diff --git a/.github/workflows/acc-test.yaml b/.github/workflows/acc-test.yaml index 89a0f16b..4537b66d 100644 --- a/.github/workflows/acc-test.yaml +++ b/.github/workflows/acc-test.yaml @@ -12,6 +12,7 @@ on: jobs: check: + timeout-minutes: 720 runs-on: ubuntu-latest steps: - name: Checking for Fork diff --git a/README.md b/README.md index 2b845abc..11ad549d 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ This Terraform module deploys a Kubernetes cluster on Azure using AKS (Azure Kub -> **NOTE:** If you have not assigned `client_id` or `client_secret`, A `SystemAssigned` identity will be created. +-> **NOTE:** If you're using AzureRM `v4`, you can use this module by setting `source` to `Azure/aks/azurerm//v4`. + ## Notice on breaking changes Please be aware that major version(e.g., from 6.8.0 to 7.0.0) update contains breaking changes that may impact your infrastructure. It is crucial to review these changes with caution before proceeding with the upgrade. @@ -283,10 +285,10 @@ No modules. | [agents\_proximity\_placement\_group\_id](#input\_agents\_proximity\_placement\_group\_id) | (Optional) The ID of the Proximity Placement Group of the default Azure AKS agentpool (nodepool). Changing this forces a new resource to be created. | `string` | `null` | no | | [agents\_size](#input\_agents\_size) | The default virtual machine size for the Kubernetes agents. Changing this without specifying `var.temporary_name_for_rotation` forces a new resource to be created. | `string` | `"Standard_D2s_v3"` | no | | [agents\_tags](#input\_agents\_tags) | (Optional) A mapping of tags to assign to the Node Pool. | `map(string)` | `{}` | no | -| [agents\_taints](#input\_agents\_taints) | (Optional) A list of the taints added to new nodes during node pool create and scale. Changing this forces a new resource to be created. | `list(string)` | `null` | no | +| [agents\_taints](#input\_agents\_taints) | DEPRECATED, (Optional) A list of the taints added to new nodes during node pool create and scale. Changing this forces a new resource to be created. | `list(string)` | `null` | no | | [agents\_type](#input\_agents\_type) | (Optional) The type of Node Pool which should be created. Possible values are AvailabilitySet and VirtualMachineScaleSets. Defaults to VirtualMachineScaleSets. | `string` | `"VirtualMachineScaleSets"` | no | | [api\_server\_authorized\_ip\_ranges](#input\_api\_server\_authorized\_ip\_ranges) | (Optional) The IP ranges to allow for incoming traffic to the server nodes. | `set(string)` | `null` | no | -| [api\_server\_subnet\_id](#input\_api\_server\_subnet\_id) | (Optional) The ID of the Subnet where the API server endpoint is delegated to. | `string` | `null` | no | +| [api\_server\_subnet\_id](#input\_api\_server\_subnet\_id) | DEPRECATED, (Optional) The ID of the Subnet where the API server endpoint is delegated to. | `string` | `null` | no | | [attached\_acr\_id\_map](#input\_attached\_acr\_id\_map) | Azure Container Registry ids that need an authentication mechanism with Azure Kubernetes Service (AKS). Map key must be static string as acr's name, the value is acr's resource id. Changing this forces some new resources to be created. | `map(string)` | `{}` | no | | [auto\_scaler\_profile\_balance\_similar\_node\_groups](#input\_auto\_scaler\_profile\_balance\_similar\_node\_groups) | Detect similar node groups and balance the number of nodes between them. Defaults to `false`. | `bool` | `false` | no | | [auto\_scaler\_profile\_empty\_bulk\_delete\_max](#input\_auto\_scaler\_profile\_empty\_bulk\_delete\_max) | Maximum number of empty nodes that can be deleted at the same time. Defaults to `10`. | `number` | `10` | no | @@ -397,10 +399,10 @@ No modules. | [rbac\_aad](#input\_rbac\_aad) | (Optional) Is Azure Active Directory integration enabled? | `bool` | `true` | no | | [rbac\_aad\_admin\_group\_object\_ids](#input\_rbac\_aad\_admin\_group\_object\_ids) | Object ID of groups with admin access. | `list(string)` | `null` | no | | [rbac\_aad\_azure\_rbac\_enabled](#input\_rbac\_aad\_azure\_rbac\_enabled) | (Optional) Is Role Based Access Control based on Azure AD enabled? | `bool` | `null` | no | -| [rbac\_aad\_client\_app\_id](#input\_rbac\_aad\_client\_app\_id) | The Client ID of an Azure Active Directory Application. | `string` | `null` | no | +| [rbac\_aad\_client\_app\_id](#input\_rbac\_aad\_client\_app\_id) | DEPRECATED, The Client ID of an Azure Active Directory Application. | `string` | `null` | no | | [rbac\_aad\_managed](#input\_rbac\_aad\_managed) | Is the Azure Active Directory integration Managed, meaning that Azure will create/manage the Service Principal used for integration. | `bool` | `false` | no | -| [rbac\_aad\_server\_app\_id](#input\_rbac\_aad\_server\_app\_id) | The Server ID of an Azure Active Directory Application. | `string` | `null` | no | -| [rbac\_aad\_server\_app\_secret](#input\_rbac\_aad\_server\_app\_secret) | The Server Secret of an Azure Active Directory Application. | `string` | `null` | no | +| [rbac\_aad\_server\_app\_id](#input\_rbac\_aad\_server\_app\_id) | DEPRECATED, The Server ID of an Azure Active Directory Application. | `string` | `null` | no | +| [rbac\_aad\_server\_app\_secret](#input\_rbac\_aad\_server\_app\_secret) | DEPRECATED, The Server Secret of an Azure Active Directory Application. | `string` | `null` | no | | [rbac\_aad\_tenant\_id](#input\_rbac\_aad\_tenant\_id) | (Optional) The Tenant ID used for Azure Active Directory Application. If this isn't specified the Tenant ID of the current Subscription is used. | `string` | `null` | no | | [resource\_group\_name](#input\_resource\_group\_name) | The resource group name to be imported | `string` | n/a | yes | | [role\_based\_access\_control\_enabled](#input\_role\_based\_access\_control\_enabled) | Enable Role Based Access Control. | `bool` | `false` | no | diff --git a/deprecated_variables.tf b/deprecated_variables.tf new file mode 100644 index 00000000..2982ffed --- /dev/null +++ b/deprecated_variables.tf @@ -0,0 +1,31 @@ +# tflint-ignore-file: terraform_standard_module_structure + +variable "agents_taints" { + type = list(string) + default = null + description = "DEPRECATED, (Optional) A list of the taints added to new nodes during node pool create and scale. Changing this forces a new resource to be created." +} + +variable "api_server_subnet_id" { + type = string + default = null + description = "DEPRECATED, (Optional) The ID of the Subnet where the API server endpoint is delegated to." +} + +variable "rbac_aad_client_app_id" { + type = string + default = null + description = "DEPRECATED, The Client ID of an Azure Active Directory Application." +} + +variable "rbac_aad_server_app_id" { + type = string + default = null + description = "DEPRECATED, The Server ID of an Azure Active Directory Application." +} + +variable "rbac_aad_server_app_secret" { + type = string + default = null + description = "DEPRECATED, The Server Secret of an Azure Active Directory Application." +} \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/data.tf b/examples/application_gateway_ingress_v4/data.tf new file mode 100644 index 00000000..012126ad --- /dev/null +++ b/examples/application_gateway_ingress_v4/data.tf @@ -0,0 +1 @@ +data "azurerm_client_config" "this" {} \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/k8s_workload.tf b/examples/application_gateway_ingress_v4/k8s_workload.tf new file mode 120000 index 00000000..3a85d4c7 --- /dev/null +++ b/examples/application_gateway_ingress_v4/k8s_workload.tf @@ -0,0 +1 @@ +../application_gateway_ingress/k8s_workload.tf \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/main.tf b/examples/application_gateway_ingress_v4/main.tf new file mode 120000 index 00000000..025a83a1 --- /dev/null +++ b/examples/application_gateway_ingress_v4/main.tf @@ -0,0 +1 @@ +../application_gateway_ingress/main.tf \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/main_override.tf b/examples/application_gateway_ingress_v4/main_override.tf new file mode 100644 index 00000000..370cc63a --- /dev/null +++ b/examples/application_gateway_ingress_v4/main_override.tf @@ -0,0 +1,5 @@ +module "aks" { + #checkov:skip=CKV_AZURE_141:We enable admin account here so we can provision K8s resources directly in this simple example + source = "../../v4" + rbac_aad_tenant_id = data.azurerm_client_config.this.tenant_id +} \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/outputs.tf b/examples/application_gateway_ingress_v4/outputs.tf new file mode 120000 index 00000000..b7f9f6ed --- /dev/null +++ b/examples/application_gateway_ingress_v4/outputs.tf @@ -0,0 +1 @@ +../application_gateway_ingress/outputs.tf \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/providers.tf b/examples/application_gateway_ingress_v4/providers.tf new file mode 120000 index 00000000..66780034 --- /dev/null +++ b/examples/application_gateway_ingress_v4/providers.tf @@ -0,0 +1 @@ +../application_gateway_ingress/providers.tf \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/providers_override.tf b/examples/application_gateway_ingress_v4/providers_override.tf new file mode 100644 index 00000000..59b6fc4e --- /dev/null +++ b/examples/application_gateway_ingress_v4/providers_override.tf @@ -0,0 +1,22 @@ +# tflint-ignore-file: terraform_required_version_declaration + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.22.0" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + time = { + source = "hashicorp/time" + version = "0.9.1" + } + } +} \ No newline at end of file diff --git a/examples/application_gateway_ingress_v4/variables.tf b/examples/application_gateway_ingress_v4/variables.tf new file mode 120000 index 00000000..7899944f --- /dev/null +++ b/examples/application_gateway_ingress_v4/variables.tf @@ -0,0 +1 @@ +../application_gateway_ingress/variables.tf \ No newline at end of file diff --git a/examples/multiple_node_pools/main.tf b/examples/multiple_node_pools/main.tf index 2cba335b..26926246 100644 --- a/examples/multiple_node_pools/main.tf +++ b/examples/multiple_node_pools/main.tf @@ -24,11 +24,10 @@ resource "azurerm_virtual_network" "test" { } resource "azurerm_subnet" "test" { - address_prefixes = ["10.52.0.0/24"] - name = "${random_id.prefix.hex}-sn" - resource_group_name = local.resource_group.name - virtual_network_name = azurerm_virtual_network.test.name - enforce_private_link_endpoint_network_policies = true + address_prefixes = ["10.52.0.0/24"] + name = "${random_id.prefix.hex}-sn" + resource_group_name = local.resource_group.name + virtual_network_name = azurerm_virtual_network.test.name } locals { diff --git a/examples/multiple_node_pools/main_override.tf b/examples/multiple_node_pools/main_override.tf new file mode 100644 index 00000000..e3a7af9f --- /dev/null +++ b/examples/multiple_node_pools/main_override.tf @@ -0,0 +1,3 @@ +resource "azurerm_subnet" "test" { + enforce_private_link_endpoint_network_policies = true +} \ No newline at end of file diff --git a/examples/multiple_node_pools_v4/main.tf b/examples/multiple_node_pools_v4/main.tf new file mode 120000 index 00000000..a320b6cf --- /dev/null +++ b/examples/multiple_node_pools_v4/main.tf @@ -0,0 +1 @@ +../multiple_node_pools/main.tf \ No newline at end of file diff --git a/examples/multiple_node_pools_v4/main_override.tf b/examples/multiple_node_pools_v4/main_override.tf new file mode 100644 index 00000000..b7c945b8 --- /dev/null +++ b/examples/multiple_node_pools_v4/main_override.tf @@ -0,0 +1,8 @@ +resource "azurerm_subnet" "test" { + private_endpoint_network_policies = "Disabled" + private_link_service_network_policies_enabled = true +} + +module "aks" { + source = "../../v4" +} \ No newline at end of file diff --git a/examples/multiple_node_pools_v4/outputs.tf b/examples/multiple_node_pools_v4/outputs.tf new file mode 120000 index 00000000..a2313596 --- /dev/null +++ b/examples/multiple_node_pools_v4/outputs.tf @@ -0,0 +1 @@ +../multiple_node_pools/outputs.tf \ No newline at end of file diff --git a/examples/multiple_node_pools_v4/providers.tf b/examples/multiple_node_pools_v4/providers.tf new file mode 120000 index 00000000..99bf33e1 --- /dev/null +++ b/examples/multiple_node_pools_v4/providers.tf @@ -0,0 +1 @@ +../multiple_node_pools/providers.tf \ No newline at end of file diff --git a/examples/multiple_node_pools_v4/providers_override.tf b/examples/multiple_node_pools_v4/providers_override.tf new file mode 100644 index 00000000..01e99a37 --- /dev/null +++ b/examples/multiple_node_pools_v4/providers_override.tf @@ -0,0 +1,14 @@ +# tflint-ignore-file: terraform_required_version_declaration + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + } +} \ No newline at end of file diff --git a/examples/multiple_node_pools_v4/variables.tf b/examples/multiple_node_pools_v4/variables.tf new file mode 120000 index 00000000..a2f1eeca --- /dev/null +++ b/examples/multiple_node_pools_v4/variables.tf @@ -0,0 +1 @@ +../multiple_node_pools/variables.tf \ No newline at end of file diff --git a/examples/named_cluster/main.tf b/examples/named_cluster/main.tf index d51a0211..bc025504 100644 --- a/examples/named_cluster/main.tf +++ b/examples/named_cluster/main.tf @@ -24,11 +24,10 @@ resource "azurerm_virtual_network" "test" { } resource "azurerm_subnet" "test" { - address_prefixes = ["10.52.0.0/24"] - name = "${random_id.prefix.hex}-sn" - resource_group_name = local.resource_group.name - virtual_network_name = azurerm_virtual_network.test.name - enforce_private_link_endpoint_network_policies = true + address_prefixes = ["10.52.0.0/24"] + name = "${random_id.prefix.hex}-sn" + resource_group_name = local.resource_group.name + virtual_network_name = azurerm_virtual_network.test.name } resource "azurerm_user_assigned_identity" "test" { diff --git a/examples/named_cluster/main_override.tf b/examples/named_cluster/main_override.tf new file mode 100644 index 00000000..e3a7af9f --- /dev/null +++ b/examples/named_cluster/main_override.tf @@ -0,0 +1,3 @@ +resource "azurerm_subnet" "test" { + enforce_private_link_endpoint_network_policies = true +} \ No newline at end of file diff --git a/examples/named_cluster_v4/disk_encryption_set.tf b/examples/named_cluster_v4/disk_encryption_set.tf new file mode 120000 index 00000000..4f249ab7 --- /dev/null +++ b/examples/named_cluster_v4/disk_encryption_set.tf @@ -0,0 +1 @@ +../named_cluster/disk_encryption_set.tf \ No newline at end of file diff --git a/examples/named_cluster_v4/key_vault.tf b/examples/named_cluster_v4/key_vault.tf new file mode 120000 index 00000000..bcc40744 --- /dev/null +++ b/examples/named_cluster_v4/key_vault.tf @@ -0,0 +1 @@ +../named_cluster/key_vault.tf \ No newline at end of file diff --git a/examples/named_cluster_v4/kms.tf b/examples/named_cluster_v4/kms.tf new file mode 120000 index 00000000..4bc974e9 --- /dev/null +++ b/examples/named_cluster_v4/kms.tf @@ -0,0 +1 @@ +../named_cluster/kms.tf \ No newline at end of file diff --git a/examples/named_cluster_v4/main.tf b/examples/named_cluster_v4/main.tf new file mode 120000 index 00000000..0245d70a --- /dev/null +++ b/examples/named_cluster_v4/main.tf @@ -0,0 +1 @@ +../named_cluster/main.tf \ No newline at end of file diff --git a/examples/named_cluster_v4/main_override.tf b/examples/named_cluster_v4/main_override.tf new file mode 100644 index 00000000..5d224163 --- /dev/null +++ b/examples/named_cluster_v4/main_override.tf @@ -0,0 +1,5 @@ + +module "aks_cluster_name" { + source = "../../v4" + rbac_aad_tenant_id = data.azurerm_client_config.current.tenant_id +} diff --git a/examples/named_cluster_v4/outputs.tf b/examples/named_cluster_v4/outputs.tf new file mode 120000 index 00000000..189e2dde --- /dev/null +++ b/examples/named_cluster_v4/outputs.tf @@ -0,0 +1 @@ +../named_cluster/outputs.tf \ No newline at end of file diff --git a/examples/named_cluster_v4/providers.tf b/examples/named_cluster_v4/providers.tf new file mode 120000 index 00000000..4d6ffa73 --- /dev/null +++ b/examples/named_cluster_v4/providers.tf @@ -0,0 +1 @@ +../named_cluster/providers.tf \ No newline at end of file diff --git a/examples/named_cluster_v4/providers_override.tf b/examples/named_cluster_v4/providers_override.tf new file mode 100644 index 00000000..0e2a6bc8 --- /dev/null +++ b/examples/named_cluster_v4/providers_override.tf @@ -0,0 +1,18 @@ +# tflint-ignore-file: terraform_required_version_declaration + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + curl = { + source = "anschoewe/curl" + version = "1.0.2" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + } +} \ No newline at end of file diff --git a/examples/named_cluster_v4/variables.tf b/examples/named_cluster_v4/variables.tf new file mode 120000 index 00000000..99dad032 --- /dev/null +++ b/examples/named_cluster_v4/variables.tf @@ -0,0 +1 @@ +../named_cluster/variables.tf \ No newline at end of file diff --git a/examples/startup/main.tf b/examples/startup/main.tf index fd6d089a..7f022322 100644 --- a/examples/startup/main.tf +++ b/examples/startup/main.tf @@ -28,11 +28,10 @@ resource "azurerm_virtual_network" "test" { } resource "azurerm_subnet" "test" { - address_prefixes = ["10.52.0.0/24"] - name = "${random_id.prefix.hex}-sn" - resource_group_name = local.resource_group.name - virtual_network_name = azurerm_virtual_network.test.name - enforce_private_link_endpoint_network_policies = true + address_prefixes = ["10.52.0.0/24"] + name = "${random_id.prefix.hex}-sn" + resource_group_name = local.resource_group.name + virtual_network_name = azurerm_virtual_network.test.name } module "aks" { diff --git a/examples/startup/main_override.tf b/examples/startup/main_override.tf new file mode 100644 index 00000000..e3a7af9f --- /dev/null +++ b/examples/startup/main_override.tf @@ -0,0 +1,3 @@ +resource "azurerm_subnet" "test" { + enforce_private_link_endpoint_network_policies = true +} \ No newline at end of file diff --git a/examples/startup_v4/disk_encryption_set.tf b/examples/startup_v4/disk_encryption_set.tf new file mode 120000 index 00000000..626703a0 --- /dev/null +++ b/examples/startup_v4/disk_encryption_set.tf @@ -0,0 +1 @@ +../startup/disk_encryption_set.tf \ No newline at end of file diff --git a/examples/startup_v4/main.tf b/examples/startup_v4/main.tf new file mode 120000 index 00000000..e6269863 --- /dev/null +++ b/examples/startup_v4/main.tf @@ -0,0 +1 @@ +../startup/main.tf \ No newline at end of file diff --git a/examples/startup_v4/main_override.tf b/examples/startup_v4/main_override.tf new file mode 100644 index 00000000..ae101007 --- /dev/null +++ b/examples/startup_v4/main_override.tf @@ -0,0 +1,4 @@ +module "aks" { + source = "../../v4" + rbac_aad_tenant_id = data.azurerm_client_config.current.tenant_id +} \ No newline at end of file diff --git a/examples/startup_v4/outputs.tf b/examples/startup_v4/outputs.tf new file mode 120000 index 00000000..9d7b14a6 --- /dev/null +++ b/examples/startup_v4/outputs.tf @@ -0,0 +1 @@ +../startup/outputs.tf \ No newline at end of file diff --git a/examples/startup_v4/providers.tf b/examples/startup_v4/providers.tf new file mode 120000 index 00000000..d6882a84 --- /dev/null +++ b/examples/startup_v4/providers.tf @@ -0,0 +1 @@ +../startup/providers.tf \ No newline at end of file diff --git a/examples/startup_v4/providers_override.tf b/examples/startup_v4/providers_override.tf new file mode 100644 index 00000000..0e2a6bc8 --- /dev/null +++ b/examples/startup_v4/providers_override.tf @@ -0,0 +1,18 @@ +# tflint-ignore-file: terraform_required_version_declaration + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + curl = { + source = "anschoewe/curl" + version = "1.0.2" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + } +} \ No newline at end of file diff --git a/examples/startup_v4/variables.tf b/examples/startup_v4/variables.tf new file mode 120000 index 00000000..fd59ac33 --- /dev/null +++ b/examples/startup_v4/variables.tf @@ -0,0 +1 @@ +../startup/variables.tf \ No newline at end of file diff --git a/examples/uai_and_assign_role_on_subnet_v4/data.tf b/examples/uai_and_assign_role_on_subnet_v4/data.tf new file mode 100644 index 00000000..012126ad --- /dev/null +++ b/examples/uai_and_assign_role_on_subnet_v4/data.tf @@ -0,0 +1 @@ +data "azurerm_client_config" "this" {} \ No newline at end of file diff --git a/examples/uai_and_assign_role_on_subnet_v4/main.tf b/examples/uai_and_assign_role_on_subnet_v4/main.tf new file mode 120000 index 00000000..b9409925 --- /dev/null +++ b/examples/uai_and_assign_role_on_subnet_v4/main.tf @@ -0,0 +1 @@ +../uai_and_assign_role_on_subnet/main.tf \ No newline at end of file diff --git a/examples/uai_and_assign_role_on_subnet_v4/main_override.tf b/examples/uai_and_assign_role_on_subnet_v4/main_override.tf new file mode 100644 index 00000000..390bc574 --- /dev/null +++ b/examples/uai_and_assign_role_on_subnet_v4/main_override.tf @@ -0,0 +1,4 @@ +module "aks" { + source = "../../v4" + rbac_aad_tenant_id = data.azurerm_client_config.this.tenant_id +} \ No newline at end of file diff --git a/examples/uai_and_assign_role_on_subnet_v4/providers.tf b/examples/uai_and_assign_role_on_subnet_v4/providers.tf new file mode 120000 index 00000000..14f87c9b --- /dev/null +++ b/examples/uai_and_assign_role_on_subnet_v4/providers.tf @@ -0,0 +1 @@ +../uai_and_assign_role_on_subnet/providers.tf \ No newline at end of file diff --git a/examples/uai_and_assign_role_on_subnet_v4/providers_override.tf b/examples/uai_and_assign_role_on_subnet_v4/providers_override.tf new file mode 100644 index 00000000..01e99a37 --- /dev/null +++ b/examples/uai_and_assign_role_on_subnet_v4/providers_override.tf @@ -0,0 +1,14 @@ +# tflint-ignore-file: terraform_required_version_declaration + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + } +} \ No newline at end of file diff --git a/examples/uai_and_assign_role_on_subnet_v4/variables.tf b/examples/uai_and_assign_role_on_subnet_v4/variables.tf new file mode 120000 index 00000000..893f5b44 --- /dev/null +++ b/examples/uai_and_assign_role_on_subnet_v4/variables.tf @@ -0,0 +1 @@ +../uai_and_assign_role_on_subnet/variables.tf \ No newline at end of file diff --git a/examples/with_acr/main.tf b/examples/with_acr/main.tf index 8b5105f7..3bf44d03 100644 --- a/examples/with_acr/main.tf +++ b/examples/with_acr/main.tf @@ -24,11 +24,10 @@ resource "azurerm_virtual_network" "test" { } resource "azurerm_subnet" "test" { - address_prefixes = ["10.52.0.0/24"] - name = "${random_id.prefix.hex}-sn" - resource_group_name = local.resource_group.name - virtual_network_name = azurerm_virtual_network.test.name - enforce_private_link_endpoint_network_policies = true + address_prefixes = ["10.52.0.0/24"] + name = "${random_id.prefix.hex}-sn" + resource_group_name = local.resource_group.name + virtual_network_name = azurerm_virtual_network.test.name } resource "random_string" "acr_suffix" { @@ -43,11 +42,6 @@ resource "azurerm_container_registry" "example" { name = "aksacrtest${random_string.acr_suffix.result}" resource_group_name = local.resource_group.name sku = "Premium" - - retention_policy { - days = 7 - enabled = true - } } module "aks" { diff --git a/examples/with_acr/main_override.tf b/examples/with_acr/main_override.tf new file mode 100644 index 00000000..35192b17 --- /dev/null +++ b/examples/with_acr/main_override.tf @@ -0,0 +1,10 @@ +resource "azurerm_subnet" "test" { + enforce_private_link_endpoint_network_policies = true +} + +resource "azurerm_container_registry" "example" { + retention_policy { + days = 7 + enabled = true + } +} \ No newline at end of file diff --git a/examples/with_acr_v4/data.tf b/examples/with_acr_v4/data.tf new file mode 100644 index 00000000..012126ad --- /dev/null +++ b/examples/with_acr_v4/data.tf @@ -0,0 +1 @@ +data "azurerm_client_config" "this" {} \ No newline at end of file diff --git a/examples/with_acr_v4/main.tf b/examples/with_acr_v4/main.tf new file mode 120000 index 00000000..a4ead1f2 --- /dev/null +++ b/examples/with_acr_v4/main.tf @@ -0,0 +1 @@ +../with_acr/main.tf \ No newline at end of file diff --git a/examples/with_acr_v4/main_override.tf b/examples/with_acr_v4/main_override.tf new file mode 100644 index 00000000..f5d3db65 --- /dev/null +++ b/examples/with_acr_v4/main_override.tf @@ -0,0 +1,8 @@ +resource "azurerm_container_registry" "example" { + retention_policy_in_days = 7 +} + +module "aks" { + source = "../../v4" + rbac_aad_tenant_id = data.azurerm_client_config.this.tenant_id +} \ No newline at end of file diff --git a/examples/with_acr_v4/outputs.tf b/examples/with_acr_v4/outputs.tf new file mode 120000 index 00000000..d7a2645e --- /dev/null +++ b/examples/with_acr_v4/outputs.tf @@ -0,0 +1 @@ +../with_acr/outputs.tf \ No newline at end of file diff --git a/examples/with_acr_v4/providers.tf b/examples/with_acr_v4/providers.tf new file mode 120000 index 00000000..f54e7d78 --- /dev/null +++ b/examples/with_acr_v4/providers.tf @@ -0,0 +1 @@ +../with_acr/providers.tf \ No newline at end of file diff --git a/examples/with_acr_v4/providers_override.tf b/examples/with_acr_v4/providers_override.tf new file mode 100644 index 00000000..01e99a37 --- /dev/null +++ b/examples/with_acr_v4/providers_override.tf @@ -0,0 +1,14 @@ +# tflint-ignore-file: terraform_required_version_declaration + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + } +} \ No newline at end of file diff --git a/examples/with_acr_v4/variables.tf b/examples/with_acr_v4/variables.tf new file mode 120000 index 00000000..4c5ec50d --- /dev/null +++ b/examples/with_acr_v4/variables.tf @@ -0,0 +1 @@ +../with_acr/variables.tf \ No newline at end of file diff --git a/examples/without_monitor/main.tf b/examples/without_monitor/main.tf index 6be8a984..d5338885 100644 --- a/examples/without_monitor/main.tf +++ b/examples/without_monitor/main.tf @@ -24,11 +24,10 @@ resource "azurerm_virtual_network" "test" { } resource "azurerm_subnet" "test" { - address_prefixes = ["10.52.0.0/24"] - name = "${random_id.prefix.hex}-sn" - resource_group_name = local.resource_group.name - virtual_network_name = azurerm_virtual_network.test.name - enforce_private_link_endpoint_network_policies = true + address_prefixes = ["10.52.0.0/24"] + name = "${random_id.prefix.hex}-sn" + resource_group_name = local.resource_group.name + virtual_network_name = azurerm_virtual_network.test.name } resource "azurerm_dns_zone" "aks_web_app_routing" { diff --git a/examples/without_monitor/main_override.tf b/examples/without_monitor/main_override.tf new file mode 100644 index 00000000..e3a7af9f --- /dev/null +++ b/examples/without_monitor/main_override.tf @@ -0,0 +1,3 @@ +resource "azurerm_subnet" "test" { + enforce_private_link_endpoint_network_policies = true +} \ No newline at end of file diff --git a/examples/without_monitor_v4/disk_encryption_set.tf b/examples/without_monitor_v4/disk_encryption_set.tf new file mode 120000 index 00000000..f8e69f92 --- /dev/null +++ b/examples/without_monitor_v4/disk_encryption_set.tf @@ -0,0 +1 @@ +../without_monitor/disk_encryption_set.tf \ No newline at end of file diff --git a/examples/without_monitor_v4/main.tf b/examples/without_monitor_v4/main.tf new file mode 120000 index 00000000..ec81c61f --- /dev/null +++ b/examples/without_monitor_v4/main.tf @@ -0,0 +1 @@ +../without_monitor/main.tf \ No newline at end of file diff --git a/examples/without_monitor_v4/main_override.tf b/examples/without_monitor_v4/main_override.tf new file mode 100644 index 00000000..2b62b4e9 --- /dev/null +++ b/examples/without_monitor_v4/main_override.tf @@ -0,0 +1,4 @@ +module "aks_without_monitor" { + source = "../../v4" + rbac_aad_tenant_id = data.azurerm_client_config.current.tenant_id +} \ No newline at end of file diff --git a/examples/without_monitor_v4/outputs.tf b/examples/without_monitor_v4/outputs.tf new file mode 120000 index 00000000..1c88a8fa --- /dev/null +++ b/examples/without_monitor_v4/outputs.tf @@ -0,0 +1 @@ +../without_monitor/outputs.tf \ No newline at end of file diff --git a/examples/without_monitor_v4/providers.tf b/examples/without_monitor_v4/providers.tf new file mode 120000 index 00000000..4dd53a7e --- /dev/null +++ b/examples/without_monitor_v4/providers.tf @@ -0,0 +1 @@ +../without_monitor/providers.tf \ No newline at end of file diff --git a/examples/without_monitor_v4/providers_override.tf b/examples/without_monitor_v4/providers_override.tf new file mode 100644 index 00000000..0e2a6bc8 --- /dev/null +++ b/examples/without_monitor_v4/providers_override.tf @@ -0,0 +1,18 @@ +# tflint-ignore-file: terraform_required_version_declaration + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + curl = { + source = "anschoewe/curl" + version = "1.0.2" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + } +} \ No newline at end of file diff --git a/examples/without_monitor_v4/variables.tf b/examples/without_monitor_v4/variables.tf new file mode 120000 index 00000000..b42da667 --- /dev/null +++ b/examples/without_monitor_v4/variables.tf @@ -0,0 +1 @@ +../without_monitor/variables.tf \ No newline at end of file diff --git a/extra_node_pool.tf b/extra_node_pool.tf index 9b732b35..9bc68313 100644 --- a/extra_node_pool.tf +++ b/extra_node_pool.tf @@ -10,10 +10,6 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_before_destroy name = "${each.value.name}${substr(md5(uuid()), 0, 4)}" vm_size = each.value.vm_size capacity_reservation_group_id = each.value.capacity_reservation_group_id - custom_ca_trust_enabled = each.value.custom_ca_trust_enabled - enable_auto_scaling = each.value.enable_auto_scaling - enable_host_encryption = each.value.enable_host_encryption - enable_node_public_ip = each.value.enable_node_public_ip eviction_policy = each.value.eviction_policy fips_enabled = each.value.fips_enabled gpu_instance = each.value.gpu_instance @@ -21,7 +17,6 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_before_destroy kubelet_disk_type = each.value.kubelet_disk_type max_count = each.value.max_count max_pods = each.value.max_pods - message_of_the_day = each.value.message_of_the_day min_count = each.value.min_count mode = each.value.mode node_count = each.value.node_count @@ -171,17 +166,12 @@ resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_after_destroy" name = each.value.name vm_size = each.value.vm_size capacity_reservation_group_id = each.value.capacity_reservation_group_id - custom_ca_trust_enabled = each.value.custom_ca_trust_enabled - enable_auto_scaling = each.value.enable_auto_scaling - enable_host_encryption = each.value.enable_host_encryption - enable_node_public_ip = each.value.enable_node_public_ip eviction_policy = each.value.eviction_policy fips_enabled = each.value.fips_enabled host_group_id = each.value.host_group_id kubelet_disk_type = each.value.kubelet_disk_type max_count = each.value.max_count max_pods = each.value.max_pods - message_of_the_day = each.value.message_of_the_day min_count = each.value.min_count mode = each.value.mode node_count = each.value.node_count diff --git a/extra_node_pool_override.tf b/extra_node_pool_override.tf new file mode 100644 index 00000000..500f27ec --- /dev/null +++ b/extra_node_pool_override.tf @@ -0,0 +1,17 @@ +# tflint-ignore-file: azurerm_resource_tag + +resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_before_destroy" { + custom_ca_trust_enabled = each.value.custom_ca_trust_enabled + enable_auto_scaling = each.value.enable_auto_scaling + enable_host_encryption = each.value.enable_host_encryption + enable_node_public_ip = each.value.enable_node_public_ip + message_of_the_day = each.value.message_of_the_day +} + +resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_after_destroy" { + custom_ca_trust_enabled = each.value.custom_ca_trust_enabled + enable_auto_scaling = each.value.enable_auto_scaling + enable_host_encryption = each.value.enable_host_encryption + enable_node_public_ip = each.value.enable_node_public_ip + message_of_the_day = each.value.message_of_the_day +} \ No newline at end of file diff --git a/main.tf b/main.tf index ed358fa2..29ef433c 100644 --- a/main.tf +++ b/main.tf @@ -18,7 +18,6 @@ resource "azurerm_kubernetes_cluster" "main" { location = coalesce(var.location, data.azurerm_resource_group.main.location) name = "${local.cluster_name}${var.cluster_name_random_suffix ? substr(md5(uuid()), 0, 4) : ""}" resource_group_name = data.azurerm_resource_group.main.name - automatic_channel_upgrade = var.automatic_channel_upgrade azure_policy_enabled = var.azure_policy_enabled cost_analysis_enabled = var.cost_analysis_enabled disk_encryption_set_id = var.disk_encryption_set_id @@ -27,7 +26,6 @@ resource "azurerm_kubernetes_cluster" "main" { image_cleaner_interval_hours = var.image_cleaner_interval_hours kubernetes_version = var.kubernetes_version local_account_disabled = var.local_account_disabled - node_os_channel_upgrade = var.node_os_channel_upgrade node_resource_group = var.node_resource_group oidc_issuer_enabled = var.oidc_issuer_enabled open_service_mesh_enabled = var.open_service_mesh_enabled diff --git a/main_override.tf b/main_override.tf new file mode 100644 index 00000000..a1f53765 --- /dev/null +++ b/main_override.tf @@ -0,0 +1,6 @@ +# tflint-ignore-file: azurerm_resource_tag + +resource "azurerm_kubernetes_cluster" "main" { + automatic_channel_upgrade = var.automatic_channel_upgrade + node_os_channel_upgrade = var.node_os_channel_upgrade +} \ No newline at end of file diff --git a/test/e2e/terraform_aks_test.go b/test/e2e/terraform_aks_test.go index de37fa0a..ec4cda2f 100644 --- a/test/e2e/terraform_aks_test.go +++ b/test/e2e/terraform_aks_test.go @@ -4,21 +4,21 @@ import ( "fmt" "io" "os" + "path/filepath" "regexp" "strings" "testing" "time" - "github.com/hashicorp/go-retryablehttp" - - "github.com/stretchr/testify/require" - test_helper "github.com/Azure/terraform-module-test-helper" "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/hashicorp/go-retryablehttp" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestExamplesStartup(t *testing.T) { + t.Parallel() vars := map[string]interface{}{ "client_id": "", "client_secret": "", @@ -46,6 +46,7 @@ func assertOutputNotEmpty(t *testing.T, output test_helper.TerraformOutput, name } func TestExamplesWithoutMonitor(t *testing.T) { + t.Parallel() vars := make(map[string]interface{}, 0) managedIdentityId := os.Getenv("MSI_ID") if managedIdentityId != "" { @@ -71,6 +72,7 @@ func TestExamplesWithoutMonitor(t *testing.T) { } func TestExamplesNamedCluster(t *testing.T) { + t.Parallel() vars := make(map[string]interface{}) managedIdentityId := os.Getenv("MSI_ID") if managedIdentityId != "" { @@ -98,6 +100,7 @@ func TestExamplesNamedCluster(t *testing.T) { } func TestExamplesWithoutAssertion(t *testing.T) { + t.Parallel() examples := []string{ "examples/with_acr", "examples/multiple_node_pools", @@ -105,6 +108,7 @@ func TestExamplesWithoutAssertion(t *testing.T) { for _, e := range examples { example := e t.Run(example, func(t *testing.T) { + t.Parallel() test_helper.RunE2ETest(t, "../../", example, terraform.Options{ Upgrade: true, }, nil) @@ -113,6 +117,7 @@ func TestExamplesWithoutAssertion(t *testing.T) { } func TestExamples_differentLocationForLogAnalyticsSolution(t *testing.T) { + t.Parallel() vars := make(map[string]any, 0) managedIdentityId := os.Getenv("MSI_ID") if managedIdentityId != "" { @@ -128,6 +133,7 @@ func TestExamples_differentLocationForLogAnalyticsSolution(t *testing.T) { } func TestExamples_applicationGatewayIngress(t *testing.T) { + t.Parallel() useExistingAppGw := []struct { useBrownFieldAppGw bool bringYourOwnVnet bool @@ -151,6 +157,7 @@ func TestExamples_applicationGatewayIngress(t *testing.T) { } for _, u := range useExistingAppGw { t.Run(fmt.Sprintf("useExistingAppGw %t %t %t", u.bringYourOwnVnet, u.useBrownFieldAppGw, u.createRoleBindingForAppGw), func(t *testing.T) { + t.Parallel() test_helper.RunE2ETest(t, "../../", "examples/application_gateway_ingress", terraform.Options{ Upgrade: true, Vars: map[string]interface{}{ @@ -178,6 +185,37 @@ func TestExamples_applicationGatewayIngress(t *testing.T) { } } +func TestExamplesForV4(t *testing.T) { + t.Parallel() + examples, err := os.ReadDir("../../examples") + require.NoError(t, err) + for _, example := range examples { + if !example.IsDir() { + continue + } + if !strings.HasSuffix(example.Name(), "_v4") { + continue + } + t.Run(example.Name(), func(t *testing.T) { + t.Parallel() + tmp, err := os.MkdirTemp("", "") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmp) + }() + tfvars := filepath.Join(tmp, "terraform.tfvars") + require.NoError(t, os.WriteFile(tfvars, []byte(` + client_id = "" + client_secret = "" +`), 0o600)) + test_helper.RunE2ETest(t, "../../", fmt.Sprintf("examples/%s", example.Name()), terraform.Options{ + Upgrade: true, + VarFiles: []string{tfvars}, + }, nil) + }) + } +} + func getHTML(url string) (string, error) { client := retryablehttp.NewClient() client.RetryMax = 10 diff --git a/test/upgrade/upgrade_test.go b/test/upgrade/upgrade_test.go index efc79a3a..ed686240 100644 --- a/test/upgrade/upgrade_test.go +++ b/test/upgrade/upgrade_test.go @@ -3,14 +3,19 @@ package upgrade import ( "fmt" "os" + "path/filepath" + "strings" "testing" "time" + "github.com/stretchr/testify/require" + test_helper "github.com/Azure/terraform-module-test-helper" "github.com/gruntwork-io/terratest/modules/terraform" ) func TestExampleUpgrade_startup(t *testing.T) { + t.Parallel() currentRoot, err := test_helper.GetCurrentModuleRootPath() if err != nil { t.FailNow() @@ -34,6 +39,7 @@ func TestExampleUpgrade_startup(t *testing.T) { } func TestExampleUpgrade_without_monitor(t *testing.T) { + t.Parallel() currentRoot, err := test_helper.GetCurrentModuleRootPath() if err != nil { t.FailNow() @@ -56,6 +62,7 @@ func TestExampleUpgrade_without_monitor(t *testing.T) { } func TestExampleUpgrade_named_cluster(t *testing.T) { + t.Parallel() currentRoot, err := test_helper.GetCurrentModuleRootPath() if err != nil { t.FailNow() @@ -78,6 +85,7 @@ func TestExampleUpgrade_named_cluster(t *testing.T) { } func TestExampleUpgrade(t *testing.T) { + t.Parallel() examples := []string{ "examples/with_acr", "examples/multiple_node_pools", @@ -85,6 +93,7 @@ func TestExampleUpgrade(t *testing.T) { for _, e := range examples { example := e t.Run(example, func(t *testing.T) { + t.Parallel() currentRoot, err := test_helper.GetCurrentModuleRootPath() if err != nil { t.FailNow() @@ -101,6 +110,7 @@ func TestExampleUpgrade(t *testing.T) { } func TestExampleUpgrade_applicationGw(t *testing.T) { + t.Parallel() useExistingAppGw := []struct { useBrownFieldAppGw bool bringYourOwnVnet bool @@ -124,6 +134,7 @@ func TestExampleUpgrade_applicationGw(t *testing.T) { } for _, u := range useExistingAppGw { t.Run(fmt.Sprintf("useExistingAppGw %t %t %t", u.bringYourOwnVnet, u.useBrownFieldAppGw, u.createRoleBindingForAppGw), func(t *testing.T) { + t.Parallel() currentRoot, err := test_helper.GetCurrentModuleRootPath() if err != nil { t.FailNow() @@ -148,3 +159,41 @@ func TestExampleUpgrade_applicationGw(t *testing.T) { }) } } + +func TestExamplesForV4(t *testing.T) { + t.Parallel() + examples, err := os.ReadDir("../../examples") + require.NoError(t, err) + currentRoot, err := test_helper.GetCurrentModuleRootPath() + if err != nil { + t.FailNow() + } + currentMajorVersion, err := test_helper.GetCurrentMajorVersionFromEnv() + if err != nil { + t.FailNow() + } + for _, example := range examples { + if !example.IsDir() { + continue + } + if !strings.HasSuffix(example.Name(), "_v4") { + continue + } + t.Run(example.Name(), func(t *testing.T) { + t.Parallel() + tmp, err := os.MkdirTemp("", "") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmp) + }() + tfvars := filepath.Join(tmp, "terraform.tfvars") + require.NoError(t, os.WriteFile(tfvars, []byte(` + client_id = "" + client_secret = "" +`), 0o600)) + test_helper.ModuleUpgradeTest(t, "Azure", "terraform-azurerm-aks", fmt.Sprintf("examples/%s", example.Name()), currentRoot, terraform.Options{ + VarFiles: []string{tfvars}, + }, currentMajorVersion) + }) + } +} diff --git a/v4/extra_node_pool.tf b/v4/extra_node_pool.tf new file mode 120000 index 00000000..9cbc2968 --- /dev/null +++ b/v4/extra_node_pool.tf @@ -0,0 +1 @@ +../extra_node_pool.tf \ No newline at end of file diff --git a/v4/extra_node_pool_override.tf b/v4/extra_node_pool_override.tf new file mode 100644 index 00000000..0d0227a6 --- /dev/null +++ b/v4/extra_node_pool_override.tf @@ -0,0 +1,13 @@ +# tflint-ignore-file: azurerm_resource_tag + +resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_before_destroy" { + auto_scaling_enabled = each.value.enable_auto_scaling + host_encryption_enabled = each.value.enable_host_encryption + node_public_ip_enabled = each.value.enable_node_public_ip +} + +resource "azurerm_kubernetes_cluster_node_pool" "node_pool_create_after_destroy" { + auto_scaling_enabled = each.value.enable_auto_scaling + host_encryption_enabled = each.value.enable_host_encryption + node_public_ip_enabled = each.value.enable_node_public_ip +} \ No newline at end of file diff --git a/v4/locals.tf b/v4/locals.tf new file mode 120000 index 00000000..1b032e65 --- /dev/null +++ b/v4/locals.tf @@ -0,0 +1 @@ +../locals.tf \ No newline at end of file diff --git a/v4/log_analytics.tf b/v4/log_analytics.tf new file mode 120000 index 00000000..639a396c --- /dev/null +++ b/v4/log_analytics.tf @@ -0,0 +1 @@ +../log_analytics.tf \ No newline at end of file diff --git a/v4/main.tf b/v4/main.tf new file mode 120000 index 00000000..6c481fa3 --- /dev/null +++ b/v4/main.tf @@ -0,0 +1 @@ +../main.tf \ No newline at end of file diff --git a/v4/main_override.tf b/v4/main_override.tf new file mode 100644 index 00000000..f82ced5f --- /dev/null +++ b/v4/main_override.tf @@ -0,0 +1,302 @@ +# tflint-ignore-file: azurerm_resource_tag + +resource "azurerm_kubernetes_cluster" "main" { + automatic_upgrade_channel = var.automatic_channel_upgrade + node_os_upgrade_channel = var.node_os_channel_upgrade + + dynamic "default_node_pool" { + for_each = var.enable_auto_scaling == true ? [] : ["default_node_pool_manually_scaled"] + + content { + name = var.agents_pool_name + vm_size = var.agents_size + auto_scaling_enabled = var.enable_auto_scaling + fips_enabled = var.default_node_pool_fips_enabled + host_encryption_enabled = var.enable_host_encryption + max_count = null + max_pods = var.agents_max_pods + min_count = null + node_count = var.agents_count + node_labels = var.agents_labels + node_public_ip_enabled = var.enable_node_public_ip + only_critical_addons_enabled = var.only_critical_addons_enabled + orchestrator_version = var.orchestrator_version + os_disk_size_gb = var.os_disk_size_gb + os_disk_type = var.os_disk_type + os_sku = var.os_sku + pod_subnet_id = var.pod_subnet_id + proximity_placement_group_id = var.agents_proximity_placement_group_id + scale_down_mode = var.scale_down_mode + snapshot_id = var.snapshot_id + tags = merge(var.tags, var.agents_tags) + temporary_name_for_rotation = var.temporary_name_for_rotation + type = var.agents_type + ultra_ssd_enabled = var.ultra_ssd_enabled + vnet_subnet_id = var.vnet_subnet_id + zones = var.agents_availability_zones + + dynamic "kubelet_config" { + for_each = var.agents_pool_kubelet_configs + + content { + allowed_unsafe_sysctls = kubelet_config.value.allowed_unsafe_sysctls + container_log_max_line = kubelet_config.value.container_log_max_line + container_log_max_size_mb = kubelet_config.value.container_log_max_size_mb + cpu_cfs_quota_enabled = kubelet_config.value.cpu_cfs_quota_enabled + cpu_cfs_quota_period = kubelet_config.value.cpu_cfs_quota_period + cpu_manager_policy = kubelet_config.value.cpu_manager_policy + image_gc_high_threshold = kubelet_config.value.image_gc_high_threshold + image_gc_low_threshold = kubelet_config.value.image_gc_low_threshold + pod_max_pid = kubelet_config.value.pod_max_pid + topology_manager_policy = kubelet_config.value.topology_manager_policy + } + } + dynamic "linux_os_config" { + for_each = var.agents_pool_linux_os_configs + + content { + swap_file_size_mb = linux_os_config.value.swap_file_size_mb + transparent_huge_page_defrag = linux_os_config.value.transparent_huge_page_defrag + transparent_huge_page_enabled = linux_os_config.value.transparent_huge_page_enabled + + dynamic "sysctl_config" { + for_each = linux_os_config.value.sysctl_configs == null ? [] : linux_os_config.value.sysctl_configs + + content { + fs_aio_max_nr = sysctl_config.value.fs_aio_max_nr + fs_file_max = sysctl_config.value.fs_file_max + fs_inotify_max_user_watches = sysctl_config.value.fs_inotify_max_user_watches + fs_nr_open = sysctl_config.value.fs_nr_open + kernel_threads_max = sysctl_config.value.kernel_threads_max + net_core_netdev_max_backlog = sysctl_config.value.net_core_netdev_max_backlog + net_core_optmem_max = sysctl_config.value.net_core_optmem_max + net_core_rmem_default = sysctl_config.value.net_core_rmem_default + net_core_rmem_max = sysctl_config.value.net_core_rmem_max + net_core_somaxconn = sysctl_config.value.net_core_somaxconn + net_core_wmem_default = sysctl_config.value.net_core_wmem_default + net_core_wmem_max = sysctl_config.value.net_core_wmem_max + net_ipv4_ip_local_port_range_max = sysctl_config.value.net_ipv4_ip_local_port_range_max + net_ipv4_ip_local_port_range_min = sysctl_config.value.net_ipv4_ip_local_port_range_min + net_ipv4_neigh_default_gc_thresh1 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh1 + net_ipv4_neigh_default_gc_thresh2 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh2 + net_ipv4_neigh_default_gc_thresh3 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh3 + net_ipv4_tcp_fin_timeout = sysctl_config.value.net_ipv4_tcp_fin_timeout + net_ipv4_tcp_keepalive_intvl = sysctl_config.value.net_ipv4_tcp_keepalive_intvl + net_ipv4_tcp_keepalive_probes = sysctl_config.value.net_ipv4_tcp_keepalive_probes + net_ipv4_tcp_keepalive_time = sysctl_config.value.net_ipv4_tcp_keepalive_time + net_ipv4_tcp_max_syn_backlog = sysctl_config.value.net_ipv4_tcp_max_syn_backlog + net_ipv4_tcp_max_tw_buckets = sysctl_config.value.net_ipv4_tcp_max_tw_buckets + net_ipv4_tcp_tw_reuse = sysctl_config.value.net_ipv4_tcp_tw_reuse + net_netfilter_nf_conntrack_buckets = sysctl_config.value.net_netfilter_nf_conntrack_buckets + net_netfilter_nf_conntrack_max = sysctl_config.value.net_netfilter_nf_conntrack_max + vm_max_map_count = sysctl_config.value.vm_max_map_count + vm_swappiness = sysctl_config.value.vm_swappiness + vm_vfs_cache_pressure = sysctl_config.value.vm_vfs_cache_pressure + } + } + } + } + dynamic "upgrade_settings" { + for_each = var.agents_pool_max_surge == null ? [] : ["upgrade_settings"] + + content { + max_surge = var.agents_pool_max_surge + drain_timeout_in_minutes = var.agents_pool_drain_timeout_in_minutes + node_soak_duration_in_minutes = var.agents_pool_node_soak_duration_in_minutes + } + } + } + } + dynamic "default_node_pool" { + for_each = var.enable_auto_scaling == true ? ["default_node_pool_auto_scaled"] : [] + + content { + name = var.agents_pool_name + vm_size = var.agents_size + auto_scaling_enabled = var.enable_auto_scaling + fips_enabled = var.default_node_pool_fips_enabled + host_encryption_enabled = var.enable_host_encryption + max_count = var.agents_max_count + max_pods = var.agents_max_pods + min_count = var.agents_min_count + node_labels = var.agents_labels + node_public_ip_enabled = var.enable_node_public_ip + only_critical_addons_enabled = var.only_critical_addons_enabled + orchestrator_version = var.orchestrator_version + os_disk_size_gb = var.os_disk_size_gb + os_disk_type = var.os_disk_type + os_sku = var.os_sku + pod_subnet_id = var.pod_subnet_id + proximity_placement_group_id = var.agents_proximity_placement_group_id + scale_down_mode = var.scale_down_mode + snapshot_id = var.snapshot_id + tags = merge(var.tags, var.agents_tags) + temporary_name_for_rotation = var.temporary_name_for_rotation + type = var.agents_type + ultra_ssd_enabled = var.ultra_ssd_enabled + vnet_subnet_id = var.vnet_subnet_id + zones = var.agents_availability_zones + + dynamic "kubelet_config" { + for_each = var.agents_pool_kubelet_configs + + content { + allowed_unsafe_sysctls = kubelet_config.value.allowed_unsafe_sysctls + container_log_max_line = kubelet_config.value.container_log_max_line + container_log_max_size_mb = kubelet_config.value.container_log_max_size_mb + cpu_cfs_quota_enabled = kubelet_config.value.cpu_cfs_quota_enabled + cpu_cfs_quota_period = kubelet_config.value.cpu_cfs_quota_period + cpu_manager_policy = kubelet_config.value.cpu_manager_policy + image_gc_high_threshold = kubelet_config.value.image_gc_high_threshold + image_gc_low_threshold = kubelet_config.value.image_gc_low_threshold + pod_max_pid = kubelet_config.value.pod_max_pid + topology_manager_policy = kubelet_config.value.topology_manager_policy + } + } + dynamic "linux_os_config" { + for_each = var.agents_pool_linux_os_configs + + content { + swap_file_size_mb = linux_os_config.value.swap_file_size_mb + transparent_huge_page_defrag = linux_os_config.value.transparent_huge_page_defrag + transparent_huge_page_enabled = linux_os_config.value.transparent_huge_page_enabled + + dynamic "sysctl_config" { + for_each = linux_os_config.value.sysctl_configs == null ? [] : linux_os_config.value.sysctl_configs + + content { + fs_aio_max_nr = sysctl_config.value.fs_aio_max_nr + fs_file_max = sysctl_config.value.fs_file_max + fs_inotify_max_user_watches = sysctl_config.value.fs_inotify_max_user_watches + fs_nr_open = sysctl_config.value.fs_nr_open + kernel_threads_max = sysctl_config.value.kernel_threads_max + net_core_netdev_max_backlog = sysctl_config.value.net_core_netdev_max_backlog + net_core_optmem_max = sysctl_config.value.net_core_optmem_max + net_core_rmem_default = sysctl_config.value.net_core_rmem_default + net_core_rmem_max = sysctl_config.value.net_core_rmem_max + net_core_somaxconn = sysctl_config.value.net_core_somaxconn + net_core_wmem_default = sysctl_config.value.net_core_wmem_default + net_core_wmem_max = sysctl_config.value.net_core_wmem_max + net_ipv4_ip_local_port_range_max = sysctl_config.value.net_ipv4_ip_local_port_range_max + net_ipv4_ip_local_port_range_min = sysctl_config.value.net_ipv4_ip_local_port_range_min + net_ipv4_neigh_default_gc_thresh1 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh1 + net_ipv4_neigh_default_gc_thresh2 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh2 + net_ipv4_neigh_default_gc_thresh3 = sysctl_config.value.net_ipv4_neigh_default_gc_thresh3 + net_ipv4_tcp_fin_timeout = sysctl_config.value.net_ipv4_tcp_fin_timeout + net_ipv4_tcp_keepalive_intvl = sysctl_config.value.net_ipv4_tcp_keepalive_intvl + net_ipv4_tcp_keepalive_probes = sysctl_config.value.net_ipv4_tcp_keepalive_probes + net_ipv4_tcp_keepalive_time = sysctl_config.value.net_ipv4_tcp_keepalive_time + net_ipv4_tcp_max_syn_backlog = sysctl_config.value.net_ipv4_tcp_max_syn_backlog + net_ipv4_tcp_max_tw_buckets = sysctl_config.value.net_ipv4_tcp_max_tw_buckets + net_ipv4_tcp_tw_reuse = sysctl_config.value.net_ipv4_tcp_tw_reuse + net_netfilter_nf_conntrack_buckets = sysctl_config.value.net_netfilter_nf_conntrack_buckets + net_netfilter_nf_conntrack_max = sysctl_config.value.net_netfilter_nf_conntrack_max + vm_max_map_count = sysctl_config.value.vm_max_map_count + vm_swappiness = sysctl_config.value.vm_swappiness + vm_vfs_cache_pressure = sysctl_config.value.vm_vfs_cache_pressure + } + } + } + } + dynamic "upgrade_settings" { + for_each = var.agents_pool_max_surge == null ? [] : ["upgrade_settings"] + + content { + max_surge = var.agents_pool_max_surge + drain_timeout_in_minutes = var.agents_pool_drain_timeout_in_minutes + node_soak_duration_in_minutes = var.agents_pool_node_soak_duration_in_minutes + } + } + } + } + dynamic "service_mesh_profile" { + for_each = var.service_mesh_profile == null ? [] : ["service_mesh_profile"] + + content { + mode = var.service_mesh_profile.mode + revisions = var.service_mesh_profile.revisions + external_ingress_gateway_enabled = var.service_mesh_profile.external_ingress_gateway_enabled + internal_ingress_gateway_enabled = var.service_mesh_profile.internal_ingress_gateway_enabled + } + } + dynamic "api_server_access_profile" { + for_each = var.api_server_authorized_ip_ranges != null ? [ + "api_server_access_profile" + ] : [] + + content { + authorized_ip_ranges = var.api_server_authorized_ip_ranges + } + } + dynamic "azure_active_directory_role_based_access_control" { + for_each = var.role_based_access_control_enabled && var.rbac_aad && var.rbac_aad_managed ? ["rbac"] : [] + + content { + admin_group_object_ids = var.rbac_aad_admin_group_object_ids + azure_rbac_enabled = var.rbac_aad_azure_rbac_enabled + tenant_id = var.rbac_aad_tenant_id + } + } + dynamic "azure_active_directory_role_based_access_control" { + for_each = var.role_based_access_control_enabled && var.rbac_aad && !var.rbac_aad_managed ? ["rbac"] : [] + + content { + tenant_id = var.rbac_aad_tenant_id + } + } + network_profile { + network_plugin = var.network_plugin + dns_service_ip = var.net_profile_dns_service_ip + load_balancer_sku = var.load_balancer_sku + network_data_plane = var.ebpf_data_plane + network_plugin_mode = var.network_plugin_mode + network_policy = var.network_policy + outbound_type = var.net_profile_outbound_type + pod_cidr = var.net_profile_pod_cidr + service_cidr = var.net_profile_service_cidr + + dynamic "load_balancer_profile" { + for_each = var.load_balancer_profile_enabled && var.load_balancer_sku == "standard" ? [ + "load_balancer_profile" + ] : [] + + content { + idle_timeout_in_minutes = var.load_balancer_profile_idle_timeout_in_minutes + managed_outbound_ip_count = var.load_balancer_profile_managed_outbound_ip_count + managed_outbound_ipv6_count = var.load_balancer_profile_managed_outbound_ipv6_count + outbound_ip_address_ids = var.load_balancer_profile_outbound_ip_address_ids + outbound_ip_prefix_ids = var.load_balancer_profile_outbound_ip_prefix_ids + outbound_ports_allocated = var.load_balancer_profile_outbound_ports_allocated + } + } + } + dynamic "storage_profile" { + for_each = var.storage_profile_enabled ? ["storage_profile"] : [] + + content { + blob_driver_enabled = var.storage_profile_blob_driver_enabled + disk_driver_enabled = var.storage_profile_disk_driver_enabled + file_driver_enabled = var.storage_profile_file_driver_enabled + snapshot_controller_enabled = var.storage_profile_snapshot_controller_enabled + } + } + dynamic "web_app_routing" { + for_each = var.web_app_routing == null ? [] : ["web_app_routing"] + + content { + dns_zone_ids = [var.web_app_routing.dns_zone_id] + } + } + + lifecycle { + ignore_changes = [ + http_application_routing_enabled, + http_proxy_config[0].no_proxy, + kubernetes_version, + # we might have a random suffix in cluster's name so we have to ignore it here, but we've traced user supplied cluster name by `null_resource.kubernetes_cluster_name_keeper` so when the name is changed we'll recreate this resource. + name, + network_profile[0].load_balancer_profile[0].outbound_ip_address_ids, + network_profile[0].load_balancer_profile[0].outbound_ip_prefix_ids, + ] + } +} \ No newline at end of file diff --git a/v4/outputs.tf b/v4/outputs.tf new file mode 120000 index 00000000..1a861df4 --- /dev/null +++ b/v4/outputs.tf @@ -0,0 +1 @@ +../outputs.tf \ No newline at end of file diff --git a/v4/role_assignments.tf b/v4/role_assignments.tf new file mode 120000 index 00000000..705ff1c9 --- /dev/null +++ b/v4/role_assignments.tf @@ -0,0 +1 @@ +../role_assignments.tf \ No newline at end of file diff --git a/v4/variables.tf b/v4/variables.tf new file mode 120000 index 00000000..3a65dccd --- /dev/null +++ b/v4/variables.tf @@ -0,0 +1 @@ +../variables.tf \ No newline at end of file diff --git a/v4/variables_override.tf b/v4/variables_override.tf new file mode 100644 index 00000000..2a35b6a7 --- /dev/null +++ b/v4/variables_override.tf @@ -0,0 +1,15 @@ +variable "service_mesh_profile" { + type = object({ + mode = string + revisions = list(string) + internal_ingress_gateway_enabled = optional(bool, true) + external_ingress_gateway_enabled = optional(bool, true) + }) + default = null + description = <<-EOT + `mode` - (Required) The mode of the service mesh. Possible value is `Istio`. + `revisions` - (Required) Specify 1 or 2 Istio control plane revisions for managing minor upgrades using the canary upgrade process. For example, create the resource with `revisions` set to `["asm-1-20"]`, or leave it empty (the `revisions` will only be known after apply). To start the canary upgrade, change `revisions` to `["asm-1-20", "asm-1-21"]`. To roll back the canary upgrade, revert to `["asm-1-20"]`. To confirm the upgrade, change to `["asm-1-21"]`. + `internal_ingress_gateway_enabled` - (Optional) Is Istio Internal Ingress Gateway enabled? Defaults to `true`. + `external_ingress_gateway_enabled` - (Optional) Is Istio External Ingress Gateway enabled? Defaults to `true`. + EOT +} \ No newline at end of file diff --git a/v4/versions.tf b/v4/versions.tf new file mode 120000 index 00000000..8bd0ff14 --- /dev/null +++ b/v4/versions.tf @@ -0,0 +1 @@ +../versions.tf \ No newline at end of file diff --git a/v4/versions_override.tf b/v4/versions_override.tf new file mode 100644 index 00000000..e7d3b24b --- /dev/null +++ b/v4/versions_override.tf @@ -0,0 +1,20 @@ +terraform { + required_providers { + azapi = { + source = "Azure/azapi" + version = ">= 1.4.0, < 2.0" + } + azurerm = { + source = "hashicorp/azurerm" + version = "~> 4.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + tls = { + source = "hashicorp/tls" + version = ">= 3.1" + } + } +} diff --git a/variables.tf b/variables.tf index fd4c25b5..9c762d95 100644 --- a/variables.tf +++ b/variables.tf @@ -210,12 +210,6 @@ variable "agents_tags" { description = "(Optional) A mapping of tags to assign to the Node Pool." } -variable "agents_taints" { - type = list(string) - default = null - description = "(Optional) A list of the taints added to new nodes during node pool create and scale. Changing this forces a new resource to be created." -} - variable "agents_type" { type = string default = "VirtualMachineScaleSets" @@ -228,12 +222,6 @@ variable "api_server_authorized_ip_ranges" { description = "(Optional) The IP ranges to allow for incoming traffic to the server nodes." } -variable "api_server_subnet_id" { - type = string - default = null - description = "(Optional) The ID of the Subnet where the API server endpoint is delegated to." -} - variable "attached_acr_id_map" { type = map(string) default = {} @@ -1285,12 +1273,6 @@ variable "rbac_aad_azure_rbac_enabled" { description = "(Optional) Is Role Based Access Control based on Azure AD enabled?" } -variable "rbac_aad_client_app_id" { - type = string - default = null - description = "The Client ID of an Azure Active Directory Application." -} - variable "rbac_aad_managed" { type = bool default = false @@ -1298,18 +1280,6 @@ variable "rbac_aad_managed" { nullable = false } -variable "rbac_aad_server_app_id" { - type = string - default = null - description = "The Server ID of an Azure Active Directory Application." -} - -variable "rbac_aad_server_app_secret" { - type = string - default = null - description = "The Server Secret of an Azure Active Directory Application." -} - variable "rbac_aad_tenant_id" { type = string default = null