Merge branch 'main' into marrobi/marrobi/issue4029

microsoft · Nov 6, 2024 · a8e14a1 · a8e14a1
2 parents d0f567c + 9140431
commit a8e14a1
Show file tree

Hide file tree

Showing 100 changed files with 815 additions and 445 deletions.
diff --git a/.github/actions/devcontainer_run_command/action.yml b/.github/actions/devcontainer_run_command/action.yml
@@ -129,6 +129,10 @@ inputs:
     description: "Firewall SKU"
     required: false
     default: ""
+  APP_GATEWAY_SKU:
+    description: "Application Gateway SKU"
+    required: false
+    default: ""
 
 runs:
   using: composite
@@ -182,6 +186,9 @@ runs:
           -e TF_INPUT="0" \
           -e TF_IN_AUTOMATION="1" \
           -e USE_ENV_VARS_NOT_FILES="true" \
+          -e ARM_STORAGE_USE_AZUREAD="true" \
+          -e ARM_USE_AZUREAD="true" \
+          -e ARM_USE_OIDC="true" \
           -e BUNDLE_TYPE="${{ inputs.BUNDLE_TYPE }}" \
           -e WORKSPACE_SERVICE_NAME="${{ inputs.WORKSPACE_SERVICE_NAME }}" \
           -e ARM_ENVIRONMENT="${{ env.ARM_ENVIRONMENT }}" \
@@ -239,6 +246,7 @@ runs:
           -e TF_VAR_resource_processor_number_processes_per_instance="${{ (inputs.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE != ''
             && inputs.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE) || 5 }}" \
           -e TF_VAR_firewall_sku=${{ inputs.FIREWALL_SKU }} \
+          -e TF_VAR_app_gateway_sku=${{ inputs.APP_GATEWAY_SKU }} \
           -e E2E_TESTS_NUMBER_PROCESSES="${{ inputs.E2E_TESTS_NUMBER_PROCESSES }}" \
           '${{ inputs.CI_CACHE_ACR_NAME }}${{ env.ACR_DOMAIN_SUFFIX }}/tredev:${{ inputs.DEVCONTAINER_TAG }}' \
         bash -c "${{ inputs.COMMAND }}"
diff --git a/.github/linters/.tflint.hcl b/.github/linters/.tflint.hcl
@@ -30,8 +30,3 @@ rule "terraform_naming_convention" {
 rule "terraform_standard_module_structure" {
   enabled = true
 }
-
-rule "azurerm_resource_missing_tags" {
-  enabled = true
-  tags = ["tre_id"]
-}
diff --git a/.github/linters/.tflint_core.hcl b/.github/linters/.tflint_core.hcl
@@ -0,0 +1,19 @@
+# This is used for TRE tags validation only.
+
+config {
+  module = true
+  force = false
+}
+
+plugin "azurerm" {
+    enabled = true
+}
+
+rule "terraform_typed_variables" {
+  enabled = false
+}
+
+rule "azurerm_resource_missing_tags" {
+  enabled = true
+  tags = ["tre_id"]
+}
diff --git a/.github/workflows/build_validation_develop.yml b/.github/workflows/build_validation_develop.yml
@@ -40,6 +40,9 @@ jobs:
             docs:
               - 'docs/**/*'
 
+            terraform_core:
+              - core/**/terraform/**/*.tf
+
             terraform_workspaces:
               - templates/workspaces/**/terraform/**/*.tf
 
@@ -95,6 +98,17 @@ jobs:
           pip install -r docs/requirements.txt
           mkdocs build --strict
 
+      - name: Core Tags
+        if: ${{ steps.filter.outputs.terraform_core == 'true' }}
+        uses: github/super-linter/slim@v5.0.0
+        env:
+          VALIDATE_ALL_CODEBASE: false
+          DEFAULT_BRANCH: main
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          VALIDATE_TERRAFORM_TFLINT: true
+          TERRAFORM_TFLINT_CONFIG_FILE: .tflint_core.hcl
+          FILTER_REGEX_INCLUDE: './core/.*'
+
       - name: Workspace Tags
         if: ${{ steps.filter.outputs.terraform_workspaces == 'true' }}
         uses: github/super-linter/slim@v5.0.0
@@ -104,7 +118,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_TERRAFORM_TFLINT: true
           TERRAFORM_TFLINT_CONFIG_FILE: .tflint_workspaces.hcl
-          LINTER_REGEX_INCLUDE: './templates/workspaces/.*'
+          FILTER_REGEX_INCLUDE: './templates/workspaces/.*'
 
       - name: Workspace Services Tags
         if: ${{ steps.filter.outputs.terraform_workspace_services == 'true' }}
@@ -115,7 +129,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_TERRAFORM_TFLINT: true
           TERRAFORM_TFLINT_CONFIG_FILE: .tflint_workspace_services.hcl
-          LINTER_REGEX_INCLUDE: './templates/workspaces/.*'
+          FILTER_REGEX_INCLUDE: './templates/workspaces/.*'
           FILTER_REGEX_EXCLUDE: '.*user_resource.*'
 
       - name: User Resources Tags
@@ -127,7 +141,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_TERRAFORM_TFLINT: true
           TERRAFORM_TFLINT_CONFIG_FILE: .tflint_user_resources.hcl
-          LINTER_REGEX_INCLUDE: './templates/workspace_services/.*/user_resources/.*'
+          FILTER_REGEX_INCLUDE: './templates/workspace_services/.*/user_resources/.*'
 
       - name: Shared Services Tags
         if: ${{ steps.filter.outputs.terraform_shared_services == 'true' }}
@@ -138,4 +152,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_TERRAFORM_TFLINT: true
           TERRAFORM_TFLINT_CONFIG_FILE: .tflint_shared_services.hcl
-          LINTER_REGEX_INCLUDE: './templates/shared_services/.*'
+          FILTER_REGEX_INCLUDE: './templates/shared_services/.*'
diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml
@@ -358,6 +358,7 @@ jobs:
           RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE: ${{ vars.RESOURCE_PROCESSOR_NUMBER_PROCESSES_PER_INSTANCE }}
           RP_BUNDLE_VALUES: ${{ vars.RP_BUNDLE_VALUES }}
           FIREWALL_SKU: ${{ vars.FIREWALL_SKU}}
+          APP_GATEWAY_SKU: ${{ vars.APP_GATEWAY_SKU }}
 
       - name: API Healthcheck
         uses: ./.github/actions/devcontainer_run_command

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,9 @@
 FEATURES:
 
 ENHANCEMENTS:
+* Split log entries with [Log chunk X of Y] for better readability. ([[#3992](https://github.com/microsoft/AzureTRE/issues/3992)
+* Expose APP_SERVICE_SKU build variable to allow enablement of App Gateway WAF ([#4111](https://github.com/microsoft/AzureTRE/pull/4111))
+* Update Terraform to use Azure AD authentication rather than storage account keys ([#4103](https://github.com/microsoft/AzureTRE/issues/4103))
 
 BUG FIXES:
 
@@ -22,6 +25,7 @@ ENHANCEMENTS:
 * Add info regarding workspace limit into docs ([#3920](https://github.com/microsoft/AzureTRE/issues/3920))
 
 BUG FIXES:
+* Add Snyk Security updates for September
 * Workspace creation blocked due to Azure API depreciation ([#4095](https://github.com/microsoft/AzureTRE/issues/4095))
 
 COMPONENTS:

diff --git a/api_app/_version.py b/api_app/_version.py
@@ -1 +1 @@
-__version__ = "0.19.2"
+__version__ = "0.19.3"
diff --git a/config.sample.yaml b/config.sample.yaml
@@ -43,6 +43,7 @@ tre:
   # Uncomment the following to disable deployment of the Web UI.
   # deploy_ui: false
   firewall_sku: Standard
+  app_gateway_sku: Standard_v2
 
   # Uncomment to deploy to a custom domain
   # custom_domain: __CHANGE_ME__

diff --git a/config_schema.json b/config_schema.json
@@ -89,6 +89,10 @@
           "description": "SKU of the Azure Firewall.",
           "type": "string"
         },
+        "app_gateway_sku": {
+          "description": "SKU of the Application Gateway.",
+          "type": "string"
+        },
         "custom_domain": {
           "description": "Custom domain name.",
           "type": "string"

diff --git a/core/terraform/.terraform.lock.hcl b/core/terraform/.terraform.lock.hcl
diff --git a/core/terraform/airlock/main.tf b/core/terraform/airlock/main.tf
@@ -7,7 +7,7 @@ terraform {
     }
     azapi = {
       source  = "Azure/azapi"
-      version = ">= 1.9.0"
+      version = ">= 1.15.0"
     }
     local = {
       source  = "hashicorp/local"

diff --git a/core/terraform/appgateway/appgateway.tf b/core/terraform/appgateway/appgateway.tf
@@ -26,11 +26,13 @@ resource "azurerm_application_gateway" "agw" {
   tags                = local.tre_core_tags
 
   sku {
-    name     = "Standard_v2"
-    tier     = "Standard_v2"
+    name     = coalesce(var.app_gateway_sku, "Standard_v2")
+    tier     = coalesce(var.app_gateway_sku, "Standard_v2")
     capacity = 1
   }
 
+  firewall_policy_id = var.app_gateway_sku == "WAF_v2" ? azurerm_web_application_firewall_policy.waf[0].id : null
+
   # User-assign managed identify id required to access certificate in KeyVault
   identity {
     type         = "UserAssigned"
@@ -120,6 +122,12 @@ resource "azurerm_application_gateway" "agw" {
     path                = "/api/ping"
     timeout             = "30"
     unhealthy_threshold = "3"
+
+    match {
+      status_code = [
+        "200-399"
+      ]
+    }
   }
 
   # Public HTTPS listener
@@ -198,6 +206,40 @@ resource "azurerm_application_gateway" "agw" {
 
 }
 
+resource "azurerm_web_application_firewall_policy" "waf" {
+
+  // only create WAF policy when App Gateway sku.tier == "WAF_v2"
+  count = var.app_gateway_sku == "WAF_v2" ? 1 : 0
+
+  name                = "wafpolicy-${var.tre_id}"
+  resource_group_name = var.resource_group_name
+  location            = var.location
+
+  policy_settings {
+    enabled = true
+    mode    = "Detection"
+  }
+
+  managed_rules {
+    managed_rule_set {
+      type    = "OWASP"
+      version = 3.2
+    }
+  }
+
+  // once created ignore policy_settings and rulesets allow to be managed outside of here
+  lifecycle { ignore_changes = [policy_settings, managed_rules] }
+
+  // terraform doesn't handle the downgrade from WAF_v2 > Standard_v2 SKU, this is required to detatch the policy from the app gateway before deletion of the policy
+  provisioner "local-exec" {
+    when    = destroy
+    command = <<EOT
+          APP_GATEWAY_ID=$(az network application-gateway waf-policy show --name ${self.name} --resource-group ${self.resource_group_name} --query applicationGateways[0].id --output tsv)
+          az network application-gateway update --ids $APP_GATEWAY_ID --set firewallPolicy=null --set sku.name=Standard_v2 --set sku.tier=Standard_v2
+        EOT
+  }
+}
+
 resource "azurerm_monitor_diagnostic_setting" "agw" {
   name                       = "diagnostics-agw-${var.tre_id}"
   target_resource_id         = azurerm_application_gateway.agw.id

diff --git a/core/terraform/appgateway/variables.tf b/core/terraform/appgateway/variables.tf
@@ -26,3 +26,6 @@ variable "static_web_dns_zone_id" {
 variable "log_analytics_workspace_id" {
   type = string
 }
+variable "app_gateway_sku" {
+  type = string
+}
diff --git a/core/terraform/main.tf b/core/terraform/main.tf
@@ -19,7 +19,7 @@ terraform {
     }
     azapi = {
       source  = "Azure/azapi"
-      version = "~> 1.13.0"
+      version = "~> 1.15.0"
     }
   }
 
@@ -99,6 +99,7 @@ module "appgateway" {
   keyvault_id                = azurerm_key_vault.kv.id
   static_web_dns_zone_id     = module.network.static_web_dns_zone_id
   log_analytics_workspace_id = module.azure_monitor.log_analytics_workspace_id
+  app_gateway_sku            = var.app_gateway_sku
 
   depends_on = [
     module.network,