Merge branches 'ross/ServicePlan' and 'ross/ServicePlan' of https://g…

…ithub.com/ross-p-smith/AzureTRE into ross/ServicePlan

.github/workflows/build_validation_develop.yml

@@ -42,17 +42,15 @@ jobs:
         if: ${{ steps.filter.outputs.terraform == 'true' }}
         run: |
           find . -type d -name 'terraform' -not -path '*cnab*' -print0 \
-          | xargs -0 -I{} sh -c 'echo "***** Validating: {} *****"; \https://github.com/github/super-linter/issues/2433
+          | xargs -0 -I{} sh -c 'echo "***** Validating: {} *****"; \
           terraform -chdir={} init -backend=false; terraform -chdir={} validate'
 
       - name: Lint code base
         # the slim image is 2GB smaller and we don't use the extra stuff
-        # Moved this after the Terraform checks above due something similar to this issue: https://github.com/github/super-linter/issues/2433
-        uses: github/super-linter/slim@v4.9.3
+        # Moved this after the Terraform checks above due something similar to this issue:
+        # https://github.com/github/super-linter/issues/2433
+        uses: github/super-linter/slim@v4.9.4
         env:
-          # Until https://github.com/github/super-linter/commit/ec0662756da93f1e3aad4df049712df7d764d143 is released
-          # we need to set the correct plugin directory (which is incorrectly set to github/home/.tflint.d/plugins by default)
-          TFLINT_PLUGIN_DIR: "/root/.tflint.d/plugins"
           VALIDATE_ALL_CODEBASE: false
           DEFAULT_BRANCH: main
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -64,7 +62,5 @@ jobs:
           JAVA_FILE_NAME: checkstyle.xml
           VALIDATE_BASH: true
           VALIDATE_BASH_EXEC: true
+          VALIDATE_GITHUB_ACTIONS: true
           VALIDATE_DOCKERFILE_HADOLINT: true
-          # https://github.com/microsoft/AzureTRE/issues/1723 tracks re-instating VALIDATE_GITHUB_ACTIONS
-          # Note: in the meantime, the `.github/scripts/run-test.sh` script includes the `actionlint` checks)
-          # VALIDATE_GITHUB_ACTIONS: true

.github/workflows/deploy_tre.yml

@@ -22,8 +22,7 @@ jobs:
     uses: ./.github/workflows/deploy_tre_reusable.yml
     with:
       ciGitRef: ${{ github.ref }}
-      runExtendedTests: true
-      runSharedServicesTests: true
+      e2eTestsCustomSelector: "extended or shared_services"
     secrets:
       AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }}
       ACR_NAME: ${{ secrets.ACR_NAME }}

.github/workflows/deploy_tre_branch.yml

@@ -9,17 +9,13 @@ name: Deploy Azure TRE (branch)
 on:  # yamllint disable-line rule:truthy
   workflow_dispatch:
     inputs:
-      runExtendedTests:
-        description: Run the extended tests as part of the deployment?
-        type: boolean
-        default: false
-        required: false
-      runSharedServicesTests:
-        description: Run the shared services tests as part of the deployment?
-        type: boolean
-        default: false
+      e2eTestsCustomSelector:
+        description: A pytest marker selector for the e2e tests to be run
+        type: string
+        default: ""
         required: false
 
+
 # This will prevent multiple runs of this entire workflow.
 # We should NOT cancel in progress runs as that can destabilize the environment.
 concurrency: "${{ github.workflow }}-${{ github.ref }}"
@@ -54,9 +50,7 @@ jobs:
     uses: ./.github/workflows/deploy_tre_reusable.yml
     with:
       ciGitRef: ${{ github.ref }}
-      # testing input against string 'true' - see https://github.com/actions/runner/issues/1483
-      runExtendedTests: ${{ github.event.inputs.runExtendedTests == 'true' }}
-      runSharedSevicesTests: ${{ github.event.inputs.runSharedServicesTests == 'true' }}
+      e2eTestsCustomSelector: ${{ github.event.inputs.e2eTestsCustomSelector }}
     secrets:
       AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }}
       ACR_NAME: ${{ format('tre{0}', needs.prepare-not-main.outputs.refid) }}

.github/workflows/deploy_tre_reusable.yml

@@ -9,22 +9,19 @@ on:  # yamllint disable-line rule:truthy
         type: string
         required: false
       prHeadSha:
-        description: For PR builds where GITHUB_REF isn't set to the PR (e.g. comment trigger), pass the PR's head SHA commit here
+        description: >-
+          For PR builds where GITHUB_REF isn't set to the PR (e.g. comment trigger),
+          pass the PR's head SHA commit here
         type: string
         required: false
       ciGitRef:
         description: The git ref to use in annotations to associate a deployment with the code that triggered it
         type: string
         required: true
-      runExtendedTests:
-        description: Controls whether to run the extended tests as part of the deployment
-        type: boolean
-        default: false
-        required: false
-      runSharedServicesTests:
-        description: Controls whether to run the shared services tests as part of the deployment
-        type: boolean
-        default: false
+      e2eTestsCustomSelector:
+        description: The pytest marker selector for the e2e tests to be run
+        type: string
+        default: ""
         required: false
     secrets:
       AAD_TENANT_ID:
@@ -212,10 +209,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        target: [
-          build-and-push-api,
-          build-and-push-resource-processor,
-          build-and-push-gitea]
+        target: [build-and-push-api, build-and-push-resource-processor]
 
     steps:
       - name: Checkout
@@ -277,18 +271,18 @@ jobs:
           TRE_ID: "${{ secrets.TRE_ID }}"
           LOCATION: ${{ secrets.LOCATION }}
           ACR_NAME: ${{ secrets.ACR_NAME }}
-          TF_VAR_terraform_state_container_name:
-            ${{ secrets.TF_STATE_CONTAINER }}
+          TF_VAR_terraform_state_container_name: ${{ secrets.TF_STATE_CONTAINER }}
           TF_VAR_mgmt_resource_group_name: ${{ secrets.MGMT_RESOURCE_GROUP }}
-          TF_VAR_mgmt_storage_account_name:
-            ${{ secrets.STATE_STORAGE_ACCOUNT_NAME }}
+          TF_VAR_mgmt_storage_account_name: ${{ secrets.STATE_STORAGE_ACCOUNT_NAME }}
           TF_VAR_core_address_space: ${{ secrets.CORE_ADDRESS_SPACE }}
           TF_VAR_tre_address_space: ${{ secrets.TRE_ADDRESS_SPACE }}
           TF_VAR_swagger_ui_client_id: "${{ secrets.SWAGGER_UI_CLIENT_ID }}"
           TF_VAR_api_client_id: "${{ secrets.API_CLIENT_ID }}"
           TF_VAR_api_client_secret: "${{ secrets.API_CLIENT_SECRET }}"
-          TF_VAR_keyvault_purge_protection_enabled: "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
-          TF_VAR_stateful_resources_locked: "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
+          TF_VAR_keyvault_purge_protection_enabled:
+            "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
+          TF_VAR_stateful_resources_locked:
+            "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
 
   deploy_tre:
     name: Deploy TRE
@@ -326,18 +320,18 @@ jobs:
           TRE_ID: "${{ secrets.TRE_ID }}"
           LOCATION: ${{ secrets.LOCATION }}
           ACR_NAME: ${{ secrets.ACR_NAME }}
-          TF_VAR_terraform_state_container_name:
-            ${{ secrets.TF_STATE_CONTAINER }}
+          TF_VAR_terraform_state_container_name: ${{ secrets.TF_STATE_CONTAINER }}
           TF_VAR_mgmt_resource_group_name: ${{ secrets.MGMT_RESOURCE_GROUP }}
-          TF_VAR_mgmt_storage_account_name:
-            ${{ secrets.STATE_STORAGE_ACCOUNT_NAME }}
+          TF_VAR_mgmt_storage_account_name: ${{ secrets.STATE_STORAGE_ACCOUNT_NAME }}
           TF_VAR_core_address_space: ${{ secrets.CORE_ADDRESS_SPACE }}
           TF_VAR_tre_address_space: ${{ secrets.TRE_ADDRESS_SPACE }}
           TF_VAR_swagger_ui_client_id: "${{ secrets.SWAGGER_UI_CLIENT_ID }}"
           TF_VAR_api_client_id: "${{ secrets.API_CLIENT_ID }}"
           TF_VAR_api_client_secret: "${{ secrets.API_CLIENT_SECRET }}"
-          TF_VAR_keyvault_purge_protection_enabled: "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
-          TF_VAR_stateful_resources_locked: "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
+          TF_VAR_keyvault_purge_protection_enabled:
+            "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
+          TF_VAR_stateful_resources_locked:
+            "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
 
       - name: API Healthcheck
         uses: ./.github/actions/devcontainer_run_command
@@ -361,18 +355,18 @@ jobs:
           TRE_ID: "${{ secrets.TRE_ID }}"
           LOCATION: ${{ secrets.LOCATION }}
           ACR_NAME: ${{ secrets.ACR_NAME }}
-          TF_VAR_terraform_state_container_name:
-            ${{ secrets.TF_STATE_CONTAINER }}
+          TF_VAR_terraform_state_container_name: ${{ secrets.TF_STATE_CONTAINER }}
           TF_VAR_mgmt_resource_group_name: ${{ secrets.MGMT_RESOURCE_GROUP }}
-          TF_VAR_mgmt_storage_account_name:
-            ${{ secrets.STATE_STORAGE_ACCOUNT_NAME }}
+          TF_VAR_mgmt_storage_account_name: ${{ secrets.STATE_STORAGE_ACCOUNT_NAME }}
           TF_VAR_core_address_space: ${{ secrets.CORE_ADDRESS_SPACE }}
           TF_VAR_tre_address_space: ${{ secrets.TRE_ADDRESS_SPACE }}
           TF_VAR_swagger_ui_client_id: "${{ secrets.SWAGGER_UI_CLIENT_ID }}"
           TF_VAR_api_client_id: "${{ secrets.API_CLIENT_ID }}"
           TF_VAR_api_client_secret: "${{ secrets.API_CLIENT_SECRET }}"
-          TF_VAR_keyvault_purge_protection_enabled: "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
-          TF_VAR_stateful_resources_locked: "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
+          TF_VAR_keyvault_purge_protection_enabled:
+            "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
+          TF_VAR_stateful_resources_locked:
+            "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
 
   publish_bundles:
     name: Publish Bundles
@@ -437,7 +431,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        target: [build-and-push-guacamole]
+        target: [build-and-push-gitea, build-and-push-guacamole]
 
     steps:
       - name: Checkout
@@ -538,7 +532,7 @@ jobs:
           # then the default checkout will apply
           ref: ${{ inputs.prRef }}
 
-      - name: Register/deploy firewall
+      - name: Deploy firewall
         uses: ./.github/actions/devcontainer_run_command
         with:
           COMMAND: "make deploy-shared-service DIR=./templates/shared_services/firewall/ BUNDLE_TYPE=shared_service"
@@ -581,18 +575,18 @@ jobs:
           TRE_ID: "${{ secrets.TRE_ID }}"
           LOCATION: ${{ secrets.LOCATION }}
           ACR_NAME: ${{ secrets.ACR_NAME }}
-          TF_VAR_terraform_state_container_name:
-            ${{ secrets.TF_STATE_CONTAINER }}
+          TF_VAR_terraform_state_container_name: ${{ secrets.TF_STATE_CONTAINER }}
           TF_VAR_mgmt_resource_group_name: ${{ secrets.MGMT_RESOURCE_GROUP }}
-          TF_VAR_mgmt_storage_account_name:
-            ${{ secrets.STATE_STORAGE_ACCOUNT_NAME }}
+          TF_VAR_mgmt_storage_account_name: ${{ secrets.STATE_STORAGE_ACCOUNT_NAME }}
           TF_VAR_core_address_space: ${{ secrets.CORE_ADDRESS_SPACE }}
           TF_VAR_tre_address_space: ${{ secrets.TRE_ADDRESS_SPACE }}
           TF_VAR_swagger_ui_client_id: "${{ secrets.SWAGGER_UI_CLIENT_ID }}"
           TF_VAR_api_client_id: "${{ secrets.API_CLIENT_ID }}"
           TF_VAR_api_client_secret: "${{ secrets.API_CLIENT_SECRET }}"
-          TF_VAR_keyvault_purge_protection_enabled: "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
-          TF_VAR_stateful_resources_locked: "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
+          TF_VAR_keyvault_purge_protection_enabled:
+            "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
+          TF_VAR_stateful_resources_locked:
+            "${{ github.ref == 'refs/heads/main' && inputs.prRef == '' && true || false }}"
 
   e2e_tests_smoke:
     name: "Run E2E Tests (Smoke)"
@@ -659,65 +653,14 @@ jobs:
         with:
           files: "./e2e_tests/pytest_e2e_smoke.xml"
 
-  e2e_tests_extended:
-    name: "Run E2E Tests (Extended)"
-    if: ${{ inputs.runExtendedTests }}
-    runs-on: ubuntu-latest
-    environment: CICD
-    needs: [deploy_shared_services, build_additional_images]
-    timeout-minutes: 50
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          persist-credentials: false
-          # if the following values are missing (i.e. not triggered via comment workflow)
-          # then the default checkout will apply
-          ref: ${{ inputs.prRef }}
-
-      - name: Run E2E Tests (Extended)
-        uses: ./.github/actions/devcontainer_run_command
-        with:
-          COMMAND: "make test-e2e-extended"
-          ACTIONS_ACR_NAME: ${{ secrets.ACTIONS_ACR_NAME }}
-          ACTIONS_ACR_URI: ${{ secrets.ACTIONS_ACR_URI }}
-          ACTIONS_ACR_PASSWORD: ${{ secrets.ACTIONS_ACR_PASSWORD }}
-          ACTIONS_DEVCONTAINER_TAG: ${{ secrets.ACTIONS_DEVCONTAINER_TAG }}
-          ARM_TENANT_ID: "${{ secrets.ARM_TENANT_ID }}"
-          ARM_CLIENT_ID: "${{ secrets.ARM_CLIENT_ID }}"
-          ARM_CLIENT_SECRET: "${{ secrets.ARM_CLIENT_SECRET }}"
-          ARM_SUBSCRIPTION_ID: "${{ secrets.ARM_SUBSCRIPTION_ID }}"
-          LOCATION: "${{ secrets.LOCATION }}"
-          API_CLIENT_ID: "${{ secrets.API_CLIENT_ID }}"
-          AAD_TENANT_ID: "${{ secrets.AAD_TENANT_ID }}"
-          TEST_APP_ID: "${{ secrets.TEST_APP_ID }}"
-          TEST_WORKSPACE_APP_ID: "${{ secrets.TEST_WORKSPACE_APP_ID }}"
-          TEST_WORKSPACE_APP_SECRET: "${{ secrets.TEST_WORKSPACE_APP_SECRET }}"
-          TEST_ACCOUNT_CLIENT_ID: "${{ secrets.TEST_ACCOUNT_CLIENT_ID }}"
-          TEST_ACCOUNT_CLIENT_SECRET: "${{ secrets.TEST_ACCOUNT_CLIENT_SECRET }}"
-          TRE_ID: "${{ secrets.TRE_ID }}"
-          IS_API_SECURED: false
-
-      - name: Upload Test Results
-        if: always()
-        uses: actions/upload-artifact@v2
-        with:
-          name: E2E Test (Extended) Results
-          path: "./e2e_tests/pytest_e2e_extended.xml"
-
-      - name: Publish Test Results
-        if: always()
-        uses: EnricoMi/publish-unit-test-result-action@v1
-        with:
-          files: "./e2e_tests/pytest_e2e_extended.xml"
 
-  e2e_tests_shared_services:
-    name: "Run E2E Tests (Shared Services)"
-    if: ${{ inputs.runSharedServicesTests }}
+  e2e_tests_custom:
+    name: "Run E2E Tests"
+    if: ${{ inputs.e2eTestsCustomSelector != '' }}
     runs-on: ubuntu-latest
     environment: CICD
     needs: [deploy_shared_services, build_additional_images]
-    timeout-minutes: 50
+    timeout-minutes: 120
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -727,10 +670,10 @@ jobs:
           # then the default checkout will apply
           ref: ${{ inputs.prRef }}
 
-      - name: Run E2E Tests (Shared Services)
+      - name: Run E2E Tests
         uses: ./.github/actions/devcontainer_run_command
         with:
-          COMMAND: "make test-e2e-shared-services"
+          COMMAND: "make test-e2e-custom SELECTOR='${{ inputs.e2eTestsCustomSelector }}'"
           ACTIONS_ACR_NAME: ${{ secrets.ACTIONS_ACR_NAME }}
           ACTIONS_ACR_URI: ${{ secrets.ACTIONS_ACR_URI }}
           ACTIONS_ACR_PASSWORD: ${{ secrets.ACTIONS_ACR_PASSWORD }}
@@ -754,18 +697,18 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v2
         with:
-          name: E2E Test (Shared Services) Results
-          path: "./e2e_tests/pytest_e2e_shared_services.xml"
+          name: E2E Test Results
+          path: "./e2e_tests/pytest_e2e_custom.xml"
 
       - name: Publish Test Results
         if: always()
         uses: EnricoMi/publish-unit-test-result-action@v1
         with:
-          files: "./e2e_tests/pytest_e2e_shared_services.xml"
+          files: "./e2e_tests/pytest_e2e_custom.xml"
 
   summary:
     name: Summary Notification
-    needs: [e2e_tests_smoke, e2e_tests_extended, e2e_tests_shared_services]
+    needs: [e2e_tests_smoke, e2e_tests_custom]
     runs-on: ubuntu-latest
     if: ${{ always() && (github.ref == 'refs/heads/main' && inputs.prRef == '') }}
     environment: CICD

.github/workflows/pr_comment_bot.yml

@@ -134,15 +134,19 @@ jobs:
   run_test:
     # Run the tests with the re-usable workflow
     needs: [pr_comment]
-    if: ${{ needs.pr_comment.outputs.command == 'run-tests' || needs.pr_comment.outputs.command == 'run-tests-extended' || needs.pr_comment.outputs.command == 'run-tests-shared-services' }}
+    if: |
+      ${{ needs.pr_comment.outputs.command == 'run-tests' ||
+      needs.pr_comment.outputs.command == 'run-tests-extended' ||
+      needs.pr_comment.outputs.command == 'run-tests-shared-services' }}
     name: Deploy PR
     uses: ./.github/workflows/deploy_tre_reusable.yml
     with:
       prRef: ${{ needs.pr_comment.outputs.prRef }}
       prHeadSha: ${{ needs.pr_comment.outputs.prHeadSha }}
       ciGitRef: ${{ needs.pr_comment.outputs.ciGitRef }}
-      runExtendedTests: ${{ needs.pr_comment.outputs.command == 'run-tests-extended' }}
-      runSharedServicesTests: ${{ needs.pr_comment.outputs.command == 'run-tests-shared-services' }}
+      e2eTestsCustomSelector: |
+        ${{ (needs.pr_comment.outputs.command == 'run-tests-extended' && 'extended') ||
+        (needs.pr_comment.outputs.command == 'run-tests-shared-services' && 'shared_sevices')}}
     secrets:
       AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }}
       ACR_NAME: ${{ format('tre{0}', needs.pr_comment.outputs.prRefId) }}

.gitignore

@@ -201,8 +201,7 @@ templates/core/terraform/scripts/validation.txt
 templates/core/terraform/plan
 
 # Test results
-e2e_tests/pytest_e2e_smoke.xml
-e2e_tests/pytest_e2e_extended.xml
+e2e_tests/pytest_e2e_*.xml
 e2e_tests/workspace_id.txt
 pytest_api_unit.xml
 pytest_api_unit_failed

Makefile

@@ -243,7 +243,7 @@ lint:
 		-e VALIDATE_GITHUB_ACTIONS=true \
 		-e VALIDATE_DOCKERFILE_HADOLINT=true \
 		-v $${LOCAL_WORKSPACE_FOLDER}:/tmp/lint \
-		github/super-linter:slim-v4
+		github/super-linter:slim-v4.9.4
 
 bundle-build:
 	$(call target_title, "Building ${DIR} bundle with Porter") \
@@ -376,6 +376,11 @@ test-e2e-shared-services:
 	cd e2e_tests && \
 	python -m pytest -m shared_services --verify $${IS_API_SECURED:-true} --junit-xml pytest_e2e_shared_services.xml
 
+test-e2e-custom:
+	$(call target_title, "Running E2E shared service tests") && \
+	cd e2e_tests && \
+	python -m pytest -m "${SELECTOR}" --verify $${IS_API_SECURED:-true} --junit-xml pytest_e2e_custom.xml
+
 setup-local-debugging:
 	$(call target_title,"Setting up the ability to debug the API and Resource Processor") \
 	&& . ${MAKEFILE_DIR}/devops/scripts/check_dependencies.sh nodocker \