From 0ffb8d1514bab11445b229a96dc2cc1a589ce369 Mon Sep 17 00:00:00 2001 From: Paulo Henrique Date: Fri, 19 Jan 2024 13:54:48 -0800 Subject: [PATCH 01/87] Adding Option to override environment URL --- .github/workflows/destroy-environment.yml | 14 ++++- .github/workflows/test-environment.yml | 29 +++++++--- .github/workflows/test-gcp-dm.yml | 1 + .github/workflows/upgrade-environment.yml | 13 ++--- .github/workflows/weekly-enviroment.yml | 7 +-- deploy/cloud/main.tf | 1 + deploy/cloud/modules/ec/variables.tf | 4 ++ deploy/cloud/variables.tf | 5 ++ deploy/test-environments/README.md | 32 +++++------ deploy/test-environments/delete_env.sh | 1 + deploy/test-environments/main.tf | 7 +-- deploy/test-environments/variables.tf | 6 +++ deploy/weekly-environment/main.tf | 1 + dev-docs/Cloud-Env-Testing.md | 66 +++++++++++------------ 14 files changed, 115 insertions(+), 72 deletions(-) diff --git a/.github/workflows/destroy-environment.yml b/.github/workflows/destroy-environment.yml index 3c9ec87329..9cd0780132 100644 --- a/.github/workflows/destroy-environment.yml +++ b/.github/workflows/destroy-environment.yml @@ -12,6 +12,14 @@ on: ignore-prefix: type: string description: "Ignore all environments starting with `ignore-prefix`" + ec-url: + required: true + default: "https://cloud.elastic.co/" + type: choice + description: Select the Environment URL to delete + options: + - https://cloud.elastic.co/ + - https://console.qa.cld.elstc.co/ ec-api-key: type: string description: "**Optional** To delete env environments on your own organization, enter your Elastic Cloud API key." @@ -31,8 +39,8 @@ jobs: timeout-minutes: 120 # Add "id-token" with the intended permissions. permissions: - contents: 'read' - id-token: 'write' + contents: "read" + id-token: "write" steps: - name: Check out the repo uses: actions/checkout@v4 @@ -45,8 +53,10 @@ jobs: if: ${{ inputs.ec-api-key != '' }} run: | ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) + ec_url=$(jq -r '.inputs["ec-url"]' $GITHUB_EVENT_PATH) echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV + echo "TF_VAR_ec_url=$ec_url" >> $GITHUB_ENV - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v2 diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index f79ad50de1..d44c1f7617 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -15,6 +15,14 @@ on: type: boolean required: true default: false + ec-url: + required: true + default: "https://cloud.elastic.co/" + type: choice + description: Select the Environment URL + options: + - https://cloud.elastic.co/ + - https://console.qa.cld.elstc.co/ elk-stack-version: required: true description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT" @@ -30,9 +38,9 @@ on: description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)" type: string run-sanity-tests: - description: "Run sanity tests after provision" - default: false - type: boolean + description: "Run sanity tests after provision" + default: false + type: boolean cleanup-env: description: "Cleanup resources after provision" default: false @@ -52,6 +60,14 @@ on: type: boolean required: true default: false + ec-url: + required: true + default: "https://cloud.elastic.co/" + type: choice + description: Select the Environment URL + options: + - https://cloud.elastic.co/ + - https://console.qa.cld.elstc.co/ elk-stack-version: required: true description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT" @@ -95,6 +111,7 @@ env: AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments" GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments" TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} + TF_VAR_ec_url: ${{ secrets.EC_URL }} jobs: Deploy: @@ -114,8 +131,8 @@ jobs: CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack" # Add "id-token" with the intended permissions. permissions: - contents: 'read' - id-token: 'write' + contents: "read" + id-token: "write" outputs: deploy-s3-bucket: ${{ steps.upload-state.outputs.s3-bucket-folder }} aws-cnvm-stack-name: ${{ steps.upload-state.outputs.aws-cnvm-stack }} @@ -192,7 +209,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: "3.9" - name: Install Poetry run: | diff --git a/.github/workflows/test-gcp-dm.yml b/.github/workflows/test-gcp-dm.yml index 0cb8a9c562..fd19e25575 100644 --- a/.github/workflows/test-gcp-dm.yml +++ b/.github/workflows/test-gcp-dm.yml @@ -15,6 +15,7 @@ on: env: WORKING_DIR: deploy/test-environments TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} + TF_VAR_ec_url: ${{ secrets.EC_URL }} TF_VAR_ess_region: gcp-us-west2 # default region for testing deployments jobs: diff --git a/.github/workflows/upgrade-environment.yml b/.github/workflows/upgrade-environment.yml index d308987b0a..5abaf2cd99 100644 --- a/.github/workflows/upgrade-environment.yml +++ b/.github/workflows/upgrade-environment.yml @@ -32,6 +32,7 @@ env: TF_VAR_stack_version: ${{ inputs.target-elk-stack-version }} TF_VAR_ess_region: gcp-us-west2 TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} + TF_VAR_ec_url: ${{ secrets.EC_URL }} DOCKER_IMAGE: ${{ inputs.docker-image-override }} jobs: @@ -73,8 +74,8 @@ jobs: needs: init # Required for the 'Deploy' job in the 'test-environment.yml' to authenticate with Google Cloud (gcloud). permissions: - contents: 'read' - id-token: 'write' + contents: "read" + id-token: "write" with: deployment_name: ${{ inputs.deployment_name }} elk-stack-version: ${{ needs.init.outputs.base-stack-version }} @@ -90,8 +91,8 @@ jobs: run: working-directory: ${{ env.WORKING_DIR }} permissions: - contents: 'read' - id-token: 'write' + contents: "read" + id-token: "write" steps: - name: Check out the repo uses: actions/checkout@v4 @@ -103,7 +104,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: "3.9" - name: Install Poetry run: | @@ -193,7 +194,7 @@ jobs: - name: Set Docker Image version if: ${{ ! inputs.docker-image-override }} env: - VERSION: 'docker.elastic.co/beats/elastic-agent:${{ inputs.target-elk-stack-version }}' + VERSION: "docker.elastic.co/beats/elastic-agent:${{ inputs.target-elk-stack-version }}" run: | echo "DOCKER_IMAGE=${{ env.VERSION }}" >> $GITHUB_ENV diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 74e8ab7c21..f660f69dbf 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -4,14 +4,14 @@ on: workflow_dispatch: inputs: environment: - description: 'Environment to deploy' + description: "Environment to deploy" type: choice options: - weekly environment logLevel: - description: 'Log level' + description: "Log level" required: true - default: 'INFO' + default: "INFO" type: choice options: - TRACE @@ -24,6 +24,7 @@ env: WORKING_DIR: deploy/weekly-environment SCRIPTS_DIR: deploy/weekly-environment/scripts/benchmarks/kspm_vanilla TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} + TF_VAR_ec_url: ${{ secrets.EC_URL }} TF_VAR_environment: ${{ github.event.inputs.logLevel }} TF_LOG: ${{ github.event.inputs.logLevel }} TF_VAR_stack_version: 8.7.0-SNAPSHOT diff --git a/deploy/cloud/main.tf b/deploy/cloud/main.tf index f40dead25d..b6c82529aa 100644 --- a/deploy/cloud/main.tf +++ b/deploy/cloud/main.tf @@ -1,5 +1,6 @@ provider "ec" { apikey = var.ec_api_key + url = var.ec_url } module "ec_deployment" { diff --git a/deploy/cloud/modules/ec/variables.tf b/deploy/cloud/modules/ec/variables.tf index ca702462c2..1a26a11475 100644 --- a/deploy/cloud/modules/ec/variables.tf +++ b/deploy/cloud/modules/ec/variables.tf @@ -2,6 +2,10 @@ variable "ec_api_key" { type = string } +variable "ec_url" { + type = string +} + variable "stack_version" { description = "Optional version of the Elastic Cloud deployment" type = string diff --git a/deploy/cloud/variables.tf b/deploy/cloud/variables.tf index 96b6323335..83259a566d 100644 --- a/deploy/cloud/variables.tf +++ b/deploy/cloud/variables.tf @@ -5,6 +5,11 @@ variable "ec_api_key" { type = string } +variable "ec_url" { + description = "Elastic cloud Environment URL" + type = string +} + variable "ess_region" { default = "gcp-us-central1" description = "Optional ESS region where the deployment will be created. Defaults to gcp-us-west2" diff --git a/deploy/test-environments/README.md b/deploy/test-environments/README.md index 43b1ddda1f..b00fcaae70 100644 --- a/deploy/test-environments/README.md +++ b/deploy/test-environments/README.md @@ -3,7 +3,6 @@ **Motivation** To provide an easy and deterministic way to set up the latest cloud environment, ensuring proper monitoring and usability - **Prerequisite** This project utilizes AWS and Elastic Cloud accounts. To ensure proper deployment and usage, it is essential to obtain appropriate licenses in compliance with the licensing terms and conditions provided by the respective service providers. @@ -20,17 +19,16 @@ To generate an Elastic Cloud token, you have two options: Choose the method that is most convenient for you to obtain the Elastic Cloud token required for deployment. - Ensure that the following AWS credentials are defined: - `AWS_ACCESS_KEY_ID`: Your AWS access key ID. - `AWS_SECRET_ACCESS_KEY`: Your AWS secret access key. - To successfully deploy the environment, ensure that the following variables are provided as deployment parameters or exported as environment variables: ```bash export TF_VAR_ec_api_key={TOKEN} # <-- should be replaced by Elastic Cloud TOKEN +export TF_VAR_ec_url=https://cloud.elastic.co # <-- should be replaced by Elastic Cloud TOKEN export TF_VAR_stack_version=8.7.2-SNAPSHOT export TF_VAR_ess_region=gcp-us-west2 ``` @@ -49,22 +47,20 @@ Please note that the customized image is currently available in the following re **Module variables (CSPM / KSPM)** -| Variable | Default Value | Comment | -|:-------------:|:-------------:|:------------| -| region | eu-west-1 | AWS EC2 deployment region | - - +| Variable | Default Value | Comment | +| :------: | :-----------: | :------------------------ | +| region | eu-west-1 | AWS EC2 deployment region | ### Elastic Cloud **ec_deployment** - This module facilitates the deployment of Elastic Cloud instance. -| Variable | Default Value | Comment | -|:-------------:|:-------------:|:------------| -| ec_api_key | None | The API key for Elastic Cloud can also be defined using the `TF_VAR_ec_api_key` environment variable | -| ess_region | gcp-us-west2 | The ESS deployment region can also be defined using the `TF_VAR_stack_version` environment variable| -| stack_version | latest | The ELK stack version can also be defined using the `TF_VAR_stack_version` environment variable | -| pin_version | None | Optional: The ELK pin version (docker tag override) can also be defined using the `TF_VAR_pin_version` environment variable | +| Variable | Default Value | Comment | +| :-----------: | :-----------: | :-------------------------------------------------------------------------------------------------------------------------- | +| ec_api_key | None | The API key for Elastic Cloud can also be defined using the `TF_VAR_ec_api_key` environment variable | +| ess_region | gcp-us-west2 | The ESS deployment region can also be defined using the `TF_VAR_stack_version` environment variable | +| stack_version | latest | The ELK stack version can also be defined using the `TF_VAR_stack_version` environment variable | +| pin_version | None | Optional: The ELK pin version (docker tag override) can also be defined using the `TF_VAR_pin_version` environment variable | ## Execution @@ -111,7 +107,6 @@ BC version terraform apply --auto-approve -var="stack_version=8.12.0" -var="pin_version=8.12.0-9f05a310" -target "module.ec_deployment" ``` - - EKS Deployment ```bash @@ -122,17 +117,16 @@ terraform apply --auto-approve -target "module.eks" To destroy local environment use -``` bash +```bash terraform destroy -var="region=eu-west-1" ``` - To destroy the environment provisioned using the Sanity job, follow these steps: 1. [Download](https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra?region=eu-west-3&tab=objects) the Terraform state file to the [test-environments](../test-environments/) folder. 2. Rename the state file, for example, `terraform-sanity.tfstate`. 3. Run the following command: -``` bash +```bash terraform destroy -var="region=eu-west-1" -state terraform-sanity.tfstate -``` \ No newline at end of file +``` diff --git a/deploy/test-environments/delete_env.sh b/deploy/test-environments/delete_env.sh index 0943f94258..a6c1a73804 100755 --- a/deploy/test-environments/delete_env.sh +++ b/deploy/test-environments/delete_env.sh @@ -88,6 +88,7 @@ done # Ensure required environment variables and parameters are set : "${ENV_PREFIX:?$(echo "Missing -p|--prefix. Please provide an environment prefix to delete" && usage && exit 1)}" : "${TF_VAR_ec_api_key:?Please set TF_VAR_ec_api_key with an Elastic Cloud API Key}" +: "${TF_VAR_ec_url:?Please set TF_VAR_url with an Elastic Cloud Environment URL}" BUCKET=s3://tf-state-bucket-test-infra ALL_ENVS=$(aws s3 ls $BUCKET/"$ENV_PREFIX" | awk '{print $2}' | sed 's/\///g') diff --git a/deploy/test-environments/main.tf b/deploy/test-environments/main.tf index 96c005dd94..25b002201d 100644 --- a/deploy/test-environments/main.tf +++ b/deploy/test-environments/main.tf @@ -10,7 +10,7 @@ locals { project = "${var.project}" owner = "${var.owner}" } - ec_url = "https://cloud.elastic.co" + ec_url = "${var.ec_url}" ec_headers = { Content-type = "application/json" Authorization = "ApiKey ${var.ec_api_key}" @@ -44,11 +44,12 @@ resource "random_string" "suffix" { provider "ec" { apikey = var.ec_api_key + url = var.ec_url } provider "restapi" { alias = "ec" - uri = local.ec_url + uri = var.ec_url write_returns_object = true headers = local.ec_headers } @@ -88,7 +89,7 @@ module "ec_project" { count = var.serverless_mode ? 1 : 0 source = "../cloud/modules/serverless" ec_apikey = var.ec_api_key - ec_url = local.ec_url + ec_url = var.ec_url project_name = "${var.deployment_name}-${random_string.suffix.result}" region_id = "aws-us-east-1" # TODO: replace with var.ess_region when more regions are supported } diff --git a/deploy/test-environments/variables.tf b/deploy/test-environments/variables.tf index 43e8a199e8..6d087ad8d4 100644 --- a/deploy/test-environments/variables.tf +++ b/deploy/test-environments/variables.tf @@ -18,6 +18,12 @@ variable "ami_map" { # Elastic Cloud variables # =========================================== +variable "ec_url" { + default = "https://cloud.elastic.co" + description = "Optional Elastic Cloud Environment URL, use export TF_VAR_ec_url={URL}, it defaults to https://cloud.elastic.co" + type = string +} + variable "ec_api_key" { description = "Provide Elastic Cloud API key or use export TF_VAR_ec_api_key={TOKEN}" type = string diff --git a/deploy/weekly-environment/main.tf b/deploy/weekly-environment/main.tf index 8493db9dcf..8677c8a2e1 100644 --- a/deploy/weekly-environment/main.tf +++ b/deploy/weekly-environment/main.tf @@ -1,6 +1,7 @@ provider "ec" { apikey = var.ec_api_key endpoint = var.endpoint + url = var.ec_url } module "ec_deployment" { diff --git a/dev-docs/Cloud-Env-Testing.md b/dev-docs/Cloud-Env-Testing.md index 12572ddd83..b7c7c1ad5c 100644 --- a/dev-docs/Cloud-Env-Testing.md +++ b/dev-docs/Cloud-Env-Testing.md @@ -18,42 +18,44 @@ Follow these steps to run the workflow: 3. Complete the required parameters: - - **`deployment_name`**: Name your environment (Allowed characters: a-zA-Z0-9 and `-`). For - instance: `john-8-7-2-June01`. + - **`deployment_name`**: Name your environment (Allowed characters: a-zA-Z0-9 and `-`). For + instance: `john-8-7-2-June01`. - - **`elk-stack-version`**: Specify the version of Elastic Cloud stack, either a SNAPSHOT or a build candidate (BC) - version. Check the available versions [here](https://artifacts-staging.elastic.co/dra-info/index.html). - For BC, enter version with additions/commit sha, e.g. `8.12.0-61156bc6`. - For SNAPSHOT, enter the full version, e.g. `8.13.0-SNAPSHOT`. + - **`elk-stack-version`**: Specify the version of Elastic Cloud stack, either a SNAPSHOT or a build candidate (BC) + version. Check the available versions [here](https://artifacts-staging.elastic.co/dra-info/index.html). + For BC, enter version with additions/commit sha, e.g. `8.12.0-61156bc6`. + For SNAPSHOT, enter the full version, e.g. `8.13.0-SNAPSHOT`. - - **`ess-region`**: Indicate the Elastic Cloud deployment region. The default value is `gcp-us-west2`, which - supports - snapshot and build candidate (BC) versions. Specify a different region only if necessary. + - **`ess-region`**: Indicate the Elastic Cloud deployment region. The default value is `gcp-us-west2`, which + supports + snapshot and build candidate (BC) versions. Specify a different region only if necessary. ![Required Parameters](https://github.com/oren-zohar/cloudbeat/assets/85433724/6159129e-6d4d-46b1-97a1-f0d3859500fd) 4. Optionally, modify other parameters if required: - - **`docker-image-override`** (**optional**): Use this to replace the default agent Docker image for build candidate (BC) or - SNAPSHOT versions. - Provide the full image path. Leave this field blank for snapshot versions. Follow this format for the image - path: `docker.elastic.co/cloud-release/elastic-agent-cloud:8.8.1-9ac7eb02`. If you're not sure where to get this - image path from, look for message like [this](https://elastic.slack.com/archives/C0JFN9HJL/p1689227472876399) in - #mission-control channel, you can see it specify the stack version and the BC commit sha in the first line, - e.g. `elastic / unified-release - staging # 8.9 - 11 - 8.9.0-c6bb8f7a Success after 4 hr 58 min`. Now just copy it - and replace it the image path: `docker.elastic.co/cloud-release/elastic-agent-cloud:8.9.0-c6bb8f7a`. + - **`docker-image-override`** (**optional**): Use this to replace the default agent Docker image for build candidate (BC) or + SNAPSHOT versions. + Provide the full image path. Leave this field blank for snapshot versions. Follow this format for the image + path: `docker.elastic.co/cloud-release/elastic-agent-cloud:8.8.1-9ac7eb02`. If you're not sure where to get this + image path from, look for message like [this](https://elastic.slack.com/archives/C0JFN9HJL/p1689227472876399) in + #mission-control channel, you can see it specify the stack version and the BC commit sha in the first line, + e.g. `elastic / unified-release - staging # 8.9 - 11 - 8.9.0-c6bb8f7a Success after 4 hr 58 min`. Now just copy it + and replace it the image path: `docker.elastic.co/cloud-release/elastic-agent-cloud:8.9.0-c6bb8f7a`. - - **`run-sanity-tests`** (**optional**): Set to `true` to run sanity tests after the environment is set up. Default: `false` + - **`run-sanity-tests`** (**optional**): Set to `true` to run sanity tests after the environment is set up. Default: `false` - - **`cleanup-env`** (**optional**): Set to `true` if you want the resources to automatically be cleaned up after - provisioning - useful if you don't want to test the env manually after deployment. - Default: `false`. + - **`cleanup-env`** (**optional**): Set to `true` if you want the resources to automatically be cleaned up after + provisioning - useful if you don't want to test the env manually after deployment. + Default: `false`. - - **`ec-api-key`** (**optional**): By default, all the new environments will be created in our EC Cloud Security organization. - If you want to create the environment on your personal org (`@elastic.co`) you can enter - your private [Elastic Cloud](https://cloud.elastic.co/home) API key. Follow the - [Cloud API Keys](https://www.elastic.co/guide/en/cloud/current/ec-api-authentication.html) documentation for - step-by-step instructions on generating the token. + - **`ec-url`** (**optional**): By default, all the new environments will be created in the Production environment (https://cloud.elastic.co/). If you want to create the environment others environment you can enter other Environment URL. + + - **`ec-api-key`** (**optional**): By default, all the new environments will be created in our EC Cloud Security organization. + If you want to create the environment on your personal org (`@elastic.co`) you can enter + your private [Elastic Cloud](https://cloud.elastic.co/home) API key. Follow the + [Cloud API Keys](https://www.elastic.co/guide/en/cloud/current/ec-api-authentication.html) documentation for + step-by-step instructions on generating the token. ![Optional Parameters](https://github.com/oren-zohar/cloudbeat/assets/85433724/17933589-ee0e-4181-a244-f501f54bda6c) @@ -61,7 +63,6 @@ Follow these steps to run the workflow: ![Run Workflow](https://github.com/oren-zohar/cloudbeat/assets/85433724/7b05bf58-cc0b-4ec9-8e49-55d117673df8) - ## Tracking Workflow Execution Monitor the progress of the workflow execution as follows: @@ -121,7 +122,7 @@ Follow these steps to connect to your Amazon Elastic Kubernetes Service (EKS) cl To configure kubectl to communicate with your EKS cluster, replace `` with your EKS cluster's name and run the following command: - ```aws eks update-kubeconfig --region eu-west-1 --name ``` + `aws eks update-kubeconfig --region eu-west-1 --name ` This command updates your ~/.kube/config file with the necessary cluster configuration. @@ -129,11 +130,10 @@ Follow these steps to connect to your Amazon Elastic Kubernetes Service (EKS) cl To verify your connectivity to the EKS cluster, run the following kubectl command: - ```kubectl get po -n kube-system``` + `kubectl get po -n kube-system` This command should list the pods in the kube-system namespace, confirming that you have successfully connected to your EKS cluster. - ## Cleanup Procedure If you wish to automatically delete the environment after the tests finish, set the `cleanup-env` input to `true`. @@ -158,14 +158,14 @@ Follow these steps to run the workflow: 3. Complete the required input fields: - - `prefix` (required): The prefix used to identify the environments to be deleted. + - `prefix` (required): The prefix used to identify the environments to be deleted. Enter Inputs 4. Optionally, modify other input value if required: - - `ignore-prefix` (optional): The prefix used to identify environments that should be excluded from deletion. - - `ec-api-key` (required): Use your own [Elastic Cloud](https://cloud.elastic.co/home) API key if you want to delete environments from your Elastic Cloud account. + - `ignore-prefix` (optional): The prefix used to identify environments that should be excluded from deletion. + - `ec-api-key` (required): Use your own [Elastic Cloud](https://cloud.elastic.co/home) API key if you want to delete environments from your Elastic Cloud account. Optional Inputs From 514da187dcb6db81476478a65b1fceb5a929cc56 Mon Sep 17 00:00:00 2001 From: Paulo Henrique Date: Fri, 19 Jan 2024 14:04:08 -0800 Subject: [PATCH 02/87] fix compatibility --- .github/workflows/test-environment.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index d44c1f7617..2e2ff9aaee 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -63,11 +63,8 @@ on: ec-url: required: true default: "https://cloud.elastic.co/" - type: choice - description: Select the Environment URL - options: - - https://cloud.elastic.co/ - - https://console.qa.cld.elstc.co/ + type: string + description: Environment URL elk-stack-version: required: true description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT" From e10a0a6545efbc5144176e05577fba187812c157 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 24 Jan 2024 08:43:52 -0500 Subject: [PATCH 03/87] create tf var def for input with ec url --- .github/workflows/test-environment.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 2e2ff9aaee..037267544d 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -32,6 +32,9 @@ on: required: true description: "Elastic Cloud deployment region" default: "gcp-us-west2" + options: + - gcp-us-west2 + - aws-eu-west-1 type: string docker-image-override: required: false @@ -126,6 +129,7 @@ jobs: S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra" DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }} CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack" + TF_VAR_EC_URL_OPTION: ${{inputs.ec-url}} # Add "id-token" with the intended permissions. permissions: contents: "read" @@ -152,8 +156,8 @@ jobs: fi # Check pattern required for cloud deployment - if ! [[ $deployment_name =~ ^[a-z][-a-z0-9]*$ ]]; then - echo "error: Deployment name doesn't match the required pattern [a-z][-a-z0-9]*" + if ! [[ $deployment_name =~ (^keep_[a-z][-a-z0-9]*|[a-z][-a-z0-9]*)$ ]]; then + echo "error: Deployment name doesn't match the required pattern QA env keep_[a-z][-a-z0-9]* or regular env [a-z][-a-z0-9]*" exit 1 fi @@ -243,6 +247,7 @@ jobs: terraform apply --auto-approve \ -var="deployment_name=${{ env.DEPLOYMENT_NAME }}" \ -var="region=${{ env.AWS_REGION }}" \ + -var="ec_url"=${{ env.TF_VAR_EC_URL_OPTION }}"\ -var="project=${{ github.actor }}" \ -var="owner=${{ github.actor }}" From 922db97bcabf59e8e692bafa36c0205e88224cf9 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 24 Jan 2024 09:09:44 -0500 Subject: [PATCH 04/87] add ec_url var to ec-deployment module --- deploy/cloud/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/cloud/main.tf b/deploy/cloud/main.tf index b6c82529aa..4ff7638b11 100644 --- a/deploy/cloud/main.tf +++ b/deploy/cloud/main.tf @@ -5,7 +5,8 @@ provider "ec" { module "ec_deployment" { source = "github.com/elastic/apm-server/testing/infra/terraform/modules/ec_deployment" - + + ec_url = var.ec_url region = var.ess_region stack_version = var.stack_version From 0166ccbf813a6bcdbec9e5f056fdf7c019c94b52 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 24 Jan 2024 09:14:44 -0500 Subject: [PATCH 05/87] populate TF_VAR_ec_url env var --- .github/workflows/test-environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 037267544d..3c57b60310 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -111,7 +111,7 @@ env: AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments" GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments" TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} - TF_VAR_ec_url: ${{ secrets.EC_URL }} + TF_VAR_ec_url: ${{ inputs.ec-url }} jobs: Deploy: @@ -365,7 +365,7 @@ jobs: working-directory: deploy/azure run: | if [[ "${{ env.STACK_VERSION }}" == *"8.11"* ]]; then - az deployment sub create --location EastUS --template-file ARM-for-single-account.json --parameters @arm_parameters.json + az deployment sub create --locatPion EastUS --template-file ARM-for-single-account.json --parameters @arm_parameters.json else az group create --location EastUS --name "${{ env.DEPLOYMENT_NAME }}" az deployment group create --resource-group "${{ env.DEPLOYMENT_NAME }}" --template-file ARM-for-single-account.json --parameters @arm_parameters.json From 11afebead2c1c695013460c8a3673caa3ee2d5ae Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 24 Jan 2024 09:49:07 -0500 Subject: [PATCH 06/87] fix typo --- .github/workflows/test-environment.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 3c57b60310..3784f68ebf 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -129,7 +129,6 @@ jobs: S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra" DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }} CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack" - TF_VAR_EC_URL_OPTION: ${{inputs.ec-url}} # Add "id-token" with the intended permissions. permissions: contents: "read" @@ -247,7 +246,7 @@ jobs: terraform apply --auto-approve \ -var="deployment_name=${{ env.DEPLOYMENT_NAME }}" \ -var="region=${{ env.AWS_REGION }}" \ - -var="ec_url"=${{ env.TF_VAR_EC_URL_OPTION }}"\ + -var="ec_url"=${{ env.TF_VAR_ec_url }}"\ -var="project=${{ github.actor }}" \ -var="owner=${{ github.actor }}" @@ -365,7 +364,7 @@ jobs: working-directory: deploy/azure run: | if [[ "${{ env.STACK_VERSION }}" == *"8.11"* ]]; then - az deployment sub create --locatPion EastUS --template-file ARM-for-single-account.json --parameters @arm_parameters.json + az deployment sub create --location EastUS --template-file ARM-for-single-account.json --parameters @arm_parameters.json else az group create --location EastUS --name "${{ env.DEPLOYMENT_NAME }}" az deployment group create --resource-group "${{ env.DEPLOYMENT_NAME }}" --template-file ARM-for-single-account.json --parameters @arm_parameters.json From d45a191f5228f5359d22c391883ccef23e32136a Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Tue, 30 Jan 2024 22:18:44 -0500 Subject: [PATCH 07/87] remove ec_url definitions --- deploy/cloud/main.tf | 2 -- deploy/cloud/modules/ec/variables.tf | 3 --- 2 files changed, 5 deletions(-) diff --git a/deploy/cloud/main.tf b/deploy/cloud/main.tf index 4ff7638b11..08fef742e1 100644 --- a/deploy/cloud/main.tf +++ b/deploy/cloud/main.tf @@ -1,12 +1,10 @@ provider "ec" { apikey = var.ec_api_key - url = var.ec_url } module "ec_deployment" { source = "github.com/elastic/apm-server/testing/infra/terraform/modules/ec_deployment" - ec_url = var.ec_url region = var.ess_region stack_version = var.stack_version diff --git a/deploy/cloud/modules/ec/variables.tf b/deploy/cloud/modules/ec/variables.tf index 1a26a11475..e66520b0c8 100644 --- a/deploy/cloud/modules/ec/variables.tf +++ b/deploy/cloud/modules/ec/variables.tf @@ -2,9 +2,6 @@ variable "ec_api_key" { type = string } -variable "ec_url" { - type = string -} variable "stack_version" { description = "Optional version of the Elastic Cloud deployment" From e8c828131965a64d14e64254a1b3a1d85528cd09 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 8 Feb 2024 08:42:01 -0500 Subject: [PATCH 08/87] fix ec url options --- .github/workflows/test-environment.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 3784f68ebf..1b18d3bedc 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -17,12 +17,12 @@ on: default: false ec-url: required: true - default: "https://cloud.elastic.co/" + default: "https://cloud.elastic.co" type: choice description: Select the Environment URL options: - - https://cloud.elastic.co/ - - https://console.qa.cld.elstc.co/ + - https://cloud.elastic.co + - https://console.qa.cld.elstc.co elk-stack-version: required: true description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT" @@ -65,9 +65,12 @@ on: default: false ec-url: required: true - default: "https://cloud.elastic.co/" - type: string + default: "https://cloud.elastic.co" + type: choice description: Environment URL + options: + - https://cloud.elastic.co + - https://console.qa.cld.elstc.co elk-stack-version: required: true description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT" From 8aa7e71d48c7aa91d4e69fe4b5925e8a24f487ff Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 8 Feb 2024 08:46:36 -0500 Subject: [PATCH 09/87] remove ec url options --- .github/workflows/test-environment.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 1b18d3bedc..8b59fafe5d 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -66,11 +66,8 @@ on: ec-url: required: true default: "https://cloud.elastic.co" - type: choice + type: string description: Environment URL - options: - - https://cloud.elastic.co - - https://console.qa.cld.elstc.co elk-stack-version: required: true description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT" From acba93df8576961efff2faac5ed4d3b659fca424 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Fri, 9 Feb 2024 13:53:14 -0500 Subject: [PATCH 10/87] fix region --- deploy/test-environments/main.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/test-environments/main.tf b/deploy/test-environments/main.tf index 25b002201d..751e92869e 100644 --- a/deploy/test-environments/main.tf +++ b/deploy/test-environments/main.tf @@ -44,7 +44,6 @@ resource "random_string" "suffix" { provider "ec" { apikey = var.ec_api_key - url = var.ec_url } provider "restapi" { @@ -91,7 +90,7 @@ module "ec_project" { ec_apikey = var.ec_api_key ec_url = var.ec_url project_name = "${var.deployment_name}-${random_string.suffix.result}" - region_id = "aws-us-east-1" # TODO: replace with var.ess_region when more regions are supported + region_id = var.ess_region } module "eks" { From a46cf8d03df9bd7a9ed35937442ae36bb6b149e7 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Fri, 9 Feb 2024 14:49:39 -0500 Subject: [PATCH 11/87] fix EOF file error --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 5192df04fd..84c4310838 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -246,7 +246,7 @@ jobs: terraform apply --auto-approve \ -var="deployment_name=${{ env.DEPLOYMENT_NAME }}" \ -var="region=${{ env.AWS_REGION }}" \ - -var="ec_url"=${{ env.TF_VAR_ec_url }}"\ + -var="ec_url=${{ env.TF_VAR_ec_url }}" \ -var="project=${{ github.actor }}" \ -var="owner=${{ github.actor }}" From a6682d696df6f3e26ba3516eb61db2467637ec74 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Mon, 12 Feb 2024 17:53:04 -0500 Subject: [PATCH 12/87] add QA env api key to provisioning process --- .github/workflows/test-environment.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 84c4310838..29ac7b502f 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -111,6 +111,7 @@ env: AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments" GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments" TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} + TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} TF_VAR_ec_url: ${{ inputs.ec-url }} jobs: @@ -161,11 +162,16 @@ jobs: fi - name: Mask Sensitive Data - if: inputs.ec-api-key != '' run: | - ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) - echo "::add-mask::$ec_api_key" - echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV + if [[ inputs.ec-api-key != '' && inputs.ec-url == 'https://cloud.elastic.co']]; then + ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) + echo "::add-mask::$ec_api_key" + echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV + else + ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) + echo "::add-mask::$ec_api_key" + echo "TF_VAR_qa_ec_api_key=$ec_api_key" >> $GITHUB_ENV + fi - name: Process Stack Version id: remove-commit-hash From a9095b70ca275da06b12305f8376fa86fd203668 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Mon, 12 Feb 2024 18:07:37 -0500 Subject: [PATCH 13/87] fix bash script syntax --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 29ac7b502f..5b851f433c 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -163,7 +163,7 @@ jobs: - name: Mask Sensitive Data run: | - if [[ inputs.ec-api-key != '' && inputs.ec-url == 'https://cloud.elastic.co']]; then + if [[ "${inputs.ec-api-key}" != '' && "${inputs.ec-url}" == 'https://cloud.elastic.co']]; then ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV From 192f7c879c18ac3cb86b3afe09f9f920a69cf8f4 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Mon, 12 Feb 2024 18:21:49 -0500 Subject: [PATCH 14/87] fix condition spacing --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 5b851f433c..e23443eee7 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -163,7 +163,7 @@ jobs: - name: Mask Sensitive Data run: | - if [[ "${inputs.ec-api-key}" != '' && "${inputs.ec-url}" == 'https://cloud.elastic.co']]; then + if [[ "${inputs.ec-api-key}" != '' && "${inputs.ec-url}" == 'https://cloud.elastic.co' ]]; then ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV From b03f13513779766098756075293dfafaa5034cd7 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Tue, 13 Feb 2024 09:44:47 -0500 Subject: [PATCH 15/87] fix bash script condition --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index e23443eee7..8139ce82f2 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -163,7 +163,7 @@ jobs: - name: Mask Sensitive Data run: | - if [[ "${inputs.ec-api-key}" != '' && "${inputs.ec-url}" == 'https://cloud.elastic.co' ]]; then + if [ $inputs.ec-api-key != '' && $inputs.ec-url == 'https://cloud.elastic.co' ]; then ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV From f75f6598845582f761e6028907810146a5784358 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Tue, 13 Feb 2024 15:00:20 -0500 Subject: [PATCH 16/87] use secrets for TF_VAR_ec_api_key --- .github/workflows/test-environment.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 8139ce82f2..0ae4f426f7 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -111,7 +111,6 @@ env: AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments" GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments" TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} - TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} TF_VAR_ec_url: ${{ inputs.ec-url }} jobs: @@ -163,14 +162,16 @@ jobs: - name: Mask Sensitive Data run: | - if [ $inputs.ec-api-key != '' && $inputs.ec-url == 'https://cloud.elastic.co' ]; then + ec_api_key='https://cloud.elastic.co' + qa_ec_api_key='https://console.qa.cld.elstc.co' + if [ $inputs.ec-api-key != '' && $inputs.ec-url == ec_api_key ]; then ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV + elif [ $inputs.ec-api-key != '' && $inputs.ec-url == qa_ec_api_key ]; + echo "TF_VAR_ec_api_key= ${{ secrets.QA_EC_API_KEY }}" >> $GITHUB_ENV else - ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) - echo "::add-mask::$ec_api_key" - echo "TF_VAR_qa_ec_api_key=$ec_api_key" >> $GITHUB_ENV + echo "error: Elastic Cloud API key is required from a valid ec url" fi - name: Process Stack Version From cbd89dab93a9ff9249e7c601b0578342252b0f08 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Tue, 13 Feb 2024 15:08:37 -0500 Subject: [PATCH 17/87] fix syntax --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 0ae4f426f7..fe7d85c28c 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -168,7 +168,7 @@ jobs: ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [ $inputs.ec-api-key != '' && $inputs.ec-url == qa_ec_api_key ]; + elif [ $inputs.ec-api-key != '' && $inputs.ec-url == qa_ec_api_key ]; then echo "TF_VAR_ec_api_key= ${{ secrets.QA_EC_API_KEY }}" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" From 228aacaeb1b03662a1c15eee96997bb6f7066b10 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Tue, 13 Feb 2024 16:00:02 -0500 Subject: [PATCH 18/87] update condition --- .github/workflows/test-environment.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index fe7d85c28c..8be76fafa1 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -162,14 +162,16 @@ jobs: - name: Mask Sensitive Data run: | - ec_api_key='https://cloud.elastic.co' - qa_ec_api_key='https://console.qa.cld.elstc.co' - if [ $inputs.ec-api-key != '' && $inputs.ec-url == ec_api_key ]; then + ec_url='https://cloud.elastic.co' + qa_ec_url='https://console.qa.cld.elstc.co' + qa_ec_api_key= "${{ secrets.EC_API_KEY }}" + + if [ $inputs.ec-api-key != '' && $inputs.ec-url == ec_url ]; then ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [ $inputs.ec-api-key != '' && $inputs.ec-url == qa_ec_api_key ]; then - echo "TF_VAR_ec_api_key= ${{ secrets.QA_EC_API_KEY }}" >> $GITHUB_ENV + elif [ $inputs.ec-url == qa_ec_url ]; then + echo "TF_VAR_ec_api_key= $qa_ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" fi From d681aad5bc79836d1e487be5d536ca87636fd658 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Tue, 13 Feb 2024 16:25:21 -0500 Subject: [PATCH 19/87] store secret in TF_VAR_ec_api_key --- .github/workflows/test-environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 8be76fafa1..d181249a58 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -111,6 +111,7 @@ env: AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments" GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments" TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} + TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} TF_VAR_ec_url: ${{ inputs.ec-url }} jobs: @@ -164,14 +165,13 @@ jobs: run: | ec_url='https://cloud.elastic.co' qa_ec_url='https://console.qa.cld.elstc.co' - qa_ec_api_key= "${{ secrets.EC_API_KEY }}" if [ $inputs.ec-api-key != '' && $inputs.ec-url == ec_url ]; then ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV elif [ $inputs.ec-url == qa_ec_url ]; then - echo "TF_VAR_ec_api_key= $qa_ec_api_key" >> $GITHUB_ENV + echo "TF_VAR_ec_api_key= $TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" fi From f32657e38b230cca67921abccf94d2b1c8457955 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 14 Feb 2024 09:47:17 -0500 Subject: [PATCH 20/87] fix syntax for if condition --- .github/workflows/test-environment.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index d181249a58..c70ddba3b8 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -163,17 +163,20 @@ jobs: - name: Mask Sensitive Data run: | + input_ec_url = ${{ inputs.ec-url }} ec_url='https://cloud.elastic.co' qa_ec_url='https://console.qa.cld.elstc.co' + input_ec_api_key=${{ inputs.ec-api-key }} - if [ $inputs.ec-api-key != '' && $inputs.ec-url == ec_url ]; then + if [[ $input_ec_api_key != '' && $inputs_ec_url == $ec_url ]]; then ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [ $inputs.ec-url == qa_ec_url ]; then + elif [ $inputs_ec_url == qa_ec_url ]; then echo "TF_VAR_ec_api_key= $TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" + exit 1 fi - name: Process Stack Version From 15b852155737be50036c5b77ccb7c246c28ec462 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 14 Feb 2024 14:12:55 -0500 Subject: [PATCH 21/87] fix syntax --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index c70ddba3b8..526903e67e 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -168,7 +168,7 @@ jobs: qa_ec_url='https://console.qa.cld.elstc.co' input_ec_api_key=${{ inputs.ec-api-key }} - if [[ $input_ec_api_key != '' && $inputs_ec_url == $ec_url ]]; then + if [[ $input_ec_api_key != '' && $input_ec_url == $ec_url ]]; then ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV From dda0bcb641378ba01c0b0280bafc30ea7cd68205 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 14 Feb 2024 14:19:36 -0500 Subject: [PATCH 22/87] fix bash script condition --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 526903e67e..9c2c48e030 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -172,7 +172,7 @@ jobs: ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [ $inputs_ec_url == qa_ec_url ]; then + elif [ $input_ec_url == qa_ec_url ]; then echo "TF_VAR_ec_api_key= $TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" From a330eb7ed29d79c7f2242ba64ee4dda3009de331 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 14 Feb 2024 14:28:29 -0500 Subject: [PATCH 23/87] fix variable values --- .github/workflows/test-environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 9c2c48e030..94790507b2 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -163,10 +163,10 @@ jobs: - name: Mask Sensitive Data run: | - input_ec_url = ${{ inputs.ec-url }} + input_ec_url="${{ inputs.ec-url }}" ec_url='https://cloud.elastic.co' qa_ec_url='https://console.qa.cld.elstc.co' - input_ec_api_key=${{ inputs.ec-api-key }} + input_ec_api_key="${{ inputs.ec-api-key }}" if [[ $input_ec_api_key != '' && $input_ec_url == $ec_url ]]; then ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) From 76cfb5bfaba63cee144f4c9cec581d45d216fea8 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 14 Feb 2024 14:34:09 -0500 Subject: [PATCH 24/87] fix variable syntax --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 94790507b2..c288539fd9 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -172,7 +172,7 @@ jobs: ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [ $input_ec_url == qa_ec_url ]; then + elif [ $input_ec_url == $qa_ec_url ]; then echo "TF_VAR_ec_api_key= $TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" From adf9e0fd57070aad03b8939b7be979a4d7575bd8 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 14 Feb 2024 16:06:36 -0500 Subject: [PATCH 25/87] fix ec_api_key in logs --- .github/workflows/test-environment.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index c288539fd9..736ae79c92 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -163,17 +163,24 @@ jobs: - name: Mask Sensitive Data run: | + # Extract the EC API key and selected EC URL input_ec_url="${{ inputs.ec-url }}" + ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) + + # Production elastic cloud url ec_url='https://cloud.elastic.co' + + # QA elastic cloud url qa_ec_url='https://console.qa.cld.elstc.co' - input_ec_api_key="${{ inputs.ec-api-key }}" - if [[ $input_ec_api_key != '' && $input_ec_url == $ec_url ]]; then - ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) + # Handles setting API KEY for each deployment mode for elastic cloud environment + if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then; echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [ $input_ec_url == $qa_ec_url ]; then - echo "TF_VAR_ec_api_key= $TF_VAR_qa_ec_api_key" >> $GITHUB_ENV + elif [[ $ec_api_key == '' && $input_ec_url == $ec_url ]]; then + echo "TF_VAR_ec_api_key" >> $GITHUB_ENV + elif [[ $input_ec_url == $qa_ec_url ]]; then + echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" exit 1 From 16c45ec35a05b4926c784a63bf307bd830413751 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 14 Feb 2024 16:11:07 -0500 Subject: [PATCH 26/87] fix yaml file --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 736ae79c92..da93c2919d 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -173,7 +173,7 @@ jobs: # QA elastic cloud url qa_ec_url='https://console.qa.cld.elstc.co' - # Handles setting API KEY for each deployment mode for elastic cloud environment + # Handles setting API KEY for each deployment mode for elastic cloud environment if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then; echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV From e98f15e7da6003a8edcf2ececce3e35833e58ee6 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 14 Feb 2024 16:30:03 -0500 Subject: [PATCH 27/87] add semi colon --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index da93c2919d..efe7665ebd 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -177,7 +177,7 @@ jobs: if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then; echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [[ $ec_api_key == '' && $input_ec_url == $ec_url ]]; then + elif [[ $ec_api_key == '' && $input_ec_url == $ec_url ]]; then; echo "TF_VAR_ec_api_key" >> $GITHUB_ENV elif [[ $input_ec_url == $qa_ec_url ]]; then echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV From 5e19766123cdc268f111cd04044d38f6463afd7e Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 14 Feb 2024 16:32:08 -0500 Subject: [PATCH 28/87] add semi colon --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index efe7665ebd..da64da1424 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -179,7 +179,7 @@ jobs: echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV elif [[ $ec_api_key == '' && $input_ec_url == $ec_url ]]; then; echo "TF_VAR_ec_api_key" >> $GITHUB_ENV - elif [[ $input_ec_url == $qa_ec_url ]]; then + elif [[ $input_ec_url == $qa_ec_url ]]; then; echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" From 317d3addf6ce82b4ce8c44e78473bc75a25dda7f Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 14 Feb 2024 16:36:26 -0500 Subject: [PATCH 29/87] remove semi from then --- .github/workflows/test-environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index da64da1424..9405a5a4fb 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -174,12 +174,12 @@ jobs: qa_ec_url='https://console.qa.cld.elstc.co' # Handles setting API KEY for each deployment mode for elastic cloud environment - if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then; + if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [[ $ec_api_key == '' && $input_ec_url == $ec_url ]]; then; + elif [[ $ec_api_key == '' && $input_ec_url == $ec_url ]]; then echo "TF_VAR_ec_api_key" >> $GITHUB_ENV - elif [[ $input_ec_url == $qa_ec_url ]]; then; + elif [[ $input_ec_url == $qa_ec_url ]]; then echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" From 1b2652b8fbb71d2f459974ace7064aafb276c690 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 15 Feb 2024 09:41:43 -0500 Subject: [PATCH 30/87] remove TF_VAR_EC_URL environment --- .github/workflows/test-environment.yml | 1 - .github/workflows/test-gcp-dm.yml | 1 - .github/workflows/upgrade-environment.yml | 1 - .github/workflows/weekly-enviroment.yml | 1 - deploy/test-environments/README.md | 1 - deploy/test-environments/delete_env.sh | 1 - 6 files changed, 6 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 9405a5a4fb..00c70784e6 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -112,7 +112,6 @@ env: GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments" TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} - TF_VAR_ec_url: ${{ inputs.ec-url }} jobs: Deploy: diff --git a/.github/workflows/test-gcp-dm.yml b/.github/workflows/test-gcp-dm.yml index 4a211c23c7..b6305e44be 100644 --- a/.github/workflows/test-gcp-dm.yml +++ b/.github/workflows/test-gcp-dm.yml @@ -15,7 +15,6 @@ on: env: WORKING_DIR: deploy/test-environments TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} - TF_VAR_ec_url: ${{ secrets.EC_URL }} TF_VAR_ess_region: gcp-us-west2 # default region for testing deployments jobs: diff --git a/.github/workflows/upgrade-environment.yml b/.github/workflows/upgrade-environment.yml index 64c3d9c0c8..b9049490cc 100644 --- a/.github/workflows/upgrade-environment.yml +++ b/.github/workflows/upgrade-environment.yml @@ -32,7 +32,6 @@ env: TF_VAR_stack_version: ${{ inputs.target-elk-stack-version }} TF_VAR_ess_region: gcp-us-west2 TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} - TF_VAR_ec_url: ${{ secrets.EC_URL }} DOCKER_IMAGE: ${{ inputs.docker-image-override }} jobs: diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 5959d5234b..834fbcdbc4 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -24,7 +24,6 @@ env: WORKING_DIR: deploy/weekly-environment SCRIPTS_DIR: deploy/weekly-environment/scripts/benchmarks/kspm_vanilla TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} - TF_VAR_ec_url: ${{ secrets.EC_URL }} TF_VAR_environment: ${{ github.event.inputs.logLevel }} TF_LOG: ${{ github.event.inputs.logLevel }} TF_VAR_stack_version: 8.7.0-SNAPSHOT diff --git a/deploy/test-environments/README.md b/deploy/test-environments/README.md index b00fcaae70..1169370427 100644 --- a/deploy/test-environments/README.md +++ b/deploy/test-environments/README.md @@ -28,7 +28,6 @@ To successfully deploy the environment, ensure that the following variables are ```bash export TF_VAR_ec_api_key={TOKEN} # <-- should be replaced by Elastic Cloud TOKEN -export TF_VAR_ec_url=https://cloud.elastic.co # <-- should be replaced by Elastic Cloud TOKEN export TF_VAR_stack_version=8.7.2-SNAPSHOT export TF_VAR_ess_region=gcp-us-west2 ``` diff --git a/deploy/test-environments/delete_env.sh b/deploy/test-environments/delete_env.sh index a6c1a73804..0943f94258 100755 --- a/deploy/test-environments/delete_env.sh +++ b/deploy/test-environments/delete_env.sh @@ -88,7 +88,6 @@ done # Ensure required environment variables and parameters are set : "${ENV_PREFIX:?$(echo "Missing -p|--prefix. Please provide an environment prefix to delete" && usage && exit 1)}" : "${TF_VAR_ec_api_key:?Please set TF_VAR_ec_api_key with an Elastic Cloud API Key}" -: "${TF_VAR_ec_url:?Please set TF_VAR_url with an Elastic Cloud Environment URL}" BUCKET=s3://tf-state-bucket-test-infra ALL_ENVS=$(aws s3 ls $BUCKET/"$ENV_PREFIX" | awk '{print $2}' | sed 's/\///g') From 16a478dad8906a10401a37d0cc701e3335801139 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 15 Feb 2024 15:17:30 -0500 Subject: [PATCH 31/87] update script to work with production env --- .github/workflows/test-environment.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 00c70784e6..3f20e50e27 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -113,6 +113,9 @@ env: TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} +variables: + qa_ec_url: "https://console.qa.cld.elstc.co" + jobs: Deploy: runs-on: ubuntu-20.04 @@ -161,6 +164,7 @@ jobs: fi - name: Mask Sensitive Data + if: inputs.ec-api-key != '' || variables.qa_ec_url == ${{ inputs.ec-url }} run: | # Extract the EC API key and selected EC URL input_ec_url="${{ inputs.ec-url }}" @@ -170,14 +174,12 @@ jobs: ec_url='https://cloud.elastic.co' # QA elastic cloud url - qa_ec_url='https://console.qa.cld.elstc.co' + qa_ec_url="${{ variables.qa_ec_url}}" # Handles setting API KEY for each deployment mode for elastic cloud environment if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [[ $ec_api_key == '' && $input_ec_url == $ec_url ]]; then - echo "TF_VAR_ec_api_key" >> $GITHUB_ENV elif [[ $input_ec_url == $qa_ec_url ]]; then echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else From 64a1f6d5e5054e62f45830dcdf7c8f6a92358f91 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 15 Feb 2024 15:48:55 -0500 Subject: [PATCH 32/87] update script with env variables --- .github/workflows/test-environment.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 3f20e50e27..687efa1cbe 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -113,8 +113,6 @@ env: TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} -variables: - qa_ec_url: "https://console.qa.cld.elstc.co" jobs: Deploy: @@ -132,6 +130,8 @@ jobs: S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra" DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }} CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack" + QA_EC_URL: "https://console.qa.cld.elstc.co" + PROD_EC_URL: "https://cloud.elastic.co" # Add "id-token" with the intended permissions. permissions: contents: "read" @@ -164,17 +164,17 @@ jobs: fi - name: Mask Sensitive Data - if: inputs.ec-api-key != '' || variables.qa_ec_url == ${{ inputs.ec-url }} + if: inputs.ec-api-key != '' || inputs.ec_url == ${{ env.QA_EC_URL }} run: | # Extract the EC API key and selected EC URL input_ec_url="${{ inputs.ec-url }}" ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) # Production elastic cloud url - ec_url='https://cloud.elastic.co' + ec_url="${{ env.PROD_EC_URL }}" # QA elastic cloud url - qa_ec_url="${{ variables.qa_ec_url}}" + qa_ec_url="${{ env.QA_EC_URL }}" # Handles setting API KEY for each deployment mode for elastic cloud environment if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then From 6efde085cb0e7226a11fe00737a9bde0418a4e98 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 15 Feb 2024 16:49:52 -0500 Subject: [PATCH 33/87] remove ec url setting --- .github/workflows/test-environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 687efa1cbe..89fcb418d6 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -266,7 +266,6 @@ jobs: terraform apply --auto-approve \ -var="deployment_name=${{ env.DEPLOYMENT_NAME }}" \ -var="region=${{ env.AWS_REGION }}" \ - -var="ec_url=${{ env.TF_VAR_ec_url }}" \ -var="project=${{ github.actor }}" \ -var="owner=${{ github.actor }}" From ad9ed6fe169de0393569f61ea60b155e946e5e22 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 15 Feb 2024 17:17:57 -0500 Subject: [PATCH 34/87] add back ec url with input --- .github/workflows/test-environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 89fcb418d6..3344e6e68d 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -266,6 +266,7 @@ jobs: terraform apply --auto-approve \ -var="deployment_name=${{ env.DEPLOYMENT_NAME }}" \ -var="region=${{ env.AWS_REGION }}" \ + -var="ec_url=${{ inputs.ec_url }}" \ -var="project=${{ github.actor }}" \ -var="owner=${{ github.actor }}" From d9fa950d8237bac58673a23c999cb41ad68be5ee Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 15 Feb 2024 17:28:52 -0500 Subject: [PATCH 35/87] fix syntax --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 3344e6e68d..dd985022a9 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -266,7 +266,7 @@ jobs: terraform apply --auto-approve \ -var="deployment_name=${{ env.DEPLOYMENT_NAME }}" \ -var="region=${{ env.AWS_REGION }}" \ - -var="ec_url=${{ inputs.ec_url }}" \ + -var="ec_url=${{ inputs.ec-url }}" \ -var="project=${{ github.actor }}" \ -var="owner=${{ github.actor }}" From d74931b407d868f73226d94657af8f5b899ac560 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 15 Feb 2024 19:26:35 -0500 Subject: [PATCH 36/87] fix inputs syntax --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index dd985022a9..9a78be108e 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -164,7 +164,7 @@ jobs: fi - name: Mask Sensitive Data - if: inputs.ec-api-key != '' || inputs.ec_url == ${{ env.QA_EC_URL }} + if: inputs.ec-api-key != '' || inputs.ec-url == ${{ env.QA_EC_URL }} run: | # Extract the EC API key and selected EC URL input_ec_url="${{ inputs.ec-url }}" From f97e6137f98bc9281e3ce5bde19d2ab2d9417dfb Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 15 Feb 2024 20:06:10 -0500 Subject: [PATCH 37/87] set EC CLOUD API KEY --- .github/workflows/test-environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 9a78be108e..584dcfaeff 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -164,7 +164,6 @@ jobs: fi - name: Mask Sensitive Data - if: inputs.ec-api-key != '' || inputs.ec-url == ${{ env.QA_EC_URL }} run: | # Extract the EC API key and selected EC URL input_ec_url="${{ inputs.ec-url }}" @@ -180,6 +179,8 @@ jobs: if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV + elif [[ $input_ec_url == $ec_url ]]; then + echo "TF_VAR_ec_api_key=$TF_VAR_ec_api_key" >> $GITHUB_ENV elif [[ $input_ec_url == $qa_ec_url ]]; then echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else From 6d55b31c09b5f1822a3b83d1f92cba0dd903481f Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Sat, 17 Feb 2024 11:52:32 -0500 Subject: [PATCH 38/87] add ability to destroy environments in qa --- .github/workflows/destroy-environment.yml | 30 ++++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/destroy-environment.yml b/.github/workflows/destroy-environment.yml index 9cd0780132..2285602339 100644 --- a/.github/workflows/destroy-environment.yml +++ b/.github/workflows/destroy-environment.yml @@ -32,6 +32,7 @@ env: ENV_PREFIX: ${{ inputs.prefix }} ENV_IGNORE_PREFIX: ${{ inputs.ignore-prefix }} TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} + TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} jobs: Destroy: @@ -50,13 +51,30 @@ jobs: working-directory: ./ - name: Mask API Key - if: ${{ inputs.ec-api-key != '' }} - run: | + run: + | + # Extract the EC API key and selected EC URL + input_ec_url="${{ inputs.ec-url }}" ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) - ec_url=$(jq -r '.inputs["ec-url"]' $GITHUB_EVENT_PATH) - echo "::add-mask::$ec_api_key" - echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - echo "TF_VAR_ec_url=$ec_url" >> $GITHUB_ENV + + # Production elastic cloud url + ec_url="${{ env.PROD_EC_URL }}" + + # QA elastic cloud url + qa_ec_url="${{ env.QA_EC_URL }}" + + # Handles setting API KEY for each deployment mode for elastic cloud environment + if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then + echo "::add-mask::$ec_api_key" + echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV + elif [[ $input_ec_url == $ec_url ]]; then + echo "TF_VAR_ec_api_key=$TF_VAR_ec_api_key" >> $GITHUB_ENV + elif [[ $input_ec_url == $qa_ec_url ]]; then + echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV + else + echo "error: Elastic Cloud API key igis required from a valid ec url" + exit 1 + fi - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v2 From be2a53bc116361a85d28695093384ab0fe40d749 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Sat, 17 Feb 2024 11:58:08 -0500 Subject: [PATCH 39/87] set env variables for for url --- .github/workflows/destroy-environment.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/destroy-environment.yml b/.github/workflows/destroy-environment.yml index 2285602339..7c196ad0f2 100644 --- a/.github/workflows/destroy-environment.yml +++ b/.github/workflows/destroy-environment.yml @@ -33,6 +33,8 @@ env: ENV_IGNORE_PREFIX: ${{ inputs.ignore-prefix }} TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} + QA_EC_URL: "https://console.qa.cld.elstc.co" + PROD_EC_URL: "https://cloud.elastic.co" jobs: Destroy: @@ -72,7 +74,7 @@ jobs: elif [[ $input_ec_url == $qa_ec_url ]]; then echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else - echo "error: Elastic Cloud API key igis required from a valid ec url" + echo "error: Elastic Cloud API key is required from a valid ec url" exit 1 fi From 3c44959af2208e2354713f6514b3488c11a23d0c Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Sat, 17 Feb 2024 12:09:25 -0500 Subject: [PATCH 40/87] remove the slash --- .github/workflows/destroy-environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/destroy-environment.yml b/.github/workflows/destroy-environment.yml index 7c196ad0f2..72c9f5197c 100644 --- a/.github/workflows/destroy-environment.yml +++ b/.github/workflows/destroy-environment.yml @@ -14,12 +14,12 @@ on: description: "Ignore all environments starting with `ignore-prefix`" ec-url: required: true - default: "https://cloud.elastic.co/" + default: "https://cloud.elastic.co" type: choice description: Select the Environment URL to delete options: - - https://cloud.elastic.co/ - - https://console.qa.cld.elstc.co/ + - https://cloud.elastic.co + - https://console.qa.cld.elstc.co ec-api-key: type: string description: "**Optional** To delete env environments on your own organization, enter your Elastic Cloud API key." From 27d7496df148fe92bfefb5a986d4f5fcd4bc3c59 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 21 Feb 2024 11:24:04 -0500 Subject: [PATCH 41/87] add TF_VAR_ec_url --- .github/workflows/test-environment.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 584dcfaeff..c6caa81dab 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -130,8 +130,7 @@ jobs: S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra" DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }} CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack" - QA_EC_URL: "https://console.qa.cld.elstc.co" - PROD_EC_URL: "https://cloud.elastic.co" + TF_VAR_ec_url: ${{inputs.ec-url}} # Add "id-token" with the intended permissions. permissions: contents: "read" @@ -164,11 +163,15 @@ jobs: fi - name: Mask Sensitive Data + env: + QA_EC_URL: "https://console.qa.cld.elstc.co" + PROD_EC_URL: "https://cloud.elastic.co" run: | # Extract the EC API key and selected EC URL - input_ec_url="${{ inputs.ec-url }}" ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) + $input_ec_url=${{ TF_VAR_ec_url }} + # Production elastic cloud url ec_url="${{ env.PROD_EC_URL }}" From 68aa9596149dc929bbcfac1b7add4bfbfe0c4752 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 21 Feb 2024 11:32:17 -0500 Subject: [PATCH 42/87] fix yaml file --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index c6caa81dab..fa7ed2c62f 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -112,6 +112,7 @@ env: GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments" TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} + TF_VAR_ec_url: ${{ inputs.ec-url }} jobs: @@ -130,7 +131,6 @@ jobs: S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra" DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }} CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack" - TF_VAR_ec_url: ${{inputs.ec-url}} # Add "id-token" with the intended permissions. permissions: contents: "read" From 34118a64c47b806b68a4877b67063af8109d37e1 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 21 Feb 2024 11:37:50 -0500 Subject: [PATCH 43/87] move inputs env down --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index fa7ed2c62f..d0d233b191 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -112,7 +112,6 @@ env: GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments" TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} - TF_VAR_ec_url: ${{ inputs.ec-url }} jobs: @@ -131,6 +130,7 @@ jobs: S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra" DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }} CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack" + TF_VAR_ec_url: ${{ inputs.ec-url }} # Add "id-token" with the intended permissions. permissions: contents: "read" From d5016da0816c507e1f47d7aad7913b8ed8d226ba Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 21 Feb 2024 11:40:08 -0500 Subject: [PATCH 44/87] add env TF_VAR_ec_url --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index d0d233b191..46767ce98f 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -170,7 +170,7 @@ jobs: # Extract the EC API key and selected EC URL ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) - $input_ec_url=${{ TF_VAR_ec_url }} + $input_ec_url=${{ env.TF_VAR_ec_url }} # Production elastic cloud url ec_url="${{ env.PROD_EC_URL }}" From b2b7fbeb9ced18f8cac728768da452e0aa0bb0b5 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 21 Feb 2024 11:46:21 -0500 Subject: [PATCH 45/87] add quotes to input var --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 46767ce98f..6367129a55 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -170,7 +170,7 @@ jobs: # Extract the EC API key and selected EC URL ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) - $input_ec_url=${{ env.TF_VAR_ec_url }} + $input_ec_url="${{ env.TF_VAR_ec_url }}" # Production elastic cloud url ec_url="${{ env.PROD_EC_URL }}" From f326b43ed4b257fa22e9835402dc9c7d00b0cabd Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 21 Feb 2024 11:58:36 -0500 Subject: [PATCH 46/87] move inputs globally --- .github/workflows/test-environment.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 6367129a55..c910d7d823 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -112,6 +112,8 @@ env: GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments" TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} + TF_VAR_ec_url: ${{ inputs.ec-url }} + jobs: @@ -130,7 +132,6 @@ jobs: S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra" DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }} CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack" - TF_VAR_ec_url: ${{ inputs.ec-url }} # Add "id-token" with the intended permissions. permissions: contents: "read" @@ -270,7 +271,7 @@ jobs: terraform apply --auto-approve \ -var="deployment_name=${{ env.DEPLOYMENT_NAME }}" \ -var="region=${{ env.AWS_REGION }}" \ - -var="ec_url=${{ inputs.ec-url }}" \ + -var="ec_url=${{ env.TF_VAR_ec_url }}" \ -var="project=${{ github.actor }}" \ -var="owner=${{ github.actor }}" From 7f3248ef1c3b5b66fa79584007966023138f6e5a Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 21 Feb 2024 12:03:09 -0500 Subject: [PATCH 47/87] fix syntax --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index c910d7d823..490a456b78 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -171,7 +171,7 @@ jobs: # Extract the EC API key and selected EC URL ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) - $input_ec_url="${{ env.TF_VAR_ec_url }}" + input_ec_url="${{ env.TF_VAR_ec_url }}" # Production elastic cloud url ec_url="${{ env.PROD_EC_URL }}" From 4075592de8aabe2c8b7d68fa9ad2c58904f95fdf Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 22 Feb 2024 11:07:09 -0500 Subject: [PATCH 48/87] revert untouched files to main --- .github/workflows/upgrade-environment.yml | 12 ++--- .github/workflows/weekly-enviroment.yml | 6 +-- deploy/cloud/main.tf | 2 +- deploy/test-environments/README.md | 29 ++++++---- dev-docs/Cloud-Env-Testing.md | 66 +++++++++++------------ 5 files changed, 61 insertions(+), 54 deletions(-) diff --git a/.github/workflows/upgrade-environment.yml b/.github/workflows/upgrade-environment.yml index b9049490cc..48bf9277ca 100644 --- a/.github/workflows/upgrade-environment.yml +++ b/.github/workflows/upgrade-environment.yml @@ -73,8 +73,8 @@ jobs: needs: init # Required for the 'Deploy' job in the 'test-environment.yml' to authenticate with Google Cloud (gcloud). permissions: - contents: "read" - id-token: "write" + contents: 'read' + id-token: 'write' with: deployment_name: ${{ inputs.deployment_name }} elk-stack-version: ${{ needs.init.outputs.base-stack-version }} @@ -90,8 +90,8 @@ jobs: run: working-directory: ${{ env.WORKING_DIR }} permissions: - contents: "read" - id-token: "write" + contents: 'read' + id-token: 'write' steps: - name: Check out the repo uses: actions/checkout@v4 @@ -103,7 +103,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: '3.9' - name: Install Poetry run: | @@ -192,7 +192,7 @@ jobs: - name: Set Docker Image version if: ${{ ! inputs.docker-image-override }} env: - VERSION: "docker.elastic.co/beats/elastic-agent:${{ inputs.target-elk-stack-version }}" + VERSION: 'docker.elastic.co/beats/elastic-agent:${{ inputs.target-elk-stack-version }}' run: | echo "DOCKER_IMAGE=${{ env.VERSION }}" >> $GITHUB_ENV diff --git a/.github/workflows/weekly-enviroment.yml b/.github/workflows/weekly-enviroment.yml index 834fbcdbc4..d43869dc01 100644 --- a/.github/workflows/weekly-enviroment.yml +++ b/.github/workflows/weekly-enviroment.yml @@ -4,14 +4,14 @@ on: workflow_dispatch: inputs: environment: - description: "Environment to deploy" + description: 'Environment to deploy' type: choice options: - weekly environment logLevel: - description: "Log level" + description: 'Log level' required: true - default: "INFO" + default: 'INFO' type: choice options: - TRACE diff --git a/deploy/cloud/main.tf b/deploy/cloud/main.tf index 08fef742e1..f40dead25d 100644 --- a/deploy/cloud/main.tf +++ b/deploy/cloud/main.tf @@ -4,7 +4,7 @@ provider "ec" { module "ec_deployment" { source = "github.com/elastic/apm-server/testing/infra/terraform/modules/ec_deployment" - + region = var.ess_region stack_version = var.stack_version diff --git a/deploy/test-environments/README.md b/deploy/test-environments/README.md index 1169370427..e26ecbc987 100644 --- a/deploy/test-environments/README.md +++ b/deploy/test-environments/README.md @@ -3,6 +3,7 @@ **Motivation** To provide an easy and deterministic way to set up the latest cloud environment, ensuring proper monitoring and usability + **Prerequisite** This project utilizes AWS and Elastic Cloud accounts. To ensure proper deployment and usage, it is essential to obtain appropriate licenses in compliance with the licensing terms and conditions provided by the respective service providers. @@ -19,11 +20,13 @@ To generate an Elastic Cloud token, you have two options: Choose the method that is most convenient for you to obtain the Elastic Cloud token required for deployment. + Ensure that the following AWS credentials are defined: - `AWS_ACCESS_KEY_ID`: Your AWS access key ID. - `AWS_SECRET_ACCESS_KEY`: Your AWS secret access key. + To successfully deploy the environment, ensure that the following variables are provided as deployment parameters or exported as environment variables: ```bash @@ -46,20 +49,22 @@ Please note that the customized image is currently available in the following re **Module variables (CSPM / KSPM)** -| Variable | Default Value | Comment | -| :------: | :-----------: | :------------------------ | -| region | eu-west-1 | AWS EC2 deployment region | +| Variable | Default Value | Comment | +|:-------------:|:-------------:|:------------| +| region | eu-west-1 | AWS EC2 deployment region | + + ### Elastic Cloud **ec_deployment** - This module facilitates the deployment of Elastic Cloud instance. -| Variable | Default Value | Comment | -| :-----------: | :-----------: | :-------------------------------------------------------------------------------------------------------------------------- | -| ec_api_key | None | The API key for Elastic Cloud can also be defined using the `TF_VAR_ec_api_key` environment variable | -| ess_region | gcp-us-west2 | The ESS deployment region can also be defined using the `TF_VAR_stack_version` environment variable | -| stack_version | latest | The ELK stack version can also be defined using the `TF_VAR_stack_version` environment variable | -| pin_version | None | Optional: The ELK pin version (docker tag override) can also be defined using the `TF_VAR_pin_version` environment variable | +| Variable | Default Value | Comment | +|:-------------:|:-------------:|:------------| +| ec_api_key | None | The API key for Elastic Cloud can also be defined using the `TF_VAR_ec_api_key` environment variable | +| ess_region | gcp-us-west2 | The ESS deployment region can also be defined using the `TF_VAR_stack_version` environment variable| +| stack_version | latest | The ELK stack version can also be defined using the `TF_VAR_stack_version` environment variable | +| pin_version | None | Optional: The ELK pin version (docker tag override) can also be defined using the `TF_VAR_pin_version` environment variable | ## Execution @@ -106,6 +111,7 @@ BC version terraform apply --auto-approve -var="stack_version=8.12.0" -var="pin_version=8.12.0-9f05a310" -target "module.ec_deployment" ``` + - EKS Deployment ```bash @@ -116,16 +122,17 @@ terraform apply --auto-approve -target "module.eks" To destroy local environment use -```bash +``` bash terraform destroy -var="region=eu-west-1" ``` + To destroy the environment provisioned using the Sanity job, follow these steps: 1. [Download](https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra?region=eu-west-3&tab=objects) the Terraform state file to the [test-environments](../test-environments/) folder. 2. Rename the state file, for example, `terraform-sanity.tfstate`. 3. Run the following command: -```bash +``` bash terraform destroy -var="region=eu-west-1" -state terraform-sanity.tfstate ``` diff --git a/dev-docs/Cloud-Env-Testing.md b/dev-docs/Cloud-Env-Testing.md index b7c7c1ad5c..12572ddd83 100644 --- a/dev-docs/Cloud-Env-Testing.md +++ b/dev-docs/Cloud-Env-Testing.md @@ -18,44 +18,42 @@ Follow these steps to run the workflow: 3. Complete the required parameters: - - **`deployment_name`**: Name your environment (Allowed characters: a-zA-Z0-9 and `-`). For - instance: `john-8-7-2-June01`. + - **`deployment_name`**: Name your environment (Allowed characters: a-zA-Z0-9 and `-`). For + instance: `john-8-7-2-June01`. - - **`elk-stack-version`**: Specify the version of Elastic Cloud stack, either a SNAPSHOT or a build candidate (BC) - version. Check the available versions [here](https://artifacts-staging.elastic.co/dra-info/index.html). - For BC, enter version with additions/commit sha, e.g. `8.12.0-61156bc6`. - For SNAPSHOT, enter the full version, e.g. `8.13.0-SNAPSHOT`. + - **`elk-stack-version`**: Specify the version of Elastic Cloud stack, either a SNAPSHOT or a build candidate (BC) + version. Check the available versions [here](https://artifacts-staging.elastic.co/dra-info/index.html). + For BC, enter version with additions/commit sha, e.g. `8.12.0-61156bc6`. + For SNAPSHOT, enter the full version, e.g. `8.13.0-SNAPSHOT`. - - **`ess-region`**: Indicate the Elastic Cloud deployment region. The default value is `gcp-us-west2`, which - supports - snapshot and build candidate (BC) versions. Specify a different region only if necessary. + - **`ess-region`**: Indicate the Elastic Cloud deployment region. The default value is `gcp-us-west2`, which + supports + snapshot and build candidate (BC) versions. Specify a different region only if necessary. ![Required Parameters](https://github.com/oren-zohar/cloudbeat/assets/85433724/6159129e-6d4d-46b1-97a1-f0d3859500fd) 4. Optionally, modify other parameters if required: - - **`docker-image-override`** (**optional**): Use this to replace the default agent Docker image for build candidate (BC) or - SNAPSHOT versions. - Provide the full image path. Leave this field blank for snapshot versions. Follow this format for the image - path: `docker.elastic.co/cloud-release/elastic-agent-cloud:8.8.1-9ac7eb02`. If you're not sure where to get this - image path from, look for message like [this](https://elastic.slack.com/archives/C0JFN9HJL/p1689227472876399) in - #mission-control channel, you can see it specify the stack version and the BC commit sha in the first line, - e.g. `elastic / unified-release - staging # 8.9 - 11 - 8.9.0-c6bb8f7a Success after 4 hr 58 min`. Now just copy it - and replace it the image path: `docker.elastic.co/cloud-release/elastic-agent-cloud:8.9.0-c6bb8f7a`. + - **`docker-image-override`** (**optional**): Use this to replace the default agent Docker image for build candidate (BC) or + SNAPSHOT versions. + Provide the full image path. Leave this field blank for snapshot versions. Follow this format for the image + path: `docker.elastic.co/cloud-release/elastic-agent-cloud:8.8.1-9ac7eb02`. If you're not sure where to get this + image path from, look for message like [this](https://elastic.slack.com/archives/C0JFN9HJL/p1689227472876399) in + #mission-control channel, you can see it specify the stack version and the BC commit sha in the first line, + e.g. `elastic / unified-release - staging # 8.9 - 11 - 8.9.0-c6bb8f7a Success after 4 hr 58 min`. Now just copy it + and replace it the image path: `docker.elastic.co/cloud-release/elastic-agent-cloud:8.9.0-c6bb8f7a`. - - **`run-sanity-tests`** (**optional**): Set to `true` to run sanity tests after the environment is set up. Default: `false` + - **`run-sanity-tests`** (**optional**): Set to `true` to run sanity tests after the environment is set up. Default: `false` - - **`cleanup-env`** (**optional**): Set to `true` if you want the resources to automatically be cleaned up after - provisioning - useful if you don't want to test the env manually after deployment. - Default: `false`. + - **`cleanup-env`** (**optional**): Set to `true` if you want the resources to automatically be cleaned up after + provisioning - useful if you don't want to test the env manually after deployment. + Default: `false`. - - **`ec-url`** (**optional**): By default, all the new environments will be created in the Production environment (https://cloud.elastic.co/). If you want to create the environment others environment you can enter other Environment URL. - - - **`ec-api-key`** (**optional**): By default, all the new environments will be created in our EC Cloud Security organization. - If you want to create the environment on your personal org (`@elastic.co`) you can enter - your private [Elastic Cloud](https://cloud.elastic.co/home) API key. Follow the - [Cloud API Keys](https://www.elastic.co/guide/en/cloud/current/ec-api-authentication.html) documentation for - step-by-step instructions on generating the token. + - **`ec-api-key`** (**optional**): By default, all the new environments will be created in our EC Cloud Security organization. + If you want to create the environment on your personal org (`@elastic.co`) you can enter + your private [Elastic Cloud](https://cloud.elastic.co/home) API key. Follow the + [Cloud API Keys](https://www.elastic.co/guide/en/cloud/current/ec-api-authentication.html) documentation for + step-by-step instructions on generating the token. ![Optional Parameters](https://github.com/oren-zohar/cloudbeat/assets/85433724/17933589-ee0e-4181-a244-f501f54bda6c) @@ -63,6 +61,7 @@ Follow these steps to run the workflow: ![Run Workflow](https://github.com/oren-zohar/cloudbeat/assets/85433724/7b05bf58-cc0b-4ec9-8e49-55d117673df8) + ## Tracking Workflow Execution Monitor the progress of the workflow execution as follows: @@ -122,7 +121,7 @@ Follow these steps to connect to your Amazon Elastic Kubernetes Service (EKS) cl To configure kubectl to communicate with your EKS cluster, replace `` with your EKS cluster's name and run the following command: - `aws eks update-kubeconfig --region eu-west-1 --name ` + ```aws eks update-kubeconfig --region eu-west-1 --name ``` This command updates your ~/.kube/config file with the necessary cluster configuration. @@ -130,10 +129,11 @@ Follow these steps to connect to your Amazon Elastic Kubernetes Service (EKS) cl To verify your connectivity to the EKS cluster, run the following kubectl command: - `kubectl get po -n kube-system` + ```kubectl get po -n kube-system``` This command should list the pods in the kube-system namespace, confirming that you have successfully connected to your EKS cluster. + ## Cleanup Procedure If you wish to automatically delete the environment after the tests finish, set the `cleanup-env` input to `true`. @@ -158,14 +158,14 @@ Follow these steps to run the workflow: 3. Complete the required input fields: - - `prefix` (required): The prefix used to identify the environments to be deleted. + - `prefix` (required): The prefix used to identify the environments to be deleted. Enter Inputs 4. Optionally, modify other input value if required: - - `ignore-prefix` (optional): The prefix used to identify environments that should be excluded from deletion. - - `ec-api-key` (required): Use your own [Elastic Cloud](https://cloud.elastic.co/home) API key if you want to delete environments from your Elastic Cloud account. + - `ignore-prefix` (optional): The prefix used to identify environments that should be excluded from deletion. + - `ec-api-key` (required): Use your own [Elastic Cloud](https://cloud.elastic.co/home) API key if you want to delete environments from your Elastic Cloud account. Optional Inputs From 8005b3af95e577ac071fa4b3576980491feb1d27 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 22 Feb 2024 11:09:20 -0500 Subject: [PATCH 49/87] revert more untouched files to main --- .github/workflows/destroy-environment.yml | 42 ++++------------------- deploy/cloud/modules/ec/variables.tf | 1 - 2 files changed, 6 insertions(+), 37 deletions(-) diff --git a/.github/workflows/destroy-environment.yml b/.github/workflows/destroy-environment.yml index 72c9f5197c..3c9ec87329 100644 --- a/.github/workflows/destroy-environment.yml +++ b/.github/workflows/destroy-environment.yml @@ -12,14 +12,6 @@ on: ignore-prefix: type: string description: "Ignore all environments starting with `ignore-prefix`" - ec-url: - required: true - default: "https://cloud.elastic.co" - type: choice - description: Select the Environment URL to delete - options: - - https://cloud.elastic.co - - https://console.qa.cld.elstc.co ec-api-key: type: string description: "**Optional** To delete env environments on your own organization, enter your Elastic Cloud API key." @@ -32,9 +24,6 @@ env: ENV_PREFIX: ${{ inputs.prefix }} ENV_IGNORE_PREFIX: ${{ inputs.ignore-prefix }} TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} - TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} - QA_EC_URL: "https://console.qa.cld.elstc.co" - PROD_EC_URL: "https://cloud.elastic.co" jobs: Destroy: @@ -42,8 +31,8 @@ jobs: timeout-minutes: 120 # Add "id-token" with the intended permissions. permissions: - contents: "read" - id-token: "write" + contents: 'read' + id-token: 'write' steps: - name: Check out the repo uses: actions/checkout@v4 @@ -53,30 +42,11 @@ jobs: working-directory: ./ - name: Mask API Key - run: - | - # Extract the EC API key and selected EC URL - input_ec_url="${{ inputs.ec-url }}" + if: ${{ inputs.ec-api-key != '' }} + run: | ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) - - # Production elastic cloud url - ec_url="${{ env.PROD_EC_URL }}" - - # QA elastic cloud url - qa_ec_url="${{ env.QA_EC_URL }}" - - # Handles setting API KEY for each deployment mode for elastic cloud environment - if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then - echo "::add-mask::$ec_api_key" - echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [[ $input_ec_url == $ec_url ]]; then - echo "TF_VAR_ec_api_key=$TF_VAR_ec_api_key" >> $GITHUB_ENV - elif [[ $input_ec_url == $qa_ec_url ]]; then - echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV - else - echo "error: Elastic Cloud API key is required from a valid ec url" - exit 1 - fi + echo "::add-mask::$ec_api_key" + echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v2 diff --git a/deploy/cloud/modules/ec/variables.tf b/deploy/cloud/modules/ec/variables.tf index 5757c2be29..797085e211 100644 --- a/deploy/cloud/modules/ec/variables.tf +++ b/deploy/cloud/modules/ec/variables.tf @@ -2,7 +2,6 @@ variable "ec_api_key" { type = string } - variable "stack_version" { description = "Optional version of the Elastic Cloud deployment" type = string From ab120c3cf9d8fcfe286cffd46c1eee6db23afe4c Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 22 Feb 2024 11:12:59 -0500 Subject: [PATCH 50/87] update the TF script logic --- .github/workflows/test-environment.yml | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 490a456b78..81134dae4f 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -112,8 +112,6 @@ env: GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments" TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} - TF_VAR_ec_url: ${{ inputs.ec-url }} - jobs: @@ -132,6 +130,7 @@ jobs: S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra" DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }} CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack" + TF_VAR_ec_url: ${{ inputs.ec-url }} # Add "id-token" with the intended permissions. permissions: contents: "read" @@ -166,26 +165,22 @@ jobs: - name: Mask Sensitive Data env: QA_EC_URL: "https://console.qa.cld.elstc.co" - PROD_EC_URL: "https://cloud.elastic.co" run: | # Extract the EC API key and selected EC URL ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) - input_ec_url="${{ env.TF_VAR_ec_url }}" - - # Production elastic cloud url - ec_url="${{ env.PROD_EC_URL }}" + $input_ec_url="${{ env.TF_VAR_ec_url }}" - # QA elastic cloud url qa_ec_url="${{ env.QA_EC_URL }}" - # Handles setting API KEY for each deployment mode for elastic cloud environment - if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then + # When user stores input ec api key, update Elastic Cloud env API KEY + if [[ $ec_api_key != '']]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [[ $input_ec_url == $ec_url ]]; then - echo "TF_VAR_ec_api_key=$TF_VAR_ec_api_key" >> $GITHUB_ENV - elif [[ $input_ec_url == $qa_ec_url ]]; then + exit 0 + + # For QA environments, sets Elastic Cloud env API Key with QA cloud API Key + if [[ $input_ec_url == $qa_ec_url ]]; then echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" From d733f360196b2d58b01f684e4ea008e1f4b39bee Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 22 Feb 2024 11:16:25 -0500 Subject: [PATCH 51/87] fix yaml syntax issue --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 81134dae4f..807ac899b5 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -179,7 +179,7 @@ jobs: echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV exit 0 - # For QA environments, sets Elastic Cloud env API Key with QA cloud API Key + # For QA environments, sets Elastic Cloud env API Key with QA cloud API Key if [[ $input_ec_url == $qa_ec_url ]]; then echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else From 2b7d3dd4ff67de4cd596680f6e057c15c808fda4 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 22 Feb 2024 12:27:41 -0500 Subject: [PATCH 52/87] add closing fi --- .github/workflows/test-environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 807ac899b5..3472121c62 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -178,6 +178,7 @@ jobs: echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV exit 0 + fi # For QA environments, sets Elastic Cloud env API Key with QA cloud API Key if [[ $input_ec_url == $qa_ec_url ]]; then From 1a61325233449aa5e335ef99da915c181db1854b Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 22 Feb 2024 12:38:43 -0500 Subject: [PATCH 53/87] use quotes syntax --- .github/workflows/test-environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 3472121c62..dbce351e83 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -174,14 +174,14 @@ jobs: qa_ec_url="${{ env.QA_EC_URL }}" # When user stores input ec api key, update Elastic Cloud env API KEY - if [[ $ec_api_key != '']]; then + if [[ "$ec_api_key" != '']]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV exit 0 fi # For QA environments, sets Elastic Cloud env API Key with QA cloud API Key - if [[ $input_ec_url == $qa_ec_url ]]; then + if [[ "$input_ec_url" == "$qa_ec_url" ]]; then echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" From 9edc4d0c64c06d923f8c9affc4e9408fa6f05bc6 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 22 Feb 2024 12:43:50 -0500 Subject: [PATCH 54/87] fix yaml syntax --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index dbce351e83..8909c4e08e 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -169,7 +169,7 @@ jobs: # Extract the EC API key and selected EC URL ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) - $input_ec_url="${{ env.TF_VAR_ec_url }}" + input_ec_url="${{ env.TF_VAR_ec_url }}" qa_ec_url="${{ env.QA_EC_URL }}" From 28a0b99b1868ce1ad781f9164d8ef26655ad1b48 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 22 Feb 2024 12:47:03 -0500 Subject: [PATCH 55/87] fix yaml syntax --- .github/workflows/test-environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 8909c4e08e..1125c26281 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -174,14 +174,14 @@ jobs: qa_ec_url="${{ env.QA_EC_URL }}" # When user stores input ec api key, update Elastic Cloud env API KEY - if [[ "$ec_api_key" != '']]; then + if [[ $ec_api_key != '']]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV exit 0 fi # For QA environments, sets Elastic Cloud env API Key with QA cloud API Key - if [[ "$input_ec_url" == "$qa_ec_url" ]]; then + if [[ $input_ec_url == $qa_ec_url ]]; then echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" From 751ce6c5ac28e8baf6f43e057cebaf2ffa8ee32c Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 22 Feb 2024 12:55:19 -0500 Subject: [PATCH 56/87] revert TF script logic --- .github/workflows/test-environment.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 1125c26281..ebd4244677 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -165,6 +165,7 @@ jobs: - name: Mask Sensitive Data env: QA_EC_URL: "https://console.qa.cld.elstc.co" + EC_URL: "https://cloud.elastic.co" run: | # Extract the EC API key and selected EC URL ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) @@ -172,16 +173,15 @@ jobs: input_ec_url="${{ env.TF_VAR_ec_url }}" qa_ec_url="${{ env.QA_EC_URL }}" + ec_url="${{ env.EC_URL }}" # When user stores input ec api key, update Elastic Cloud env API KEY if [[ $ec_api_key != '']]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - exit 0 - fi - - # For QA environments, sets Elastic Cloud env API Key with QA cloud API Key - if [[ $input_ec_url == $qa_ec_url ]]; then + elif [[ $input_ec_url == $ec_url ]]; then + echo "TF_VAR_ec_api_key=$TF_VAR_ec_api_key" >> $GITHUB_ENV + elif [[ $input_ec_url == $qa_ec_url ]]; then echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" From b7958f3f82da870ff158169ecb108206133870e1 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 22 Feb 2024 13:00:03 -0500 Subject: [PATCH 57/87] remove comma --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index ebd4244677..a4dfc1e5ca 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -175,7 +175,7 @@ jobs: qa_ec_url="${{ env.QA_EC_URL }}" ec_url="${{ env.EC_URL }}" - # When user stores input ec api key, update Elastic Cloud env API KEY + # When user stores input ec api key then update Elastic Cloud env API KEY if [[ $ec_api_key != '']]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV From 20f51aafd855e0a1a206c90e8076a2b5a506e7a5 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 22 Feb 2024 13:03:52 -0500 Subject: [PATCH 58/87] update condition --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index a4dfc1e5ca..812e78930e 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -176,7 +176,7 @@ jobs: ec_url="${{ env.EC_URL }}" # When user stores input ec api key then update Elastic Cloud env API KEY - if [[ $ec_api_key != '']]; then + if [[ $ec_api_key != '' ]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV elif [[ $input_ec_url == $ec_url ]]; then From 2dd5578dc25d1913d5b5a129551878569f541dc0 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 22 Feb 2024 13:06:23 -0500 Subject: [PATCH 59/87] test tf logic --- .github/workflows/test-environment.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 812e78930e..7b4a431138 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -179,9 +179,10 @@ jobs: if [[ $ec_api_key != '' ]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [[ $input_ec_url == $ec_url ]]; then - echo "TF_VAR_ec_api_key=$TF_VAR_ec_api_key" >> $GITHUB_ENV - elif [[ $input_ec_url == $qa_ec_url ]]; then + exit 0 + fi + + if [[ $input_ec_url == $qa_ec_url ]]; then echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" From 8a03c7173b43902f57b26135a8f7ff818c789415 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 7 Mar 2024 07:43:19 -0500 Subject: [PATCH 60/87] address pr comments --- .github/workflows/test-environment.yml | 4 ++-- deploy/test-environments/main.tf | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 7b4a431138..c6a99ea402 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -32,10 +32,10 @@ on: required: true description: "Elastic Cloud deployment region" default: "gcp-us-west2" + type: choice options: - gcp-us-west2 - aws-eu-west-1 - type: string docker-image-override: required: false description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)" @@ -179,7 +179,7 @@ jobs: if [[ $ec_api_key != '' ]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - exit 0 + exit 0 fi if [[ $input_ec_url == $qa_ec_url ]]; then diff --git a/deploy/test-environments/main.tf b/deploy/test-environments/main.tf index 751e92869e..1256786d8e 100644 --- a/deploy/test-environments/main.tf +++ b/deploy/test-environments/main.tf @@ -10,7 +10,6 @@ locals { project = "${var.project}" owner = "${var.owner}" } - ec_url = "${var.ec_url}" ec_headers = { Content-type = "application/json" Authorization = "ApiKey ${var.ec_api_key}" From 9952eda62f2b9fb9e4b4ff4dc05de9c748408073 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 7 Mar 2024 11:27:31 -0500 Subject: [PATCH 61/87] remove ec_url var --- .github/workflows/test-environment.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index c6a99ea402..d723768df8 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -268,7 +268,6 @@ jobs: terraform apply --auto-approve \ -var="deployment_name=${{ env.DEPLOYMENT_NAME }}" \ -var="region=${{ env.AWS_REGION }}" \ - -var="ec_url=${{ env.TF_VAR_ec_url }}" \ -var="project=${{ github.actor }}" \ -var="owner=${{ github.actor }}" From 6e12489c72bfc4214116476756707edebae5e672 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 7 Mar 2024 12:22:46 -0500 Subject: [PATCH 62/87] update destroy script --- deploy/test-environments/delete_env.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deploy/test-environments/delete_env.sh b/deploy/test-environments/delete_env.sh index 0943f94258..9b72f7b5bd 100755 --- a/deploy/test-environments/delete_env.sh +++ b/deploy/test-environments/delete_env.sh @@ -17,6 +17,8 @@ AWS_REGION="eu-west-1" # Add your desired default AWS region here DELETED_ENVS=() FAILED_ENVS=() +: "${TF_VAR_qa_ec_api_key:?Please set TF_VAR_qa_ec_api_key with an Elastic QA Cloud API Key}" + # Function to delete Terraform environment function delete_environment() { local ENV=$1 @@ -29,6 +31,10 @@ function delete_environment() { # Check if the resource aws_auth exists in the local state file and remove it terraform state rm -state "$tfstate" "$(terraform state list -state "$tfstate" | grep "kubernetes_config_map_v1_data.aws_auth")" || true + echo "KIBANA_URL=$(terraform output -raw kibana_url)" + if [[ "$KIBANA_URL" == *"qa.elastic"* ]]; then + echo TF_VAR_ec_api_key = $TF_VAR_qa_ec_api_key + fi # Destroy environment and remove environment data from S3 if terraform destroy -var="region=$AWS_REGION" -state "$tfstate" --auto-approve && aws s3 rm "$BUCKET/$ENV" --recursive; then From f79f676c3e1e279b732887c628d163018b9981f0 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 7 Mar 2024 12:24:50 -0500 Subject: [PATCH 63/87] add TF_VAR_qa_ec_api_key env var --- .github/workflows/destroy-environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/destroy-environment.yml b/.github/workflows/destroy-environment.yml index 3c9ec87329..92fc0c9b5e 100644 --- a/.github/workflows/destroy-environment.yml +++ b/.github/workflows/destroy-environment.yml @@ -24,6 +24,7 @@ env: ENV_PREFIX: ${{ inputs.prefix }} ENV_IGNORE_PREFIX: ${{ inputs.ignore-prefix }} TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} + TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} jobs: Destroy: From b67835697e21bf54e58b8de1f950edd2f157a701 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Mon, 11 Mar 2024 10:24:04 -0400 Subject: [PATCH 64/87] fix build --- deploy/test-environments/delete_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/test-environments/delete_env.sh b/deploy/test-environments/delete_env.sh index 9b72f7b5bd..a23bcf02d6 100755 --- a/deploy/test-environments/delete_env.sh +++ b/deploy/test-environments/delete_env.sh @@ -33,7 +33,7 @@ function delete_environment() { terraform state rm -state "$tfstate" "$(terraform state list -state "$tfstate" | grep "kubernetes_config_map_v1_data.aws_auth")" || true echo "KIBANA_URL=$(terraform output -raw kibana_url)" if [[ "$KIBANA_URL" == *"qa.elastic"* ]]; then - echo TF_VAR_ec_api_key = $TF_VAR_qa_ec_api_key + echo TF_VAR_ec_api_key = "$TF_VAR_qa_ec_api_key" fi # Destroy environment and remove environment data from S3 if terraform destroy -var="region=$AWS_REGION" -state "$tfstate" --auto-approve && From fcf6025bf7f12e22fd69d4b5bf633ceb65db309b Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Tue, 12 Mar 2024 10:32:48 -0400 Subject: [PATCH 65/87] fix exit strategy --- .github/workflows/test-environment.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index d723768df8..f9e789bee9 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -112,6 +112,7 @@ env: GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments" TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} + TF_VAR_ec_url: ${{ inputs.ec-url }} jobs: @@ -130,7 +131,6 @@ jobs: S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra" DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }} CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack" - TF_VAR_ec_url: ${{ inputs.ec-url }} # Add "id-token" with the intended permissions. permissions: contents: "read" @@ -179,14 +179,13 @@ jobs: if [[ $ec_api_key != '' ]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - exit 0 - fi - - if [[ $input_ec_url == $qa_ec_url ]]; then - echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else - echo "error: Elastic Cloud API key is required from a valid ec url" - exit 1 + if [[ $input_ec_url == $qa_ec_url ]]; then + echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV + else + echo "error: Elastic Cloud API key is required from a valid ec url" + exit 1 + fi fi - name: Process Stack Version From 83151733e2d1f45ac0174c68936a65c15b3ff63d Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 14 Mar 2024 11:01:27 -0400 Subject: [PATCH 66/87] fix env var syntax --- .github/workflows/test-environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index f9e789bee9..cfa3c2942c 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -181,7 +181,7 @@ jobs: echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV else if [[ $input_ec_url == $qa_ec_url ]]; then - echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV + echo "TF_VAR_ec_api_key=${{env.TF_VAR_qa_ec_api_key}}" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" exit 1 From f121cabd759f55b1eff95d56c435d8f2183dbebb Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 14 Mar 2024 11:35:32 -0400 Subject: [PATCH 67/87] update script with local secrets --- .github/workflows/test-environment.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index cfa3c2942c..4b7a8de510 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -110,8 +110,8 @@ env: INTEGRATIONS_SETUP_DIR: tests/integrations_setup AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments" GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments" - TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} - TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} + # TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} + # TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} TF_VAR_ec_url: ${{ inputs.ec-url }} @@ -166,6 +166,8 @@ jobs: env: QA_EC_URL: "https://console.qa.cld.elstc.co" EC_URL: "https://cloud.elastic.co" + local_ec_api_key: ${{ secrets.EC_API_KEY }} + local_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} run: | # Extract the EC API key and selected EC URL ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) @@ -181,7 +183,9 @@ jobs: echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV else if [[ $input_ec_url == $qa_ec_url ]]; then - echo "TF_VAR_ec_api_key=${{env.TF_VAR_qa_ec_api_key}}" >> $GITHUB_ENV + echo "TF_VAR_ec_api_key=${{env.local_qa_ec_api_key}}" >> $GITHUB_ENV + elif [[ $input_ec_url == $ec_url ]]; then + echo "TF_VAR_ec_api_key=${{env.local_ec_api_key}}" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" exit 1 From 952bb4bfd6590495f6edc36b777eb3034d69ef44 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 14 Mar 2024 12:11:12 -0400 Subject: [PATCH 68/87] try another syntax pattern to update api key --- .github/workflows/test-environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 4b7a8de510..2b77ce68e1 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -111,7 +111,7 @@ env: AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments" GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments" # TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} - # TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} + TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} TF_VAR_ec_url: ${{ inputs.ec-url }} @@ -183,9 +183,9 @@ jobs: echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV else if [[ $input_ec_url == $qa_ec_url ]]; then - echo "TF_VAR_ec_api_key=${{env.local_qa_ec_api_key}}" >> $GITHUB_ENV + echo "TF_VAR_ec_api_key=${local_qa_ec_api_key}" >> $GITHUB_ENV elif [[ $input_ec_url == $ec_url ]]; then - echo "TF_VAR_ec_api_key=${{env.local_ec_api_key}}" >> $GITHUB_ENV + echo "TF_VAR_ec_api_key=${local_ec_api_key}" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" exit 1 From 43cbb5d377734e814cbed381fd8e2c64f1a23207 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 14 Mar 2024 12:28:07 -0400 Subject: [PATCH 69/87] update script condition --- .github/workflows/test-environment.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 2b77ce68e1..9e7e3cdd46 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -178,18 +178,16 @@ jobs: ec_url="${{ env.EC_URL }}" # When user stores input ec api key then update Elastic Cloud env API KEY - if [[ $ec_api_key != '' ]]; then + if [[ $ec_api_key != '' && $input_ec_url == $ec_url]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV + elif [[ $input_ec_url == $ec_url && $ec_api_key == '' ]]; then + echo "TF_VAR_ec_api_key=${local_ec_api_key}" >> $GITHUB_ENV + elif [[ $input_ec_url == $qa_ec_url ]]; then + echo "TF_VAR_ec_api_key=${local_qa_ec_api_key}" >> $GITHUB_ENV else - if [[ $input_ec_url == $qa_ec_url ]]; then - echo "TF_VAR_ec_api_key=${local_qa_ec_api_key}" >> $GITHUB_ENV - elif [[ $input_ec_url == $ec_url ]]; then - echo "TF_VAR_ec_api_key=${local_ec_api_key}" >> $GITHUB_ENV - else echo "error: Elastic Cloud API key is required from a valid ec url" exit 1 - fi fi - name: Process Stack Version From 74127c7666647139822ee80a95da5397d067eaff Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 14 Mar 2024 14:46:25 -0400 Subject: [PATCH 70/87] fix spacing for syntax --- .github/workflows/test-environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 9e7e3cdd46..9fda9a048a 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -178,7 +178,7 @@ jobs: ec_url="${{ env.EC_URL }}" # When user stores input ec api key then update Elastic Cloud env API KEY - if [[ $ec_api_key != '' && $input_ec_url == $ec_url]]; then + if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV elif [[ $input_ec_url == $ec_url && $ec_api_key == '' ]]; then @@ -186,8 +186,8 @@ jobs: elif [[ $input_ec_url == $qa_ec_url ]]; then echo "TF_VAR_ec_api_key=${local_qa_ec_api_key}" >> $GITHUB_ENV else - echo "error: Elastic Cloud API key is required from a valid ec url" - exit 1 + echo "error: Elastic Cloud API key is required from a valid ec url" + exit 1 fi - name: Process Stack Version From 0797e767103d829c541409aadb4ceafe56cbaa68 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 20 Mar 2024 11:33:02 -0400 Subject: [PATCH 71/87] add another condition for qa env --- .github/workflows/test-environment.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 9fda9a048a..e940877832 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -110,7 +110,7 @@ env: INTEGRATIONS_SETUP_DIR: tests/integrations_setup AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments" GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments" - # TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} + TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} TF_VAR_ec_url: ${{ inputs.ec-url }} @@ -181,10 +181,13 @@ jobs: if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [[ $input_ec_url == $ec_url && $ec_api_key == '' ]]; then + elif [[ $ec_api_key == '' && $input_ec_url == $ec_url ]]; then echo "TF_VAR_ec_api_key=${local_ec_api_key}" >> $GITHUB_ENV - elif [[ $input_ec_url == $qa_ec_url ]]; then + elif [[ $ec_api_key == '' && $input_ec_url == $qa_ec_url ]]; then echo "TF_VAR_ec_api_key=${local_qa_ec_api_key}" >> $GITHUB_ENV + elif [[ $ec_api_key != '' && $input_ec_url == $qa_ec_url ]]; then + echo "::add-mask::$ec_api_key" + echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" exit 1 From 193b9ce76a9990556417679fefafd50c0f65453b Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 20 Mar 2024 14:50:41 -0400 Subject: [PATCH 72/87] remove condition --- .github/workflows/test-environment.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index e940877832..ab90f36bed 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -181,10 +181,8 @@ jobs: if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [[ $ec_api_key == '' && $input_ec_url == $ec_url ]]; then - echo "TF_VAR_ec_api_key=${local_ec_api_key}" >> $GITHUB_ENV elif [[ $ec_api_key == '' && $input_ec_url == $qa_ec_url ]]; then - echo "TF_VAR_ec_api_key=${local_qa_ec_api_key}" >> $GITHUB_ENV + echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV elif [[ $ec_api_key != '' && $input_ec_url == $qa_ec_url ]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV From 4a95fcc9660a95651b05e6b309b20f0b6b7fcf47 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 21 Mar 2024 13:16:55 -0400 Subject: [PATCH 73/87] update ec url to terraform apply --- .github/workflows/test-environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index ab90f36bed..c9aa0903ad 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -271,7 +271,8 @@ jobs: -var="deployment_name=${{ env.DEPLOYMENT_NAME }}" \ -var="region=${{ env.AWS_REGION }}" \ -var="project=${{ github.actor }}" \ - -var="owner=${{ github.actor }}" + -var="owner=${{ github.actor }}" \ + -var="ec_url=${{ env.TF_VAR_ec_url }}" - name: Set Environment Output id: env-output From 34f434d28b26dda686139e090062cfda819992f6 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 21 Mar 2024 13:50:30 -0400 Subject: [PATCH 74/87] update original build passing logic --- .github/workflows/test-environment.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index c9aa0903ad..0ebd68b623 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -181,11 +181,10 @@ jobs: if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV - elif [[ $ec_api_key == '' && $input_ec_url == $qa_ec_url ]]; then + elif [[ $input_ec_url == $ec_url ]]; then + echo "TF_VAR_ec_api_key=$TF_VAR_ec_api_key" >> $GITHUB_ENV + elif [[ $input_ec_url == $qa_ec_url ]]; then echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV - elif [[ $ec_api_key != '' && $input_ec_url == $qa_ec_url ]]; then - echo "::add-mask::$ec_api_key" - echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" exit 1 @@ -272,7 +271,7 @@ jobs: -var="region=${{ env.AWS_REGION }}" \ -var="project=${{ github.actor }}" \ -var="owner=${{ github.actor }}" \ - -var="ec_url=${{ env.TF_VAR_ec_url }}" + - name: Set Environment Output id: env-output From 5e700861cffcba7427877cf56c65e8cdacd0f23c Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Mon, 25 Mar 2024 09:25:52 -0400 Subject: [PATCH 75/87] add region to cloud env --- .github/workflows/test-environment.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 0ebd68b623..c781cee5e3 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -36,6 +36,7 @@ on: options: - gcp-us-west2 - aws-eu-west-1 + - aws-us-east-1 docker-image-override: required: false description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)" @@ -166,8 +167,6 @@ jobs: env: QA_EC_URL: "https://console.qa.cld.elstc.co" EC_URL: "https://cloud.elastic.co" - local_ec_api_key: ${{ secrets.EC_API_KEY }} - local_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }} run: | # Extract the EC API key and selected EC URL ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) From 22a8be42f6365150c1e00e44fbe7be54513d14b4 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Tue, 26 Mar 2024 10:44:27 -0400 Subject: [PATCH 76/87] enable support for keep_ --- deploy/test-environments/main.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deploy/test-environments/main.tf b/deploy/test-environments/main.tf index 1256786d8e..4bcc56e85a 100644 --- a/deploy/test-environments/main.tf +++ b/deploy/test-environments/main.tf @@ -10,6 +10,7 @@ locals { project = "${var.project}" owner = "${var.owner}" } + deployment_name = replace(var.deployment_name, "keep-", "keep_") ec_headers = { Content-type = "application/json" Authorization = "ApiKey ${var.ec_api_key}" @@ -63,7 +64,7 @@ module "ec_deployment" { tags = local.common_tags deployment_template = var.deployment_template - deployment_name_prefix = "${var.deployment_name}-${random_string.suffix.result}" + deployment_name_prefix = "${local.deployment_name}-${random_string.suffix.result}" elasticsearch_autoscale = true elasticsearch_size = var.elasticsearch_size @@ -88,7 +89,7 @@ module "ec_project" { source = "../cloud/modules/serverless" ec_apikey = var.ec_api_key ec_url = var.ec_url - project_name = "${var.deployment_name}-${random_string.suffix.result}" + project_name = "${local.deployment_name}-${random_string.suffix.result}" region_id = var.ess_region } From d567e3d7cdd83189be04e93ccfaf8977ed5231cb Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 27 Mar 2024 09:41:42 -0400 Subject: [PATCH 77/87] remove keep prefix and update api key logic --- .github/workflows/test-environment.yml | 6 +++--- deploy/test-environments/main.tf | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index c781cee5e3..3bfae3be77 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -158,8 +158,8 @@ jobs: fi # Check pattern required for cloud deployment - if ! [[ $deployment_name =~ (^keep_[a-z][-a-z0-9]*|[a-z][-a-z0-9]*)$ ]]; then - echo "error: Deployment name doesn't match the required pattern QA env keep_[a-z][-a-z0-9]* or regular env [a-z][-a-z0-9]*" + if ! [[ $deployment_name =~ ([a-z][-a-z0-9]*)$ ]]; then + echo "error: Deployment name doesn't match the required pattern regular env [a-z][-a-z0-9]*" exit 1 fi @@ -177,7 +177,7 @@ jobs: ec_url="${{ env.EC_URL }}" # When user stores input ec api key then update Elastic Cloud env API KEY - if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then + if [[ $ec_api_key != '' ]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV elif [[ $input_ec_url == $ec_url ]]; then diff --git a/deploy/test-environments/main.tf b/deploy/test-environments/main.tf index 4bcc56e85a..1256786d8e 100644 --- a/deploy/test-environments/main.tf +++ b/deploy/test-environments/main.tf @@ -10,7 +10,6 @@ locals { project = "${var.project}" owner = "${var.owner}" } - deployment_name = replace(var.deployment_name, "keep-", "keep_") ec_headers = { Content-type = "application/json" Authorization = "ApiKey ${var.ec_api_key}" @@ -64,7 +63,7 @@ module "ec_deployment" { tags = local.common_tags deployment_template = var.deployment_template - deployment_name_prefix = "${local.deployment_name}-${random_string.suffix.result}" + deployment_name_prefix = "${var.deployment_name}-${random_string.suffix.result}" elasticsearch_autoscale = true elasticsearch_size = var.elasticsearch_size @@ -89,7 +88,7 @@ module "ec_project" { source = "../cloud/modules/serverless" ec_apikey = var.ec_api_key ec_url = var.ec_url - project_name = "${local.deployment_name}-${random_string.suffix.result}" + project_name = "${var.deployment_name}-${random_string.suffix.result}" region_id = var.ess_region } From 0b709ab2c84c52f4987b48ba4ca946dbc002c0a9 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Wed, 27 Mar 2024 10:15:57 -0400 Subject: [PATCH 78/87] rever logic and updated required fields docs --- .github/workflows/test-environment.yml | 8 ++------ dev-docs/Cloud-Env-Testing.md | 6 +++++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 3bfae3be77..f354863644 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -32,11 +32,7 @@ on: required: true description: "Elastic Cloud deployment region" default: "gcp-us-west2" - type: choice - options: - - gcp-us-west2 - - aws-eu-west-1 - - aws-us-east-1 + type: string docker-image-override: required: false description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)" @@ -177,7 +173,7 @@ jobs: ec_url="${{ env.EC_URL }}" # When user stores input ec api key then update Elastic Cloud env API KEY - if [[ $ec_api_key != '' ]]; then + if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV elif [[ $input_ec_url == $ec_url ]]; then diff --git a/dev-docs/Cloud-Env-Testing.md b/dev-docs/Cloud-Env-Testing.md index 12572ddd83..cc2674cd9e 100644 --- a/dev-docs/Cloud-Env-Testing.md +++ b/dev-docs/Cloud-Env-Testing.md @@ -20,15 +20,19 @@ Follow these steps to run the workflow: - **`deployment_name`**: Name your environment (Allowed characters: a-zA-Z0-9 and `-`). For instance: `john-8-7-2-June01`. + - **`serverless_mode`**: Set to `true` to enable serverless mode. Default: `false`. - **`elk-stack-version`**: Specify the version of Elastic Cloud stack, either a SNAPSHOT or a build candidate (BC) version. Check the available versions [here](https://artifacts-staging.elastic.co/dra-info/index.html). For BC, enter version with additions/commit sha, e.g. `8.12.0-61156bc6`. For SNAPSHOT, enter the full version, e.g. `8.13.0-SNAPSHOT`. + - **`ec_url`**: Indicate the Elastic Cloud URL. The default value is `https://cloud.elastic.co`. For QA environments, + use `https://console.qa.cld.elstc.co/`. + - **`ess-region`**: Indicate the Elastic Cloud deployment region. The default value is `gcp-us-west2`, which supports - snapshot and build candidate (BC) versions. Specify a different region only if necessary. + snapshot and build candidate (BC) versions. However, for QA environments, use `aws-eu-west-1`. Specify a different region only if necessary. ![Required Parameters](https://github.com/oren-zohar/cloudbeat/assets/85433724/6159129e-6d4d-46b1-97a1-f0d3859500fd) From 3d8feff18dcbe4e5e0de78a8e4e66296a67351f4 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Mon, 1 Apr 2024 13:23:43 -0400 Subject: [PATCH 79/87] updates docs with manual notes for qa --- dev-docs/Cloud-Env-Testing.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/dev-docs/Cloud-Env-Testing.md b/dev-docs/Cloud-Env-Testing.md index cc2674cd9e..89f10758c1 100644 --- a/dev-docs/Cloud-Env-Testing.md +++ b/dev-docs/Cloud-Env-Testing.md @@ -20,6 +20,7 @@ Follow these steps to run the workflow: - **`deployment_name`**: Name your environment (Allowed characters: a-zA-Z0-9 and `-`). For instance: `john-8-7-2-June01`. + - **`serverless_mode`**: Set to `true` to enable serverless mode. Default: `false`. - **`elk-stack-version`**: Specify the version of Elastic Cloud stack, either a SNAPSHOT or a build candidate (BC) @@ -27,7 +28,7 @@ Follow these steps to run the workflow: For BC, enter version with additions/commit sha, e.g. `8.12.0-61156bc6`. For SNAPSHOT, enter the full version, e.g. `8.13.0-SNAPSHOT`. - - **`ec_url`**: Indicate the Elastic Cloud URL. The default value is `https://cloud.elastic.co`. For QA environments, + - **`ec_url`**: Indicate the Elastic Cloud URL. The default value is `https://cloud.elastic.co` for production environments. For QA environments, use `https://console.qa.cld.elstc.co/`. - **`ess-region`**: Indicate the Elastic Cloud deployment region. The default value is `gcp-us-west2`, which @@ -65,6 +66,17 @@ Follow these steps to run the workflow: ![Run Workflow](https://github.com/oren-zohar/cloudbeat/assets/85433724/7b05bf58-cc0b-4ec9-8e49-55d117673df8) +## Manual QA Environments Updates + +For qa environments, to avoid [automatic deployment deletion](https://docs.elastic.dev/serverless/qa) be sure to update the deployment name with the prefix `keep_`. + +1. Go to [`Projects > Project-Deployment Name`](https://console.qa.cld.elstc.co/) + ![Screenshot 2024-04-01 at 12.24.25 PM.png](..%2F..%2F..%2FDesktop%2FScreenshot%202024-04-01%20at%2012.24.25%E2%80%AFPM.png) + +2. Update the *Project name* field value with the prefix `keep_` +![Screenshot 2024-04-01 at 12.24.25 PM.png](..%2F..%2F..%2FDesktop%2FScreenshot%202024-04-01%20at%2012.24.25%E2%80%AFPM.png) + +Note that when you go to destroy the environment, be sure to remove `keep_` from the deployment name. ## Tracking Workflow Execution From 90499eb74243979b95c3834e221026f28cf273ff Mon Sep 17 00:00:00 2001 From: Lola Date: Mon, 1 Apr 2024 13:34:01 -0400 Subject: [PATCH 80/87] Update Cloud-Env-Testing.md with images --- dev-docs/Cloud-Env-Testing.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dev-docs/Cloud-Env-Testing.md b/dev-docs/Cloud-Env-Testing.md index 89f10758c1..99abb5b40d 100644 --- a/dev-docs/Cloud-Env-Testing.md +++ b/dev-docs/Cloud-Env-Testing.md @@ -70,11 +70,13 @@ Follow these steps to run the workflow: For qa environments, to avoid [automatic deployment deletion](https://docs.elastic.dev/serverless/qa) be sure to update the deployment name with the prefix `keep_`. -1. Go to [`Projects > Project-Deployment Name`](https://console.qa.cld.elstc.co/) - ![Screenshot 2024-04-01 at 12.24.25 PM.png](..%2F..%2F..%2FDesktop%2FScreenshot%202024-04-01%20at%2012.24.25%E2%80%AFPM.png) + +1. Go to [`Projects > Project > "Project Deployment Name"`](https://console.qa.cld.elstc.co/) + + Screenshot 2024-04-01 at 12 24 25 PM 2. Update the *Project name* field value with the prefix `keep_` -![Screenshot 2024-04-01 at 12.24.25 PM.png](..%2F..%2F..%2FDesktop%2FScreenshot%202024-04-01%20at%2012.24.25%E2%80%AFPM.png) +Screenshot 2024-04-01 at 12 24 54 PM Note that when you go to destroy the environment, be sure to remove `keep_` from the deployment name. @@ -147,7 +149,7 @@ Follow these steps to connect to your Amazon Elastic Kubernetes Service (EKS) cl ```kubectl get po -n kube-system``` - This command should list the pods in the kube-system namespace, confirming that you have successfully connected to your EKS cluster. + This command should list the pods in the kube-system namespace, confirming that you have successfully connected to your EKS cluster. ## Cleanup Procedure From 028a2a35b14e86816fae0b707a9ac1fc9e2b943e Mon Sep 17 00:00:00 2001 From: Lola Date: Mon, 1 Apr 2024 15:04:47 -0400 Subject: [PATCH 81/87] update required parameters docs --- dev-docs/Cloud-Env-Testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-docs/Cloud-Env-Testing.md b/dev-docs/Cloud-Env-Testing.md index 99abb5b40d..c54db0a1ae 100644 --- a/dev-docs/Cloud-Env-Testing.md +++ b/dev-docs/Cloud-Env-Testing.md @@ -34,8 +34,8 @@ Follow these steps to run the workflow: - **`ess-region`**: Indicate the Elastic Cloud deployment region. The default value is `gcp-us-west2`, which supports snapshot and build candidate (BC) versions. However, for QA environments, use `aws-eu-west-1`. Specify a different region only if necessary. + Screenshot 2024-04-01 at 1 35 27 PM - ![Required Parameters](https://github.com/oren-zohar/cloudbeat/assets/85433724/6159129e-6d4d-46b1-97a1-f0d3859500fd) 4. Optionally, modify other parameters if required: From 186b4f62bbab48b58c4674545a188a6c6c38c7d2 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Mon, 1 Apr 2024 15:08:21 -0400 Subject: [PATCH 82/87] remove ec url var from weekly --- deploy/weekly-environment/main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/weekly-environment/main.tf b/deploy/weekly-environment/main.tf index 8677c8a2e1..8493db9dcf 100644 --- a/deploy/weekly-environment/main.tf +++ b/deploy/weekly-environment/main.tf @@ -1,7 +1,6 @@ provider "ec" { apikey = var.ec_api_key endpoint = var.endpoint - url = var.ec_url } module "ec_deployment" { From b2fa1dc9c1044d67f307531f9b4df3ed844ef0ab Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 4 Apr 2024 09:22:52 -0400 Subject: [PATCH 83/87] test delete script without TF_VARupdate --- deploy/test-environments/delete_env.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/deploy/test-environments/delete_env.sh b/deploy/test-environments/delete_env.sh index 92736b723d..1f0ae9e26d 100755 --- a/deploy/test-environments/delete_env.sh +++ b/deploy/test-environments/delete_env.sh @@ -32,9 +32,7 @@ function delete_environment() { # Check if the resource aws_auth exists in the local state file and remove it terraform state rm -state "$tfstate" "$(terraform state list -state "$tfstate" | grep "kubernetes_config_map_v1_data.aws_auth")" || true echo "KIBANA_URL=$(terraform output -raw kibana_url)" - if [[ "$KIBANA_URL" == *"qa.elastic"* ]]; then - echo TF_VAR_ec_api_key = "$TF_VAR_qa_ec_api_key" - fi + # Destroy environment and remove environment data from S3 if terraform destroy -var="region=$AWS_REGION" -state "$tfstate" --auto-approve && aws s3 rm "$BUCKET/$ENV" --recursive; then From 73b73a8a6fa77365cc63c4f67d2bd417043309c6 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 4 Apr 2024 09:57:18 -0400 Subject: [PATCH 84/87] use echo instead of export --- deploy/test-environments/delete_env.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deploy/test-environments/delete_env.sh b/deploy/test-environments/delete_env.sh index 1f0ae9e26d..aab16b4de7 100755 --- a/deploy/test-environments/delete_env.sh +++ b/deploy/test-environments/delete_env.sh @@ -32,7 +32,10 @@ function delete_environment() { # Check if the resource aws_auth exists in the local state file and remove it terraform state rm -state "$tfstate" "$(terraform state list -state "$tfstate" | grep "kubernetes_config_map_v1_data.aws_auth")" || true echo "KIBANA_URL=$(terraform output -raw kibana_url)" - + if [[ "$KIBANA_URL" == *"qa.elastic"* ]]; then + echo "TF_VAR_ec_api_key QA Elastic Cloud API Key" + export TF_VAR_ec_api_key="$TF_VAR_qa_ec_api_key" + fi # Destroy environment and remove environment data from S3 if terraform destroy -var="region=$AWS_REGION" -state "$tfstate" --auto-approve && aws s3 rm "$BUCKET/$ENV" --recursive; then From 0afb9d92b5b82221749a57f5bf1605502543113f Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 4 Apr 2024 11:00:16 -0400 Subject: [PATCH 85/87] revert regex condition and update logging --- .github/workflows/test-environment.yml | 7 +++++-- deploy/test-environments/delete_env.sh | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml index 0e2d8108d7..1f103c9ab3 100644 --- a/.github/workflows/test-environment.yml +++ b/.github/workflows/test-environment.yml @@ -154,8 +154,8 @@ jobs: fi # Check pattern required for cloud deployment - if ! [[ $deployment_name =~ ([a-z][-a-z0-9]*)$ ]]; then - echo "error: Deployment name doesn't match the required pattern regular env [a-z][-a-z0-9]*" + if ! [[ $deployment_name =~ ^[a-z][-a-z0-9]*$ ]]; then + echo "error: Deployment name doesn't match the required pattern [a-z][-a-z0-9]*" exit 1 fi @@ -174,11 +174,14 @@ jobs: # When user stores input ec api key then update Elastic Cloud env API KEY if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then + echo "Set TF_VAR_ec_api_key from the input ec api key" echo "::add-mask::$ec_api_key" echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV elif [[ $input_ec_url == $ec_url ]]; then + echo "Set TF_VAR_ec_api_key from the PROD github secret api key" echo "TF_VAR_ec_api_key=$TF_VAR_ec_api_key" >> $GITHUB_ENV elif [[ $input_ec_url == $qa_ec_url ]]; then + echo "Set TF_VAR_ec_api_key from the QA github secret api key" echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV else echo "error: Elastic Cloud API key is required from a valid ec url" diff --git a/deploy/test-environments/delete_env.sh b/deploy/test-environments/delete_env.sh index aab16b4de7..250eed5852 100755 --- a/deploy/test-environments/delete_env.sh +++ b/deploy/test-environments/delete_env.sh @@ -33,7 +33,7 @@ function delete_environment() { terraform state rm -state "$tfstate" "$(terraform state list -state "$tfstate" | grep "kubernetes_config_map_v1_data.aws_auth")" || true echo "KIBANA_URL=$(terraform output -raw kibana_url)" if [[ "$KIBANA_URL" == *"qa.elastic"* ]]; then - echo "TF_VAR_ec_api_key QA Elastic Cloud API Key" + echo "Set TF_VAR_ec_api_key with QA Elastic Cloud API Key" export TF_VAR_ec_api_key="$TF_VAR_qa_ec_api_key" fi # Destroy environment and remove environment data from S3 From 83792df6fb5339922b848eee22832494daeb4132 Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 4 Apr 2024 12:57:01 -0400 Subject: [PATCH 86/87] update kibana url --- deploy/test-environments/delete_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/test-environments/delete_env.sh b/deploy/test-environments/delete_env.sh index 250eed5852..30b150aa52 100755 --- a/deploy/test-environments/delete_env.sh +++ b/deploy/test-environments/delete_env.sh @@ -31,7 +31,7 @@ function delete_environment() { # Check if the resource aws_auth exists in the local state file and remove it terraform state rm -state "$tfstate" "$(terraform state list -state "$tfstate" | grep "kubernetes_config_map_v1_data.aws_auth")" || true - echo "KIBANA_URL=$(terraform output -raw kibana_url)" + echo "KIBANA_URL=$(terraform output -raw kibana_url)" >> $GITHUB_ENV if [[ "$KIBANA_URL" == *"qa.elastic"* ]]; then echo "Set TF_VAR_ec_api_key with QA Elastic Cloud API Key" export TF_VAR_ec_api_key="$TF_VAR_qa_ec_api_key" From 6f0b37ce3861ee67baa3b6a544069beee35fd2cb Mon Sep 17 00:00:00 2001 From: Omolola Akinleye Date: Thu, 4 Apr 2024 16:04:45 -0400 Subject: [PATCH 87/87] add logging --- deploy/test-environments/delete_env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/test-environments/delete_env.sh b/deploy/test-environments/delete_env.sh index 30b150aa52..cf4ad50f90 100755 --- a/deploy/test-environments/delete_env.sh +++ b/deploy/test-environments/delete_env.sh @@ -32,6 +32,7 @@ function delete_environment() { # Check if the resource aws_auth exists in the local state file and remove it terraform state rm -state "$tfstate" "$(terraform state list -state "$tfstate" | grep "kubernetes_config_map_v1_data.aws_auth")" || true echo "KIBANA_URL=$(terraform output -raw kibana_url)" >> $GITHUB_ENV + echo "Value of KIBANA_URL: $KIBANA_URL" if [[ "$KIBANA_URL" == *"qa.elastic"* ]]; then echo "Set TF_VAR_ec_api_key with QA Elastic Cloud API Key" export TF_VAR_ec_api_key="$TF_VAR_qa_ec_api_key"