diff --git a/.github/workflows/destroy-environment.yml b/.github/workflows/destroy-environment.yml
index 8762664b91..ef81246137 100644
--- a/.github/workflows/destroy-environment.yml
+++ b/.github/workflows/destroy-environment.yml
@@ -24,6 +24,7 @@ env:
ENV_PREFIX: ${{ inputs.prefix }}
ENV_IGNORE_PREFIX: ${{ inputs.ignore-prefix }}
TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
+ TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }}
jobs:
Destroy:
diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml
index e85e56878f..1f103c9ab3 100644
--- a/.github/workflows/test-environment.yml
+++ b/.github/workflows/test-environment.yml
@@ -15,6 +15,14 @@ on:
type: boolean
required: true
default: false
+ ec-url:
+ required: true
+ default: "https://cloud.elastic.co"
+ type: choice
+ description: Select the Environment URL
+ options:
+ - https://cloud.elastic.co
+ - https://console.qa.cld.elstc.co
elk-stack-version:
required: true
description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT"
@@ -30,9 +38,9 @@ on:
description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
type: string
run-sanity-tests:
- description: "Run sanity tests after provision"
- default: false
- type: boolean
+ description: "Run sanity tests after provision"
+ default: false
+ type: boolean
cleanup-env:
description: "Cleanup resources after provision"
default: false
@@ -52,6 +60,11 @@ on:
type: boolean
required: true
default: false
+ ec-url:
+ required: true
+ default: "https://cloud.elastic.co"
+ type: string
+ description: Environment URL
elk-stack-version:
required: true
description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT"
@@ -95,6 +108,9 @@ env:
AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments"
GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments"
TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
+ TF_VAR_qa_ec_api_key: ${{ secrets.QA_EC_API_KEY }}
+ TF_VAR_ec_url: ${{ inputs.ec-url }}
+
jobs:
Deploy:
@@ -114,8 +130,8 @@ jobs:
CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack"
# Add "id-token" with the intended permissions.
permissions:
- contents: 'read'
- id-token: 'write'
+ contents: "read"
+ id-token: "write"
outputs:
deploy-s3-bucket: ${{ steps.upload-state.outputs.s3-bucket-folder }}
aws-cnvm-stack-name: ${{ steps.upload-state.outputs.aws-cnvm-stack }}
@@ -144,11 +160,33 @@ jobs:
fi
- name: Mask Sensitive Data
- if: inputs.ec-api-key != ''
+ env:
+ QA_EC_URL: "https://console.qa.cld.elstc.co"
+ EC_URL: "https://cloud.elastic.co"
run: |
+ # Extract the EC API key and selected EC URL
ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH)
- echo "::add-mask::$ec_api_key"
- echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV
+
+ input_ec_url="${{ env.TF_VAR_ec_url }}"
+
+ qa_ec_url="${{ env.QA_EC_URL }}"
+ ec_url="${{ env.EC_URL }}"
+
+ # When user stores input ec api key then update Elastic Cloud env API KEY
+ if [[ $ec_api_key != '' && $input_ec_url == $ec_url ]]; then
+ echo "Set TF_VAR_ec_api_key from the input ec api key"
+ echo "::add-mask::$ec_api_key"
+ echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV
+ elif [[ $input_ec_url == $ec_url ]]; then
+ echo "Set TF_VAR_ec_api_key from the PROD github secret api key"
+ echo "TF_VAR_ec_api_key=$TF_VAR_ec_api_key" >> $GITHUB_ENV
+ elif [[ $input_ec_url == $qa_ec_url ]]; then
+ echo "Set TF_VAR_ec_api_key from the QA github secret api key"
+ echo "TF_VAR_ec_api_key=$TF_VAR_qa_ec_api_key" >> $GITHUB_ENV
+ else
+ echo "error: Elastic Cloud API key is required from a valid ec url"
+ exit 1
+ fi
- name: Process Stack Version
id: remove-commit-hash
@@ -192,7 +230,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
- python-version: '3.9'
+ python-version: "3.9"
- name: Install Poetry
run: |
@@ -230,7 +268,8 @@ jobs:
-var="deployment_name=${{ env.DEPLOYMENT_NAME }}" \
-var="region=${{ env.AWS_REGION }}" \
-var="project=${{ github.actor }}" \
- -var="owner=${{ github.actor }}"
+ -var="owner=${{ github.actor }}" \
+
- name: Set Environment Output
id: env-output
diff --git a/deploy/cloud/variables.tf b/deploy/cloud/variables.tf
index 96b6323335..83259a566d 100644
--- a/deploy/cloud/variables.tf
+++ b/deploy/cloud/variables.tf
@@ -5,6 +5,11 @@ variable "ec_api_key" {
type = string
}
+variable "ec_url" {
+ description = "Elastic cloud Environment URL"
+ type = string
+}
+
variable "ess_region" {
default = "gcp-us-central1"
description = "Optional ESS region where the deployment will be created. Defaults to gcp-us-west2"
diff --git a/deploy/test-environments/delete_env.sh b/deploy/test-environments/delete_env.sh
index 563139a9b7..d45afc170c 100755
--- a/deploy/test-environments/delete_env.sh
+++ b/deploy/test-environments/delete_env.sh
@@ -17,6 +17,8 @@ AWS_REGION="eu-west-1" # Add your desired default AWS region here
DELETED_ENVS=()
FAILED_ENVS=()
+: "${TF_VAR_qa_ec_api_key:?Please set TF_VAR_qa_ec_api_key with an Elastic QA Cloud API Key}"
+
# Function to delete Terraform environment
function delete_environment() {
local ENV=$1
@@ -29,6 +31,12 @@ function delete_environment() {
# Check if the resource aws_auth exists in the local state file and remove it
terraform state rm -state "$tfstate" "$(terraform state list -state "$tfstate" | grep "kubernetes_config_map_v1_data.aws_auth")" || true
+ echo "KIBANA_URL=$(terraform output -raw kibana_url)" >> $GITHUB_ENV
+ echo "Value of KIBANA_URL: $KIBANA_URL"
+ if [[ "$KIBANA_URL" == *"qa.elastic"* ]]; then
+ echo "Set TF_VAR_ec_api_key with QA Elastic Cloud API Key"
+ export TF_VAR_ec_api_key="$TF_VAR_qa_ec_api_key"
+ fi
# Destroy environment and remove environment data from S3
if terraform destroy -var="region=$AWS_REGION" -state "$tfstate" --auto-approve &&
aws s3 rm "$BUCKET/$ENV" --recursive; then
diff --git a/deploy/test-environments/main.tf b/deploy/test-environments/main.tf
index 76dbc48dd6..619d85a949 100644
--- a/deploy/test-environments/main.tf
+++ b/deploy/test-environments/main.tf
@@ -11,7 +11,6 @@ locals {
owner = "${var.owner}"
deployment = "${var.deployment_name}"
}
- ec_url = "https://cloud.elastic.co"
ec_headers = {
Content-type = "application/json"
Authorization = "ApiKey ${var.ec_api_key}"
@@ -49,7 +48,7 @@ provider "ec" {
provider "restapi" {
alias = "ec"
- uri = local.ec_url
+ uri = var.ec_url
write_returns_object = true
headers = local.ec_headers
}
@@ -89,9 +88,9 @@ module "ec_project" {
count = var.serverless_mode ? 1 : 0
source = "../cloud/modules/serverless"
ec_apikey = var.ec_api_key
- ec_url = local.ec_url
+ ec_url = var.ec_url
project_name = "${var.deployment_name}-${random_string.suffix.result}"
- region_id = "aws-us-east-1" # TODO: replace with var.ess_region when more regions are supported
+ region_id = var.ess_region
}
module "eks" {
diff --git a/deploy/test-environments/variables.tf b/deploy/test-environments/variables.tf
index 43e8a199e8..6d087ad8d4 100644
--- a/deploy/test-environments/variables.tf
+++ b/deploy/test-environments/variables.tf
@@ -18,6 +18,12 @@ variable "ami_map" {
# Elastic Cloud variables
# ===========================================
+variable "ec_url" {
+ default = "https://cloud.elastic.co"
+ description = "Optional Elastic Cloud Environment URL, use export TF_VAR_ec_url={URL}, it defaults to https://cloud.elastic.co"
+ type = string
+}
+
variable "ec_api_key" {
description = "Provide Elastic Cloud API key or use export TF_VAR_ec_api_key={TOKEN}"
type = string
diff --git a/dev-docs/Cloud-Env-Testing.md b/dev-docs/Cloud-Env-Testing.md
index 12572ddd83..c54db0a1ae 100644
--- a/dev-docs/Cloud-Env-Testing.md
+++ b/dev-docs/Cloud-Env-Testing.md
@@ -21,16 +21,21 @@ Follow these steps to run the workflow:
- **`deployment_name`**: Name your environment (Allowed characters: a-zA-Z0-9 and `-`). For
instance: `john-8-7-2-June01`.
+ - **`serverless_mode`**: Set to `true` to enable serverless mode. Default: `false`.
+
- **`elk-stack-version`**: Specify the version of Elastic Cloud stack, either a SNAPSHOT or a build candidate (BC)
version. Check the available versions [here](https://artifacts-staging.elastic.co/dra-info/index.html).
For BC, enter version with additions/commit sha, e.g. `8.12.0-61156bc6`.
For SNAPSHOT, enter the full version, e.g. `8.13.0-SNAPSHOT`.
+ - **`ec_url`**: Indicate the Elastic Cloud URL. The default value is `https://cloud.elastic.co` for production environments. For QA environments,
+ use `https://console.qa.cld.elstc.co/`.
+
- **`ess-region`**: Indicate the Elastic Cloud deployment region. The default value is `gcp-us-west2`, which
supports
- snapshot and build candidate (BC) versions. Specify a different region only if necessary.
+ snapshot and build candidate (BC) versions. However, for QA environments, use `aws-eu-west-1`. Specify a different region only if necessary.
+
- ![Required Parameters](https://github.com/oren-zohar/cloudbeat/assets/85433724/6159129e-6d4d-46b1-97a1-f0d3859500fd)
4. Optionally, modify other parameters if required:
@@ -61,6 +66,19 @@ Follow these steps to run the workflow:
![Run Workflow](https://github.com/oren-zohar/cloudbeat/assets/85433724/7b05bf58-cc0b-4ec9-8e49-55d117673df8)
+## Manual QA Environments Updates
+
+For qa environments, to avoid [automatic deployment deletion](https://docs.elastic.dev/serverless/qa) be sure to update the deployment name with the prefix `keep_`.
+
+
+1. Go to [`Projects > Project > "Project Deployment Name"`](https://console.qa.cld.elstc.co/)
+
+
+
+2. Update the *Project name* field value with the prefix `keep_`
+
+
+Note that when you go to destroy the environment, be sure to remove `keep_` from the deployment name.
## Tracking Workflow Execution
@@ -131,7 +149,7 @@ Follow these steps to connect to your Amazon Elastic Kubernetes Service (EKS) cl
```kubectl get po -n kube-system```
- This command should list the pods in the kube-system namespace, confirming that you have successfully connected to your EKS cluster.
+ This command should list the pods in the kube-system namespace, confirming that you have successfully connected to your EKS cluster.
## Cleanup Procedure