Add AWS tests to Jenkinsfile (#17480) (#18163)

Add execution of AWS integration tests for Metricbeat to Jenkinsfile.
For that, a simple terraform scenario is created that seems to be
enough to pass the AWS module tests, this scenario is started by
Jenkins, and destroyed as a cleanup step. With this approach terraform
scenarios are defined per module, this is consequent with other efforts
we are doing with other integrations, where integration test scenarios
are defined at the module level. Similar approach will be possibly
followed for input integration tests.

Most of the logic is added in the Jenkinsfile and as scripts. Some things
could be moved to mage when we modify our targets to start scenarios
depending on the type of provisioner. Some parts are going to continue
being needed in Jenkinsfile in my opinion, as the archive of Terraform
states for manual cleanups.

(cherry picked from commit c94d4bf)

.ci/scripts/install-terraform.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+MSG="parameter missing."
+TERRAFORM_VERSION=${TERRAFORM_VERSION:?$MSG}
+HOME=${HOME:?$MSG}
+TERRAFORM_CMD="${HOME}/bin/terraform"
+
+OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo - "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${OS}_amd64.zip" > ${TERRAFORM_CMD}.zip
+unzip -o ${TERRAFORM_CMD}.zip -d $(dirname ${TERRAFORM_CMD})
+rm ${TERRAFORM_CMD}.zip
+
+chmod +x "${TERRAFORM_CMD}"

.ci/scripts/terraform-cleanup.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+DIRECTORY=${1:-.}
+
+FAILED=0
+for tfstate in $(find $DIRECTORY -name terraform.tfstate); do
+  cd $(dirname $tfstate)
+  if ! terraform destroy -auto-approve; then
+    FAILED=1
+  fi
+  cd -
+done
+
+exit $FAILED

.gitignore

@@ -39,3 +39,7 @@ x-pack/dockerlogbeat/temproot.tar
 *.test
 *.prof
 *.pyc
+
+# Terraform
+*.terraform
+*.tfstate*

Jenkinsfile

@@ -15,9 +15,11 @@ pipeline {
     BASE_DIR = 'src/github.com/elastic/beats'
     GOX_FLAGS = "-arch amd64"
     DOCKER_COMPOSE_VERSION = "1.21.0"
+    TERRAFORM_VERSION = "0.12.24"
     PIPELINE_LOG_LEVEL = "INFO"
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_REGISTRY = 'docker.elastic.co'
+    AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
     RUNBLD_DISABLE_NOTIFICATIONS = 'true'
   }
   options {
@@ -36,6 +38,11 @@ pipeline {
     booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.')
     booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.')
     booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.')
+
+    booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.')
+    booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.')
+    string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.')
+
     booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
     booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
   }
@@ -352,8 +359,30 @@ pipeline {
               return env.BUILD_METRICBEAT_XPACK != "false"
             }
           }
-          steps {
-            mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test")
+          stages {
+            stage('Prepare cloud integration tests environments'){
+              agent { label 'ubuntu && immutable' }
+              options { skipDefaultCheckout() }
+              steps {
+                startCloudTestEnv('x-pack-metricbeat', [
+                   [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'],
+                ])
+              }
+            }
+            stage('Metricbeat x-pack'){
+              agent { label 'ubuntu && immutable' }
+              options { skipDefaultCheckout() }
+              steps {
+                withCloudTestEnv() {
+                  mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test")
+                }
+              }
+            }
+          }
+          post {
+            cleanup {
+              terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat')
+            }
           }
         }
         stage('Metricbeat crosscompile'){
@@ -671,7 +700,7 @@ def withBeatsEnv(boolean archive, Closure body) {
     "TEST_COVERAGE=true",
     "RACE_DETECTOR=true",
     "PYTHON_ENV=${WORKSPACE}/python-env",
-    "TEST_TAGS=oracle",
+    "TEST_TAGS=${env.TEST_TAGS},oracle",
     "DOCKER_PULL=0",
   ]) {
     deleteDir()
@@ -738,6 +767,7 @@ def installTools() {
   if(isUnix()) {
     retry(i) { sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") }
     retry(i) { sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") }
+    retry(i) { sh(label: "Install Terraform ${TERRAFORM_VERSION}", script: ".ci/scripts/install-terraform.sh") }
     retry(i) { sh(label: "Install Mage", script: "make mage") }
   } else {
     retry(i) { bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") }
@@ -809,6 +839,7 @@ def dumpFilteredEnvironment(){
   echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}"
   echo "STRESS_TESTS: ${env.STRESS_TESTS}"
   echo "STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS}"
+  echo "TEST_TAGS: ${env.TEST_TAGS}"
   echo "GOX_OS: ${env.GOX_OS}"
   echo "GOX_OSARCH: ${env.GOX_OSARCH}"
   echo "GOX_FLAGS: ${env.GOX_FLAGS}"
@@ -895,6 +926,98 @@ def isChangedXPackCode(patterns) {
   return isChanged(allPatterns)
 }
 
+// withCloudTestEnv executes a closure with credentials for cloud test
+// environments.
+def withCloudTestEnv(Closure body) {
+  def maskedVars = []
+  def testTags = "${env.TEST_TAGS}"
+
+  // AWS
+  if (params.allCloudTests || params.awsCloudTests) {
+    testTags = "${testTags},aws"
+    def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data
+    if (!aws.containsKey('access_key')) {
+      error("${AWS_ACCOUNT_SECRET} doesn't contain 'access_key'")
+    }
+    if (!aws.containsKey('secret_key')) {
+      error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'")
+    }
+    maskedVars.addAll([
+      [var: "AWS_REGION", password: params.awsRegion],
+      [var: "AWS_ACCESS_KEY_ID", password: aws.access_key],
+      [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key],
+    ])
+  }
+
+  withEnv([
+    "TEST_TAGS=${testTags}",
+  ]) {
+    withEnvMask(vars: maskedVars) {
+      body()
+    }
+  }
+}
+
+def terraformInit(String directory) {
+  dir(directory) {
+    sh(label: "Terraform Init on ${directory}", script: "terraform init")
+  }
+}
+
+def terraformApply(String directory) {
+  terraformInit(directory)
+  dir(directory) {
+    sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve")
+  }
+}
+
+// Start testing environment on cloud using terraform. Terraform files are
+// stashed so they can be used by other stages. They are also archived in
+// case manual cleanup is needed.
+//
+// Example:
+//   startCloudTestEnv('x-pack-metricbeat', [
+//     [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'],
+//   ])
+//   ...
+//   terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat')
+def startCloudTestEnv(String name, environments = []) {
+  withCloudTestEnv() {
+    withBeatsEnv(false) {
+      def runAll = params.runAllCloudTests
+      try {
+        for (environment in environments) {
+          if (environment.cond || runAll) {
+            retry(2) {
+              terraformApply(environment.dir)
+            }
+          }
+        }
+      } finally {
+        // Archive terraform states in case manual cleanup is needed.
+        archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate')
+      }
+      stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**')
+    }
+  }
+}
+
+
+// Looks for all terraform states in directory and runs terraform destroy for them,
+// it uses terraform states previously stashed by startCloudTestEnv.
+def terraformCleanup(String stashName, String directory) {
+  stage("Remove cloud scenarios in ${directory}"){
+    withCloudTestEnv() {
+      withBeatsEnv(false) {
+        unstash "terraform-${stashName}"
+        retry(2) {
+          sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}")
+        }
+      }
+    }
+  }
+}
+
 def loadConfigEnvVars(){
   def empty = []
   env.GO_VERSION = readFile(".go-version").trim()

dev-tools/mage/common.go

@@ -833,3 +833,28 @@ func ListMatchingEnvVars(prefixes ...string) []string {
 	}
 	return vars
 }
+
+// IntegrationTestEnvVars returns the names of environment variables needed to configure
+// connections to integration test environments.
+func IntegrationTestEnvVars() []string {
+	// Environment variables that can be configured with paths to files
+	// with authentication information.
+	vars := []string{
+		"AWS_SHARED_CREDENTIAL_FILE",
+		"AZURE_AUTH_LOCATION",
+		"GOOGLE_APPLICATION_CREDENTIALS",
+	}
+	// Environment variables with authentication information.
+	prefixes := []string{
+		"AWS_",
+		"AZURE_",
+
+		// Accepted by terraform, but not by many clients, including Beats
+		"GOOGLE_",
+		"GCLOUD_",
+	}
+	for _, prefix := range prefixes {
+		vars = append(vars, ListMatchingEnvVars(prefix)...)
+	}
+	return vars
+}

dev-tools/mage/gotest.go

@@ -156,7 +156,9 @@ func GoTestIntegrationForModule(ctx context.Context) error {
 		foundModule = true
 
 		// Set MODULE because only want that modules tests to run inside the testing environment.
-		runners, err := NewIntegrationRunners(path.Join("./module", fi.Name()), map[string]string{"MODULE": fi.Name()})
+		env := map[string]string{"MODULE": fi.Name()}
+		passThroughEnvs(env, IntegrationTestEnvVars()...)
+		runners, err := NewIntegrationRunners(path.Join("./module", fi.Name()), env)
 		if err != nil {
 			return errors.Wrapf(err, "test setup failed for module %s", fi.Name())
 		}

docs/devguide/terraform.asciidoc

@@ -0,0 +1,101 @@
+[[terraform-beats]]
+== Terraform in Beats
+
+Terraform is used to provision scenarios for integration testing of some cloud
+features. Features implementing integration tests that require the presence of
+cloud resources should have their own Terraform configuration, this configuration
+can be used when developing locally to create (and destroy) resources that allow
+to test these features.
+
+Tests requiring access to cloud providers should be disabled by default with the
+use of build tags.
+
+[[installing-terraform]]
+=== Installing Terraform
+
+Terraform is available in https://www.terraform.io/downloads.html
+
+Download it and place it in some directory in your PATH.
+
+`terraform` is the main command for Terraform and the only one that is usually
+needed to manage configurations. Terraform will also download other plugins that
+implement the specific functionality for each provider. These plugins are
+automatically managed and stored in the working copy, if you want to share the
+plugins between multiple working copies you can manually install them in the
+user the user plugins directory located at `~/.terraform.d/plugins`,
+or `%APPDATA%\terraform.d\plugins on Windows`.
+
+Plugins are available in https://registry.terraform.io/
+
+[[using-terraform]]
+=== Using Terraform
+
+The most important commands when using Terraform are:
+* `terraform init` to do some initial checks and install the required plugins.
+* `terraform apply` to create the resources defined in the configuration.
+* `terraform destroy` to destroy resources previously created.
+
+Cloud providers use to require credentials, they can be provided with the usual
+methods supported by these providers, using environment variables and/or
+credential files.
+
+Terraform stores the last known state of the resources managed by a
+configuration in a `terraform.tfstate` file. It is important to keep this file
+as it is used as input by `terraform destroy`. This file is created in the same
+directory where `terraform apply` is executed.
+
+Please take a look to Terraform documentation for more details: https://www.terraform.io/intro/index.html
+
+[[terraform-configurations]]
+=== Terraform configuration guidelines
+
+The main purpouse of Terraform in Beats is to create and destroy cloud resources
+required by integration tests. For these configurations there are some things to
+take into account:
+* Apply should work without additional inputs or files. Only input will be the
+  required for specific providers, using environment variables or credential
+  files.
+* You must be able to apply the same configuration multiple times in the same
+  account. This will allow to have multiple builds using the same configuration
+  but with different instances of the resources. Some resources are already
+  created with unique identifiers (as EC2 instances), some others have to be
+  explicitly created with unique names (e.g. S3 buckets). For these cases random
+  suffixes can be added to identifiers.
+* Destroy must work without additional input, and should be able to destroy all
+  the resources created by the configuration. There are some resources that need
+  specific flags to be destroyed by `terraform destroy`. For example S3 buckets
+  need a flag to force to empty the bucket before deleting it, or RDS instances
+  need a flag to disable snapshots on deletion.
+
+[[terraform-in-ci]]
+=== Terraform in CI
+
+Integration tests that need the presence of certain resources to work can be
+executed in CI if they provide a Terraform configuration to start these
+resources. These tests are disabled by default in CI.
+
+Terraform states are archived as artifacrs of builds, this allows to manually
+destroy resources created by builds that were not able to do a proper cleanup.
+
+Here is a checklist to add support for a cloud feature in Jenkins:
+* In the feature code:
+  * Tests have a build tag so they are disabled by default. When run from mage,
+    its execution can be selected using the `TEST_TAGS` environment variable, e.g:
+    `TEST_TAGS=aws` for AWS tests.
+  * There is some Terraform configuration that defines a cloud scenario where
+    tests pass. This configuration should be in the directory of the feature.
+* In the Jenkinsfile:
+  * Add a boolean parameter to run the tests on this environment, e.g.
+    `awsCloudTests`. This parameter should be set to false by default.
+  * Add a conditional block in `withCloudTestEnv` that:
+     * Will be executed if the previously added boolean parameter, or `allCloudTests`
+       are set to true.
+     * Adds the tag to `TEST_TAGS` (as comma separated values), so tests are
+       selected.
+     * Defines how to obtain the credentials and provide them to the tests.
+  * In the stage of the specific beat:
+    * Add a stage that calls to `startCloudTestEnv`, if there isn't anyone.
+    * Add a post cleanup step that calls to `terraformCleanup`, if there isn't anyone.
+    * Add a environment to the list of environments started by `startCloudEnv`,
+      with the condition to start the scenario, and the path to the directory
+      with its definition, e.g. `[cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws']`