[CI] 2.0 pipeline for 6.8 (#21859)

* cherry-pick aad07ad
* Add stages to Jenkins pipeline (#14484)
* ci: avoid to modify go.mod file (#16842)
* Ensure install scripts only install if needed (#20349)
* ci: fix warnings with wildcards and archive system-tests (#18695)
* ci: run test on Windows (#15570)
* [CI] fail if not possible to install python3 (#19164)
* [CI] lint stage doesn't produce test reports (#21888)
* [CI] Add stage name in the step (#21887)
* [CI] kind setup fails sometimes (#21857)
* [CI] refactor runbld post build action (#21256)
* [CI] Archive build reasons (#21347)
* [CI] Fix runbld when workspace does not exist (#21350)
* [CI] Send slack message with build status (#21428)
* [CI] Setup git config globally (#21562)
* [CI] fix 'no matches found within 10000' (#21466)
* [CI] Change notification channel (#21559)
* [CI] Support skip-ci label (#21377)
* [BUILD][CI] fetch dependencies with retry (#21614)
* Disable x-pack/metricbeat in linux since build is broken #21962
* Disable tests for windows in auditbeat. See #21958
* Disable integration testing for libbeat. See #21959
* Disable integration testing for metricbeat. See #21961
* Disable windows build for auditbeat. See #21958
* Disable build for libbeat. See #21959
* Disable build for metricbeat. See #21961
* Disable windows build for metricbeat. See #22257
* Disable build for x-pack/metricbeat. See #21962
* Disable windows build for x-pack/winlogbeat. See #22263
* Disable windows build for x-pack/filebeat. See #21963
* integration-tests-environment is not available for winlogbeat. see #22265
* integration-tests-environment is not available for packetbeat. see #22266
* Build not available for x-pack/metricbeats in 6.8. See #22269
* Cherry-pick 96c60d0

.ci/scripts/generate_build_table.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+import os
+import yaml
+
+if __name__ == "__main__":
+
+    print("| Beat  | Stage  | Command  | MODULE  | Platforms  | When |")
+    print("|-------|--------|----------|---------|------------|------|")
+    for root, dirs, files in os.walk("."):
+        dirs.sort()
+        for file in files:
+            if file.endswith("Jenkinsfile.yml") and root != ".":
+                with open(os.path.join(root, file), 'r') as f:
+                    doc = yaml.load(f, Loader=yaml.FullLoader)
+                module = root.replace(".{}".format(os.sep), '')
+                platforms = [doc["platform"]]
+                when = ""
+                if "branches" in doc["when"]:
+                    when = f"{when}/:palm_tree:"
+                if "changeset" in doc["when"]:
+                    when = f"{when}/:file_folder:"
+                if "comments" in doc["when"]:
+                    when = f"{when}/:speech_balloon:"
+                if "labels" in doc["when"]:
+                    when = f"{when}/:label:"
+                if "parameters" in doc["when"]:
+                    when = f"{when}/:smiley:"
+                if "tags" in doc["when"]:
+                    when = f"{when}/:taco:"
+                for stage in doc["stages"]:
+                    withModule = False
+                    if "make" in doc["stages"][stage]:
+                        command = doc["stages"][stage]["make"]
+                    if "mage" in doc["stages"][stage]:
+                        command = doc["stages"][stage]["mage"]
+                    if "platforms" in doc["stages"][stage]:
+                        platforms = doc["stages"][stage]["platforms"]
+                    if "withModule" in doc["stages"][stage]:
+                        withModule = doc["stages"][stage]["withModule"]
+                    if "when" in doc["stages"][stage]:
+                        when = f"{when}/:star:"
+                    print("| {} | {} | `{}` | {} | `{}` | {} |".format(
+                        module, stage, command, withModule, platforms, when))
+
+print("> :palm_tree: -> Git Branch based")
+print("> :label: -> GitHub Pull Request Label based")
+print("> :file_folder: -> Changeset based")
+print("> :speech_balloon: -> GitHub Pull Request comment based")
+print("> :taco: -> Git tag based")
+print("> :smiley: -> Manual UI interaction based")
+print("> :star: -> More specific cases based")

.ci/scripts/get-vendor-dependencies.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Given the go module it will list all the dependencies that will be later on
+# used by the CI to enable/disable specific stages as long as the changeset
+# matches any of those patterns.
+#
+
+GO_VERSION=${GO_VERSION:?"GO_VERSION environment variable is not set"}
+BEATS=${1:?"parameter missing."}
+eval "$(gvm "${GO_VERSION}")"
+
+go list -deps ./"${BEATS}" \
+| grep 'elastic/beats' \
+| sort \
+| sed -e "s#github.com/elastic/beats/v7/##g" \
+| awk '{print "^" $1 "/.*"}'

.ci/scripts/install-docker-compose.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+MSG="environment variable missing: DOCKER_COMPOSE_VERSION."
+DOCKER_COMPOSE_VERSION=${DOCKER_COMPOSE_VERSION:?$MSG}
+HOME=${HOME:?$MSG}
+
+if command -v docker-compose
+then
+    echo "Found docker-compose. Checking version.."
+    FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version|awk '{print $3}'|sed s/\,//)
+    if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]
+    then
+        echo "Versions match. No need to install docker-compose. Exiting."
+        exit 0
+    fi
+fi
+
+echo "UNMET DEP: Installing docker-compose"
+
+DC_CMD="${HOME}/bin/docker-compose"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${DC_CMD}" "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)"
+chmod +x "${DC_CMD}"

.ci/scripts/install-go.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+MSG="environment variable missing"
+GO_VERSION=${GO_VERSION:?$MSG}
+PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"}
+HOME=${HOME:?$MSG}
+ARCH=$(uname -s| tr '[:upper:]' '[:lower:]')
+GVM_CMD="${HOME}/bin/gvm"
+
+if command -v go
+then
+    echo "Found Go. Checking version.."
+    FOUND_GO_VERSION=$(go version|awk '{print $3}'|sed s/go//)
+    if [ $FOUND_GO_VERSION == $GO_VERSION ]
+    then
+        echo "Versions match. No need to install Go. Exiting."
+        exit 0
+    fi
+fi
+
+echo "UNMET DEP: Installing Go"
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.1/gvm-${ARCH}-amd64"
+chmod +x "${GVM_CMD}"
+
+gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE}
+
+eval $(gvm ${GO_VERSION})

.ci/scripts/install-kind.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+MSG="environment variable missing."
+DEFAULT_HOME="/usr/local"
+KIND_VERSION=${KIND_VERSION:?$MSG}
+HOME=${HOME:?$DEFAULT_HOME}
+KIND_CMD="${HOME}/bin/kind"
+
+if command -v kind
+then
+    echo "Found Kind. Checking version.."
+    FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}')
+    if [ $FOUND_KIND_VERSION == $KIND_VERSION ]
+    then
+        echo "Versions match. No need to install Kind. Exiting."
+        exit 0
+    fi
+fi
+
+echo "UNMET DEP: Installing Kind"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${KIND_CMD}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64"
+chmod +x "${KIND_CMD}"

.ci/scripts/install-terraform.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+MSG="environment variable missing."
+TERRAFORM_VERSION=${TERRAFORM_VERSION:?$MSG}
+HOME=${HOME:?$MSG}
+TERRAFORM_CMD="${HOME}/bin/terraform"
+
+OS=$(uname -s | tr '[:upper:]' '[:lower:]')
+
+if command -v terraform
+then
+    echo "Found Terraform. Checking version.."
+    FOUND_TERRAFORM_VERSION=$(terraform --version | awk '{print $2}' | sed s/v//)
+    if [ $FOUND_TERRAFORM_VERSION == $TERRAFORM_VERSION ]
+    then
+        echo "Versions match. No need to install Terraform. Exiting."
+        exit 0
+    fi
+fi
+
+echo "UNMET DEP: Installing Terraform"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo - "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${OS}_amd64.zip" > ${TERRAFORM_CMD}.zip
+unzip -o ${TERRAFORM_CMD}.zip -d $(dirname ${TERRAFORM_CMD})
+rm ${TERRAFORM_CMD}.zip
+
+chmod +x "${TERRAFORM_CMD}"

.ci/scripts/install-tools.bat

@@ -0,0 +1,25 @@
+set GOPATH=%WORKSPACE%
+set MAGEFILE_CACHE=%WORKSPACE%\.magefile
+set PATH=%WORKSPACE%\bin;C:\ProgramData\chocolatey\bin;%PATH%
+
+where /q curl
+IF ERRORLEVEL 1 (
+ choco install curl -y --no-progress --skipdownloadcache
+)
+mkdir %WORKSPACE%\bin
+where /q gvm
+IF ERRORLEVEL 1 (
+ curl -sL -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.1/gvm-windows-amd64.exe
+)
+FOR /f "tokens=*" %%i IN ('"gvm.exe" use %GO_VERSION% --format=batch') DO %%i
+
+go install github.com/elastic/beats/vendor/github.com/magefile/mage
+mage -version
+where mage
+
+if not exist C:\Python38\python.exe (
+    REM Install python 3.8.
+    choco install python -y -r --no-progress --version 3.8.2 || echo ERROR && exit /b
+)
+python --version
+where python

.ci/scripts/install-tools.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+.ci/scripts/install-go.sh
+.ci/scripts/install-docker-compose.sh
+.ci/scripts/install-terraform.sh
+make mage

.ci/scripts/kind-setup.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+MSG="parameter missing."
+K8S_VERSION=${K8S_VERSION:?$MSG}
+HOME=${HOME:?$MSG}
+KBC_CMD="${HOME}/bin/kubectl"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
+chmod +x "${KBC_CMD}"
+
+GO111MODULE="on" go get sigs.k8s.io/kind@v0.5.1
+kind create cluster --image kindest/node:${K8S_VERSION}
+
+export KUBECONFIG="$(kind get kubeconfig-path)"
+kubectl cluster-info

.ci/scripts/minikube-setup.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+MSG="parameter missing."
+K8S_VERSION=${K8S_VERSION:?$MSG}
+MINIKUBE_VERSION=${MINIKUBE_VERSION:?$MSG}
+HOME=${HOME:?$MSG}
+
+KBC_CMD="${HOME}/bin/kubectl"
+MKB_CMD="${HOME}/bin/minikube"
+
+export CHANGE_MINIKUBE_NONE_USER=true
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
+chmod +x "${KBC_CMD}"
+
+curl -sSLo "${MKB_CMD}" "https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64"
+chmod +x "${MKB_CMD}"
+
+mkdir -p "${HOME}/.kube" "${HOME}/.minikube"
+touch "${HOME}/.kube/config"
+
+minikube start --vm-driver=none --kubernetes-version=${K8S_VERSION} --logtostderr
+minikube update-context
+
+JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
+until kubectl get nodes -o jsonpath="${JSONPATH}" 2>&1 | grep -q "Ready=True"
+do
+  echo "waiting for Minikube..."
+  sleep 5
+done

.ci/scripts/pre_archive_test.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+
+import os
+import distutils
+from distutils import dir_util
+
+
+if __name__ == "__main__":
+
+    if not os.path.exists('build'):
+        os.makedirs('build')
+
+    # Top level folders to be excluded
+    EXCLUDE = set(['.ci', '.git', '.github', 'vendor', 'dev-tools'])
+    for root, dirs, files in os.walk('.'):
+        dirs[:] = [d for d in dirs if d not in EXCLUDE]
+        if root.endswith(('build')) and not root.startswith((".{}build".format(os.sep))):
+            dest = os.path.join('build', root.replace(".{}".format(os.sep), ''))
+            print("Copy {} into {}".format(root, dest))
+            distutils.dir_util.copy_tree(root, dest, preserve_symlinks=1)

.ci/scripts/search_system_tests.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+
+import os
+
+
+if __name__ == "__main__":
+
+    for root, dirs, files in os.walk('build'):
+        if root.endswith(('system-tests')):
+            print(root.replace(".{}".format(os.sep), ''))