Merge branch 'elastic:main' into main

agithomas · May 13, 2024 · 3d1d720 · 3d1d720
2 parents db95d41 + c773028
commit 3d1d720
Show file tree

Hide file tree

Showing 96 changed files with 2,078 additions and 1,879 deletions.
diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -22,6 +22,10 @@ env:
   # Other deps
   ASDF_MAGE_VERSION: 1.15.0
 
+  # Unit tests
+  RACE_DETECTOR: "true"
+  TEST_COVERAGE: "true"
+
 steps:
   - group: "Auditbeat Mandatory Testing"
     key: "auditbeat-mandatory-tests"
@@ -34,7 +38,7 @@ steps:
           mage build unitTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -53,7 +57,7 @@ steps:
           mage build unitTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
@@ -71,7 +75,7 @@ steps:
           mage build unitTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2016}"
@@ -91,7 +95,7 @@ steps:
           mage build unitTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2022}"
@@ -112,7 +116,7 @@ steps:
           GOX_FLAGS: "-arch amd64"
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -135,7 +139,7 @@ steps:
           mage build integTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -156,7 +160,7 @@ steps:
           mage build integTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -177,7 +181,7 @@ steps:
           mage build unitTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -198,7 +202,7 @@ steps:
           mage build unitTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -218,7 +222,7 @@ steps:
           mage build unitTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
@@ -241,7 +245,7 @@ steps:
           mage build unitTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -262,7 +266,7 @@ steps:
           mage build unitTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
@@ -283,7 +287,7 @@ steps:
           mage build unitTest
         retry:
           automatic:
-           - limit: 3
+            - limit: 3
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
@@ -316,6 +320,10 @@ steps:
           set -euo pipefail
           cd auditbeat
           mage package
+        retry:
+          automatic:
+            - limit: 3
+        timeout_in_minutes: 20
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -334,6 +342,10 @@ steps:
           set -euo pipefail
           cd auditbeat
           mage package
+        retry:
+          automatic:
+            - limit: 3
+        timeout_in_minutes: 20
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"

diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml
@@ -3,78 +3,113 @@
 env:
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  MODULE: "kubernetes"
+
+  # Other deps
+  ASDF_KIND_VERSION: "0.20.0"
 
 steps:
   - group: "Deploy/K8S"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") =~ /.*kubernetes.*/
 
     steps:
     - label: "Checks"
-      command: ".buildkite/deploy/kubernetes/scripts/make.sh"
+      command: |
+        set -euo pipefail
+        make -C deploy/kubernetes all
+        make check-no-changes
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "Deploy/k8s-checks"
+            context: "deploy/k8s checks"
 
     - label: "K8S Test/K8S version: v1.29.0"
       key: "k8s-test-129"
       env:
         K8S_VERSION: "v1.29.0"
-      commands:
-        - "MODULE=kubernetes make -C metricbeat integration-tests"
-        - "make -C deploy/kubernetes test"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
+      retry:
+        automatic:
+          - limit: 3
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "Deploy/k8s-test v1.29.0"
+            context: "deploy/k8s test v1.29.0"
 
     - label: "K8S Test/K8S version: v1.28.0"
       key: "k8s-test-128"
       env:
         K8S_VERSION: "v1.28.0"
-      commands:
-        - "MODULE=kubernetes make -C metricbeat integration-tests"
-        - "make -C deploy/kubernetes test"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
+      retry:
+        automatic:
+          - limit: 3
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "Deploy/k8s-test v1.28.0"
+            context: "deploy/k8s test v1.28.0"
 
     - label: "K8S Test/K8S version: v1.27.3"
       key: "k8s-test-1273"
       env:
         K8S_VERSION: "v1.27.3"
-      commands:
-        - "MODULE=kubernetes make -C metricbeat integration-tests"
-        - "make -C deploy/kubernetes test"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
+      retry:
+        automatic:
+          - limit: 3
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "Deploy/k8s-test v1.27.3"
+            context: "deploy/k8s test v1.27.3"
 
     - label: "K8S Test/K8S version: v1.26.6"
       key: "k8s-test-1266"
       env:
         K8S_VERSION: "v1.26.6"
-      commands:
-        - "MODULE=kubernetes make -C metricbeat integration-tests"
-        - "make -C deploy/kubernetes test"
+        MODULE: "${MODULE}"
+      commands: |
+        set -euo pipefail
+        source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
+        echo "--- Executing Tests"
+        make -C metricbeat integration-tests
+        make -C deploy/kubernetes test
+      retry:
+        automatic:
+          - limit: 3
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
       notify:
         - github_commit_status:
-            context: "Deploy/k8s-test v1.26.6"
+            context: "deploy/k8s test v1.26.6"
diff --git a/.buildkite/deploy/kubernetes/scripts/install-kind.sh b/.buildkite/deploy/kubernetes/scripts/install-kind.sh
diff --git a/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh b/.buildkite/deploy/kubernetes/scripts/install-kubectl.sh
diff --git a/.buildkite/deploy/kubernetes/scripts/make.sh b/.buildkite/deploy/kubernetes/scripts/make.sh
diff --git a/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh b/.buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
@@ -4,11 +4,15 @@ set -euo pipefail
 
 source .buildkite/env-scripts/util.sh
 
+export KUBECONFIG="${WORKSPACE}/kubecfg"
+export BIN="${WORKSPACE}/bin"
+add_bin_path
+
 echo "--- Installing kind & kubectl"
-retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kind.sh
-retry_with_count 5 .buildkite/deploy/kubernetes/scripts/install-kubectl.sh
+asdf plugin add kind
+asdf install kind $ASDF_KIND_VERSION
 
-echo "--- Setting up kind"
+echo "~~~ Setting up kind"
 max_retries=3
 timeout=5
 retries=0