Migrate packetbeat to static Buildkite pipeline (#39087)

This commit refactors the packetbeat pipeline to the pipeline selector
approach as laid out in #38783.

Relates elastic/ingest-dev#3072

.buildkite/hooks/pre-command

@@ -59,7 +59,6 @@ ENABLED_BEATS_PIPELINES_SLUGS=(
   "heartbeat"
   "filebeat"
   "beats-metricbeat"
-  "beats-packetbeat"
   "beats-winlogbeat"
   "beats-winlogbeat"
   "beats-xpack-packetbeat"

.buildkite/packetbeat/pipeline.packetbeat.yml

@@ -2,12 +2,12 @@
 name: "beats-packetbeat"
 
 env:
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
-  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
-  BEATS_PROJECT_NAME: "packetbeat"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+  IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
   IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
@@ -17,52 +17,228 @@ env:
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
 
-  #Packaging
-  PACKAGING_ARM_PLATFORMS: "linux/arm64"
-  PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-
   #Deps
   ASDF_MAGE_VERSION: 1.15.0
 
 steps:
-  - input: "Input Parameters"
-    key: "input-run-all-stages"
-    fields:
-      - select: "Packetbeat - run_packetbeat"
-        key: "run_packetbeat"
-        options:
-          - label: "True"
-            value: "true"
-          - label: "False"
-            value: "false"
-        default: "false"
-      - select: "Packetbeat - run_packetbeat_arm_tests"
-        key: "run_packetbeat_arm_tests"
-        options:
-          - label: "True"
-            value: "true"
-          - label: "False"
-            value: "false"
-        default: "false"
-      - select: "Packetbeat - run_packetbeat_macos_tests"
-        key: "run_packetbeat_macos_tests"
-        options:
-          - label: "True"
-            value: "true"
-          - label: "False"
-            value: "false"
-        default: "false"
-    if: "build.source == 'ui'"
+  - group: "packetbeat Mandatory Tests"
+    key: "packetbeat-mandatory-tests"
+    steps:
+      - label: ":linux: Ubuntu Unit Tests"
+        command: |
+          cd packetbeat
+          mage build unitTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths:
+          - "packetbeat/build/*.xml"
+          - "packetbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "packetbeat: Ubuntu Unit Tests"
+
+      - label: ":rhel: RHEL9 Unit Tests"
+        command: |
+          cd packetbeat
+          mage build unitTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_RHEL9_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths:
+          - "packetbeat/build/*.xml"
+          - "packetbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "packetbeat: RHEL9 Unit Tests"
+
+      - label: ":windows: Windows 2016 Unit Tests"
+        command: |
+          Set-Location -Path packetbeat
+          mage build unitTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2016}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "packetbeat/build/*.xml"
+          - "packetbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "packetbeat: Windows 2016 Unit Tests"
+
+      - label: ":windows: Windows 2022 Unit Tests"
+        command: |
+          Set-Location -Path packetbeat
+          mage build unitTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2022}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "packetbeat/build/*.xml"
+          - "packetbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "packetbeat: Windows 2022 Unit Tests"
+
+  - group: "Extended Windows Tests"
+    key: "packetbeat-extended-windows-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+    steps:
+      - label: ":windows: Windows 10 Unit Tests"
+        command: |
+          Set-Location -Path packetbeat
+          mage build unitTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_10}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "packetbeat/build/*.xml"
+          - "packetbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "packetbeat: Extended Windows 10 Unit Tests"
+
+      - label: ":windows: Windows 11 Unit Tests"
+        command: |
+          Set-Location -Path packetbeat
+          mage build unitTest
+        key: "extended-win-11-unit-tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_11}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "packetbeat/build/*.xml"
+          - "packetbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "packetbeat: Extended Windows 11 Unit Tests"
+
+      - label: ":windows: Windows 2019 Unit Tests"
+        command: |
+          Set-Location -Path packetbeat
+          mage build unitTest
+        key: "extended-win-2019-unit-tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2019}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "packetbeat/build/*.xml"
+          - "packetbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "packetbeat: Extended Windows 2019 Unit Tests"
+
+  - group: "Extended Tests"
+    key: "extended-tests"
+    steps:
+      - label: ":mac: MacOS x86_64 Unit Tests"
+        key: "macos-x86-64-unit-tests-extended"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        command: |
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
+          cd packetbeat
+          mage build unitTest
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        artifact_paths:
+          - "packetbeat/build/*.xml"
+          - "packetbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "packetbeat: Extended MacOS x86_64 Unit Tests"
+
+      - label: ":mac: MacOS arm64 Unit Tests"
+        key: "macos-arm64-unit-tests-extended"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/
+        command: |
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
+          cd packetbeat
+          mage build unitTest
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"
+        artifact_paths:
+          - "packetbeat/build/*.xml"
+          - "packetbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "packetbeat: Extended MacOS arm64 Unit Tests"
+
+      - label: ":linux: Ubuntu ARM Unit Tests"
+        key: "linux-arm64-unit-tests-extended"
+        command: "cd packetbeat && mage build unitTest"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - "packetbeat/build/*.xml"
+          - "packetbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "packetbeat: Extended Ubuntu ARM Unit Tests"
+
 
   - wait: ~
-    if: "build.source == 'ui'"
-    allow_dependency_failure: false
-
-  - label: ":linux: Load dynamic packetbeat pipeline"
-    key: "packetbeat-pipeline"
-    command: ".buildkite/scripts/generate_packetbeat_pipeline.sh"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest"
-    notify:
-      - github_commit_status:
-          context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"
+    # with PRs, we want to run packaging only if mandatory tests succeed
+    # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in mandatory tests
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "packetbeat-mandatory-tests"
+
+  - group: "Packetbeat Packaging"
+    key: "packaging"
+    steps:
+      - label: ":linux: Packaging Linux"
+        key: "packaging-linux"
+        command: |
+          cd packetbeat
+          mage package
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+        notify:
+          - github_commit_status:
+              context: "packetbeat: Packaging Linux"
+
+      - label: ":linux: Packaging ARM"
+        key: "packaging-arm"
+        command: |
+          cd packetbeat
+          mage package
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
+        notify:
+          - github_commit_status:
+              context: "packetbeat: Packaging Linux ARM"

.buildkite/pipeline.yml

@@ -392,3 +392,29 @@ steps:
                     - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
                     - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
                     - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+  - label: "Trigger Packetbeat"
+    plugins:
+      - monorepo-diff#v1.0.1:
+          diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+          watch:
+            - path:
+                - packetbeat/
+                - .buildkite/packetbeat/pipeline.packetbeat.yml
+                - .buildkite/scripts/
+                - .buildkite/hooks/
+                #OSS
+                - go.mod
+                - pytest.ini
+                - dev-tools/
+                - libbeat/
+                - testing/
+              config:
+                trigger: "beats-packetbeat"
+                build:
+                  commit: "${BUILDKITE_COMMIT}"
+                  branch: "${BUILDKITE_BRANCH}"
+                  env:
+                    - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                    - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                    - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}

.buildkite/scripts/common.sh

@@ -13,27 +13,20 @@ OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
 XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
 # define if needed run the whole pipeline for the particular beat
 [ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")"
-[ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")"
 [ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")"
 [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")"
 
 # define if needed run ARM platform-specific tests for the particular beat
 [ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")"
-[ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")"
 [ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")"
 
 # define if needed run MacOS platform-specific tests for the particular beat
-[ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")"
 [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")"
 [ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")"
 
 # define if needed run cloud-specific tests for the particular beat
 [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")"
 
-packetbeat_changeset=(
-  "^packetbeat/.*"
-  )
-
 winlogbeat_changeset=(
   "^winlogbeat/.*"
   )
@@ -77,9 +70,6 @@ packaging_changeset=(
   )
 
 case "${BUILDKITE_PIPELINE_SLUG}" in
-  "beats-packetbeat")
-    BEAT_CHANGESET_REFERENCE=${packetbeat_changeset[@]}
-    ;;
   "beats-xpack-metricbeat")
     BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]}
     ;;
@@ -323,20 +313,9 @@ are_conditions_met_mandatory_tests() {
   return 1
 }
 
-are_conditions_met_arm_tests() {
-  if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
-    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
-      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then
-        return 0
-      fi
-    fi
-  fi
-  return 1
-}
-
 are_conditions_met_macos_tests() {
   if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
-    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
       if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
         return 0
       fi