Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.14](backport #39087) Migrate packetbeat to static Buildkite pipeline #39108

Merged
merged 4 commits into from
Apr 22, 2024
Merged

[8.14](backport #39087) Migrate packetbeat to static Buildkite pipeline #39108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c4182e7
Migrate packetbeat to static Buildkite pipeline (#39087)
dliappis Apr 22, 2024
0f969a9
fix conflicts
dliappis Apr 22, 2024
3152f8f
fix another conflict
dliappis Apr 22, 2024
4374de8
bring back accidental removal
dliappis Apr 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .buildkite/hooks/pre-command
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ ENABLED_BEATS_PIPELINES_SLUGS=(
"heartbeat"
"filebeat"
"beats-metricbeat"
"beats-packetbeat"
"beats-winlogbeat"
"beats-winlogbeat"
"beats-xpack-packetbeat"
Expand Down
268 changes: 222 additions & 46 deletions .buildkite/packetbeat/pipeline.packetbeat.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@
name: "beats-packetbeat"

env:
AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
BEATS_PROJECT_NAME: "packetbeat"
AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
GCP_WIN_MACHINE_TYPE: "n2-standard-8"
IMAGE_MACOS_ARM: "generic-13-ventura-arm"
IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9"
IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
Expand All @@ -17,52 +17,228 @@ env:
IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"

#Packaging
PACKAGING_ARM_PLATFORMS: "linux/arm64"
PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"

#Deps
ASDF_MAGE_VERSION: 1.15.0

steps:
- input: "Input Parameters"
key: "input-run-all-stages"
fields:
- select: "Packetbeat - run_packetbeat"
key: "run_packetbeat"
options:
- label: "True"
value: "true"
- label: "False"
value: "false"
default: "false"
- select: "Packetbeat - run_packetbeat_arm_tests"
key: "run_packetbeat_arm_tests"
options:
- label: "True"
value: "true"
- label: "False"
value: "false"
default: "false"
- select: "Packetbeat - run_packetbeat_macos_tests"
key: "run_packetbeat_macos_tests"
options:
- label: "True"
value: "true"
- label: "False"
value: "false"
default: "false"
if: "build.source == 'ui'"
- group: "packetbeat Mandatory Tests"
key: "packetbeat-mandatory-tests"
steps:
- label: ":linux: Ubuntu Unit Tests"
command: |
cd packetbeat
mage build unitTest
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
artifact_paths:
- "packetbeat/build/*.xml"
- "packetbeat/build/*.json"
notify:
- github_commit_status:
context: "packetbeat: Ubuntu Unit Tests"

- label: ":rhel: RHEL9 Unit Tests"
command: |
cd packetbeat
mage build unitTest
agents:
provider: "gcp"
image: "${IMAGE_RHEL9_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
artifact_paths:
- "packetbeat/build/*.xml"
- "packetbeat/build/*.json"
notify:
- github_commit_status:
context: "packetbeat: RHEL9 Unit Tests"

- label: ":windows: Windows 2016 Unit Tests"
command: |
Set-Location -Path packetbeat
mage build unitTest
agents:
provider: "gcp"
image: "${IMAGE_WIN_2016}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
artifact_paths:
- "packetbeat/build/*.xml"
- "packetbeat/build/*.json"
notify:
- github_commit_status:
context: "packetbeat: Windows 2016 Unit Tests"

- label: ":windows: Windows 2022 Unit Tests"
command: |
Set-Location -Path packetbeat
mage build unitTest
agents:
provider: "gcp"
image: "${IMAGE_WIN_2022}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
artifact_paths:
- "packetbeat/build/*.xml"
- "packetbeat/build/*.json"
notify:
- github_commit_status:
context: "packetbeat: Windows 2022 Unit Tests"

- group: "Extended Windows Tests"
key: "packetbeat-extended-windows-tests"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
steps:
- label: ":windows: Windows 10 Unit Tests"
command: |
Set-Location -Path packetbeat
mage build unitTest
agents:
provider: "gcp"
image: "${IMAGE_WIN_10}"
machineType: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
artifact_paths:
- "packetbeat/build/*.xml"
- "packetbeat/build/*.json"
notify:
- github_commit_status:
context: "packetbeat: Extended Windows 10 Unit Tests"

- label: ":windows: Windows 11 Unit Tests"
command: |
Set-Location -Path packetbeat
mage build unitTest
key: "extended-win-11-unit-tests"
agents:
provider: "gcp"
image: "${IMAGE_WIN_11}"
machineType: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
artifact_paths:
- "packetbeat/build/*.xml"
- "packetbeat/build/*.json"
notify:
- github_commit_status:
context: "packetbeat: Extended Windows 11 Unit Tests"

- label: ":windows: Windows 2019 Unit Tests"
command: |
Set-Location -Path packetbeat
mage build unitTest
key: "extended-win-2019-unit-tests"
agents:
provider: "gcp"
image: "${IMAGE_WIN_2019}"
machineType: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
artifact_paths:
- "packetbeat/build/*.xml"
- "packetbeat/build/*.json"
notify:
- github_commit_status:
context: "packetbeat: Extended Windows 2019 Unit Tests"

- group: "Extended Tests"
key: "extended-tests"
steps:
- label: ":mac: MacOS x86_64 Unit Tests"
key: "macos-x86-64-unit-tests-extended"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
command: |
set -euo pipefail
source .buildkite/scripts/install_macos_tools.sh
cd packetbeat
mage build unitTest
agents:
provider: "orka"
imagePrefix: "${IMAGE_MACOS_X86_64}"
artifact_paths:
- "packetbeat/build/*.xml"
- "packetbeat/build/*.json"
notify:
- github_commit_status:
context: "packetbeat: Extended MacOS x86_64 Unit Tests"

- label: ":mac: MacOS arm64 Unit Tests"
key: "macos-arm64-unit-tests-extended"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/
command: |
set -euo pipefail
source .buildkite/scripts/install_macos_tools.sh
cd packetbeat
mage build unitTest
agents:
provider: "orka"
imagePrefix: "${IMAGE_MACOS_ARM}"
artifact_paths:
- "packetbeat/build/*.xml"
- "packetbeat/build/*.json"
notify:
- github_commit_status:
context: "packetbeat: Extended MacOS arm64 Unit Tests"

- label: ":linux: Ubuntu ARM Unit Tests"
key: "linux-arm64-unit-tests-extended"
command: "cd packetbeat && mage build unitTest"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
agents:
provider: "aws"
imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
artifact_paths:
- "packetbeat/build/*.xml"
- "packetbeat/build/*.json"
notify:
- github_commit_status:
context: "packetbeat: Extended Ubuntu ARM Unit Tests"


- wait: ~
if: "build.source == 'ui'"
allow_dependency_failure: false

- label: ":linux: Load dynamic packetbeat pipeline"
key: "packetbeat-pipeline"
command: ".buildkite/scripts/generate_packetbeat_pipeline.sh"
agents:
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest"
notify:
- github_commit_status:
context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"
# with PRs, we want to run packaging only if mandatory tests succeed
# for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
# this allows building DRA artifacts even if there is flakiness in mandatory tests
if: build.env("BUILDKITE_PULL_REQUEST") != "false"
depends_on: "packetbeat-mandatory-tests"

- group: "Packetbeat Packaging"
key: "packaging"
steps:
- label: ":linux: Packaging Linux"
key: "packaging-linux"
command: |
cd packetbeat
mage package
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
env:
PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
notify:
- github_commit_status:
context: "packetbeat: Packaging Linux"

- label: ":linux: Packaging ARM"
key: "packaging-arm"
command: |
cd packetbeat
mage package
agents:
provider: "aws"
imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
env:
PLATFORMS: "linux/arm64"
PACKAGES: "docker"
notify:
- github_commit_status:
context: "packetbeat: Packaging Linux ARM"
26 changes: 26 additions & 0 deletions .buildkite/pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -392,3 +392,29 @@ steps:
- BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
- BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
- GITHUB_PR_LABELS=${GITHUB_PR_LABELS}

- label: "Trigger Packetbeat"
plugins:
- monorepo-diff#v1.0.1:
diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
watch:
- path:
- packetbeat/
- .buildkite/packetbeat/pipeline.packetbeat.yml
- .buildkite/scripts/
- .buildkite/hooks/
#OSS
- go.mod
- pytest.ini
- dev-tools/
- libbeat/
- testing/
config:
trigger: "beats-packetbeat"
build:
commit: "${BUILDKITE_COMMIT}"
branch: "${BUILDKITE_BRANCH}"
env:
- BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
- BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
- GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
23 changes: 1 addition & 22 deletions .buildkite/scripts/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,27 +13,20 @@ OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
# define if needed run the whole pipeline for the particular beat
[ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")"
[ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")"
[ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")"
[ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")"

# define if needed run ARM platform-specific tests for the particular beat
[ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")"
[ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")"
[ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")"

# define if needed run MacOS platform-specific tests for the particular beat
[ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")"
[ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")"
[ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")"

# define if needed run cloud-specific tests for the particular beat
[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")"

packetbeat_changeset=(
"^packetbeat/.*"
)

winlogbeat_changeset=(
"^winlogbeat/.*"
)
Expand Down Expand Up @@ -81,9 +74,6 @@ packaging_changeset=(
)

case "${BUILDKITE_PIPELINE_SLUG}" in
"beats-packetbeat")
BEAT_CHANGESET_REFERENCE=${packetbeat_changeset[@]}
;;
"beats-winlogbeat")
BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]}
;;
Expand Down Expand Up @@ -330,20 +320,9 @@ are_conditions_met_mandatory_tests() {
return 1
}

are_conditions_met_arm_tests() {
if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then
return 0
fi
fi
fi
return 1
}

are_conditions_met_macos_tests() {
if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]] ; then
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
return 0
fi
Expand Down
Loading
Loading