Skip to content

Commit

Permalink
[7.17](backport #39007) Buildkite Xpack/Winlogbeat pipeline fixes (#3…
Browse files Browse the repository at this point in the history
…9101)

* Buildkite Xpack/Winlogbeat pipeline fixes (#39007)

* updated common.sh

* fixed typo

* updated common.sh

* added changeset ps1 script and updated xpack-winlogbeat pipeline

* pr fixes

* handle backslashes in changeset directories

(cherry picked from commit 243f9c3)

* resolved conflicts

* moved win-2016 step to extended

---------

Co-authored-by: Olga Naydyonock <olga.naidjonoka@elastic.co>
Co-authored-by: Olga Naidjonoka <olga_naidjonoka@epam.com>
  • Loading branch information
3 people authored Apr 22, 2024
1 parent 7b5bfe4 commit 1614b93
Show file tree
Hide file tree
Showing 3 changed files with 102 additions and 34 deletions.
72 changes: 72 additions & 0 deletions .buildkite/scripts/changesets.psm1
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
function ArePathsChanged($patterns) {
$changedlist = @()
foreach ($pattern in $patterns) {
$changedFiles = & git diff --name-only "HEAD@{1}" HEAD | Select-String -Pattern $pattern -SimpleMatch
if ($changedFiles) {
$changedlist += $changedFiles
}
}
if ($changedlist) {
Write-Host "--- Files changed: $changedlist"
return $true
}
else {
Write-Host "--- No files changed within specified changeset: $patterns"
return $false
}
}

function AreChangedOnlyPaths($patterns) {
$changedFiles = & git diff --name-only "HEAD@{1}" HEAD
Write-Host "--- Git Diff result:"
Write-Host "$changedFiles"

$matchedFiles = @()
foreach ($pattern in $patterns) {
$matched = $changedFiles | Select-String -Pattern $pattern -SimpleMatch
if ($matched) {
$matchedFiles += $matched
}
}
if (($matchedFiles.Count -eq $changedFiles.Count) -or ($changedFiles.Count -eq 0)) {
return $true
}
return $false
}

# This function sets a `MODULE` env var, required by IT tests, containing a comma separated list of modules for a given beats project (specified via the first argument).
# The list is built depending on directories that have changed under `modules/` excluding anything else such as asciidoc and png files.
# `MODULE` will empty if no changes apply.
function DefineModuleFromTheChangeSet($projectPath) {
$projectPathTransformed = $projectPath -replace '/', '\\'
$projectPathExclusion = "((?!^$projectPathTransformed\\\/).)*\$"
$exclude = @("^($projectPathExclusion|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)")

$changedModules = ''

$moduleDirs = Get-ChildItem -Directory "$projectPath\module"
foreach($moduleDir in $moduleDirs) {
if((ArePathsChanged($moduleDir)) -and !(AreChangedOnlyPaths($exclude))) {
if(!$changedModules) {
$changedModules = $moduleDir.Name
}
else {
$changedModules += ',' + $moduleDir.Name
}
}
}

# TODO: remove this conditional when issue https://github.com/elastic/ingest-dev/issues/2993 gets resolved
if(!$changedModules) {
if($Env:BUILDKITE_PIPELINE_SLUG -eq 'beats-xpack-metricbeat') {
$Env:MODULE = "aws"
}
else {
$Env:MODULE = "kubernetes"
}
}
else {
# TODO: once https://github.com/elastic/ingest-dev/issues/2993 gets resolved, this should be the only thing we export
$Env:MODULE = $changedModules
}
}
9 changes: 1 addition & 8 deletions .buildkite/scripts/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,6 @@ xpack_packetbeat_changeset=(
"^x-pack/packetbeat/.*"
)

xpack_winlogbeat_changeset=(
"^x-pack/winlogbeat/.*"
)

ci_changeset=(
"^.buildkite/.*"
)
Expand Down Expand Up @@ -73,9 +69,6 @@ packaging_changeset=(
)

case "${BUILDKITE_PIPELINE_SLUG}" in
"beats-winlogbeat")
BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]}
;;
"beats-xpack-metricbeat")
BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]}
;;
Expand Down Expand Up @@ -492,7 +485,7 @@ if are_paths_changed "${packaging_changeset[@]}" ; then
export PACKAGING_CHANGES="true"
fi

if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then
if [[ "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then
# Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines.
defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}"
fi
Expand Down
55 changes: 29 additions & 26 deletions .buildkite/x-pack/pipeline.xpack.winlogbeat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,16 @@ env:
ASDF_MAGE_VERSION: 1.15.0

steps:
- group: "Mandatory Tests"
key: "xpack-winlogbeat-mandatory-tests"
- group: "x-pack/Winlogbeat Mandatory Tests"
key: "x-pack-winlogbeat-mandatory-tests"
steps:

- label: ":windows: Xpack/Winlogbeat Win-2019 Unit (MODULE) Tests"
- label: ":windows: x-pack/Winlogbeat Win-2019 Unit (MODULE) Tests"
key: "mandatory-win-2019-module-unit-tests"
command: |
Import-Module ./.buildkite/scripts/changesets.psm1
defineModuleFromTheChangeSet 'x-pack/winlogbeat'
Write-Output "~~~ Will run tests with env var MODULE=$$Env:MODULE"
Set-Location -Path x-pack/winlogbeat
mage build unitTest
env:
Expand All @@ -38,16 +41,16 @@ steps:
- "x-pack/winlogbeat/build/*.json"
notify:
- github_commit_status:
context: "Xpack/Winlogbeat Win-2019 Unit (MODULE) Tests"
context: "x-pack/Winlogbeat Win-2019 Unit (MODULE) Tests"

- label: ":windows: Xpack/Winlogbeat Win-2016 Unit Tests"
- label: ":windows: x-pack/Winlogbeat Win-2022 Unit Tests"
command: |
Set-Location -Path x-pack/winlogbeat
mage build unitTest
key: "mandatory-win-2016-unit-tests"
key: "mandatory-win-2022-unit-tests"
agents:
provider: "gcp"
image: "${IMAGE_WIN_2016}"
image: "${IMAGE_WIN_2022}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
Expand All @@ -56,16 +59,21 @@ steps:
- "x-pack/winlogbeat/build/*.json"
notify:
- github_commit_status:
context: "Xpack/Winlogbeat Win-2016 Unit Tests"
context: "x-pack/Winlogbeat Win-2022 Unit Tests"

- label: ":windows: Xpack/Winlogbeat Win-2022 Unit Tests"
- group: "Extended Windows Tests"
key: "extended-win-tests"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/

steps:
- label: ":windows: x-pack/Winlogbeat Win-2016 Unit Tests"
command: |
Set-Location -Path x-pack/winlogbeat
mage build unitTest
key: "mandatory-win-2022-unit-tests"
key: "mandatory-win-2016-unit-tests"
agents:
provider: "gcp"
image: "${IMAGE_WIN_2022}"
image: "${IMAGE_WIN_2016}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
Expand All @@ -74,14 +82,9 @@ steps:
- "x-pack/winlogbeat/build/*.json"
notify:
- github_commit_status:
context: "Xpack/Winlogbeat Win-2022 Unit Tests"
context: "x-pack/Winlogbeat Win-2016 Unit Tests"

- group: "Extended Windows Tests"
key: "extended-win-tests"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/

steps:
- label: ":windows: Xpack/Winlogbeat Win-10 Unit Tests"
- label: ":windows: x-pack/Winlogbeat Win-10 Unit Tests"
command: |
Set-Location -Path x-pack/winlogbeat
mage build unitTest
Expand All @@ -97,9 +100,9 @@ steps:
- "x-pack/winlogbeat/build/*.json"
notify:
- github_commit_status:
context: "Xpack/Winlogbeat Win-10 Unit Tests"
context: "x-pack/Winlogbeat Win-10 Unit Tests"

- label: ":windows: Xpack/Winlogbeat Win-11 Unit Tests"
- label: ":windows: x-pack/Winlogbeat Win-11 Unit Tests"
command: |
Set-Location -Path x-pack/winlogbeat
mage build unitTest
Expand All @@ -115,9 +118,9 @@ steps:
- "x-pack/winlogbeat/build/*.json"
notify:
- github_commit_status:
context: "Xpack/Winlogbeat Win-11 Unit Tests"
context: "x-pack/Winlogbeat Win-11 Unit Tests"

- label: ":windows: Xpack/Winlogbeat Win-2019 Unit Tests"
- label: ":windows: x-pack/Winlogbeat Win-2019 Unit Tests"
command: |
Set-Location -Path x-pack/winlogbeat
mage build unitTest
Expand All @@ -141,10 +144,10 @@ steps:
# this allows building DRA artifacts even if there is flakiness in mandatory tests
if: build.env("BUILDKITE_PULL_REQUEST") != "false"
depends_on:
- "xpack-winlogbeat-mandatory-tests"
- "x-pack-winlogbeat-mandatory-tests"

- group: "Xpack/Winlogbeat Packaging"
key: "xpack-winlogbeat-packaging"
- group: "x-pack/Winlogbeat Packaging"
key: "x-pack-winlogbeat-packaging"

steps:
- label: ":ubuntu: Packaging Linux"
Expand All @@ -160,4 +163,4 @@ steps:
PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64"
notify:
- github_commit_status:
context: "Xpack/Winlogbeat Packaging"
context: "x-pack/Winlogbeat Packaging"

0 comments on commit 1614b93

Please sign in to comment.