Skip to content

Commit

Permalink
Buildkite Xpack/Winlogbeat pipeline fixes (#39007)
Browse files Browse the repository at this point in the history
* updated common.sh

* fixed typo

* updated common.sh

* added changeset ps1 script and updated xpack-winlogbeat pipeline

* pr fixes

* handle backslashes in changeset directories

(cherry picked from commit 243f9c3)
  • Loading branch information
oakrizan committed Apr 22, 2024
1 parent de78c51 commit e2e1899
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 26 deletions.
72 changes: 72 additions & 0 deletions .buildkite/scripts/changesets.psm1
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
function ArePathsChanged($patterns) {
$changedlist = @()
foreach ($pattern in $patterns) {
$changedFiles = & git diff --name-only "HEAD@{1}" HEAD | Select-String -Pattern $pattern -SimpleMatch
if ($changedFiles) {
$changedlist += $changedFiles
}
}
if ($changedlist) {
Write-Host "--- Files changed: $changedlist"
return $true
}
else {
Write-Host "--- No files changed within specified changeset: $patterns"
return $false
}
}

function AreChangedOnlyPaths($patterns) {
$changedFiles = & git diff --name-only "HEAD@{1}" HEAD
Write-Host "--- Git Diff result:"
Write-Host "$changedFiles"

$matchedFiles = @()
foreach ($pattern in $patterns) {
$matched = $changedFiles | Select-String -Pattern $pattern -SimpleMatch
if ($matched) {
$matchedFiles += $matched
}
}
if (($matchedFiles.Count -eq $changedFiles.Count) -or ($changedFiles.Count -eq 0)) {
return $true
}
return $false
}

# This function sets a `MODULE` env var, required by IT tests, containing a comma separated list of modules for a given beats project (specified via the first argument).
# The list is built depending on directories that have changed under `modules/` excluding anything else such as asciidoc and png files.
# `MODULE` will empty if no changes apply.
function DefineModuleFromTheChangeSet($projectPath) {
$projectPathTransformed = $projectPath -replace '/', '\\'
$projectPathExclusion = "((?!^$projectPathTransformed\\\/).)*\$"
$exclude = @("^($projectPathExclusion|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)")

$changedModules = ''

$moduleDirs = Get-ChildItem -Directory "$projectPath\module"
foreach($moduleDir in $moduleDirs) {
if((ArePathsChanged($moduleDir)) -and !(AreChangedOnlyPaths($exclude))) {
if(!$changedModules) {
$changedModules = $moduleDir.Name
}
else {
$changedModules += ',' + $moduleDir.Name
}
}
}

# TODO: remove this conditional when issue https://github.com/elastic/ingest-dev/issues/2993 gets resolved
if(!$changedModules) {
if($Env:BUILDKITE_PIPELINE_SLUG -eq 'beats-xpack-metricbeat') {
$Env:MODULE = "aws"
}
else {
$Env:MODULE = "kubernetes"
}
}
else {
# TODO: once https://github.com/elastic/ingest-dev/issues/2993 gets resolved, this should be the only thing we export
$Env:MODULE = $changedModules
}
}
9 changes: 1 addition & 8 deletions .buildkite/scripts/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,6 @@ xpack_packetbeat_changeset=(
"^x-pack/packetbeat/.*"
)

xpack_winlogbeat_changeset=(
"^x-pack/winlogbeat/.*"
)

ci_changeset=(
"^.buildkite/.*"
)
Expand Down Expand Up @@ -74,9 +70,6 @@ packaging_changeset=(
)

case "${BUILDKITE_PIPELINE_SLUG}" in
"beats-winlogbeat")
BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]}
;;
"beats-xpack-metricbeat")
BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]}
;;
Expand Down Expand Up @@ -498,7 +491,7 @@ if are_paths_changed "${packaging_changeset[@]}" ; then
export PACKAGING_CHANGES="true"
fi

if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then
if [[ "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then
# Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines.
defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}"
fi
Expand Down
38 changes: 20 additions & 18 deletions .buildkite/x-pack/pipeline.xpack.winlogbeat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,16 @@ env:
ASDF_MAGE_VERSION: 1.15.0

steps:
- group: "Mandatory Tests"
key: "xpack-winlogbeat-mandatory-tests"
- group: "x-pack/Winlogbeat Mandatory Tests"
key: "x-pack-winlogbeat-mandatory-tests"
steps:

- label: ":windows: Xpack/Winlogbeat Win-2019 Unit (MODULE) Tests"
- label: ":windows: x-pack/Winlogbeat Win-2019 Unit (MODULE) Tests"
key: "mandatory-win-2019-module-unit-tests"
command: |
Import-Module ./.buildkite/scripts/changesets.psm1
defineModuleFromTheChangeSet 'x-pack/winlogbeat'
Write-Output "~~~ Will run tests with env var MODULE=$$Env:MODULE"
Set-Location -Path x-pack/winlogbeat
mage build unitTest
env:
Expand All @@ -37,9 +40,9 @@ steps:
- "x-pack/winlogbeat/build/*.json"
notify:
- github_commit_status:
context: "Xpack/Winlogbeat Win-2019 Unit (MODULE) Tests"
context: "x-pack/Winlogbeat Win-2019 Unit (MODULE) Tests"

- label: ":windows: Xpack/Winlogbeat Win-2016 Unit Tests"
- label: ":windows: x-pack/Winlogbeat Win-2016 Unit Tests"
command: |
Set-Location -Path x-pack/winlogbeat
mage build unitTest
Expand All @@ -55,9 +58,9 @@ steps:
- "x-pack/winlogbeat/build/*.json"
notify:
- github_commit_status:
context: "Xpack/Winlogbeat Win-2016 Unit Tests"
context: "x-pack/Winlogbeat Win-2016 Unit Tests"

- label: ":windows: Xpack/Winlogbeat Win-2022 Unit Tests"
- label: ":windows: x-pack/Winlogbeat Win-2022 Unit Tests"
command: |
Set-Location -Path x-pack/winlogbeat
mage build unitTest
Expand All @@ -73,14 +76,14 @@ steps:
- "x-pack/winlogbeat/build/*.json"
notify:
- github_commit_status:
context: "Xpack/Winlogbeat Win-2022 Unit Tests"
context: "x-pack/Winlogbeat Win-2022 Unit Tests"

- group: "Extended Windows Tests"
key: "extended-win-tests"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/

steps:
- label: ":windows: Xpack/Winlogbeat Win-10 Unit Tests"
- label: ":windows: x-pack/Winlogbeat Win-10 Unit Tests"
command: |
Set-Location -Path x-pack/winlogbeat
mage build unitTest
Expand All @@ -96,9 +99,9 @@ steps:
- "x-pack/winlogbeat/build/*.json"
notify:
- github_commit_status:
context: "Xpack/Winlogbeat Win-10 Unit Tests"
context: "x-pack/Winlogbeat Win-10 Unit Tests"

- label: ":windows: Xpack/Winlogbeat Win-11 Unit Tests"
- label: ":windows: x-pack/Winlogbeat Win-11 Unit Tests"
command: |
Set-Location -Path x-pack/winlogbeat
mage build unitTest
Expand All @@ -114,9 +117,9 @@ steps:
- "x-pack/winlogbeat/build/*.json"
notify:
- github_commit_status:
context: "Xpack/Winlogbeat Win-11 Unit Tests"
context: "x-pack/Winlogbeat Win-11 Unit Tests"

- label: ":windows: Xpack/Winlogbeat Win-2019 Unit Tests"
- label: ":windows: x-pack/Winlogbeat Win-2019 Unit Tests"
command: |
Set-Location -Path x-pack/winlogbeat
mage build unitTest
Expand All @@ -140,10 +143,10 @@ steps:
# this allows building DRA artifacts even if there is flakiness in mandatory tests
if: build.env("BUILDKITE_PULL_REQUEST") != "false"
depends_on:
- "xpack-winlogbeat-mandatory-tests"
- "x-pack-winlogbeat-mandatory-tests"

- group: "Xpack/Winlogbeat Packaging"
key: "xpack-winlogbeat-packaging"
- group: "x-pack/Winlogbeat Packaging"
key: "x-pack-winlogbeat-packaging"

steps:
- label: ":ubuntu: Packaging Linux"
Expand All @@ -159,5 +162,4 @@ steps:
PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
notify:
- github_commit_status:
context: "Xpack/Winlogbeat Packaging"

context: "x-pack/Winlogbeat Packaging"

0 comments on commit e2e1899

Please sign in to comment.