From 1614b930655050d2af89c72a7ad1d66c900b8975 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 17:34:37 +0300 Subject: [PATCH] [7.17](backport #39007) Buildkite Xpack/Winlogbeat pipeline fixes (#39101) * Buildkite Xpack/Winlogbeat pipeline fixes (#39007) * updated common.sh * fixed typo * updated common.sh * added changeset ps1 script and updated xpack-winlogbeat pipeline * pr fixes * handle backslashes in changeset directories (cherry picked from commit 243f9c33d8aab9e5aba7584d7b3eb64cb28bc18c) * resolved conflicts * moved win-2016 step to extended --------- Co-authored-by: Olga Naydyonock Co-authored-by: Olga Naidjonoka --- .buildkite/scripts/changesets.psm1 | 72 +++++++++++++++++++ .buildkite/scripts/common.sh | 9 +-- .../x-pack/pipeline.xpack.winlogbeat.yml | 55 +++++++------- 3 files changed, 102 insertions(+), 34 deletions(-) create mode 100644 .buildkite/scripts/changesets.psm1 diff --git a/.buildkite/scripts/changesets.psm1 b/.buildkite/scripts/changesets.psm1 new file mode 100644 index 000000000000..15fabd3eba5d --- /dev/null +++ b/.buildkite/scripts/changesets.psm1 @@ -0,0 +1,72 @@ +function ArePathsChanged($patterns) { + $changedlist = @() + foreach ($pattern in $patterns) { + $changedFiles = & git diff --name-only "HEAD@{1}" HEAD | Select-String -Pattern $pattern -SimpleMatch + if ($changedFiles) { + $changedlist += $changedFiles + } + } + if ($changedlist) { + Write-Host "--- Files changed: $changedlist" + return $true + } + else { + Write-Host "--- No files changed within specified changeset: $patterns" + return $false + } +} + +function AreChangedOnlyPaths($patterns) { + $changedFiles = & git diff --name-only "HEAD@{1}" HEAD + Write-Host "--- Git Diff result:" + Write-Host "$changedFiles" + + $matchedFiles = @() + foreach ($pattern in $patterns) { + $matched = $changedFiles | Select-String -Pattern $pattern -SimpleMatch + if ($matched) { + $matchedFiles += $matched + } + } + if (($matchedFiles.Count -eq $changedFiles.Count) -or ($changedFiles.Count -eq 0)) { + return $true + } + return $false +} + +# This function sets a `MODULE` env var, required by IT tests, containing a comma separated list of modules for a given beats project (specified via the first argument). +# The list is built depending on directories that have changed under `modules/` excluding anything else such as asciidoc and png files. +# `MODULE` will empty if no changes apply. +function DefineModuleFromTheChangeSet($projectPath) { + $projectPathTransformed = $projectPath -replace '/', '\\' + $projectPathExclusion = "((?!^$projectPathTransformed\\\/).)*\$" + $exclude = @("^($projectPathExclusion|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + + $changedModules = '' + + $moduleDirs = Get-ChildItem -Directory "$projectPath\module" + foreach($moduleDir in $moduleDirs) { + if((ArePathsChanged($moduleDir)) -and !(AreChangedOnlyPaths($exclude))) { + if(!$changedModules) { + $changedModules = $moduleDir.Name + } + else { + $changedModules += ',' + $moduleDir.Name + } + } + } + + # TODO: remove this conditional when issue https://github.com/elastic/ingest-dev/issues/2993 gets resolved + if(!$changedModules) { + if($Env:BUILDKITE_PIPELINE_SLUG -eq 'beats-xpack-metricbeat') { + $Env:MODULE = "aws" + } + else { + $Env:MODULE = "kubernetes" + } + } + else { + # TODO: once https://github.com/elastic/ingest-dev/issues/2993 gets resolved, this should be the only thing we export + $Env:MODULE = $changedModules + } +} diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 129b116b5be4..fba1ae6166ee 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -38,10 +38,6 @@ xpack_packetbeat_changeset=( "^x-pack/packetbeat/.*" ) -xpack_winlogbeat_changeset=( - "^x-pack/winlogbeat/.*" - ) - ci_changeset=( "^.buildkite/.*" ) @@ -73,9 +69,6 @@ packaging_changeset=( ) case "${BUILDKITE_PIPELINE_SLUG}" in - "beats-winlogbeat") - BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]} - ;; "beats-xpack-metricbeat") BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} ;; @@ -492,7 +485,7 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi -if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then +if [[ "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" fi diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index 4f034a05a6d1..c41dad93e442 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -16,13 +16,16 @@ env: ASDF_MAGE_VERSION: 1.15.0 steps: - - group: "Mandatory Tests" - key: "xpack-winlogbeat-mandatory-tests" + - group: "x-pack/Winlogbeat Mandatory Tests" + key: "x-pack-winlogbeat-mandatory-tests" steps: - - label: ":windows: Xpack/Winlogbeat Win-2019 Unit (MODULE) Tests" + - label: ":windows: x-pack/Winlogbeat Win-2019 Unit (MODULE) Tests" key: "mandatory-win-2019-module-unit-tests" command: | + Import-Module ./.buildkite/scripts/changesets.psm1 + defineModuleFromTheChangeSet 'x-pack/winlogbeat' + Write-Output "~~~ Will run tests with env var MODULE=$$Env:MODULE" Set-Location -Path x-pack/winlogbeat mage build unitTest env: @@ -38,16 +41,16 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-2019 Unit (MODULE) Tests" + context: "x-pack/Winlogbeat Win-2019 Unit (MODULE) Tests" - - label: ":windows: Xpack/Winlogbeat Win-2016 Unit Tests" + - label: ":windows: x-pack/Winlogbeat Win-2022 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest - key: "mandatory-win-2016-unit-tests" + key: "mandatory-win-2022-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_2016}" + image: "${IMAGE_WIN_2022}" machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" @@ -56,16 +59,21 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-2016 Unit Tests" + context: "x-pack/Winlogbeat Win-2022 Unit Tests" - - label: ":windows: Xpack/Winlogbeat Win-2022 Unit Tests" + - group: "Extended Windows Tests" + key: "extended-win-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + + steps: + - label: ":windows: x-pack/Winlogbeat Win-2016 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest - key: "mandatory-win-2022-unit-tests" + key: "mandatory-win-2016-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_2022}" + image: "${IMAGE_WIN_2016}" machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" @@ -74,14 +82,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-2022 Unit Tests" + context: "x-pack/Winlogbeat Win-2016 Unit Tests" - - group: "Extended Windows Tests" - key: "extended-win-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ - - steps: - - label: ":windows: Xpack/Winlogbeat Win-10 Unit Tests" + - label: ":windows: x-pack/Winlogbeat Win-10 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -97,9 +100,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-10 Unit Tests" + context: "x-pack/Winlogbeat Win-10 Unit Tests" - - label: ":windows: Xpack/Winlogbeat Win-11 Unit Tests" + - label: ":windows: x-pack/Winlogbeat Win-11 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -115,9 +118,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-11 Unit Tests" + context: "x-pack/Winlogbeat Win-11 Unit Tests" - - label: ":windows: Xpack/Winlogbeat Win-2019 Unit Tests" + - label: ":windows: x-pack/Winlogbeat Win-2019 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -141,10 +144,10 @@ steps: # this allows building DRA artifacts even if there is flakiness in mandatory tests if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: - - "xpack-winlogbeat-mandatory-tests" + - "x-pack-winlogbeat-mandatory-tests" - - group: "Xpack/Winlogbeat Packaging" - key: "xpack-winlogbeat-packaging" + - group: "x-pack/Winlogbeat Packaging" + key: "x-pack-winlogbeat-packaging" steps: - label: ":ubuntu: Packaging Linux" @@ -160,4 +163,4 @@ steps: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" notify: - github_commit_status: - context: "Xpack/Winlogbeat Packaging" + context: "x-pack/Winlogbeat Packaging"