From 9b18b56e1f4dfea9951f07c5a6641379ea0c9708 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sat, 2 Nov 2024 15:12:18 +0100 Subject: [PATCH] install diff-filter-build --- pipelines/main/launch_unsigned_jobs.yml | 203 +++++++++++-------- pipelines/scheduled/launch_unsigned_jobs.yml | 70 ++++--- 2 files changed, 160 insertions(+), 113 deletions(-) diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index bf0d393a..b823613a 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -12,6 +12,19 @@ # and only need to touch the webui configuration when we need to alter # something about the privileged steps. +common: + - diff-filter-build_plugin: &diff-filter-build + https://github.com/fatteneder/diff-filter-buildkite-plugin#0cbc1b8fbf86a568fe50fa679d666e40122e2ab8: + name: "build" + ignore: + - "*.md" + - "*.json" + - "CITATION.*" + - ".gitignore" + - ".clangd" + - ".mailmap" + target_branch: "master" + steps: - group: "Build" notify: @@ -23,29 +36,32 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build commands: | - # Launch Linux build jobs - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_linux.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - # Launch macOS packaging jobs - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_macos.arches \ - .buildkite/pipelines/main/platforms/build_macos.yml - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_freebsd.arches \ - .buildkite/pipelines/main/platforms/build_freebsd.yml - GROUP="Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_windows.arches \ - .buildkite/pipelines/main/platforms/build_windows.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + # Launch Linux build jobs + GROUP="Build" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_linux.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + # Launch macOS packaging jobs + GROUP="Build" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_macos.arches \ + .buildkite/pipelines/main/platforms/build_macos.yml + GROUP="Build" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_freebsd.arches \ + .buildkite/pipelines/main/platforms/build_freebsd.yml + GROUP="Build" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_windows.arches \ + .buildkite/pipelines/main/platforms/build_windows.yml + fi agents: queue: "julia" os: "linux" @@ -59,16 +75,19 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build commands: | - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml - # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml + # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml + fi agents: queue: "julia" os: "linux" @@ -82,36 +101,39 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build commands: | export ALLOW_FAIL="false" - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml - # Launch Linux test jobs - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml - # Launch i686-linux-gnu test jobs - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.i686.arches \ - .buildkite/pipelines/main/platforms/test_linux.i686.yml - # Launch macOS test jobs - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_macos.arches \ - .buildkite/pipelines/main/platforms/test_macos.yml - # Launch FreeBSD test jobs - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_freebsd.arches \ - .buildkite/pipelines/main/platforms/test_freebsd.yml - # Launch windows test jobs - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_windows.arches \ - .buildkite/pipelines/main/platforms/test_windows.yml + # Launch Linux test jobs + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_linux.arches \ + .buildkite/pipelines/main/platforms/test_linux.yml + # Launch i686-linux-gnu test jobs + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_linux.i686.arches \ + .buildkite/pipelines/main/platforms/test_linux.i686.yml + # Launch macOS test jobs + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_macos.arches \ + .buildkite/pipelines/main/platforms/test_macos.yml + # Launch FreeBSD test jobs + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_freebsd.arches \ + .buildkite/pipelines/main/platforms/test_freebsd.yml + # Launch windows test jobs + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_windows.arches \ + .buildkite/pipelines/main/platforms/test_windows.yml + fi agents: queue: "julia" os: "linux" @@ -122,18 +144,21 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build commands: | export ALLOW_FAIL="true" - # Launch Linux allowed-to-fail build jobs - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_linux.soft_fail.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - # Launch macOS allowed-to-fail build jobs - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/build_macos.soft_fail.arches \ - .buildkite/pipelines/main/platforms/build_macos.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + # Launch Linux allowed-to-fail build jobs + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_linux.soft_fail.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + # Launch macOS allowed-to-fail build jobs + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/build_macos.soft_fail.arches \ + .buildkite/pipelines/main/platforms/build_macos.yml + fi agents: queue: "julia" os: "linux" @@ -142,29 +167,31 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build commands: | export ALLOW_FAIL="true" - - # Launch Linux allowed-to-fail test jobs - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml - # Launch macOS allowed-to-fail test jobs - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_macos.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_macos.yml - # Launch FreeBSD allowed-to-fail jobs - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_freebsd.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_freebsd.yml - # Launch windows allowed-to-fail test jobs - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_windows.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_windows.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + # Launch Linux allowed-to-fail test jobs + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_linux.soft_fail.arches \ + .buildkite/pipelines/main/platforms/test_linux.yml + # Launch macOS allowed-to-fail test jobs + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_macos.soft_fail.arches \ + .buildkite/pipelines/main/platforms/test_macos.yml + # Launch FreeBSD allowed-to-fail jobs + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_freebsd.soft_fail.arches \ + .buildkite/pipelines/main/platforms/test_freebsd.yml + # Launch windows allowed-to-fail test jobs + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_windows.soft_fail.arches \ + .buildkite/pipelines/main/platforms/test_windows.yml + fi agents: queue: "julia" os: "linux" diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_unsigned_jobs.yml index e4de50f4..770bbdad 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_unsigned_jobs.yml @@ -12,6 +12,19 @@ # and only need to touch the webui configuration when we need to alter # something about the privileged steps. +common: + - diff-filter-build_plugin: &diff-filter-build + https://github.com/fatteneder/diff-filter-buildkite-plugin#0cbc1b8fbf86a568fe50fa679d666e40122e2ab8: + name: "build" + ignore: + - "*.md" + - "*.json" + - "CITATION.*" + - ".gitignore" + - ".clangd" + - ".mailmap" + target_branch: "master" + steps: - group: "Source Build" steps: @@ -20,12 +33,15 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build commands: | - GROUP="Source Build" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/build_linux.schedule.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + GROUP="Source Build" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/build_linux.schedule.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + fi agents: queue: "julia" os: "linux" @@ -37,11 +53,13 @@ steps: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" commands: | - GROUP="Source Tests (Allow Fail)" \ - ALLOW_FAIL="true" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/test_linux.schedule.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + GROUP="Source Tests (Allow Fail)" \ + ALLOW_FAIL="true" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/test_linux.schedule.arches \ + .buildkite/pipelines/main/platforms/test_linux.yml + fi agents: queue: "julia" os: "linux" @@ -53,20 +71,22 @@ steps: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" commands: | - GROUP="no_GPL" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/build_linux.no_gpl.arches \ - .buildkite/pipelines/main/platforms/build_linux.yml - GROUP="no_GPL" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/build_macos.no_gpl.arches \ - .buildkite/pipelines/main/platforms/build_macos.yml - GROUP="no_GPL" \ - ALLOW_FAIL="false" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/build_windows.no_gpl.arches \ - .buildkite/pipelines/main/platforms/build_windows.yml + if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then + GROUP="no_GPL" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/build_linux.no_gpl.arches \ + .buildkite/pipelines/main/platforms/build_linux.yml + GROUP="no_GPL" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/build_macos.no_gpl.arches \ + .buildkite/pipelines/main/platforms/build_macos.yml + GROUP="no_GPL" \ + ALLOW_FAIL="false" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/build_windows.no_gpl.arches \ + .buildkite/pipelines/main/platforms/build_windows.yml + fi agents: queue: "julia"