diff --git a/.azurepipelines/BaseTools-Build-For-Publication.yml b/.azurepipelines/BaseTools-Build-For-Publication.yml index 036ec037112..502d57b6b59 100644 --- a/.azurepipelines/BaseTools-Build-For-Publication.yml +++ b/.azurepipelines/BaseTools-Build-For-Publication.yml @@ -1,11 +1,19 @@ -# File BaseTools-Build-For-Publication.yml -# +## @file # Azure Pipeline to construct the final Nuget package for a binary release # of BaseTools. # # Copyright (C) Microsoft Corporation # SPDX-License-Identifier: BSD-2-Clause-Patent ## + +resources: + repositories: + - repository: mu_devops + type: github + endpoint: microsoft + name: microsoft/mu_devops + ref: main + name: BaseTools_Bin_$(release_version)_$(Rev:r) variables: @@ -54,14 +62,15 @@ jobs: filename: stuart_update arguments: -c .pytool/CISettings.py - - template: templates/basetools-build-steps.yml + - template: Steps/BuildBaseTools.yml@mu_devops parameters: - tool_chain_tag: 'GCC5' extra_parameters: '--skip_path_env' + tool_chain_tag: 'GCC5' - publish: $(linux_build_output_path) artifact: $(linux_artifact_name) displayName: Publish To Pipeline + condition: SucceededOrFailed() - job: windows_build displayName: Windows Build @@ -97,14 +106,15 @@ jobs: filename: stuart_update arguments: -c .pytool/CISettings.py - - template: templates/basetools-build-steps.yml + - template: Steps/BuildBaseTools.yml@mu_devops parameters: - tool_chain_tag: 'VS2019' extra_parameters: '--skip_path_env' + tool_chain_tag: 'VS2022' - publish: $(windows_build_output_path) artifact: $(windows_artifact_name) displayName: Publish To Pipeline + condition: SucceededOrFailed() - job: assemble_package displayName: Assemble Release Package @@ -127,13 +137,13 @@ jobs: architecture: 'x64' displayName: Install correct Python - - template: templates/github-file-fetch.yml + - template: Steps/FetchGitHubFile.yml@mu_devops parameters: + dest_file_path: $(Build.StagingDirectory)/pip-requirements.txt + display_name: Fetch Pip Requirements github_repo: microsoft/mu_basecore source_branch: $(Build.SourceBranch) source_file_path: pip-requirements.txt - dest_file_path: $(Build.StagingDirectory)/pip-requirements.txt - display_name: Fetch Pip Requirements - script: pip install -r $(Build.StagingDirectory)/pip-requirements.txt --upgrade displayName: 'Install/Upgrade Pip Modules' @@ -141,7 +151,7 @@ jobs: - download: current displayName: Grab Artifacts from Builds - - template: templates/github-file-fetch.yml + - template: Steps/FetchGitHubFile.yml@mu_devops parameters: github_repo: microsoft/mu_basecore source_branch: $(Build.SourceBranch) diff --git a/.azurepipelines/Matrix-Build-Job-Clang.yml b/.azurepipelines/Matrix-Build-Job-Clang.yml new file mode 100644 index 00000000000..f290ef3b758 --- /dev/null +++ b/.azurepipelines/Matrix-Build-Job-Clang.yml @@ -0,0 +1,81 @@ +## @file +# Template file for generating a CLANGPDB build job matrix for the packages currently supported for +# build in Mu Basecore. +# +# Copyright (c) Microsoft Corporation. +# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +parameters: +- name: arch_list + displayName: Architectures (e.g. IA32, X64) + type: string + default: '' +- name: extra_build_args + displayName: Extra Build Command Arguments + type: string + default: '' +- name: extra_install_step + displayName: Extra Install Steps + type: stepList + default: + - script: echo No extra steps provided +- name: tool_chain_tag + displayName: Tool Chain (e.g. VS2022) + type: string + default: '' +- name: vm_image + displayName: Virtual Machine Image (e.g. windows-latest) + type: string + default: 'windows-latest' + +jobs: + +- job: Build + timeoutInMinutes: 120 + # Use matrix to speed up the build process + strategy: + matrix: + TARGET_MDE_CPU: + Build.Pkgs: 'MdePkg,UefiCpuPkg' + Build.Targets: 'DEBUG,RELEASE,NO-TARGET,NOOPT' + TARGET_MDEMODULE_DEBUG: + Build.Pkgs: 'MdeModulePkg' + Build.Targets: 'DEBUG,NOOPT' + TARGET_MDEMODULE_RELEASE: + Build.Pkgs: 'MdeModulePkg' + Build.Targets: 'RELEASE,NO-TARGET' + TARGET_NETWORK: + Build.Pkgs: 'NetworkPkg' + Build.Targets: 'DEBUG,RELEASE,NO-TARGET,NOOPT' + TARGET_OTHER: + Build.Pkgs: 'PcAtChipsetPkg,ShellPkg,StandaloneMmPkg,BaseTools' + Build.Targets: 'DEBUG,RELEASE,NO-TARGET,NOOPT' + TARGET_TEST_POLICY: + Build.Pkgs: 'UnitTestFrameworkPkg,PolicyServicePkg' + Build.Targets: 'DEBUG,RELEASE,NO-TARGET,NOOPT' + TARGET_CRYPTO: + Build.Pkgs: 'CryptoPkg' + Build.Targets: 'DEBUG,RELEASE,NO-TARGET,NOOPT' + + workspace: + clean: all + + pool: + vmImage: ${{ parameters.vm_image }} + + steps: + - template: Steps/PrGate.yml@mu_devops + parameters: + build_archs: ${{ parameters.arch_list }} + build_pkgs: $(Build.Pkgs) + build_targets: $(Build.Targets) + do_ci_build: true + do_ci_setup: false + do_non_ci_build: false + do_non_ci_setup: true + do_pr_eval: true + extra_build_args: ${{ parameters.extra_build_args }} + extra_install_step: ${{ parameters.extra_install_step }} + tool_chain_tag: ${{ parameters.tool_chain_tag }} diff --git a/.azurepipelines/templates/pr-gate-build-job.yml b/.azurepipelines/Matrix-Build-Job.yml similarity index 54% rename from .azurepipelines/templates/pr-gate-build-job.yml rename to .azurepipelines/Matrix-Build-Job.yml index 27d8e647894..7dea8865014 100644 --- a/.azurepipelines/templates/pr-gate-build-job.yml +++ b/.azurepipelines/Matrix-Build-Job.yml @@ -1,7 +1,6 @@ ## @file -# File templates/pr-gate-build-job.yml -# -# template file used to build supported packages. +# Template file for generating a build job matrix for the packages currently supported for +# build in Mu Basecore. # # Copyright (c) Microsoft Corporation. # Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.
@@ -9,18 +8,33 @@ ## parameters: - tool_chain_tag: '' - vm_image: '' - arch_list: '' - extra_steps: [] # MU_CHANGE - build_args: '' # MU_CHANGE - Allow environment level build args. +- name: arch_list + displayName: Architectures (e.g. IA32, X64) + type: string + default: '' +- name: extra_build_args + displayName: Extra Build Command Arguments + type: string + default: '' +- name: extra_install_step + displayName: Extra Install Steps + type: stepList + default: + - script: echo No extra steps provided +- name: tool_chain_tag + displayName: Tool Chain (e.g. VS2022) + type: string + default: '' +- name: vm_image + displayName: Virtual Machine Image (e.g. windows-latest) + type: string + default: 'windows-latest' -# Build step jobs: -- job: Build_${{ parameters.tool_chain_tag }} +- job: Build timeoutInMinutes: 120 - #Use matrix to speed up the build process + # Use matrix to speed up the build process strategy: matrix: TARGET_MDE_CPU: @@ -36,10 +50,10 @@ jobs: Build.Pkgs: 'NetworkPkg' Build.Targets: 'DEBUG,RELEASE,NO-TARGET,NOOPT' TARGET_OTHER: - Build.Pkgs: 'PcAtChipsetPkg,ShellPkg,StandaloneMmPkg,BaseTools' # MU_CHANGE add BaseTools + Build.Pkgs: 'PcAtChipsetPkg,ShellPkg,StandaloneMmPkg,BaseTools' Build.Targets: 'DEBUG,RELEASE,NO-TARGET,NOOPT' TARGET_TEST_POLICY: - Build.Pkgs: 'UnitTestFrameworkPkg,PolicyServicePkg' # MU_CHANGE add policy package + Build.Pkgs: 'UnitTestFrameworkPkg,PolicyServicePkg' Build.Targets: 'DEBUG,RELEASE,NO-TARGET,NOOPT' TARGET_CRYPTO_DEBUG: Build.Pkgs: 'CryptoPkg' @@ -55,11 +69,16 @@ jobs: vmImage: ${{ parameters.vm_image }} steps: - - template: pr-gate-steps.yml + - template: Steps/PrGate.yml@mu_devops parameters: - tool_chain_tag: ${{ parameters.tool_chain_tag }} + build_archs: ${{ parameters.arch_list }} build_pkgs: $(Build.Pkgs) build_targets: $(Build.Targets) - build_archs: ${{ parameters.arch_list }} - extra_steps: ${{ parameters.extra_steps }} # MU_CHANGE - build_args: ${{ parameters.build_args }} # MU_CHANGE + do_ci_build: true + do_ci_setup: false + do_non_ci_build: false + do_non_ci_setup: true + do_pr_eval: true + extra_build_args: ${{ parameters.extra_build_args }} + extra_install_step: ${{ parameters.extra_install_step }} + tool_chain_tag: ${{ parameters.tool_chain_tag }} diff --git a/.azurepipelines/ReadMe.md b/.azurepipelines/ReadMe.md deleted file mode 100644 index cf57d282c19..00000000000 --- a/.azurepipelines/ReadMe.md +++ /dev/null @@ -1,50 +0,0 @@ -# Azure DevOps Pipelines - -These yml files are used to provide CI builds using the Azure DevOps Pipeline Service. -Most of the CI leverages edk2-pytools to support cross platform building and execution. - -## Core CI - -Focused on building and testing all packages in Edk2 without an actual target platform. - -See `.pytools/ReadMe.py` for more details - -## Platform CI - -Focused on building a single target platform and confirming functionality on that platform. - -## Conventions - -* Files extension should be *.yml. *.yaml is also supported but in Edk2 we use those for our package configuration. -* Platform CI files should be in the `/.azurepipelines` folder. -* Core CI files are in the root folder. -* Shared templates are in the `templates` folder. -* Top level CI files should be named `-.yml` - -## Links - -* Basic Azure Landing Site - https://docs.microsoft.com/en-us/azure/devops/pipelines/?view=azure-devops -* Pipeline jobs - https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml -* Pipeline yml scheme - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=azure-devops&tabs=schema%2Cparameter-schema -* Pipeline expression - https://docs.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops -* PyTools - https://github.com/tianocore/edk2-pytool-extensions and https://github.com/tianocore/edk2-pytool-library - -## Lessons Learned - -### Templates and parameters - -They are great but evil. If they are used as part of determining the steps of a build they must resolve before the build starts. They can not use variables set in a yml or determined as part of a matrix. If they are used in a step then they can be bound late. - -### File matching patterns - -On Linux this can hang if there are too many files in the search list. - -### Templates and file splitting - -Suggestion is to do one big yaml file that does what you want for one of your targets. Then do the second one and find the deltas. From that you can start to figure out the right split of files, steps, jobs. - -### Conditional steps - -If you want the step to show up in the log but not run, use a step conditional. This is great when a platform doesn't currently support a feature but you want the builders to know that the features exists and maybe someday it will. - -If you want the step to not show up use a template step conditional wrapper. Beware this will be evaluated early (at build start). This can hide things not needed on a given OS for example. diff --git a/.azurepipelines/Ubuntu-CLANGPDB.yml b/.azurepipelines/Ubuntu-CLANGPDB.yml index 15881c26b4b..ab15cbe2921 100644 --- a/.azurepipelines/Ubuntu-CLANGPDB.yml +++ b/.azurepipelines/Ubuntu-CLANGPDB.yml @@ -1,37 +1,29 @@ ## @file -# Azure Pipeline build file for a build using ubuntu and CLANGPDB +# Azure Pipeline build file for a build using Ubuntu and CLANGPDB. # # Copyright (c) Microsoft Corporation. -# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent ## -# MU_CHANGE BEGIN -trigger: -- dev/* -- release/* -schedules: -- cron: "30 9 * * 0,3" # Sun/Wed at 2:30AM Pacific - displayName: Sun/Wed Build - branches: - include: - - dev/* - - release/* - always: true # Always build, even if no changes +resources: + repositories: + - repository: mu_devops + type: github + endpoint: microsoft + name: microsoft/mu_devops + ref: main -pr: -- dev/* -- release/* -# MU_CHANGE END +variables: +- group: architectures-x86-64 +- group: tool-chain-clangpdb +- group: tool-chain-clangpdb-windows-vm-image jobs: -- template: templates/pr-gate-build-job-clang.yml +- template: Matrix-Build-Job-Clang.yml parameters: - tool_chain_tag: 'CLANGPDB' - vm_image: 'ubuntu-latest' - arch_list: "IA32,X64" - # MU_CHANGE + arch_list: $(arch_list) extra_steps: - - script: sudo apt-get install mingw-w64 - displayName: Install Windows Resource Compiler for Linux - + - script: sudo apt-get install mingw-w64 + displayName: Install Windows Resource Compiler for Linux + tool_chain_tag: $(tool_chain_tag) + vm_image: $(vm_image) diff --git a/.azurepipelines/Ubuntu-GCC5.yml b/.azurepipelines/Ubuntu-GCC5.yml index fd632b7efbd..dc8da5651d9 100644 --- a/.azurepipelines/Ubuntu-GCC5.yml +++ b/.azurepipelines/Ubuntu-GCC5.yml @@ -1,39 +1,31 @@ ## @file -# Azure Pipeline build file for a build using ubuntu and GCC5 +# Azure Pipeline build file for a build using Ubuntu and GCC. # # Copyright (c) Microsoft Corporation. -# Copyright (c) 2020, Hewlett Packard Enterprise Development LP. All rights reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent ## -# MU_CHANGE [BEGIN] -trigger: -- dev/* -- release/* -schedules: -- cron: "30 9 * * 0,3" # Sun/Wed at 2:30AM Pacific - displayName: Sun/Wed Build - branches: - include: - - dev/* - - release/* - always: true # Always build, even if no changes +resources: + repositories: + - repository: mu_devops + type: github + endpoint: microsoft + name: microsoft/mu_devops + ref: main -pr: -- dev/* -- release/* -# MU_CHANGE [END] +variables: +- group: architectures-arm-64-x86-64 +- group: tool-chain-ubuntu-gcc jobs: -- template: templates/pr-gate-build-job.yml +- template: Matrix-Build-Job.yml parameters: - tool_chain_tag: 'GCC5' - vm_image: 'ubuntu-latest' - arch_list: "IA32,X64,ARM,AARCH64" - build_args: "CODE_COVERAGE=TRUE CC_HTML=TRUE" # MU_CHANGE - # MU_CHANGE - extra_steps: + arch_list: $(arch_list) + extra_build_args: CODE_COVERAGE=TRUE CC_HTML=TRUE + extra_install_step: - script: | sudo apt-get install -y mingw-w64 lcov pip install lcov_cobertura pycobertura displayName: Install Windows Resource Compiler for Linux & Code Coverage Tools + tool_chain_tag: $(tool_chain_tag) + vm_image: $(vm_image) diff --git a/.azurepipelines/Ubuntu-PatchCheck.yml b/.azurepipelines/Ubuntu-PatchCheck.yml deleted file mode 100644 index 3301fa6fe1a..00000000000 --- a/.azurepipelines/Ubuntu-PatchCheck.yml +++ /dev/null @@ -1,38 +0,0 @@ -## @file -# Azure Pipielines YML file that evalues the patch series in a PR using the -# python script BaseTools/Scripts/PatchCheck.py. -# -# NOTE: This example monitors pull requests against the edk2-ci branch. Most -# environments would replace 'edk2-ci' with 'master'. -# -# Copyright (c) 2019 - 2020, Intel Corporation. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent -# -# https://github.com/tianocore -# -## - -trigger: none - -# MU_CHANGE [BEGIN] -pr: -- dev/* -- release/* -# MU_CHANGE [END] - -pool: - vmImage: 'ubuntu-latest' - -steps: -- checkout: self - clean: true - -- task: UsePythonVersion@0 - inputs: - versionSpec: '>=3.10.6' - architecture: 'x64' - -- script: | - git fetch origin $(System.PullRequest.TargetBranch):$(System.PullRequest.TargetBranch) - python BaseTools/Scripts/PatchCheck.py $(System.PullRequest.TargetBranch)..$(System.PullRequest.SourceCommitId) - displayName: 'Use PatchCheck.py to verify patch series in pull request' diff --git a/.azurepipelines/Windows-CLANGPDB.yml b/.azurepipelines/Windows-CLANGPDB.yml index 301fb9acbae..5e28310dc85 100644 --- a/.azurepipelines/Windows-CLANGPDB.yml +++ b/.azurepipelines/Windows-CLANGPDB.yml @@ -1,35 +1,30 @@ ## @file -# Azure Pipeline build file for a build using Windows and clangpdb +# Azure Pipeline build file for a build using Windows and CLANGPDB. # # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: BSD-2-Clause-Patent ## -# MU_CHANGE BEGIN -trigger: -- dev/* -- release/* -schedules: -- cron: "0 8 * * 0,3" # Sun/Wed at 1AM Pacific - displayName: Sun/Wed Build - branches: - include: - - dev/* - - release/* - always: true # Always build, even if no changes +resources: + repositories: + - repository: mu_devops + type: github + endpoint: microsoft + name: microsoft/mu_devops + ref: main -pr: -- dev/* -- release/* -# MU_CHANGE END +variables: +- group: architectures-x86-64 +- group: tool-chain-clangpdb +- group: tool-chain-clangpdb-ubuntu-vm-image jobs: -- template: templates/pr-gate-build-job-clang.yml +- template: Matrix-Build-Job-Clang.yml parameters: - tool_chain_tag: 'CLANGPDB' - vm_image: 'windows-latest' - arch_list: "IA32,X64" - extra_steps: - - powershell: choco install llvm - displayName: Install LLVM and CLANG # friendly name displayed in the UI - condition: and(gt(variables.pkg_count, 0), succeeded()) + arch_list: $(arch_list) + extra_install_step: + - powershell: choco install llvm + displayName: Install LLVM and CLANG + condition: and(gt(variables.pkg_count, 0), succeeded()) + tool_chain_tag: $(tool_chain_tag) + vm_image: $(vm_image) diff --git a/.azurepipelines/Windows-VS.yml b/.azurepipelines/Windows-VS.yml new file mode 100644 index 00000000000..9fd87fd7285 --- /dev/null +++ b/.azurepipelines/Windows-VS.yml @@ -0,0 +1,25 @@ +## @file +# Azure Pipeline build file for a build using Windows and the latest Visual Studio toolchain supported. +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +resources: + repositories: + - repository: mu_devops + type: github + endpoint: microsoft + name: microsoft/mu_devops + ref: main + +variables: +- group: architectures-x86-64 +- group: tool-chain-windows-visual-studio-latest + +jobs: +- template: Matrix-Build-Job.yml + parameters: + arch_list: $(arch_list) + tool_chain_tag: $(tool_chain_tag) + vm_image: $(vm_image) diff --git a/.azurepipelines/Windows-VS2022.yml b/.azurepipelines/Windows-VS2022.yml deleted file mode 100644 index 4a0936d3314..00000000000 --- a/.azurepipelines/Windows-VS2022.yml +++ /dev/null @@ -1,40 +0,0 @@ -## @file -# Azure Pipeline build file for a build using Windows and VS2022 -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## -# MU_CHANGE [BEGIN] -trigger: -- dev/* -- release/* - -schedules: -- cron: "0 8 * * 0,3" # Sun/Wed at 1AM Pacific - displayName: Sun/Wed Build - branches: - include: - - dev/* - - release/* - always: true # Always build, even if no changes - -pr: -- dev/* -- release/* -# MU_CHANGE [END] - -variables: - - template: templates/defaults.yml - -jobs: -- template: templates/pr-gate-build-job.yml - parameters: - tool_chain_tag: 'VS2022' - vm_image: 'windows-2022' - arch_list: "IA32,X64" - usePythonVersion: ${{ variables.default_python_version }} - extra_install_step: - - powershell: choco install opencppcoverage; Write-Host "##vso[task.prependpath]C:\Program Files\OpenCppCoverage" - displayName: Install Code Coverage Tool - condition: and(gt(variables.pkg_count, 0), succeeded()) - diff --git a/.azurepipelines/templates/ReadMe.md b/.azurepipelines/templates/ReadMe.md deleted file mode 100644 index 3a6e4669f73..00000000000 --- a/.azurepipelines/templates/ReadMe.md +++ /dev/null @@ -1,70 +0,0 @@ -# CI Templates - -This folder contains azure pipeline yml templates for "Core" and "Platform" Continuous Integration and PR validation. - -## Common CI templates - -### basetools-build-steps.yml - -This template compiles the Edk2 basetools from source. The steps in this template are -conditional and will only run if variable `pkg_count` is greater than 0. - -It also has two conditional steps only used when the toolchain contains GCC. These two steps -use `apt` to update the system packages and add those necessary for Edk2 builds. - -## Core CI templates - -### pr-gate-build-job.yml - -This template contains the jobs and most importantly the matrix of which packages and -targets to run for Core CI. - -### pr-gate-build-job-clang.yml - -This template contains the jobs and most importantly the matrix of which packages and -targets to run for Core CI for clang toolchain. Since GCC and VS already ran with the NO-TARGET option it is -not needed in this list. - -### pr-gate-steps.yml - -This template is the main Core CI template. It controls all the steps run and is responsible for most functionality of the Core CI process. This template sets -the `pkg_count` variable using the `stuart_pr_eval` tool when the -build type is "pull request" - -### spell-check-prereq-steps.yml - -This template installs the node based tools used by the spell checker plugin. The steps -in this template are conditional and will only run if variable `pkg_count` is greater than 0. - -### markdownlint-check-prereq-steps.yml - -This template installs the markdown lint tools used by the markdown linter plugin. The steps -in this template are conditional and will only run if variable `pkg_count` is greater than 0. - -## Platform CI templates - -### platform-build-run-steps.yml - -This template makes heavy use of pytools to build and run a platform in the Edk2 repo - -Also uses basetools-build-steps.yml to compile basetools - -#### Special Notes - -* For a build type of pull request it will conditionally build if the patches change files that impact the platform. - * uses `stuart_pr_eval` to determine impact -* For manual builds or CI builds it will always build the platform -* It compiles basetools from source -* Will use `stuart_build --FlashOnly` to attempt to run the built image if the `Run` parameter is set. -* See the parameters block for expected configuration options -* Parameter `extra_install_step` allows the caller to insert extra steps. This is useful if additional dependencies, tools, or other things need to be installed. Here is an example of installing qemu on Windows. - - ``` yaml - steps: - - template: ../../.azurepipelines/templates/build-run-steps.yml - parameters: - extra_install_step: - - powershell: choco install qemu; Write-Host "##vso[task.prependpath]c:\Program Files\qemu" - displayName: Install QEMU and Set QEMU on path # friendly name displayed in the UI - condition: and(gt(variables.pkg_count, 0), succeeded()) - ``` diff --git a/.azurepipelines/templates/basetools-build-steps.yml b/.azurepipelines/templates/basetools-build-steps.yml deleted file mode 100644 index 9ac6e9b51d8..00000000000 --- a/.azurepipelines/templates/basetools-build-steps.yml +++ /dev/null @@ -1,30 +0,0 @@ -## @file -# File templates/basetools-build-job.yml -# -# template file to build basetools -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## -parameters: - tool_chain_tag: '' - extra_parameters: '' # MU_CHANGE - -steps: -- task: CmdLine@1 - displayName: Build Base Tools from source - inputs: - filename: python - # MU_CHANGE - arguments: BaseTools/Edk2ToolsBuild.py -t ${{ parameters.tool_chain_tag }} ${{ parameters.extra_parameters }} - condition: and(gt(variables.pkg_count, 0), succeeded()) - -- task: CopyFiles@2 - displayName: "Copy base tools build log" - inputs: - targetFolder: '$(Build.ArtifactStagingDirectory)' - SourceFolder: 'BaseTools/BaseToolsBuild' - contents: | - BASETOOLS_BUILD*.* - flattenFolders: true - condition: and(gt(variables.pkg_count, 0), succeededOrFailed()) diff --git a/.azurepipelines/templates/defaults.yml b/.azurepipelines/templates/defaults.yml deleted file mode 100644 index 8006495580f..00000000000 --- a/.azurepipelines/templates/defaults.yml +++ /dev/null @@ -1,12 +0,0 @@ -## @file -# File templates/default.yml -# -# template file containing common default values -# -# Copyright (c) Red Hat, Inc. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -variables: - default_python_version: ">=3.10.6" - default_linux_image: "ghcr.io/tianocore/containers/fedora-37-test:a0dd931" diff --git a/.azurepipelines/templates/github-file-fetch.yml b/.azurepipelines/templates/github-file-fetch.yml deleted file mode 100644 index 86601a99eee..00000000000 --- a/.azurepipelines/templates/github-file-fetch.yml +++ /dev/null @@ -1,18 +0,0 @@ -# File templates/github-file-fetch.yml -# -# Azure Pipelines template to fetch a single file from the top of a given -# branch in a public GitHub repo. -# -# Copyright (C) Microsoft Corporation -# SPDX-License-Identifier: BSD-2-Clause-Patent -## -parameters: - display_name: 'Fetch GitHub File' - -steps: - - powershell: - $branch_url = '${{ parameters.source_branch }}'.replace('refs/heads/', ''); - $fetch_source = 'https://raw.githubusercontent.com/${{ parameters.github_repo }}/'+$branch_url+'/${{ parameters.source_file_path }}'; - Write-Host $fetch_source; - (New-Object System.Net.WebClient).DownloadFile($fetch_source, '${{ parameters.dest_file_path }}'); - displayName: ${{ parameters.display_name }} diff --git a/.azurepipelines/templates/markdownlint-check-prereq-steps.yml b/.azurepipelines/templates/markdownlint-check-prereq-steps.yml deleted file mode 100644 index 5af643faf51..00000000000 --- a/.azurepipelines/templates/markdownlint-check-prereq-steps.yml +++ /dev/null @@ -1,18 +0,0 @@ -## @file -# File templates/markdownlint-check-prereq-steps.yml -# -# template file used to install markdownlint checking prerequisite -# depends on spell-check-prereq-steps to run first to set the node version -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -parameters: - none: '' - -steps: - -- script: npm install -g markdownlint-cli@0.31.1 - displayName: "Install markdown linter" - condition: and(gt(variables.pkg_count, 0), succeeded()) diff --git a/.azurepipelines/templates/platform-build-run-steps.yml b/.azurepipelines/templates/platform-build-run-steps.yml deleted file mode 100644 index 3fa7075999b..00000000000 --- a/.azurepipelines/templates/platform-build-run-steps.yml +++ /dev/null @@ -1,142 +0,0 @@ - -## @file -# File steps.yml -# -# template file containing the steps to build -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -parameters: -- name: tool_chain_tag - type: string - default: '' -- name: build_pkg - type: string - default: '' -- name: build_target - type: string - default: '' -- name: build_arch - type: string - default: '' -- name: build_file - type: string - default: '' -- name: build_flags - type: string - default: '' -- name: run_flags - type: string - default: '' - -- name: extra_install_step - type: stepList - default: [] - -steps: -- checkout: self - clean: true - fetchDepth: 1 - -- task: UsePythonVersion@0 - inputs: - versionSpec: "3.9.x" # MU_CHANGE - architecture: "x64" - -- script: pip install -r pip-requirements.txt --upgrade - displayName: 'Install/Upgrade pip modules' - -# Set default -- bash: echo "##vso[task.setvariable variable=pkg_count]${{ 1 }}" - -# MU_CHANGE [BEGIN] - this doesn't work on devops repos -# Fetch the target branch so that pr_eval can diff them. -# Seems like azure pipelines/github changed checkout process in nov 2020. -#- script: git fetch origin $(System.PullRequest.targetBranch) -# displayName: fetch target branch -# condition: eq(variables['Build.Reason'], 'PullRequest') -# MU_CHANGE [END] - -# trim the package list if this is a PR -- task: CmdLine@1 - displayName: Check if ${{ parameters.build_pkg }} need testing - inputs: - filename: stuart_pr_eval - arguments: -c ${{ parameters.build_file }} -t ${{ parameters.build_target}} -a ${{ parameters.build_arch}} --pr-target origin/$(System.PullRequest.targetBranch) --output-count-format-string "##vso[task.setvariable variable=pkg_count;isOutpout=true]{pkgcount}" - condition: eq(variables['Build.Reason'], 'PullRequest') - - # Setup repo -- task: CmdLine@1 - displayName: Setup - inputs: - filename: stuart_setup - arguments: -c ${{ parameters.build_file }} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}} -t ${{ parameters.build_target}} -a ${{ parameters.build_arch}} ${{ parameters.build_flags}} - condition: and(gt(variables.pkg_count, 0), succeeded()) - -# Stuart Update -- task: CmdLine@1 - displayName: Update - inputs: - filename: stuart_update - arguments: -c ${{ parameters.build_file }} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}} -t ${{ parameters.build_target}} -a ${{ parameters.build_arch}} ${{ parameters.build_flags}} - condition: and(gt(variables.pkg_count, 0), succeeded()) - -# build basetools -# do this after setup and update so that code base dependencies -# are all resolved. -- template: basetools-build-steps.yml - parameters: - tool_chain_tag: ${{ parameters.tool_chain_tag }} - -# Potential Extra steps -- ${{ parameters.extra_install_step }} - -# Build -- task: CmdLine@1 - displayName: Build - inputs: - filename: stuart_build - arguments: -c ${{ parameters.build_file }} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}} TARGET=${{ parameters.build_target}} -a ${{ parameters.build_arch}} ${{ parameters.build_flags}} - condition: and(gt(variables.pkg_count, 0), succeeded()) - -# Run -- task: CmdLine@1 - displayName: Run to shell - inputs: - filename: stuart_build - arguments: -c ${{ parameters.build_file }} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}} TARGET=${{ parameters.build_target}} -a ${{ parameters.build_arch}} ${{ parameters.build_flags}} ${{ parameters.run_flags }} --FlashOnly - condition: and(and(gt(variables.pkg_count, 0), succeeded()), eq(variables['Run'], true)) - timeoutInMinutes: 2 - -# Copy the build logs to the artifact staging directory -- task: CopyFiles@2 - displayName: "Copy build logs" - inputs: - targetFolder: "$(Build.ArtifactStagingDirectory)" - SourceFolder: "Build" - contents: | - BUILDLOG_*.txt - BUILDLOG_*.md - CI_*.txt - CI_*.md - CISETUP.txt - SETUPLOG.txt - UPDATE_LOG.txt - PREVALLOG.txt - TestSuites.xml - **/BUILD_TOOLS_REPORT.html - **/OVERRIDELOG.TXT - BASETOOLS_BUILD*.* - flattenFolders: true - condition: succeededOrFailed() - -# Publish build artifacts to Azure Artifacts/TFS or a file share -- task: PublishBuildArtifacts@1 - continueOnError: true - displayName: "Publish build logs" - inputs: - pathtoPublish: "$(Build.ArtifactStagingDirectory)" - artifactName: "Build Logs $(System.JobName)" - condition: succeededOrFailed() diff --git a/.azurepipelines/templates/pr-gate-build-job-clang.yml b/.azurepipelines/templates/pr-gate-build-job-clang.yml deleted file mode 100644 index a0176f656c8..00000000000 --- a/.azurepipelines/templates/pr-gate-build-job-clang.yml +++ /dev/null @@ -1,60 +0,0 @@ -## @file -# File templates/pr-gate-build-job-clang.yml -# -# template file used to build supported packages. -# -# Copyright (c) Microsoft Corporation. -# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.
-# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -parameters: - tool_chain_tag: '' - vm_image: '' - arch_list: '' - extra_steps: [] # MU_CHANGE - -# Build step -jobs: - -- job: Build_${{ parameters.tool_chain_tag }} - - #Use matrix to speed up the build process - strategy: - matrix: - TARGET_MDE_CPU: - Build.Pkgs: 'MdePkg,UefiCpuPkg' - Build.Targets: 'DEBUG,RELEASE' - TARGET_MDEMODULE_DEBUG: - Build.Pkgs: 'MdeModulePkg' - Build.Targets: 'DEBUG' #,NOOPT' - TARGET_MDEMODULE_RELEASE: - Build.Pkgs: 'MdeModulePkg' - Build.Targets: 'RELEASE' - TARGET_NETWORK: - Build.Pkgs: 'NetworkPkg' - Build.Targets: 'DEBUG,RELEASE' - TARGET_OTHER: - Build.Pkgs: 'PcAtChipsetPkg,ShellPkg,StandaloneMmPkg,BaseTools' # MU_CHANGE add BaseTools - Build.Targets: 'DEBUG,RELEASE' - TARGET_TEST: - Build.Pkgs: 'UnitTestFrameworkPkg' - Build.Targets: 'DEBUG,RELEASE' #,NOOPT' - TARGET_CRYPTO: - Build.Pkgs: 'CryptoPkg' - Build.Targets: 'DEBUG,RELEASE' #,NOOPT' - - workspace: - clean: all - - pool: - vmImage: ${{ parameters.vm_image }} - - steps: - - template: pr-gate-steps.yml - parameters: - tool_chain_tag: ${{ parameters.tool_chain_tag }} - build_pkgs: $(Build.Pkgs) - build_targets: $(Build.Targets) - build_archs: ${{ parameters.arch_list }} - extra_steps: ${{ parameters.extra_steps }} # MU_CHANGE diff --git a/.azurepipelines/templates/pr-gate-steps.yml b/.azurepipelines/templates/pr-gate-steps.yml deleted file mode 100644 index 631fcea12b2..00000000000 --- a/.azurepipelines/templates/pr-gate-steps.yml +++ /dev/null @@ -1,185 +0,0 @@ -## @file -# File templates/pr-gate-steps.yml -# -# template file containing the steps to build -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -parameters: - tool_chain_tag: '' - build_pkgs: '' - build_targets: '' - build_archs: '' - has_ci_dependencies: false # MU_CHANGE - Enable toggle for stuart_ci_build - extra_steps: [] # MU_CHANGE - needed for clang - build_args: '' # MU_CHANGE - add build arguments to support code coverage - -steps: -- checkout: self - clean: true - # fetchDepth: 1 # MU_CHANGE - -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.10.x' # MU_CHANGE - architecture: 'x64' - -- script: pip install -r pip-requirements.txt --upgrade - displayName: 'Install/Upgrade pip modules' - -# Set default -- bash: | - echo "##vso[task.setvariable variable=pkgs_to_build]${{ parameters.build_pkgs }}" - echo "##vso[task.setvariable variable=pkg_count]${{ 1 }}" - -# MU_CHANGE [BEGIN] - this doesn't work on devops repos -# Fetch the target branch so that pr_eval can diff them. -# Seems like azure pipelines/github changed checkout process in nov 2020. -#- script: git fetch origin $(System.PullRequest.targetBranch) -# displayName: fetch target branch -# MU_CHANGE [END] - -# MU_CHANGE [BEGIN] - Workaround an azure pipelines bug. -# $System.PullRequest.targetBranch looks like "dev/whatever" on GitHub, -# but looks like "refs/heads/dev/whatever" on DevOps. The DevOps version -# can't be used for comparison for the PR Eval. -- powershell: - $TargetBranch = "$(System.PullRequest.targetBranch)".replace('refs/heads/', ''); - Write-Host "##vso[task.setvariable variable=pr_compare_branch]origin/$TargetBranch"; - displayName: Workaround for Branch Names - condition: eq(variables['Build.Reason'], 'PullRequest') -# MU_CHANGE [END] - -# trim the package list if this is a PR -- task: CmdLine@1 - displayName: Check if ${{ parameters.build_pkgs }} need testing - inputs: - filename: stuart_pr_eval - # MU_CHANGE - Workaround an azure pipelines bug. - arguments: -c .pytool/CISettings.py -p ${{ parameters.build_pkgs }} --pr-target $(pr_compare_branch) --output-csv-format-string "##vso[task.setvariable variable=pkgs_to_build;isOutpout=true]{pkgcsv}" --output-count-format-string "##vso[task.setvariable variable=pkg_count;isOutpout=true]{pkgcount}" - condition: eq(variables['Build.Reason'], 'PullRequest') - -# install spell check prereqs -- template: spell-check-prereq-steps.yml - -# MU_CHANGE - Add MarkdownLint -# install markdownlint check prereqs -- template: markdownlint-check-prereq-steps.yml -# MU_CHANGE - [start] need support for extra steps for clang -# Extra steps if needed -- ${{ parameters.extra_steps }} -# MU_CHANGE [end] - - -# Build repo -# MU_CHANGE - We use CI dependencies as well as submodules. -- task: CmdLine@1 - displayName: Dependency Setup ${{ parameters.build_pkgs }} ${{ parameters.build_archs}} - inputs: - filename: stuart_ci_setup - arguments: -c .pytool/CISettings.py -p $(pkgs_to_build) --force-git -t ${{ parameters.build_targets}} -a ${{ parameters.build_archs}} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}} - condition: and(gt(variables.pkg_count, 0), succeeded(), eq('${{ parameters.has_ci_dependencies }}', 'true')) - -# Build repo -- task: CmdLine@1 - displayName: Setup ${{ parameters.build_pkgs }} ${{ parameters.build_archs}} - inputs: - filename: stuart_setup - arguments: -c .pytool/CISettings.py -p $(pkgs_to_build) -t ${{ parameters.build_targets}} -a ${{ parameters.build_archs}} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}} - condition: and(gt(variables.pkg_count, 0), succeeded()) - -- task: CmdLine@1 - displayName: Update ${{ parameters.build_pkgs }} ${{ parameters.build_archs}} - inputs: - filename: stuart_update - arguments: -c .pytool/CISettings.py -p $(pkgs_to_build) -t ${{ parameters.build_targets}} -a ${{ parameters.build_archs}} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}} - condition: and(gt(variables.pkg_count, 0), succeeded()) - -# MU_CHANGE [BEGIN] - We use binary BaseTools -# build basetools -# do this after setup and update so that code base dependencies -# are all resolved. -# - template: basetools-build-steps.yml -# parameters: -# tool_chain_tag: ${{ parameters.tool_chain_tag }} -# MU_CHANGE [END] - -- task: CmdLine@1 - displayName: Build and Test ${{ parameters.build_pkgs }} ${{ parameters.build_archs}} - inputs: - filename: stuart_ci_build - # MU_CHANGE - Add build arguments parameter - arguments: -c .pytool/CISettings.py -p $(pkgs_to_build) -t ${{ parameters.build_targets}} -a ${{ parameters.build_archs}} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}} ${{ parameters.build_args}} - condition: and(gt(variables.pkg_count, 0), succeeded()) - -# Publish Test Results to Azure Pipelines/TFS -- task: PublishTestResults@2 - displayName: 'Publish junit test results' - continueOnError: true - condition: and( succeededOrFailed(),gt(variables.pkg_count, 0)) - inputs: - testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit - testResultsFiles: 'Build/TestSuites.xml' - #searchFolder: '$(System.DefaultWorkingDirectory)' # Optional - mergeTestResults: true # Optional - testRunTitle: $(System.JobName) # Optional - #buildPlatform: # Optional - #buildConfiguration: # Optional - publishRunAttachments: true # Optional - -# Publish Test Results to Azure Pipelines/TFS -- task: PublishTestResults@2 - displayName: 'Publish host based test results for $(System.JobName)' - continueOnError: true - condition: and( succeededOrFailed(), gt(variables.pkg_count, 0)) - inputs: - testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit - testResultsFiles: 'Build/**/*.result.xml' - #searchFolder: '$(System.DefaultWorkingDirectory)' # Optional - mergeTestResults: false # Optional - testRunTitle: ${{ parameters.build_pkgs }} # Optional - #buildPlatform: # Optional - #buildConfiguration: # Optional - publishRunAttachments: true # Optional - -# Copy the build logs to the artifact staging directory -- task: CopyFiles@2 - displayName: "Copy build logs" - inputs: - targetFolder: '$(Build.ArtifactStagingDirectory)' - SourceFolder: 'Build' - contents: | - BUILDLOG_*.txt - BUILDLOG_*.md - CI_*.txt - CI_*.md - CISETUP.txt - SETUPLOG.txt - UPDATE_LOG.txt - PREVALLOG.txt - TestSuites.xml - **/BUILD_TOOLS_REPORT.html - **/OVERRIDELOG.TXT - coverage.xml - coverage.html - flattenFolders: true - condition: succeededOrFailed() - -# Publish build artifacts to Azure Artifacts/TFS or a file share -- task: PublishBuildArtifacts@1 - continueOnError: true - displayName: "Publish build logs" - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: 'Build Logs $(System.JobName)' - condition: succeededOrFailed() - -# MU_CHANGE - Publish the LCOV coverage files. -- task: PublishCodeCoverageResults@1 - condition: and(succeededOrFailed(), and(gt(variables.pkg_count, 0), contains('${{ parameters.build_args }}', 'CODE_COVERAGE=TRUE'))) - inputs: - codeCoverageTool: 'Cobertura' - summaryFileLocation: 'Build/coverage.xml' - pathToSources: '$(Build.SourcesDirectory)' diff --git a/.azurepipelines/templates/spell-check-prereq-steps.yml b/.azurepipelines/templates/spell-check-prereq-steps.yml deleted file mode 100644 index 3cb57c92757..00000000000 --- a/.azurepipelines/templates/spell-check-prereq-steps.yml +++ /dev/null @@ -1,22 +0,0 @@ -## @file -# File templates/spell-check-prereq-steps.yml -# -# template file used to install spell checking prerequisits -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -parameters: - none: '' - -steps: -- task: NodeTool@0 - inputs: - versionSpec: '17.x' # MU_CHANGE - #checkLatest: false # Optional - condition: and(gt(variables.pkg_count, 0), succeeded()) - -- script: npm install -g cspell@5.20.0 - displayName: 'Install cspell npm' - condition: and(gt(variables.pkg_count, 0), succeeded())