From 657bb22362c5d3367f819356272e478f3b3f405f Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 22 Feb 2024 14:24:49 -0800 Subject: [PATCH 01/79] Remove unused testBuildPlatforms property and some official-build-only properties from various AzDO scripts --- .../build-coreclr-and-libraries-job.yml | 2 - .../coreclr/libraries-gcstress-extra.yml | 5 --- .../libraries-gcstress0x3-gcstress0xc.yml | 5 --- .../coreclr/libraries-jitstress-random.yml | 5 --- eng/pipelines/coreclr/libraries-jitstress.yml | 5 --- .../libraries-jitstress2-jitstressregs.yml | 5 --- .../coreclr/libraries-jitstressregs.yml | 5 --- eng/pipelines/coreclr/libraries-pgo.yml | 5 --- eng/pipelines/coreclr/superpmi-collect.yml | 4 -- eng/pipelines/coreclr/templates/build-job.yml | 40 +++++-------------- 10 files changed, 9 insertions(+), 72 deletions(-) diff --git a/eng/pipelines/common/build-coreclr-and-libraries-job.yml b/eng/pipelines/common/build-coreclr-and-libraries-job.yml index 71cefc5dbbd07..5bbdb17b930e6 100644 --- a/eng/pipelines/common/build-coreclr-and-libraries-job.yml +++ b/eng/pipelines/common/build-coreclr-and-libraries-job.yml @@ -12,7 +12,6 @@ parameters: variables: {} pool: '' platform: '' - testBuildPlatforms: [] disableComponentGovernance: false jobs: @@ -49,5 +48,4 @@ jobs: pool: ${{ parameters.pool }} liveRuntimeBuildConfig: ${{ parameters.buildConfig }} platform: ${{ parameters.platform }} - testBuildPlatforms: ${{ parameters.testBuildPlatforms }} disableComponentGovernance: ${{ parameters.disableComponentGovernance }} diff --git a/eng/pipelines/coreclr/libraries-gcstress-extra.yml b/eng/pipelines/coreclr/libraries-gcstress-extra.yml index c87a99b56c1d2..5720cddd6ea17 100644 --- a/eng/pipelines/coreclr/libraries-gcstress-extra.yml +++ b/eng/pipelines/coreclr/libraries-gcstress-extra.yml @@ -27,11 +27,6 @@ extends: jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml buildConfig: checked platformGroup: gcstress - jobParameters: - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 # # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes diff --git a/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml b/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml index 1acc98df23927..ca04549c65e34 100644 --- a/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml +++ b/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml @@ -27,11 +27,6 @@ extends: jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml buildConfig: checked platformGroup: gcstress - jobParameters: - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 # # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes diff --git a/eng/pipelines/coreclr/libraries-jitstress-random.yml b/eng/pipelines/coreclr/libraries-jitstress-random.yml index eba6fb360f948..859bff6a949b0 100644 --- a/eng/pipelines/coreclr/libraries-jitstress-random.yml +++ b/eng/pipelines/coreclr/libraries-jitstress-random.yml @@ -32,11 +32,6 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - jobParameters: - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 # # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes diff --git a/eng/pipelines/coreclr/libraries-jitstress.yml b/eng/pipelines/coreclr/libraries-jitstress.yml index 4efcd59377237..ba921b4f1fa65 100644 --- a/eng/pipelines/coreclr/libraries-jitstress.yml +++ b/eng/pipelines/coreclr/libraries-jitstress.yml @@ -32,11 +32,6 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - jobParameters: - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 # # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes diff --git a/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml b/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml index 2a80b2757be90..6d8145098125f 100644 --- a/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml +++ b/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml @@ -32,11 +32,6 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - jobParameters: - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 # # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes diff --git a/eng/pipelines/coreclr/libraries-jitstressregs.yml b/eng/pipelines/coreclr/libraries-jitstressregs.yml index 57ab5bae5326f..b4ad87ff13579 100644 --- a/eng/pipelines/coreclr/libraries-jitstressregs.yml +++ b/eng/pipelines/coreclr/libraries-jitstressregs.yml @@ -32,11 +32,6 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - jobParameters: - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 # # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes diff --git a/eng/pipelines/coreclr/libraries-pgo.yml b/eng/pipelines/coreclr/libraries-pgo.yml index 46643af71d2a2..addaf68306559 100644 --- a/eng/pipelines/coreclr/libraries-pgo.yml +++ b/eng/pipelines/coreclr/libraries-pgo.yml @@ -32,11 +32,6 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - jobParameters: - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 # # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 336125d43bc13..1a9f70c46d51a 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -48,10 +48,6 @@ extends: - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: testGroup: outerloop - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 disableComponentGovernance: true # No shipping artifacts produced by this pipeline # superpmi-collect-job that targets macOS/arm64 depends on coreclr binaries produced by the macOS/x64 job diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index 9ed21ff433cd9..74e71a2342879 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -29,7 +29,6 @@ jobs: runtimeVariant: ${{ parameters.runtimeVariant }} testGroup: ${{ parameters.testGroup }} helixType: 'build/product/' - enableMicrobuild: true pool: ${{ parameters.pool }} condition: ${{ parameters.condition }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} @@ -90,9 +89,6 @@ jobs: - name: clrRuntimeComponentsBuildArg value: '-component runtime -component alljits -component nativeaot -component spmi ' - - name: SignType - value: $[ coalesce(variables.OfficialSignType, 'real') ] - # Set a default empty argument for the pgo path. # This will be set during the 'native prerequisites' step if PGO optimization is enabled. - name: CoreClrPgoDataArg @@ -122,17 +118,6 @@ jobs: - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - template: /eng/pipelines/common/restore-internal-tools.yml - # Install MicroBuild for signing the DAC and DBI - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.signBinaries, true), eq(parameters.osGroup, 'windows')) }}: - - task: MicroBuildSigningPlugin@2 - displayName: Install MicroBuild plugin for Signing - inputs: - signType: $(SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - continueOnError: false - condition: and(succeeded(), in(variables['SignType'], 'real', 'test')) - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - ${{ if ne(parameters.osGroup, 'windows') }}: - task: Bash@3 @@ -184,7 +169,7 @@ jobs: displayName: Build managed product components and packages # Build native test components - - ${{ if and(ne(parameters.isOfficialBuild, true), ne(parameters.disableClrTest, true)) }}: + - ${{ if ne(parameters.disableClrTest, true) }}: - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(osArg) $(priorityArg) $(compilerArg) displayName: Build native test components @@ -229,22 +214,15 @@ jobs: - ${{ if and(ne(parameters.testGroup, ''), ne(parameters.disableClrTest, true)) }}: # Publish test native components for consumption by test execution. - - ${{ if ne(parameters.isOfficialBuild, true) }}: - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(nativeTestArtifactRootFolderPath) - includeRootFolder: false - archiveType: $(archiveType) - tarCompression: $(tarCompression) - archiveExtension: $(archiveExtension) - artifactName: $(nativeTestArtifactName) - displayName: 'native test components' - - # Save packages using the prepare-signed-artifacts format. - - ${{ if eq(parameters.isOfficialBuild, true) }}: - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml parameters: - name: ${{ parameters.platform }} + rootFolder: $(nativeTestArtifactRootFolderPath) + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: $(nativeTestArtifactName) + displayName: 'native test components' - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log From ca1de74b47280b0feeda6e0267800d40ee590382 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 22 Feb 2024 14:30:08 -0800 Subject: [PATCH 02/79] Remove unused compilerArg parameter/variable (we don't use it now that our GCC leg uses the global build pipeline and our CI images automatically select the correct compiler) --- .../common/templates/runtimes/build-test-job.yml | 5 +---- eng/pipelines/coreclr/templates/build-jit-job.yml | 5 +---- eng/pipelines/coreclr/templates/build-job.yml | 10 ++++------ 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index e249e8ac922a7..31b769cac28dc 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -37,7 +37,6 @@ jobs: testGroup: ${{ parameters.testGroup }} pool: ${{ parameters.pool }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - compilerArg: '' # Test jobs should continue on error for internal builds ${{ if eq(variables['System.TeamProject'], 'internal') }}: @@ -69,8 +68,6 @@ jobs: - ${{ variable }} - name: liveRuntimeBuildParams value: 'libs+clr.iltools -c Release -ci' - - name: compilerArg - value: '' - name: runtimeFlavorArgs value: '' @@ -99,7 +96,7 @@ jobs: displayName: Install native dependencies # Build core/libraries dependencies of test build - - script: $(Build.SourcesDirectory)/build$(scriptExt) $(liveRuntimeBuildParams) $(crossArg) $(compilerArg) + - script: $(Build.SourcesDirectory)/build$(scriptExt) $(liveRuntimeBuildParams) $(crossArg) displayName: Build coreclr/libs components needed by test build - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index 1c131f3fc8f14..324fd3a3fbde0 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -53,9 +53,6 @@ jobs: - name: uploadAs value: ${{ parameters.uploadAs }} - - name: compilerArg - value: '' - - ${{ if eq(parameters.osGroup, 'windows') }}: - name: PythonSetupScript value: 'py -3 -m venv $(Build.SourcesDirectory)\venv' @@ -96,7 +93,7 @@ jobs: # Build CoreCLR JIT - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) -ci $(compilerArg) -component alljits -component spmi + - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) -ci -component alljits -component spmi displayName: Build CoreCLR JIT - ${{ if eq(parameters.osGroup, 'windows') }}: - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -component alljits -component spmi diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index 74e71a2342879..27f7eff9cee68 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -61,8 +61,6 @@ jobs: value: ${{ parameters.osGroup }} - name: osSubgroup value: ${{ parameters.osSubgroup }} - - name: compilerArg - value: '' - name: publishLogsArtifactPrefix value: 'BuildLogs_CoreCLR' - name: officialBuildIdArg @@ -148,14 +146,14 @@ jobs: # Build CoreCLR Runtime - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) $(nativeSymbols) + - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(clrRuntimeComponentsBuildArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) $(nativeSymbols) displayName: Build CoreCLR Runtime - ${{ if eq(parameters.osGroup, 'windows') }}: - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) displayName: Build CoreCLR Runtime - ${{ if or(eq(parameters.crossBuild, 'true'), ne(parameters.archType, 'x64')) }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -hostarch x64 $(osArg) -ci $(compilerArg) -component crosscomponents -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" $(officialBuildIdArg) + - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -hostarch x64 $(osArg) -ci -component crosscomponents -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" $(officialBuildIdArg) displayName: Build CoreCLR Cross-Arch Tools (Tools that run on x64 targeting x86) - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: @@ -165,12 +163,12 @@ jobs: displayName: Disk Usage after Build # Build CoreCLR Managed Components - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages $(crossArg) $(compilerArg) -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) -ci + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) -ci displayName: Build managed product components and packages # Build native test components - ${{ if ne(parameters.disableClrTest, true) }}: - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(osArg) $(priorityArg) $(compilerArg) + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(osArg) $(priorityArg) displayName: Build native test components # Sign and add entitlements to these MacOS binaries From 11b9929144f8c92f41f1558e7514c2ab12140a36 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 22 Feb 2024 15:53:41 -0800 Subject: [PATCH 03/79] Move riscv and freebsd build jobs to use the global-build-job template and remove parameters that were only used in these scenarios --- .../build-coreclr-and-libraries-job.yml | 2 - eng/pipelines/common/platform-matrix.yml | 3 - eng/pipelines/coreclr/clrinterpreter.yml | 4 +- eng/pipelines/coreclr/templates/build-job.yml | 83 ++++--------------- .../coreclr/templates/xplat-pipeline-job.yml | 7 -- eng/pipelines/installer/jobs/build-job.yml | 1 - eng/pipelines/runtime.yml | 50 +++++++++-- 7 files changed, 61 insertions(+), 89 deletions(-) diff --git a/eng/pipelines/common/build-coreclr-and-libraries-job.yml b/eng/pipelines/common/build-coreclr-and-libraries-job.yml index 5bbdb17b930e6..147a346b9bf4e 100644 --- a/eng/pipelines/common/build-coreclr-and-libraries-job.yml +++ b/eng/pipelines/common/build-coreclr-and-libraries-job.yml @@ -8,7 +8,6 @@ parameters: testGroup: '' crossBuild: false timeoutInMinutes: '' - signBinaries: false variables: {} pool: '' platform: '' @@ -26,7 +25,6 @@ jobs: testGroup: ${{ parameters.testGroup }} crossBuild: ${{ parameters.crossBuild }} timeoutInminutes: ${{ parameters.timeoutInMinutes }} - signBinaries: ${{ parameters.signBinaries }} variables: ${{ parameters.variables }} pool: ${{ parameters.pool }} disableComponentGovernance: ${{ parameters.disableComponentGovernance }} diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 776cdae314c42..90dbfece6688f 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -248,7 +248,6 @@ jobs: buildConfig: ${{ parameters.buildConfig }} helixQueueGroup: ${{ parameters.helixQueueGroup }} crossBuild: true - disableClrTest: true ${{ insert }}: ${{ parameters.jobParameters }} # Runtime-dev-innerloop build @@ -439,7 +438,6 @@ jobs: buildConfig: ${{ parameters.buildConfig }} helixQueueGroup: ${{ parameters.helixQueueGroup }} crossBuild: true - disableClrTest: true ${{ insert }}: ${{ parameters.jobParameters }} # WASI WebAssembly @@ -896,7 +894,6 @@ jobs: buildConfig: ${{ parameters.buildConfig }} helixQueueGroup: ${{ parameters.helixQueueGroup }} crossBuild: true - disableClrTest: true ${{ insert }}: ${{ parameters.jobParameters }} # Windows x64 diff --git a/eng/pipelines/coreclr/clrinterpreter.yml b/eng/pipelines/coreclr/clrinterpreter.yml index eac505a9f7072..8524867ee3ffe 100644 --- a/eng/pipelines/coreclr/clrinterpreter.yml +++ b/eng/pipelines/coreclr/clrinterpreter.yml @@ -21,7 +21,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_arm @@ -32,7 +32,7 @@ extends: - windows_arm64 - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: clrinterpreter + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release /p:CMakeArgs="-DFEATURE_INTERPRETER=1" - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index 27f7eff9cee68..ee00aa86674ab 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -6,14 +6,12 @@ parameters: crossBuild: false dependOnEvaluatePaths: false disableComponentGovernance: false - disableClrTest: false isOfficialBuild: false osGroup: '' osSubgroup: '' platform: '' pool: '' runtimeVariant: '' - signBinaries: false testGroup: '' timeoutInMinutes: '' variables: {} @@ -33,7 +31,6 @@ jobs: condition: ${{ parameters.condition }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - disableClrTest: ${{ parameters.disableClrTest }} # Compute job name from template parameters name: ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', @@ -75,12 +72,6 @@ jobs: - name: enforcePgoArg value: '-enforcepgo' - - name: clrInterpreterBuildArg - value: '' - - ${{ if eq(parameters.testGroup, 'clrinterpreter') }}: - - name: clrInterpreterBuildArg - value: '-cmakeargs "-DFEATURE_INTERPRETER=1"' - - name: clrRuntimeComponentsBuildArg value: '' - ${{ if ne(parameters.testGroup, 'innerloop') }}: @@ -141,19 +132,19 @@ jobs: displayName: Disk Usage before Build # Build/Generate native prerequisites - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.nativeprereqs $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) -ci /bl:$(Build.SourcesDirectory)artifacts/log/$(buildConfig)/CoreCLRNativePrereqs.binlog + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.nativeprereqs $(crossArg) -arch $(archType) -c $(buildConfig) $(officialBuildIdArg) -ci /bl:$(Build.SourcesDirectory)artifacts/log/$(buildConfig)/CoreCLRNativePrereqs.binlog displayName: Build and generate native prerequisites # Build CoreCLR Runtime - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(clrRuntimeComponentsBuildArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) $(nativeSymbols) + - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) -ci $(clrRuntimeComponentsBuildArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) $(nativeSymbols) displayName: Build CoreCLR Runtime - ${{ if eq(parameters.osGroup, 'windows') }}: - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) displayName: Build CoreCLR Runtime - ${{ if or(eq(parameters.crossBuild, 'true'), ne(parameters.archType, 'x64')) }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -hostarch x64 $(osArg) -ci -component crosscomponents -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" $(officialBuildIdArg) + - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -hostarch x64 -ci -component crosscomponents -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" $(officialBuildIdArg) displayName: Build CoreCLR Cross-Arch Tools (Tools that run on x64 targeting x86) - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: @@ -163,71 +154,25 @@ jobs: displayName: Disk Usage after Build # Build CoreCLR Managed Components - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) -ci + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages $(crossArg) -arch $(archType) -c $(buildConfig) $(officialBuildIdArg) -ci displayName: Build managed product components and packages - # Build native test components - - ${{ if ne(parameters.disableClrTest, true) }}: - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(osArg) $(priorityArg) - displayName: Build native test components - - # Sign and add entitlements to these MacOS binaries - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(parameters.osGroup, 'osx') }}: - - - template: /eng/pipelines/common/macos-sign-with-entitlements.yml - parameters: - filesToSign: - - name: createdump - path: $(buildProductRootFolderPath) - - name: corerun - path: $(buildProductRootFolderPath) - - - task: CopyFiles@2 - displayName: 'Copy signed createdump to sharedFramework' - inputs: - contents: createdump - sourceFolder: $(buildProductRootFolderPath) - targetFolder: $(buildProductRootFolderPath)/sharedFramework - overWrite: true - - - ${{ if and(eq(parameters.osGroup, 'windows'), eq(parameters.signBinaries, true)) }}: - - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml - parameters: - basePath: $(buildProductRootFolderPath) - isOfficialBuild: ${{ parameters.signBinaries }} - timeoutInMinutes: 30 - - - ${{ if ne(parameters.disableClrTest, true) }}: - # Publish product output directory for consumption by tests. - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(buildProductRootFolderPath) - includeRootFolder: false - archiveType: $(archiveType) - tarCompression: $(tarCompression) - archiveExtension: $(archiveExtension) - artifactName: $(buildProductArtifactName) - displayName: 'product build' - - - ${{ if and(ne(parameters.testGroup, ''), ne(parameters.disableClrTest, true)) }}: - # Publish test native components for consumption by test execution. - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(nativeTestArtifactRootFolderPath) - includeRootFolder: false - archiveType: $(archiveType) - tarCompression: $(tarCompression) - archiveExtension: $(archiveExtension) - artifactName: $(nativeTestArtifactName) - displayName: 'native test components' + # Publish product output directory for consumption by tests. + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(buildProductRootFolderPath) + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: $(buildProductArtifactName) + displayName: 'product build' - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log displayName: Collect vslogs on exit condition: always() - # Publish Logs - task: PublishPipelineArtifact@1 displayName: Publish Logs diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index b6c57be0c7fba..f4c913b3db51a 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -122,11 +122,4 @@ jobs: - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} - - name: osArg - value: '' - - - ${{ if eq(parameters.osGroup, 'freebsd') }}: - - name: osArg - value: -os freebsd - steps: ${{ parameters.steps }} diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml index 3bc803b2ba87a..bd68b5f81e28f 100644 --- a/eng/pipelines/installer/jobs/build-job.yml +++ b/eng/pipelines/installer/jobs/build-job.yml @@ -56,7 +56,6 @@ jobs: condition: ${{ parameters.condition }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - disableClrTest: ${{ parameters.disableClrTest }} # Compute job name from template parameters name: ${{ format('installer_{0}_{1}_{2}_{3}_', parameters.runtimeFlavor, parameters.runtimeVariant, coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index be2870611e617..85a272aa3c49c 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -72,13 +72,12 @@ extends: # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_x64 - linux_arm - linux_arm64 - - linux_riscv64 - linux_musl_arm - linux_musl_arm64 - linux_musl_x64 @@ -88,6 +87,8 @@ extends: - windows_arm64 jobParameters: testGroup: innerloop + nameSuffix: CoreCLR_With_Libs + buildArgs: -s clr+libs -c $(_BuildConfig) condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), @@ -160,7 +161,6 @@ extends: - windows_x64 - windows_x86 - windows_arm64 - - freebsd_x64 jobParameters: testGroup: innerloop # Mono/runtimetests also need this, but skip for wasm @@ -234,6 +234,48 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), eq(variables['isRollingBuild'], true)) + # + # Build CoreCLR without building test assets + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: checked + runtimeFlavor: coreclr + platforms: + - linux_riscv64 + jobParameters: + testScope: innerloop + nameSuffix: CoreCLR + buildArgs: -s clr.native+clr.tools+clr.corelib+clr.nativecorelib+clr.aot+clr.packages -c $(_BuildConfig) + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + # + # Build CoreCLR + Libs + Host + Packs + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + runtimeFlavor: coreclr + platforms: + - freebsd_x64 + jobParameters: + testScope: innerloop + nameSuffix: CoreCLR + buildArgs: -s clr+libs+host+packs -c $(_BuildConfig) -rc Checked + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + # # CoreCLR NativeAOT debug build and smoke tests # Only when CoreCLR is changed @@ -1071,7 +1113,6 @@ extends: - osx_arm64 - osx_x64 - windows_x64 - - freebsd_x64 jobParameters: testScope: innerloop condition: @@ -1195,7 +1236,6 @@ extends: - linux_arm64 - linux_musl_x64 - windows_x64 - - freebsd_x64 jobParameters: liveRuntimeBuildConfig: release liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} From 39072d3d6cc89bbbe9d7148e9cf1bab36e16f371 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 22 Feb 2024 16:17:21 -0800 Subject: [PATCH 04/79] Refactor out some of the test variables into a variable template --- .../templates/runtimes/build-test-job.yml | 6 -- .../runtimes/native-test-assets-variables.yml | 61 ++++++++++++++++++ .../coreclr/templates/xplat-pipeline-job.yml | 48 +------------- .../mono/templates/xplat-pipeline-job.yml | 63 ++----------------- 4 files changed, 68 insertions(+), 110 deletions(-) create mode 100644 eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index 31b769cac28dc..1b8ad3138a78c 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -18,12 +18,6 @@ parameters: ### Build managed test components (native components are getting built as part ### of the product build job). -### TODO: As of today, build of managed test components requires the product build -### as a prerequisite due to dependency on System.Private.Corelib. After switching -### over to its reference assembly we should be able to remove this dependency and -### run managed test builds in parallel with the product build job. - - jobs: - template: /eng/pipelines/${{ parameters.runtimeFlavor }}/templates/xplat-pipeline-job.yml parameters: diff --git a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml new file mode 100644 index 0000000000000..df82f552f654b --- /dev/null +++ b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -0,0 +1,61 @@ +# Variables to resolve the paths and artifact names for the test assets used and produced to run the tests +# in the src/tests tree. +parameters: + runtimeFlavor: 'coreclr' + runtimeVariant: '' + +variables: + - name: binTestsPath + value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr' + + # Build product defines what we are trying to build, either coreclr or mono + - ${{ if ne(parameters.runtimeFlavor, 'mono') }}: + - name: buildProductArtifactName + value: 'CoreCLRProduct___$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + + - name: buildProductRootFolderPath + value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' + - ${{ else }}: + - name: buildProductArtifactName + value: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + + # minijit and monointerpreter do not use separate product builds. + - ${{ if or(eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter')) }}: + - name : buildProductArtifactName + value : 'MonoProduct__$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + + - ${{ if eq(parameters.runtimeVariant, 'llvmfullaot') }}: + - name : buildProductArtifactName + value : 'MonoProduct_llvmaot_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + + - name: buildProductRootFolderPath + value: '$(Build.SourcesDirectory)/artifacts/bin/mono/$(osGroup).$(archType).$(buildConfigUpper)' + + # We need this because both mono and coreclr build currently depends on CoreClr + - name: coreClrProductArtifactName + value: 'CoreCLRProduct___$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + + - name: coreClrProductRootFolderPath + value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' + + - ${{ if ne(parameters.runtimeFlavor, 'mono') }}: + - name: managedGenericTestArtifactName + value: 'CoreCLRManagedTestArtifacts_AnyOS_AnyCPU_$(buildConfig)' + - ${{ else }}: + - name: managedGenericTestArtifactName + value: 'MonoManagedTestArtifacts_AnyOS_AnyCPU_$(buildConfig)' + + - name: managedTestArtifactRootFolderPath + value: '$(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper)' + + - name: nativeTestArtifactName + value: 'CoreCLRNativeTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + + - name: nativeTestArtifactRootFolderPath + value: '$(binTestsPath)/obj/$(osGroup).$(archType).$(buildConfigUpper)' + + - name: microsoftNetSdkIlFolderPath + value: '$(Build.SourcesDirectory)/.packages/microsoft.net.sdk.il' + + - name: microsoftNetSdkIlArtifactName + value: 'MicrosoftNetSdkIlPackage_AnyOS_AnyCPU_$(buildConfig)' diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index f4c913b3db51a..4757fb602db17 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -51,51 +51,9 @@ jobs: disableComponentGovernance: ${{ parameters.disableComponentGovernance }} variables: - - ${{ if ne(parameters.testGroup, '') }}: - - name: testArtifactRootName - value: ${{ parameters.Group }}${{ parameters.Subgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.testGroup }} - - - ${{ if eq(parameters.testGroup, '') }}: - - name: testArtifactRootName - value: ${{ parameters.Group }}${{ parameters.Subgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }} - - - name: binTestsPath - value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr' - - # Build product defines what we are trying to build, either coreclr or mono - - name: buildProductArtifactName - value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: buildProductRootFolderPath - value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' - - # We need this because both mono and coreclr build currently depends on CoreClr - - name: coreClrProductArtifactName - value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: coreClrProductRootFolderPath - value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' - - - name: corelibProductArtifactName - value: 'CoreLib_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: managedGenericTestArtifactName - value: 'CoreCLRManagedTestArtifacts_AnyOS_AnyCPU_$(buildConfig)' - - - name: managedTestArtifactRootFolderPath - value: '$(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper)' - - - name: nativeTestArtifactName - value: 'CoreCLRNativeTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: nativeTestArtifactRootFolderPath - value: '$(binTestsPath)/obj/$(osGroup).$(archType).$(buildConfigUpper)' - - - name: microsoftNetSdkIlFolderPath - value: '$(Build.SourcesDirectory)/.packages/microsoft.net.sdk.il' - - - name: microsoftNetSdkIlArtifactName - value: 'MicrosoftNetSdkIlPackage_AnyOS_AnyCPU_$(buildConfig)' + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr - name: priorityArg value: '' diff --git a/eng/pipelines/mono/templates/xplat-pipeline-job.yml b/eng/pipelines/mono/templates/xplat-pipeline-job.yml index 1ca84d9caac1b..b4e030695d949 100644 --- a/eng/pipelines/mono/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/mono/templates/xplat-pipeline-job.yml @@ -11,7 +11,6 @@ parameters: strategy: '' pool: '' runtimeVariant: '' - liveRuntimeBuildConfig: 'release' # arcade-specific parameters condition: true @@ -51,64 +50,10 @@ jobs: gatherAssetManifests: ${{ parameters.gatherAssetManifests }} variables: - - name: coreClrProductArtifactName - value: 'CoreCLRProduct__$(osGroup)$(osSubgroup)_$(archType)_${{ parameters.liveRuntimeBuildConfig }}' - - - name: coreClrProductRootFolderPath - value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(liveRuntimeBuildConfigUpper)' - - - name: buildProductArtifactName - value: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - # minijit and monointerpreter do not use separate product builds. - - ${{ if or(eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter')) }}: - - name : buildProductArtifactName - value : 'MonoProduct__$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - ${{ if eq(parameters.runtimeVariant, 'llvmfullaot') }}: - - name : buildProductArtifactName - value : 'MonoProduct_llvmaot_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: binTestsPath - value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr' - - - name: buildProductRootFolderPath - value: '$(Build.SourcesDirectory)/artifacts/bin/mono/$(osGroup).$(archType).$(buildConfigUpper)' - - - name: managedTestArtifactRootFolderPath - value: '$(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper)' - - - name: managedGenericTestArtifactName - value: 'MonoManagedTestArtifacts_AnyOS_AnyCPU_$(buildConfig)' - - - name: microsoftNetSdkIlFolderPath - value: '$(Build.SourcesDirectory)/.packages/microsoft.net.sdk.il' - - - name: microsoftNetSdkIlArtifactName - value: 'MicrosoftNetSdkIlPackage_AnyOS_AnyCPU_$(buildConfig)' - - - name: monoRepoRoot - value: '$(Build.SourcesDirectory)/src/mono' - - - name: nativeTestArtifactName - value: 'CoreCLRNativeTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: nativeTestArtifactRootFolderPath - value: '$(binTestsPath)/obj/$(osGroup).$(archType).$(buildConfigUpper)' - - - name: workloadPackagesPath - value: $(Build.SourcesDirectory)/artifacts/workloadPackages - - - name: workloadArtifactsPath - value: $(Build.SourcesDirectory)/artifacts/workloads - - - name: liveRuntimeBuildConfigUpper - ${{ if eq(parameters.liveRuntimeBuildConfig, 'release') }}: - value: 'Release' - ${{ if eq(parameters.liveRuntimeBuildConfig, 'checked') }}: - value: 'Checked' - ${{ if eq(parameters.liveRuntimeBuildConfig, 'debug') }}: - value: 'Debug' + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: mono + runtimeVariant: ${{ parameters.runtimeVariant }} - name: priorityArg value: '' From d6c53573f8ddfce53160fcbb0fc5842f18901cd7 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 22 Feb 2024 16:46:37 -0800 Subject: [PATCH 05/79] Move over clrinterpreter.yml pipeline as example. --- .../runtimes/native-test-assets-variables.yml | 10 ++ .../templates/runtimes/run-test-job.yml | 98 ++++++++++--------- eng/pipelines/coreclr/clrinterpreter.yml | 17 ++++ .../build-native-test-assets-step.yml | 13 +++ .../coreclr/templates/xplat-pipeline-job.yml | 10 +- .../mono/templates/xplat-pipeline-job.yml | 10 +- 6 files changed, 96 insertions(+), 62 deletions(-) create mode 100644 eng/pipelines/coreclr/templates/build-native-test-assets-step.yml diff --git a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml index df82f552f654b..1431c2571cbaf 100644 --- a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml +++ b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -3,6 +3,7 @@ parameters: runtimeFlavor: 'coreclr' runtimeVariant: '' + liveLibrariesBuildConfig: '' variables: - name: binTestsPath @@ -59,3 +60,12 @@ variables: - name: microsoftNetSdkIlArtifactName value: 'MicrosoftNetSdkIlPackage_AnyOS_AnyCPU_$(buildConfig)' + + - librariesBuildArtifactName: '' + - librariesOverrideArg: '' + - librariesDownloadDir: '' + + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - librariesBuildArtifactName: ${{ format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - librariesDownloadDir: $(Build.SourcesDirectory)/artifacts + - librariesOverrideArg: ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 28a421c105512..6363521bc1d52 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -9,6 +9,7 @@ parameters: readyToRun: false hotColdSplitting: false liveLibrariesBuildConfig: '' + unifiedArtifactsName: '' compositeBuildMode: false useCodeFlowEnforcement: '' helixQueues: '' @@ -199,62 +200,71 @@ jobs: steps: - # Optionally download live-built libraries - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - ${{ if ne(parameters.unifiedArtifactsName, '') }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(librariesDownloadDir) - cleanUnpackFolder: false - artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' - artifactName: '$(librariesBuildArtifactName)' - displayName: 'live-built libraries' + unpackFolder: $(Build.SourcesDirectory)/artifacts + artifactFileName: '$(unifiedArtifactsName)$(archiveExtension)' + artifactName: '$(unifiedArtifactsName)' + displayName: 'unified artifacts' + - ${{ else }}: - # Download and unzip managed test artifacts - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: '$(managedTestArtifactRootFolderPath)' - artifactFileName: '$(managedGenericTestArtifactName).tar.gz' - artifactName: '$(managedGenericTestArtifactName)' - displayName: 'generic managed test artifacts' + # Optionally download live-built libraries + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir) + cleanUnpackFolder: false + artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' + artifactName: '$(librariesBuildArtifactName)' + displayName: 'live-built libraries' - # Download product binaries directory - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' - displayName: 'product build' + # Download and unzip managed test artifacts + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: '$(managedTestArtifactRootFolderPath)' + artifactFileName: '$(managedGenericTestArtifactName).tar.gz' + artifactName: '$(managedGenericTestArtifactName)' + displayName: 'generic managed test artifacts' - - ${{ if eq(parameters.runtimeFlavor, 'mono') }}: - # We need to explicitly download CoreCLR for Mono + # Download product binaries directory - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(coreClrProductRootFolderPath) - artifactFileName: '$(coreClrProductArtifactName)$(archiveExtension)' - artifactName: '$(coreClrProductArtifactName)' - displayName: 'CoreCLR product download for Mono' - - # Download and unzip the Microsoft.NET.Sdk.IL package needed for traversing - # ilproj test projects during copynativeonly. - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: '$(microsoftNetSdkIlFolderPath)' - artifactFileName: '$(microsoftNetSdkIlArtifactName).tar.gz' - artifactName: '$(microsoftNetSdkIlArtifactName)' - displayName: 'Microsoft.NET.Sdk.IL package' - + unpackFolder: $(buildProductRootFolderPath) + artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' + artifactName: '$(buildProductArtifactName)' + displayName: 'product build' + + + - ${{ if eq(parameters.runtimeFlavor, 'mono') }}: + # We need to explicitly download CoreCLR for Mono + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(coreClrProductRootFolderPath) + artifactFileName: '$(coreClrProductArtifactName)$(archiveExtension)' + artifactName: '$(coreClrProductArtifactName)' + displayName: 'CoreCLR product download for Mono' + + # Download and unzip the Microsoft.NET.Sdk.IL package needed for traversing + # ilproj test projects during copynativeonly. + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: '$(microsoftNetSdkIlFolderPath)' + artifactFileName: '$(microsoftNetSdkIlArtifactName).tar.gz' + artifactName: '$(microsoftNetSdkIlArtifactName)' + displayName: 'Microsoft.NET.Sdk.IL package' - # Download and unzip native test artifacts - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: '$(nativeTestArtifactRootFolderPath)' - artifactFileName: '$(nativeTestArtifactName)$(archiveExtension)' - artifactName: '$(nativeTestArtifactName)' - displayName: 'native test artifacts' + # Download and unzip native test artifacts + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: '$(nativeTestArtifactRootFolderPath)' + artifactFileName: '$(nativeTestArtifactName)$(archiveExtension)' + artifactName: '$(nativeTestArtifactName)' + displayName: 'native test artifacts' # SuperPMI collection: Download x64 coreclr if running on non-x64 configuration (needed for mcs.exe) - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}: diff --git a/eng/pipelines/coreclr/clrinterpreter.yml b/eng/pipelines/coreclr/clrinterpreter.yml index 8524867ee3ffe..47adaeda8d754 100644 --- a/eng/pipelines/coreclr/clrinterpreter.yml +++ b/eng/pipelines/coreclr/clrinterpreter.yml @@ -33,6 +33,19 @@ extends: - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release /p:CMakeArgs="-DFEATURE_INTERPRETER=1" + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.ArtifactStagingDirectory)/artifacts + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreCLRInterpreterBuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -59,3 +72,7 @@ extends: jobParameters: testGroup: clrinterpreter liveLibrariesBuildConfig: Release + dependsOn: + - build_$(osGroup)$(osSubgroup)_$(archType)_checked_ + - CoreCLR_common_test_build_pri0_AnyOS_AnyCPU_checked + unifiedArtifactsName: CoreCLRInterpreterBuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/templates/build-native-test-assets-step.yml b/eng/pipelines/coreclr/templates/build-native-test-assets-step.yml new file mode 100644 index 0000000000000..55e8388141499 --- /dev/null +++ b/eng/pipelines/coreclr/templates/build-native-test-assets-step.yml @@ -0,0 +1,13 @@ +# Build the native assets for the tests in the src/tests +steps: + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(compilerArg) + displayName: Build native test components + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(nativeTestArtifactRootFolderPath) + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: $(nativeTestArtifactName) + displayName: 'native test components' diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index 4757fb602db17..83574df12c80e 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -54,6 +54,7 @@ jobs: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml parameters: runtimeFlavor: coreclr + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} - name: priorityArg value: '' @@ -68,15 +69,6 @@ jobs: - name: priorityArg value: '-priority 1' - - librariesBuildArtifactName: '' - - librariesOverrideArg: '' - - librariesDownloadDir: '' - - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - librariesBuildArtifactName: ${{ format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - - librariesDownloadDir: $(Build.SourcesDirectory)/artifacts - - librariesOverrideArg: ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' - - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} diff --git a/eng/pipelines/mono/templates/xplat-pipeline-job.yml b/eng/pipelines/mono/templates/xplat-pipeline-job.yml index b4e030695d949..6fac928c278c6 100644 --- a/eng/pipelines/mono/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/mono/templates/xplat-pipeline-job.yml @@ -54,19 +54,11 @@ jobs: parameters: runtimeFlavor: mono runtimeVariant: ${{ parameters.runtimeVariant }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} - name: priorityArg value: '' - - librariesBuildArtifactName: '' - - librariesOverrideArg: '' - - librariesDownloadDir: '' - - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - librariesBuildArtifactName: ${{ format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - - librariesDownloadDir: $(Build.SourcesDirectory)/artifacts - - librariesOverrideArg: ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' - - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} From fcda8f6c6ab347f32bb5d27ef5747a2650f15dae Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 23 Feb 2024 10:34:27 -0800 Subject: [PATCH 06/79] Clean up some of the clrinterpreter changes. --- .../templates/runtimes/run-test-job.yml | 50 +++++++++---------- eng/pipelines/coreclr/clrinterpreter.yml | 2 +- eng/pipelines/coreclr/templates/build-job.yml | 4 +- 3 files changed, 26 insertions(+), 30 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 6363521bc1d52..c4a878ca26c56 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -203,7 +203,7 @@ jobs: - ${{ if ne(parameters.unifiedArtifactsName, '') }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(Build.SourcesDirectory)/artifacts + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin artifactFileName: '$(unifiedArtifactsName)$(archiveExtension)' artifactName: '$(unifiedArtifactsName)' displayName: 'unified artifacts' @@ -220,16 +220,6 @@ jobs: artifactName: '$(librariesBuildArtifactName)' displayName: 'live-built libraries' - - # Download and unzip managed test artifacts - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: '$(managedTestArtifactRootFolderPath)' - artifactFileName: '$(managedGenericTestArtifactName).tar.gz' - artifactName: '$(managedGenericTestArtifactName)' - displayName: 'generic managed test artifacts' - - # Download product binaries directory - template: /eng/pipelines/common/download-artifact-step.yml parameters: @@ -238,7 +228,6 @@ jobs: artifactName: '$(buildProductArtifactName)' displayName: 'product build' - - ${{ if eq(parameters.runtimeFlavor, 'mono') }}: # We need to explicitly download CoreCLR for Mono - template: /eng/pipelines/common/download-artifact-step.yml @@ -248,23 +237,30 @@ jobs: artifactName: '$(coreClrProductArtifactName)' displayName: 'CoreCLR product download for Mono' - # Download and unzip the Microsoft.NET.Sdk.IL package needed for traversing - # ilproj test projects during copynativeonly. - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: '$(microsoftNetSdkIlFolderPath)' - artifactFileName: '$(microsoftNetSdkIlArtifactName).tar.gz' - artifactName: '$(microsoftNetSdkIlArtifactName)' - displayName: 'Microsoft.NET.Sdk.IL package' + # Download and unzip the Microsoft.NET.Sdk.IL package needed for traversing + # ilproj test projects during copynativeonly. + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: '$(microsoftNetSdkIlFolderPath)' + artifactFileName: '$(microsoftNetSdkIlArtifactName).tar.gz' + artifactName: '$(microsoftNetSdkIlArtifactName)' + displayName: 'Microsoft.NET.Sdk.IL package' + # Download and unzip managed test artifacts + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: '$(managedTestArtifactRootFolderPath)' + artifactFileName: '$(managedGenericTestArtifactName).tar.gz' + artifactName: '$(managedGenericTestArtifactName)' + displayName: 'generic managed test artifacts' - # Download and unzip native test artifacts - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: '$(nativeTestArtifactRootFolderPath)' - artifactFileName: '$(nativeTestArtifactName)$(archiveExtension)' - artifactName: '$(nativeTestArtifactName)' - displayName: 'native test artifacts' + # Download and unzip native test artifacts + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: '$(nativeTestArtifactRootFolderPath)' + artifactFileName: '$(nativeTestArtifactName)$(archiveExtension)' + artifactName: '$(nativeTestArtifactName)' + displayName: 'native test artifacts' # SuperPMI collection: Download x64 coreclr if running on non-x64 configuration (needed for mcs.exe) - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}: diff --git a/eng/pipelines/coreclr/clrinterpreter.yml b/eng/pipelines/coreclr/clrinterpreter.yml index 47adaeda8d754..0caf145bf75d1 100644 --- a/eng/pipelines/coreclr/clrinterpreter.yml +++ b/eng/pipelines/coreclr/clrinterpreter.yml @@ -37,7 +37,7 @@ extends: - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml - template: /eng/pipelines/common/upload-artifact-step.yml parameters: - rootFolder: $(Build.ArtifactStagingDirectory)/artifacts + rootFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin includeRootFolder: false archiveType: $(archiveType) archiveExtension: $(archiveExtension) diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index ee00aa86674ab..f1a2791862f1e 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -137,10 +137,10 @@ jobs: # Build CoreCLR Runtime - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) -ci $(clrRuntimeComponentsBuildArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) $(nativeSymbols) + - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) -ci $(clrRuntimeComponentsBuildArg) $(officialBuildIdArg) $(CoreClrPgoDataArg) $(nativeSymbols) displayName: Build CoreCLR Runtime - ${{ if eq(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) + - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(officialBuildIdArg) $(CoreClrPgoDataArg) displayName: Build CoreCLR Runtime - ${{ if or(eq(parameters.crossBuild, 'true'), ne(parameters.archType, 'x64')) }}: From aab46ee8a90e3e59dec43813cf86f5e7da94eb88 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 23 Feb 2024 10:49:10 -0800 Subject: [PATCH 07/79] Update runtime-cet pipeline as it's a simple one and easy to validate. --- eng/pipelines/runtime-cet.yml | 52 ++++++++++++++--------------------- 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/eng/pipelines/runtime-cet.yml b/eng/pipelines/runtime-cet.yml index c756a3c83d05c..dba73a61505a8 100644 --- a/eng/pipelines/runtime-cet.yml +++ b/eng/pipelines/runtime-cet.yml @@ -38,45 +38,29 @@ extends: jobs: # - # Build CoreCLR checked - # Only when CoreCLR is changed + # Build CoreCLR and Libraries # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - windows_x64 jobParameters: - testGroup: innerloop - - # - # Build CoreCLR release - # Always as they are needed by Installer and we always build and test the Installer. - # - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: release - platforms: - - windows_x64 - jobParameters: - testScope: innerloop - - # - # CoreCLR Test builds using live libraries release build - # Only when CoreCLR is changed - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: innerloop - + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreCLRInterpreterBuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml # # CoreCLR Test executions using live libraries @@ -95,3 +79,7 @@ extends: testGroup: innerloop liveLibrariesBuildConfig: release useCodeFlowEnforcement: true + dependsOn: + - build_$(osGroup)$(osSubgroup)_$(archType)_checked_ + - CoreCLR_common_test_build_pri0_AnyOS_AnyCPU_checked + unifiedArtifactsName: CoreCLRInterpreterBuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) From dfaa1533f7d8cf80001427763af524d8b301670d Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 23 Feb 2024 11:03:12 -0800 Subject: [PATCH 08/79] Use name/value consistently --- .../runtimes/native-test-assets-variables.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml index 1431c2571cbaf..a57b17907fa77 100644 --- a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml +++ b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -61,11 +61,17 @@ variables: - name: microsoftNetSdkIlArtifactName value: 'MicrosoftNetSdkIlPackage_AnyOS_AnyCPU_$(buildConfig)' - - librariesBuildArtifactName: '' - - librariesOverrideArg: '' - - librariesDownloadDir: '' + - name: librariesBuildArtifactName + value : '' + - name: librariesOverrideArg + value : '' + - name: librariesDownloadDir + value : '' - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - librariesBuildArtifactName: ${{ format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - - librariesDownloadDir: $(Build.SourcesDirectory)/artifacts - - librariesOverrideArg: ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' + - name: librariesBuildArtifactName + value : ${{ format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - name: librariesOverrideArg + value : $(Build.SourcesDirectory)/artifacts + - name: librariesDownloadDir + value : ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' From 3229aa65e773405eb8995120a15d1b1be4d5ba24 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 23 Feb 2024 11:17:41 -0800 Subject: [PATCH 09/79] Revert change in runtime pipeline (we'll move this pipeline last). --- eng/pipelines/runtime.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 85a272aa3c49c..2f43741e572d5 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -72,7 +72,7 @@ extends: # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml + jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml buildConfig: checked platforms: - linux_x64 @@ -87,8 +87,6 @@ extends: - windows_arm64 jobParameters: testGroup: innerloop - nameSuffix: CoreCLR_With_Libs - buildArgs: -s clr+libs -c $(_BuildConfig) condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), From 376aea3b15de865f4afc00e76777cbed43b37e76 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 23 Feb 2024 11:21:56 -0800 Subject: [PATCH 10/79] Set variables in run-test-job directly --- eng/pipelines/common/templates/runtimes/run-test-job.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index c4a878ca26c56..4230e764154f6 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -88,6 +88,13 @@ jobs: variables: + - name: osGroup + value: ${{ parameters.osGroup }} + - name: osSubgroup + value: ${{ parameters.osSubgroup }} + - name: archType + value: ${{ parameters.archType }} + - name: monoAotBuildshCommand value: '' From 2a5feb670cb6c970ed71d09c0da063cf6eb1acd3 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 23 Feb 2024 11:22:40 -0800 Subject: [PATCH 11/79] Update job dependency --- eng/pipelines/coreclr/clrinterpreter.yml | 2 +- eng/pipelines/runtime-cet.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/clrinterpreter.yml b/eng/pipelines/coreclr/clrinterpreter.yml index 0caf145bf75d1..7540a8c4eadcf 100644 --- a/eng/pipelines/coreclr/clrinterpreter.yml +++ b/eng/pipelines/coreclr/clrinterpreter.yml @@ -74,5 +74,5 @@ extends: liveLibrariesBuildConfig: Release dependsOn: - build_$(osGroup)$(osSubgroup)_$(archType)_checked_ - - CoreCLR_common_test_build_pri0_AnyOS_AnyCPU_checked + - coreclr_common_test_build_pri0_AnyOS_AnyCPU_checked unifiedArtifactsName: CoreCLRInterpreterBuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/runtime-cet.yml b/eng/pipelines/runtime-cet.yml index dba73a61505a8..f06a1a4edde29 100644 --- a/eng/pipelines/runtime-cet.yml +++ b/eng/pipelines/runtime-cet.yml @@ -81,5 +81,5 @@ extends: useCodeFlowEnforcement: true dependsOn: - build_$(osGroup)$(osSubgroup)_$(archType)_checked_ - - CoreCLR_common_test_build_pri0_AnyOS_AnyCPU_checked + - coreclr_common_test_build_pri0_AnyOS_AnyCPU_checked unifiedArtifactsName: CoreCLRInterpreterBuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) From 83e77cc23e936194b1b92bbb48c12a8373c5f90f Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 23 Feb 2024 11:31:20 -0800 Subject: [PATCH 12/79] Avoid manually specifying dependencies with variables, AzDO doesn't allow variables here. --- .../templates/runtimes/run-test-job.yml | 29 ++++++++++--------- eng/pipelines/runtime-cet.yml | 7 ++--- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 4230e764154f6..af950b6b3f91a 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -61,19 +61,22 @@ jobs: - '${{ parameters.runtimeFlavor }}_common_test_build_p0_AnyOS_AnyCPU_${{parameters.buildConfig }}' - ${{ if notIn(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: - '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}' - - ${{ if or( eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter'), eq(parameters.runtimeVariant, 'llvmaot'), eq(parameters.runtimeVariant, 'llvmfullaot')) }}: - # This is needed for creating a CORE_ROOT in the current design. - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', '', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if or( eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter')) }} : - # minijit and mono interpreter runtimevariants do not require any special build of the runtime - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, '', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if not(or(eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter'))) }}: - - ${{ if eq(parameters.runtimeVariant, 'llvmfullaot') }}: - - ${{ format('{0}_llvmaot_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.runtimeVariant, 'llvmfullaot') }}: - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - ${{ if ne(parameters.unifiedArtifactsName, '')}}: + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_' + - ${{ else }}: + - ${{ if or( eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter'), eq(parameters.runtimeVariant, 'llvmaot'), eq(parameters.runtimeVariant, 'llvmfullaot')) }}: + # This is needed for creating a CORE_ROOT in the current design. + - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', '', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if or( eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter')) }} : + # minijit and mono interpreter runtimevariants do not require any special build of the runtime + - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, '', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if not(or(eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter'))) }}: + - ${{ if eq(parameters.runtimeVariant, 'llvmfullaot') }}: + - ${{ format('{0}_llvmaot_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if ne(parameters.runtimeVariant, 'llvmfullaot') }}: + - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} # SuperPMI collection needs to run mcs.exe on the AzDO machine. Assume that's an x64 machine, and download an x64 product build if needed. - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}: - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', '', parameters.osGroup, parameters.osSubgroup, 'x64', parameters.buildConfig) }} diff --git a/eng/pipelines/runtime-cet.yml b/eng/pipelines/runtime-cet.yml index f06a1a4edde29..81f04a1b44f3d 100644 --- a/eng/pipelines/runtime-cet.yml +++ b/eng/pipelines/runtime-cet.yml @@ -57,7 +57,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: CoreCLRInterpreterBuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -79,7 +79,4 @@ extends: testGroup: innerloop liveLibrariesBuildConfig: release useCodeFlowEnforcement: true - dependsOn: - - build_$(osGroup)$(osSubgroup)_$(archType)_checked_ - - coreclr_common_test_build_pri0_AnyOS_AnyCPU_checked - unifiedArtifactsName: CoreCLRInterpreterBuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) From 0beb97a247bbeababad5d40e3fa48a3f0adae590 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 23 Feb 2024 11:32:34 -0800 Subject: [PATCH 13/79] Add back test build --- eng/pipelines/runtime-cet.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/runtime-cet.yml b/eng/pipelines/runtime-cet.yml index 81f04a1b44f3d..9ff606205df34 100644 --- a/eng/pipelines/runtime-cet.yml +++ b/eng/pipelines/runtime-cet.yml @@ -62,9 +62,20 @@ extends: extraVariablesTemplates: - /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + # + # CoreCLR Test build + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml + buildConfig: checked + platforms: + - CoreClrTestBuildHost # Either osx_x64 or linux_x64 + jobParameters: + testGroup: innerloop + # # CoreCLR Test executions using live libraries - # Only when CoreCLR is changed # - template: /eng/pipelines/common/platform-matrix.yml From a49533f0e6cdbf76414c1823439b937407da12ab Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 23 Feb 2024 13:22:38 -0800 Subject: [PATCH 14/79] Pass artifacts name as parameter, not variable --- eng/pipelines/common/templates/runtimes/run-test-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index af950b6b3f91a..0b7550f60786d 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -214,8 +214,8 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - artifactFileName: '$(unifiedArtifactsName)$(archiveExtension)' - artifactName: '$(unifiedArtifactsName)' + artifactFileName: '${{ parameters.unifiedArtifactsName }} $(archiveExtension)' + artifactName: '${{ parameters.unifiedArtifactsName }}' displayName: 'unified artifacts' - ${{ else }}: From 0049e9ba3921d7df1fac0c2ea26803991c2c2fce Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 23 Feb 2024 14:18:10 -0800 Subject: [PATCH 15/79] Fix indentation so our post-build steps actually run --- eng/pipelines/runtime-cet.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/runtime-cet.yml b/eng/pipelines/runtime-cet.yml index 9ff606205df34..b2fae3976a22f 100644 --- a/eng/pipelines/runtime-cet.yml +++ b/eng/pipelines/runtime-cet.yml @@ -48,17 +48,17 @@ extends: - windows_x64 jobParameters: buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release - postBuildSteps: - - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin - includeRootFolder: false - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) - displayName: Build Assets + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets extraVariablesTemplates: - /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml From bd2f6a63f8ca1a129b91af13d755fa2ab0684c2c Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 23 Feb 2024 15:36:28 -0800 Subject: [PATCH 16/79] Move priority arg to variable template and fix variable template usage --- .../runtimes/native-test-assets-variables.yml | 13 +++++++++ eng/pipelines/coreclr/clrinterpreter.yml | 28 ++++++++++--------- .../build-native-test-assets-step.yml | 5 +++- .../coreclr/templates/xplat-pipeline-job.yml | 13 --------- eng/pipelines/runtime-cet.yml | 6 ++-- 5 files changed, 36 insertions(+), 29 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml index a57b17907fa77..628530a0b9d50 100644 --- a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml +++ b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -75,3 +75,16 @@ variables: value : $(Build.SourcesDirectory)/artifacts - name: librariesDownloadDir value : ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' + + - name: priorityArg + value: '' + + # 'innerloop' and 'clrinterpreter' jobs run the Priority 0 tests; everything else runs the Priority 1 tests. + # 'gc-standalone' is forced to run pri0 as well to start with. + - ${{ if and(ne(parameters.testGroup, 'innerloop'), ne(parameters.testGroup, 'clrinterpreter'), ne(parameters.testGroup, 'gc-standalone'), ne(parameters.testGroup, 'gc-standalone-server') ) }}: + - ${{ if ne(parameters.osGroup, 'windows') }}: + - name: priorityArg + value: 'priority1' + - ${{ if eq(parameters.osGroup, 'windows') }}: + - name: priorityArg + value: '-priority 1' diff --git a/eng/pipelines/coreclr/clrinterpreter.yml b/eng/pipelines/coreclr/clrinterpreter.yml index 7540a8c4eadcf..2a244651bacb2 100644 --- a/eng/pipelines/coreclr/clrinterpreter.yml +++ b/eng/pipelines/coreclr/clrinterpreter.yml @@ -33,19 +33,21 @@ extends: - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release /p:CMakeArgs="-DFEATURE_INTERPRETER=1" - postBuildSteps: - - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin - includeRootFolder: false - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: CoreCLRInterpreterBuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) - displayName: Build Assets - extraVariablesTemplates: - - /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreCLRInterpreterBuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: clrinterpreter - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/coreclr/templates/build-native-test-assets-step.yml b/eng/pipelines/coreclr/templates/build-native-test-assets-step.yml index 55e8388141499..9e65b9875ff31 100644 --- a/eng/pipelines/coreclr/templates/build-native-test-assets-step.yml +++ b/eng/pipelines/coreclr/templates/build-native-test-assets-step.yml @@ -1,6 +1,9 @@ # Build the native assets for the tests in the src/tests +parameters: + compiler: '' + steps: - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(compilerArg) + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(_BuildConfig) $(archType) $(crossArg) $(priorityArg) ${{ parameters.compiler }} displayName: Build native test components - template: /eng/pipelines/common/upload-artifact-step.yml parameters: diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index 83574df12c80e..2bb667b8b512b 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -56,19 +56,6 @@ jobs: runtimeFlavor: coreclr liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} - - name: priorityArg - value: '' - - # 'innerloop' and 'clrinterpreter' jobs run the Priority 0 tests; everything else runs the Priority 1 tests. - # 'gc-standalone' is forced to run pri0 as well to start with. - - ${{ if and(ne(parameters.testGroup, 'innerloop'), ne(parameters.testGroup, 'clrinterpreter'), ne(parameters.testGroup, 'gc-standalone'), ne(parameters.testGroup, 'gc-standalone-server') ) }}: - - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: priorityArg - value: 'priority1' - - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: priorityArg - value: '-priority 1' - - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} diff --git a/eng/pipelines/runtime-cet.yml b/eng/pipelines/runtime-cet.yml index b2fae3976a22f..198cd012a3c68 100644 --- a/eng/pipelines/runtime-cet.yml +++ b/eng/pipelines/runtime-cet.yml @@ -59,8 +59,10 @@ extends: tarCompression: $(tarCompression) artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets - extraVariablesTemplates: - - /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: innerloop # # CoreCLR Test build From 8657e90d0c2bc78df802c4f437e4015c0d05da8c Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 26 Feb 2024 11:14:50 -0800 Subject: [PATCH 17/79] Fix artifacts upload folder. --- eng/pipelines/coreclr/clrinterpreter.yml | 2 +- eng/pipelines/runtime-cet.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/clrinterpreter.yml b/eng/pipelines/coreclr/clrinterpreter.yml index 2a244651bacb2..06d8863fbb5a1 100644 --- a/eng/pipelines/coreclr/clrinterpreter.yml +++ b/eng/pipelines/coreclr/clrinterpreter.yml @@ -37,7 +37,7 @@ extends: - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml - template: /eng/pipelines/common/upload-artifact-step.yml parameters: - rootFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin + rootFolder: $(Build.SourcesDirectory)/artifacts/bin includeRootFolder: false archiveType: $(archiveType) archiveExtension: $(archiveExtension) diff --git a/eng/pipelines/runtime-cet.yml b/eng/pipelines/runtime-cet.yml index 198cd012a3c68..120f7ef031344 100644 --- a/eng/pipelines/runtime-cet.yml +++ b/eng/pipelines/runtime-cet.yml @@ -52,7 +52,7 @@ extends: - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml - template: /eng/pipelines/common/upload-artifact-step.yml parameters: - rootFolder: $(Build.ArtifactStagingDirectory)/artifacts/bin + rootFolder: $(Build.SourcesDirectory)/artifacts/bin includeRootFolder: false archiveType: $(archiveType) archiveExtension: $(archiveExtension) From 1a13131d931010c06a9f94b476b99c09c7b4af25 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 26 Feb 2024 12:33:56 -0800 Subject: [PATCH 18/79] Make sure both "non-uppercase build config" variable we use in the global build today are set in both places. --- .../common/templates/runtimes/native-test-assets-variables.yml | 2 +- eng/pipelines/common/templates/runtimes/xplat-job.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml index 628530a0b9d50..01acd2fa6b00f 100644 --- a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml +++ b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -50,7 +50,7 @@ variables: value: '$(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper)' - name: nativeTestArtifactName - value: 'CoreCLRNativeTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + value: 'CoreCLRNativeTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)' - name: nativeTestArtifactRootFolderPath value: '$(binTestsPath)/obj/$(osGroup).$(archType).$(buildConfigUpper)' diff --git a/eng/pipelines/common/templates/runtimes/xplat-job.yml b/eng/pipelines/common/templates/runtimes/xplat-job.yml index 23e74c70e57ac..f97ca08e5162e 100644 --- a/eng/pipelines/common/templates/runtimes/xplat-job.yml +++ b/eng/pipelines/common/templates/runtimes/xplat-job.yml @@ -77,6 +77,9 @@ jobs: - name: buildConfig value: ${{ parameters.buildConfig }} + - name: _BuildConfig + value: ${{ parameters.buildConfig }} + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - name: _HelixSource value: official/dotnet/runtime/$(Build.SourceBranch) From 532396f86573eea37321930aa8b822ee9b262bc6 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 26 Feb 2024 18:22:07 -0800 Subject: [PATCH 19/79] Fix argument order --- .../templates/runtimes/native-test-assets-variables.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml index 01acd2fa6b00f..e7759778d8bd0 100644 --- a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml +++ b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -72,9 +72,9 @@ variables: - name: librariesBuildArtifactName value : ${{ format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - name: librariesOverrideArg - value : $(Build.SourcesDirectory)/artifacts - - name: librariesDownloadDir value : ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' + - name: librariesDownloadDir + value : $(Build.SourcesDirectory)/artifacts - name: priorityArg value: '' From c9ca09957e337c2b4399b41daf66c39aca542538 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 26 Feb 2024 20:46:50 -0800 Subject: [PATCH 20/79] Remove extraneous space --- eng/pipelines/common/templates/runtimes/run-test-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 0b7550f60786d..b8964485b9030 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -214,7 +214,7 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - artifactFileName: '${{ parameters.unifiedArtifactsName }} $(archiveExtension)' + artifactFileName: '${{ parameters.unifiedArtifactsName }}$(archiveExtension)' artifactName: '${{ parameters.unifiedArtifactsName }}' displayName: 'unified artifacts' From 9234db8ca6dd30cabd8115e150ab77f90f83cfeb Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 27 Feb 2024 11:50:48 -0800 Subject: [PATCH 21/79] Convert various pipelines to use the global build design --- eng/pipelines/coreclr/crossgen2-composite.yml | 21 ++++++++++++++--- eng/pipelines/coreclr/crossgen2-gcstress.yml | 21 ++++++++++++++--- eng/pipelines/coreclr/crossgen2.yml | 22 +++++++++++++++--- eng/pipelines/coreclr/gc-longrunning.yml | 21 ++++++++++++++--- eng/pipelines/coreclr/gc-simulator.yml | 21 ++++++++++++++--- eng/pipelines/coreclr/gc-standalone.yml | 23 ++++++++++++++++--- eng/pipelines/coreclr/gcstress-extra.yml | 21 ++++++++++++++--- .../coreclr/gcstress0x3-gcstress0xc.yml | 21 ++++++++++++++--- eng/pipelines/coreclr/ilasm.yml | 21 ++++++++++++++--- eng/pipelines/coreclr/jit-cfg.yml | 21 ++++++++++++++--- eng/pipelines/coreclr/jit-experimental.yml | 21 ++++++++++++++--- 11 files changed, 201 insertions(+), 33 deletions(-) diff --git a/eng/pipelines/coreclr/crossgen2-composite.yml b/eng/pipelines/coreclr/crossgen2-composite.yml index 2830cdc47ded1..328f394d39419 100644 --- a/eng/pipelines/coreclr/crossgen2-composite.yml +++ b/eng/pipelines/coreclr/crossgen2-composite.yml @@ -20,7 +20,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_arm @@ -31,9 +31,23 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: innerloop + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: innerloop - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -64,3 +78,4 @@ extends: compositeBuildMode: true displayNameArgs: Composite liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/crossgen2-gcstress.yml b/eng/pipelines/coreclr/crossgen2-gcstress.yml index a004e7904a824..b43c129db79e3 100644 --- a/eng/pipelines/coreclr/crossgen2-gcstress.yml +++ b/eng/pipelines/coreclr/crossgen2-gcstress.yml @@ -20,7 +20,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_x64 @@ -30,9 +30,23 @@ extends: # - osx_x64 - windows_x64 - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: gcstress-extra + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: gcstress-extra - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -62,3 +76,4 @@ extends: compositeBuildMode: true displayNameArgs: Composite liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/crossgen2.yml b/eng/pipelines/coreclr/crossgen2.yml index 53ea5878d32ce..e2bdb4781ddd0 100644 --- a/eng/pipelines/coreclr/crossgen2.yml +++ b/eng/pipelines/coreclr/crossgen2.yml @@ -20,7 +20,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_x64 @@ -29,9 +29,23 @@ extends: - osx_x64 - windows_x64 - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: innerloop + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: innerloop - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -59,6 +73,7 @@ extends: readyToRun: true displayNameArgs: R2R_CG2 liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -100,3 +115,4 @@ extends: hotColdSplitting: true displayNameArgs: R2R_CG2_HotColdSplitting liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/gc-longrunning.yml b/eng/pipelines/coreclr/gc-longrunning.yml index c58e6bd0ab5b1..40aeab2c3e19b 100644 --- a/eng/pipelines/coreclr/gc-longrunning.yml +++ b/eng/pipelines/coreclr/gc-longrunning.yml @@ -20,7 +20,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - linux_x64 @@ -28,9 +28,23 @@ extends: - windows_x64 - windows_arm64 - osx_x64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: gc-longrunning + buildArgs: -s clr+libs -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: gc-longrunning - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -55,3 +69,4 @@ extends: jobParameters: testGroup: gc-longrunning liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/gc-simulator.yml b/eng/pipelines/coreclr/gc-simulator.yml index fd4b700a7053a..dd7f08c353fa3 100644 --- a/eng/pipelines/coreclr/gc-simulator.yml +++ b/eng/pipelines/coreclr/gc-simulator.yml @@ -20,7 +20,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: # disable Linux x64 for now until OOMs are resolved. @@ -29,9 +29,23 @@ extends: - windows_x64 - windows_arm64 - osx_x64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: gc-simulator + buildArgs: -s clr+libs -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: gc-simulator - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -57,3 +71,4 @@ extends: jobParameters: testGroup: gc-simulator liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/gc-standalone.yml b/eng/pipelines/coreclr/gc-standalone.yml index 7ebf86566f3b4..9da4a15957041 100644 --- a/eng/pipelines/coreclr/gc-standalone.yml +++ b/eng/pipelines/coreclr/gc-standalone.yml @@ -20,15 +20,30 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_arm64 + - linux_x64 - windows_arm64 - windows_x64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: gc-standalone + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: gc-standalone - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -54,6 +69,7 @@ extends: testGroup: gc-standalone displayNameArgs: GCStandAlone liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -70,3 +86,4 @@ extends: testGroup: gc-standalone-server displayNameArgs: GCStandAloneServer liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/gcstress-extra.yml b/eng/pipelines/coreclr/gcstress-extra.yml index 61e5f3651f08b..7076bb9a8795b 100644 --- a/eng/pipelines/coreclr/gcstress-extra.yml +++ b/eng/pipelines/coreclr/gcstress-extra.yml @@ -20,16 +20,30 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platformGroup: gcstress platforms: # It is too early to include osx_arm64 in platform group gcstress # Adding it here will enable it also - osx_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: gcstress-extra + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: gcstress-extra - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -54,3 +68,4 @@ extends: jobParameters: testGroup: gcstress-extra liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml b/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml index 2e501b2e18abf..5e6bbe82cc524 100644 --- a/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml +++ b/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml @@ -20,16 +20,30 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platformGroup: gcstress platforms: # It is too early to include osx_arm64 in platform group gcstress # Adding it here will enable it also - osx_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: gcstress0x3-gcstress0xc + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: gcstress0x3-gcstress0xc - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -54,3 +68,4 @@ extends: jobParameters: testGroup: gcstress0x3-gcstress0xc liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/ilasm.yml b/eng/pipelines/coreclr/ilasm.yml index c57709a37d1b5..cc0c655607dec 100644 --- a/eng/pipelines/coreclr/ilasm.yml +++ b/eng/pipelines/coreclr/ilasm.yml @@ -29,7 +29,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - osx_arm64 @@ -40,9 +40,23 @@ extends: - windows_x64 - windows_x86 - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: ilasm + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: ilasm - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -71,3 +85,4 @@ extends: jobParameters: testGroup: ilasm liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/jit-cfg.yml b/eng/pipelines/coreclr/jit-cfg.yml index 87ae66fbe4329..3eda410b02035 100644 --- a/eng/pipelines/coreclr/jit-cfg.yml +++ b/eng/pipelines/coreclr/jit-cfg.yml @@ -20,16 +20,30 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_arm64 - linux_x64 - windows_arm64 - windows_x64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: jit-cfg + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: jit-cfg - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -54,3 +68,4 @@ extends: jobParameters: testGroup: jit-cfg liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/jit-experimental.yml b/eng/pipelines/coreclr/jit-experimental.yml index a599f343e310f..2c07876fdfc4b 100644 --- a/eng/pipelines/coreclr/jit-experimental.yml +++ b/eng/pipelines/coreclr/jit-experimental.yml @@ -20,7 +20,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - osx_arm64 @@ -29,9 +29,23 @@ extends: - linux_x64 - windows_arm64 - windows_x64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: jit-experimental + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: jit-experimental - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -58,3 +72,4 @@ extends: jobParameters: testGroup: jit-experimental liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) From 137cfdfc07e7ebc79cda224a841fdc411041e43c Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 27 Feb 2024 12:48:58 -0800 Subject: [PATCH 22/79] Create new jit outerloop pipeline template and move remaining simple (one product-build/test-build/test-run set of jobs) runtime test pipelines to use the global build approach. --- eng/pipelines/coreclr/ilasm.yml | 76 +++---------------- eng/pipelines/coreclr/jit-cfg.yml | 64 ++-------------- eng/pipelines/coreclr/jit-experimental.yml | 69 +++-------------- eng/pipelines/coreclr/jitstress-isas-arm.yml | 45 ++--------- .../coreclr/jitstress-isas-avx512.yml | 45 ++--------- eng/pipelines/coreclr/jitstress-isas-x86.yml | 48 ++---------- eng/pipelines/coreclr/jitstress-random.yml | 60 +++------------ eng/pipelines/coreclr/jitstress.yml | 60 +++------------ .../coreclr/jitstress2-jitstressregs.yml | 60 +++------------ eng/pipelines/coreclr/jitstressregs-x86.yml | 45 ++--------- eng/pipelines/coreclr/jitstressregs.yml | 60 +++------------ eng/pipelines/coreclr/pgo.yml | 57 +++----------- eng/pipelines/coreclr/pgostress.yml | 57 +++----------- eng/pipelines/coreclr/r2r-extra.yml | 21 ++++- eng/pipelines/coreclr/r2r.yml | 21 ++++- eng/pipelines/coreclr/release-tests.yml | 22 +++++- eng/pipelines/coreclr/runincontext.yml | 21 ++++- .../templates/jit-outerloop-pipeline.yml | 65 ++++++++++++++++ eng/pipelines/coreclr/tieringtest.yml | 21 ++++- 19 files changed, 272 insertions(+), 645 deletions(-) create mode 100644 eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml diff --git a/eng/pipelines/coreclr/ilasm.yml b/eng/pipelines/coreclr/ilasm.yml index cc0c655607dec..4747727284393 100644 --- a/eng/pipelines/coreclr/ilasm.yml +++ b/eng/pipelines/coreclr/ilasm.yml @@ -19,70 +19,16 @@ schedules: variables: - template: /eng/pipelines/common/variables.yml - extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - jobParameters: - buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release - postBuildSteps: - - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/bin - includeRootFolder: false - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) - displayName: Build Assets - extraVariablesTemplates: - - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml - parameters: - testGroup: ilasm - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: ilasm - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: ilasm - liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + platforms: + - osx_arm64 + - osx_x64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + testGroup: ilasm diff --git a/eng/pipelines/coreclr/jit-cfg.yml b/eng/pipelines/coreclr/jit-cfg.yml index 3eda410b02035..80fe44da119d0 100644 --- a/eng/pipelines/coreclr/jit-cfg.yml +++ b/eng/pipelines/coreclr/jit-cfg.yml @@ -10,62 +10,12 @@ schedules: variables: - template: /eng/pipelines/common/variables.yml - extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: checked - platforms: - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - jobParameters: - buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release - postBuildSteps: - - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/bin - includeRootFolder: false - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) - displayName: Build Assets - extraVariablesTemplates: - - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml - parameters: - testGroup: jit-cfg - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jit-cfg - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jit-cfg - liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + platforms: + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_arm64 + testGroup: jit-cfg \ No newline at end of file diff --git a/eng/pipelines/coreclr/jit-experimental.yml b/eng/pipelines/coreclr/jit-experimental.yml index 2c07876fdfc4b..42b2a607ac9f4 100644 --- a/eng/pipelines/coreclr/jit-experimental.yml +++ b/eng/pipelines/coreclr/jit-experimental.yml @@ -12,64 +12,13 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - jobParameters: - buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release - postBuildSteps: - - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/bin - includeRootFolder: false - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) - displayName: Build Assets - extraVariablesTemplates: - - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml - parameters: - testGroup: jit-experimental - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jit-experimental - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jit-experimental - liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + platforms: + - osx_arm64 + - osx_x64 + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_arm64 + testGroup: jitstress-random \ No newline at end of file diff --git a/eng/pipelines/coreclr/jitstress-isas-arm.yml b/eng/pipelines/coreclr/jitstress-isas-arm.yml index 7e178b4f42595..0733c08ab8b49 100644 --- a/eng/pipelines/coreclr/jitstress-isas-arm.yml +++ b/eng/pipelines/coreclr/jitstress-isas-arm.yml @@ -12,43 +12,10 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - linux_arm64 - - osx_arm64 - - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-isas-arm - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-isas-arm - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_arm64 - - osx_arm64 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstress-isas-arm - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - linux_arm64 + - windows_arm64 + testGroup: jitstress-isas-arm diff --git a/eng/pipelines/coreclr/jitstress-isas-avx512.yml b/eng/pipelines/coreclr/jitstress-isas-avx512.yml index 4074c2b91358a..87b33c8479529 100644 --- a/eng/pipelines/coreclr/jitstress-isas-avx512.yml +++ b/eng/pipelines/coreclr/jitstress-isas-avx512.yml @@ -27,43 +27,10 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - linux_x64 - - windows_x64 - - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-isas-avx512 - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-isas-avx512 - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_x64 - - windows_x64 - - windows_x86 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstress-isas-avx512 - liveLibrariesBuildConfig: Release + platforms: + - linux_x64 + - windows_x64 + - windows_x86 + testGroup: jitstress-isas-avx512 diff --git a/eng/pipelines/coreclr/jitstress-isas-x86.yml b/eng/pipelines/coreclr/jitstress-isas-x86.yml index b5a1b9764fc51..0951fd572772e 100644 --- a/eng/pipelines/coreclr/jitstress-isas-x86.yml +++ b/eng/pipelines/coreclr/jitstress-isas-x86.yml @@ -12,45 +12,11 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - linux_x64 - - osx_x64 - - windows_x64 - - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-isas-x86 - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-isas-x86 - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_x64 - - osx_x64 - - windows_x64 - - windows_x86 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstress-isas-x86 - liveLibrariesBuildConfig: Release + platforms: + - osx_x64 + - linux_x64 + - windows_x64 + - windows_x86 + testGroup: jitstress-isas-x86 diff --git a/eng/pipelines/coreclr/jitstress-random.yml b/eng/pipelines/coreclr/jitstress-random.yml index f9437db5ea5cb..ffbda2acb4801 100644 --- a/eng/pipelines/coreclr/jitstress-random.yml +++ b/eng/pipelines/coreclr/jitstress-random.yml @@ -12,53 +12,15 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-random - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-random - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstress-random - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - osx_x64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + testGroup: jitstress-random diff --git a/eng/pipelines/coreclr/jitstress.yml b/eng/pipelines/coreclr/jitstress.yml index 71263872c0929..0e9e0d052dfe8 100644 --- a/eng/pipelines/coreclr/jitstress.yml +++ b/eng/pipelines/coreclr/jitstress.yml @@ -12,53 +12,15 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstress - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - osx_x64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + testGroup: jitstress diff --git a/eng/pipelines/coreclr/jitstress2-jitstressregs.yml b/eng/pipelines/coreclr/jitstress2-jitstressregs.yml index ee672273e5647..0e2c961e4def2 100644 --- a/eng/pipelines/coreclr/jitstress2-jitstressregs.yml +++ b/eng/pipelines/coreclr/jitstress2-jitstressregs.yml @@ -12,53 +12,15 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress2-jitstressregs - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: checked - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstress2-jitstressregs - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - osx_x64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + testGroup: jitstress2-jitstressregs diff --git a/eng/pipelines/coreclr/jitstressregs-x86.yml b/eng/pipelines/coreclr/jitstressregs-x86.yml index 8b8009e63cf55..63ba958217f47 100644 --- a/eng/pipelines/coreclr/jitstressregs-x86.yml +++ b/eng/pipelines/coreclr/jitstressregs-x86.yml @@ -12,43 +12,10 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - linux_x64 - - windows_x64 - - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstressregs-x86 - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstressregs-x86 - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_x64 - - windows_x64 - - windows_x86 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstressregs-x86 - liveLibrariesBuildConfig: Release + platforms: + - linux_x64 + - windows_x64 + - windows_x86 + testGroup: jitstressregs-x86 diff --git a/eng/pipelines/coreclr/jitstressregs.yml b/eng/pipelines/coreclr/jitstressregs.yml index d947e48efa2ee..25ab3ead6ba47 100644 --- a/eng/pipelines/coreclr/jitstressregs.yml +++ b/eng/pipelines/coreclr/jitstressregs.yml @@ -12,53 +12,15 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstressregs - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstressregs - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstressregs - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - osx_x64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + testGroup: jitstressregs \ No newline at end of file diff --git a/eng/pipelines/coreclr/pgo.yml b/eng/pipelines/coreclr/pgo.yml index f6d00e6fea502..26b2846d1a196 100644 --- a/eng/pipelines/coreclr/pgo.yml +++ b/eng/pipelines/coreclr/pgo.yml @@ -12,51 +12,14 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - osx_arm64 - - windows_arm64 - - windows_x64 - - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: pgo - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: pgo - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - osx_arm64 - - windows_arm64 - - windows_x64 - - windows_x86 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: pgo - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + testGroup: pgo diff --git a/eng/pipelines/coreclr/pgostress.yml b/eng/pipelines/coreclr/pgostress.yml index 228b3b582e5d6..fdd8e964082dd 100644 --- a/eng/pipelines/coreclr/pgostress.yml +++ b/eng/pipelines/coreclr/pgostress.yml @@ -12,51 +12,14 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - osx_arm64 - - windows_arm64 - - windows_x64 - - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: pgostress - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: pgostress - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - osx_arm64 - - windows_arm64 - - windows_x64 - - windows_x86 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: pgostress - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + testGroup: pgostress diff --git a/eng/pipelines/coreclr/r2r-extra.yml b/eng/pipelines/coreclr/r2r-extra.yml index acc1c6a962079..f538edfcde62f 100644 --- a/eng/pipelines/coreclr/r2r-extra.yml +++ b/eng/pipelines/coreclr/r2r-extra.yml @@ -20,16 +20,30 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platformGroup: gcstress platforms: # It is too early to include osx_arm64 in platform group gcstress # Adding it here will enable it also - osx_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: r2r-extra + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: r2r-extra - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -56,3 +70,4 @@ extends: readyToRun: true displayNameArgs: R2R liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/r2r.yml b/eng/pipelines/coreclr/r2r.yml index 4f94eb6a1570b..f3d1f1063ba59 100644 --- a/eng/pipelines/coreclr/r2r.yml +++ b/eng/pipelines/coreclr/r2r.yml @@ -20,7 +20,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_arm @@ -30,9 +30,23 @@ extends: - windows_arm64 - windows_x64 - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: outerloop + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -62,3 +76,4 @@ extends: readyToRun: true displayNameArgs: R2R liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/release-tests.yml b/eng/pipelines/coreclr/release-tests.yml index 9ccf725463912..9a21c4664b887 100644 --- a/eng/pipelines/coreclr/release-tests.yml +++ b/eng/pipelines/coreclr/release-tests.yml @@ -23,7 +23,7 @@ extends: # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platformGroup: all platforms: @@ -31,8 +31,22 @@ extends: # Adding it here will enable it also - osx_arm64 jobParameters: - testGroup: outerloop - isOfficialBuild: false + buildArgs: -s clr+libs -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop # # Release test builds @@ -63,6 +77,7 @@ extends: jobParameters: testGroup: outerloop liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) # # Release R2R test runs @@ -79,4 +94,5 @@ extends: liveLibrariesBuildConfig: Release readyToRun: true displayNameArgs: R2R + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/runincontext.yml b/eng/pipelines/coreclr/runincontext.yml index 9135532fd5f93..acc6d76c48758 100644 --- a/eng/pipelines/coreclr/runincontext.yml +++ b/eng/pipelines/coreclr/runincontext.yml @@ -20,15 +20,29 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_x64 - windows_x64 - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: outerloop + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -54,3 +68,4 @@ extends: runInUnloadableContext: true displayNameArgs: RunInContext liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml b/eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml new file mode 100644 index 0000000000000..86965e2f310b7 --- /dev/null +++ b/eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml @@ -0,0 +1,65 @@ +parameters: + - name: platforms + type: object + - name: testGroup + type: string + default: outerloop + +extends: + template: /eng/pipelines/common/templates/pipeline-with-resources.yml + parameters: + stages: + - stage: Build + jobs: + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + platforms: ${{ parameters.platforms }} + jobParameters: + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: ${{ parameters.testGroup }} + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml + buildConfig: checked + platforms: + - CoreClrTestBuildHost # Either osx_x64 or linux_x64 + jobParameters: + testGroup: ${{ parameters.testGroup }} + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - osx_x64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/tieringtest.yml b/eng/pipelines/coreclr/tieringtest.yml index 117fac1b5ded7..dd426c37d4c33 100644 --- a/eng/pipelines/coreclr/tieringtest.yml +++ b/eng/pipelines/coreclr/tieringtest.yml @@ -20,15 +20,29 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_x64 - windows_x64 - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: outerloop + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -54,3 +68,4 @@ extends: tieringTest: true displayNameArgs: TieringTest liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) From d2570a283b89f5060e29d781781e6b50467a1956 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 27 Feb 2024 14:11:05 -0800 Subject: [PATCH 23/79] Convert the JIT exploratory pipeline to be based on the global build job + extra steps instead of using separate jobs. --- eng/pipelines/coreclr/exploratory.yml | 40 ++-- .../coreclr/templates/jit-exploratory-job.yml | 75 ------- .../templates/jit-exploratory-steps.yml | 87 ++++++++ .../templates/jit-exploratory-variables.yml | 58 ++++++ .../templates/jit-run-exploratory-job.yml | 185 ------------------ 5 files changed, 161 insertions(+), 284 deletions(-) delete mode 100644 eng/pipelines/coreclr/templates/jit-exploratory-job.yml create mode 100644 eng/pipelines/coreclr/templates/jit-exploratory-steps.yml create mode 100644 eng/pipelines/coreclr/templates/jit-exploratory-variables.yml delete mode 100644 eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml diff --git a/eng/pipelines/coreclr/exploratory.yml b/eng/pipelines/coreclr/exploratory.yml index af76dec9da249..9c3fc9a343b81 100644 --- a/eng/pipelines/coreclr/exploratory.yml +++ b/eng/pipelines/coreclr/exploratory.yml @@ -21,27 +21,11 @@ extends: stages: - stage: Build jobs: - - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - # Linux tests are built on the OSX machines. - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - - osx_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: outerloop - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/jit-exploratory-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml buildConfig: checked platforms: # Linux tests are built on the OSX machines. @@ -52,9 +36,17 @@ extends: - windows_x86 - windows_arm64 - osx_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - toolName: ${{ variables.toolName }} + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + timeoutInMinutes: 360 + postBuildSteps: + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly + displayName: Create Core_Root + condition: succeeded() + - template: /eng/pipelines/coreclr/templates/jit-exploratory-steps.yml + parameters: + toolName: ${{ variables.toolName }} + extraVariablesTemplates: + - template: /eng/pipelines/coreclr/templates/jit-exploratory-variables.yml + parameters: + toolName: ${{ variables.toolName }} diff --git a/eng/pipelines/coreclr/templates/jit-exploratory-job.yml b/eng/pipelines/coreclr/templates/jit-exploratory-job.yml deleted file mode 100644 index 94771613c6668..0000000000000 --- a/eng/pipelines/coreclr/templates/jit-exploratory-job.yml +++ /dev/null @@ -1,75 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osSubgroup: '' - runtimeVariant: '' - testGroup: '' - framework: net6.0 # Specify the appropriate framework when running release branches (ie netcoreapp3.0 for release/3.0) - liveLibrariesBuildConfig: '' - variables: {} - helixQueues: '' - runtimeType: 'coreclr' - pool: '' - codeGenType: 'JIT' - runJobTemplate: '/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml' - additionalSetupParameters: '' - toolName: '' # Antigen or Fuzzlyn - -### Exploratory job - -### Each exploratory job depends on a corresponding build job with the same -### buildConfig and archType. - -jobs: -- template: ${{ parameters.runJobTemplate }} - parameters: - # Compute job name from template parameters - jobName: ${{ format('exploratory_{0}{1}_{2}_{3}_{4}_{5}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType) }} - displayName: ${{ format('Exploratory {0}{1} {2} {3} {4} {5}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType) }} - pool: ${{ parameters.pool }} - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - runtimeVariant: ${{ parameters.runtimeVariant }} - liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} - runtimeType: ${{ parameters.runtimeType }} - codeGenType: ${{ parameters.codeGenType }} - testGroup: ${{ parameters.testGroup }} - helixQueues: ${{ parameters.helixQueues }} - additionalSetupParameters: ${{ parameters.additionalSetupParameters }} - toolName: ${{ parameters.toolName }} - # Test job depends on the corresponding build job - dependsOn: - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - - variables: ${{ parameters.variables }} - - frameworks: - - ${{ parameters.framework }} - steps: - # Extra steps that will be passed to the exploratory template and run before sending the job to helix (all of which is done in the template) - - # Optionally download live-built libraries - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir) - cleanUnpackFolder: false - artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' - artifactName: '$(librariesBuildArtifactName)' - displayName: 'live-built libraries' - - # Download coreclr - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' - displayName: 'Coreclr product build' - - # Create Core_Root - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) - displayName: Create Core_Root - condition: succeeded() diff --git a/eng/pipelines/coreclr/templates/jit-exploratory-steps.yml b/eng/pipelines/coreclr/templates/jit-exploratory-steps.yml new file mode 100644 index 0000000000000..7cf7757653a18 --- /dev/null +++ b/eng/pipelines/coreclr/templates/jit-exploratory-steps.yml @@ -0,0 +1,87 @@ +parameters: + osGroup: '' # required -- operating system for the job + osSubgroup: '' # optional -- operating system subgroup + archType: '' # required -- targeting CPU architecture + buildConfig: '' # required -- build configuration + helixQueues: '' # required -- Helix queues + toolName: '' # required -- which tool to use: Antigen or Fuzzlyn? + +steps: +- script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/fuzzer_setup.py -tool_name $(toolName) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) + displayName: ${{ format('{0} setup ({1}-{2})', parameters.toolName, parameters.osGroup, parameters.archType) }} + + # Run exploratory tool in helix +- template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml + parameters: + displayName: 'Send job to Helix' + helixBuild: $(Build.BuildNumber) + helixSource: $(_HelixSource) + helixType: 'build/tests/' + helixQueues: ${{ join(',', parameters.helixQueues) }} + creator: dotnet-bot + WorkItemTimeout: 2:30 # 2.5 hours + WorkItemDirectory: '$(WorkItemDirectory)' + CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)' + helixProjectArguments: '$(Build.SourcesDirectory)/src/coreclr/scripts/exploratory.proj' + BuildConfig: ${{ parameters.buildConfig }} + osGroup: ${{ parameters.osGroup }} + RunConfiguration: '$(RunConfiguration)' + ToolName: ${{ parameters.toolName }} + RunReason: '$(RunReason)' + +# Always upload the available issues-summary.txt files +- task: CopyFiles@2 + displayName: Copying issues-summary.txt of all partitions + inputs: + sourceFolder: '$(HelixResults)' + contents: '**/issues-summary-*.txt' + targetFolder: '$(IssuesLocation)' + continueOnError: true + condition: always() + +# Always upload the available AllIssues-*.zip files to be used for summarization +- task: CopyFiles@2 + displayName: Copying AllIssues.zip of all partitions + inputs: + sourceFolder: '$(HelixResults)' + contents: '**/AllIssues-*.zip' + targetFolder: '$(IssuesLocation)' + continueOnError: true + condition: always() + +- task: PublishPipelineArtifact@1 + displayName: Publish issue files + inputs: + targetPath: $(IssuesLocation) + artifactName: 'Issues_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)' + continueOnError: true + condition: always() + +# Always upload the available log files +- task: CopyFiles@2 + displayName: Copying fuzzer logs of all partitions + inputs: + sourceFolder: '$(HelixResults)' + contents: '**/*.log' + targetFolder: '$(FuzzerLogsLocation)' + continueOnError: true + condition: always() + +- task: PublishPipelineArtifact@1 + displayName: Publish fuzzer log files + inputs: + targetPath: '$(FuzzerLogsLocation)' + artifactName: '$(toolName)_Logs_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)' + continueOnError: true + condition: always() + +- script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/$(SummarizeScript) -issues_directory $(IssuesLocation) -arch $(archType) -platform $(osGroup)$(osSubgroup) -build_config $(_BuildConfig) + displayName: ${{ format('Summarize ({0}{1} {2})', parameters.osGroup, parameters.osSubgroup, parameters.archType) }} + condition: always() + +- task: PublishPipelineArtifact@1 + displayName: ${{ format('Publish {0} build logs', parameters.toolName) }} + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + artifactName: '$(toolName)_BuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)' + condition: always() diff --git a/eng/pipelines/coreclr/templates/jit-exploratory-variables.yml b/eng/pipelines/coreclr/templates/jit-exploratory-variables.yml new file mode 100644 index 0000000000000..7ed2596fe87cd --- /dev/null +++ b/eng/pipelines/coreclr/templates/jit-exploratory-variables.yml @@ -0,0 +1,58 @@ +parameters: + osGroup: '' # required -- operating system for the job + osSubgroup: '' # optional -- operating system subgroup + archType: '' # required -- targeting CPU architecture + buildConfig: '' # required -- build configuration + toolName: '' # required -- which tool to use: Antigen or Fuzzlyn? + +variables: + +- ${{ if in(variables['Build.Reason'], 'Schedule') }}: + - name: RunReason + value: 'Scheduled' +- ${{ if notin(variables['Build.Reason'], 'Schedule') }}: + - name: RunReason + value: 'PR' +- ${{ if eq(parameters.osGroup, 'windows') }}: + - name: PythonScript + value: 'py -3' + - name: PipScript + value: 'py -3 -m pip' + - name: Core_Root_Dir + value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' + - name: HelixResults + value: '$(Build.SourcesDirectory)\artifacts\helixresults\' + - name: IssuesLocation + value: '$(Build.SourcesDirectory)\artifacts\issues\' + - name: AntigenLogsLocation + value: '$(Build.SourcesDirectory)\artifacts\antigen_logs\' + - name: FuzzlynLogsLocation + value: '$(Build.SourcesDirectory)\artifacts\fuzzlyn_logs\' + +- ${{ if ne(parameters.osGroup, 'windows') }}: + - name: PythonScript + value: 'python3' + - name: PipScript + value: 'pip3' + - name: Core_Root_Dir + value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Tests/Core_Root' + - name: HelixResults + value: '$(Build.SourcesDirectory)/artifacts/helixresults/' + - name: IssuesLocation + value: '$(Build.SourcesDirectory)/artifacts/issues/' + - name: AntigenLogsLocation + value: '$(Build.SourcesDirectory)/artifacts/antigen_logs/' + - name: FuzzlynLogsLocation + value: '$(Build.SourcesDirectory)/artifacts/fuzzlyn_logs/' + +- ${{ if eq(parameters.toolName, 'Antigen') }}: + - name: SummarizeScript + value: 'antigen_summarize.py' + - name: FuzzerLogsLocation + value: '$(AntigenLogsLocation)' + +- ${{ if eq(parameters.toolName, 'Fuzzlyn') }}: + - name: SummarizeScript + value: 'fuzzlyn_summarize.py' + - name: FuzzerLogsLocation + value: '$(FuzzlynLogsLocation)' diff --git a/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml b/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml deleted file mode 100644 index 91472d75b57de..0000000000000 --- a/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml +++ /dev/null @@ -1,185 +0,0 @@ -parameters: - steps: [] # optional -- any additional steps that need to happen before pulling down the jitutils repo and sending the jitutils to helix (ie building your repo) - variables: [] # optional -- list of additional variables to send to the template - jobName: '' # required -- job name - displayName: '' # optional -- display name for the job. Will use jobName if not passed - pool: '' # required -- name of the Build pool - container: '' # required -- name of the container - buildConfig: '' # required -- build configuration - archType: '' # required -- targeting CPU architecture - osGroup: '' # required -- operating system for the job - osSubgroup: '' # optional -- operating system subgroup - continueOnError: 'false' # optional -- determines whether to continue the build if the step errors - dependsOn: '' # optional -- dependencies of the job - timeoutInMinutes: 320 # optional -- timeout for the job - enableTelemetry: false # optional -- enable for telemetry - liveLibrariesBuildConfig: '' # optional -- live-live libraries configuration to use for the run - runtimeType: 'coreclr' # optional -- Sets the runtime as coreclr or mono - codeGenType: 'JIT' # optional -- Decides on the codegen technology if running on mono - helixQueues: '' # required -- Helix queues - dependOnEvaluatePaths: false - toolName: '' # required -- which tool to use: Antigen or Fuzzlyn? - -jobs: -- template: xplat-pipeline-job.yml - parameters: - dependsOn: ${{ parameters.dependsOn }} - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} - enableTelemetry: ${{ parameters.enableTelemetry }} - enablePublishBuildArtifacts: true - continueOnError: ${{ parameters.continueOnError }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: '${{ parameters.displayName }}' - ${{ if eq(parameters.displayName, '') }}: - displayName: '${{ parameters.jobName }}' - - variables: - - ${{ each variable in parameters.variables }}: - - ${{insert}}: ${{ variable }} - - - HelixApiAccessToken: '' - - HelixPreCommand: '' - - - ${{ if in(variables['Build.Reason'], 'Schedule') }}: - - name: RunReason - value: 'Scheduled' - - ${{ if notin(variables['Build.Reason'], 'Schedule') }}: - - name: RunReason - value: 'PR' - - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' - - name: Core_Root_Dir - value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' - - name: HelixResults - value: '$(Build.SourcesDirectory)\artifacts\helixresults\' - - name: IssuesLocation - value: '$(Build.SourcesDirectory)\artifacts\issues\' - - name: AntigenLogsLocation - value: '$(Build.SourcesDirectory)\artifacts\antigen_logs\' - - name: FuzzlynLogsLocation - value: '$(Build.SourcesDirectory)\artifacts\fuzzlyn_logs\' - - - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'python3' - - name: PipScript - value: 'pip3' - - name: Core_Root_Dir - value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Tests/Core_Root' - - name: HelixResults - value: '$(Build.SourcesDirectory)/artifacts/helixresults/' - - name: IssuesLocation - value: '$(Build.SourcesDirectory)/artifacts/issues/' - - name: AntigenLogsLocation - value: '$(Build.SourcesDirectory)/artifacts/antigen_logs/' - - name: FuzzlynLogsLocation - value: '$(Build.SourcesDirectory)/artifacts/fuzzlyn_logs/' - - - ${{ if eq(parameters.toolName, 'Antigen') }}: - - name: SummarizeScript - value: 'antigen_summarize.py' - - name: FuzzerLogsLocation - value: '$(AntigenLogsLocation)' - - - ${{ if eq(parameters.toolName, 'Fuzzlyn') }}: - - name: SummarizeScript - value: 'fuzzlyn_summarize.py' - - name: FuzzerLogsLocation - value: '$(FuzzlynLogsLocation)' - - workspace: - clean: all - pool: - ${{ parameters.pool }} - container: ${{ parameters.container }} - steps: - - ${{ parameters.steps }} - - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/fuzzer_setup.py -tool_name $(toolName) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) - displayName: ${{ format('{0} setup ({1}-{2})', parameters.toolName, parameters.osGroup, parameters.archType) }} - - # Run exploratory tool in helix - - template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml - parameters: - displayName: 'Send job to Helix' - helixBuild: $(Build.BuildNumber) - helixSource: $(_HelixSource) - helixType: 'build/tests/' - helixQueues: ${{ join(',', parameters.helixQueues) }} - creator: dotnet-bot - WorkItemTimeout: 2:30 # 2.5 hours - WorkItemDirectory: '$(WorkItemDirectory)' - CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)' - helixProjectArguments: '$(Build.SourcesDirectory)/src/coreclr/scripts/exploratory.proj' - BuildConfig: ${{ parameters.buildConfig }} - osGroup: ${{ parameters.osGroup }} - RunConfiguration: '$(RunConfiguration)' - ToolName: ${{ parameters.toolName }} - RunReason: '$(RunReason)' - - # Always upload the available issues-summary.txt files - - task: CopyFiles@2 - displayName: Copying issues-summary.txt of all partitions - inputs: - sourceFolder: '$(HelixResults)' - contents: '**/issues-summary-*.txt' - targetFolder: '$(IssuesLocation)' - continueOnError: true - condition: always() - - # Always upload the available AllIssues-*.zip files to be used for summarization - - task: CopyFiles@2 - displayName: Copying AllIssues.zip of all partitions - inputs: - sourceFolder: '$(HelixResults)' - contents: '**/AllIssues-*.zip' - targetFolder: '$(IssuesLocation)' - continueOnError: true - condition: always() - - - task: PublishPipelineArtifact@1 - displayName: Publish issue files - inputs: - targetPath: $(IssuesLocation) - artifactName: 'Issues_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - continueOnError: true - condition: always() - - # Always upload the available log files - - task: CopyFiles@2 - displayName: Copying fuzzer logs of all partitions - inputs: - sourceFolder: '$(HelixResults)' - contents: '**/*.log' - targetFolder: '$(FuzzerLogsLocation)' - continueOnError: true - condition: always() - - - task: PublishPipelineArtifact@1 - displayName: Publish fuzzer log files - inputs: - targetPath: '$(FuzzerLogsLocation)' - artifactName: '$(toolName)_Logs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - continueOnError: true - condition: always() - - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/$(SummarizeScript) -issues_directory $(IssuesLocation) -arch $(archType) -platform $(osGroup)$(osSubgroup) -build_config $(buildConfig) - displayName: ${{ format('Summarize ({0}{1} {2})', parameters.osGroup, parameters.osSubgroup, parameters.archType) }} - condition: always() - - - task: PublishPipelineArtifact@1 - displayName: ${{ format('Publish {0} build logs', parameters.toolName) }} - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: '$(toolName)_BuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - condition: always() From 0e4a90d5a0109f37087ca38fbc920cef25ff365c Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 27 Feb 2024 14:41:31 -0800 Subject: [PATCH 24/79] Fix build config. --- eng/pipelines/coreclr/exploratory.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/exploratory.yml b/eng/pipelines/coreclr/exploratory.yml index 9c3fc9a343b81..1c0b6a7680bd4 100644 --- a/eng/pipelines/coreclr/exploratory.yml +++ b/eng/pipelines/coreclr/exploratory.yml @@ -40,7 +40,7 @@ extends: buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release timeoutInMinutes: 360 postBuildSteps: - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(_BuildConfig) $(archType) generatelayoutonly displayName: Create Core_Root condition: succeeded() - template: /eng/pipelines/coreclr/templates/jit-exploratory-steps.yml From a980cac4e368af57392660fd8b038586b807fb21 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 27 Feb 2024 15:54:19 -0800 Subject: [PATCH 25/79] Convert all usages of built-jit-job.yml to use the global build script. Unify all python usages to use the same python variable template (and all use venv as well) --- eng/pipelines/common/global-build-job.yml | 3 + .../runtimes/native-test-assets-variables.yml | 1 + .../templates/runtimes/run-test-job.yml | 14 +- eng/pipelines/coreclr/jitrollingbuild.yml | 15 +- .../superpmi-asmdiffs-checked-release.yml | 30 +++- eng/pipelines/coreclr/superpmi-diffs.yml | 28 +++- eng/pipelines/coreclr/superpmi-replay.yml | 14 +- .../coreclr/templates/build-jit-job.yml | 141 ------------------ .../coreclr/templates/format-job.yml | 14 +- .../templates/jit-exploratory-steps.yml | 3 + .../templates/jit-exploratory-variables.yml | 11 +- .../templates/jit-python-variables.yml | 20 +++ ...-superpmi-asmdiffs-checked-release-job.yml | 10 +- .../templates/run-superpmi-collect-job.yml | 16 +- .../templates/run-superpmi-diffs-job.yml | 15 +- .../templates/run-superpmi-replay-job.yml | 11 +- .../superpmi-asmdiffs-checked-release-job.yml | 23 ++- .../coreclr/templates/superpmi-diffs-job.yml | 20 +-- .../coreclr/templates/superpmi-replay-job.yml | 9 +- .../coreclr/templates/upload-jits-steps.yml | 19 +++ eng/pipelines/libraries/run-test-job.yml | 4 + .../libraries/superpmi-collect-variables.yml | 8 - .../libraries/superpmi-postprocess-step.yml | 6 +- 23 files changed, 193 insertions(+), 242 deletions(-) delete mode 100644 eng/pipelines/coreclr/templates/build-jit-job.yml create mode 100644 eng/pipelines/coreclr/templates/jit-python-variables.yml create mode 100644 eng/pipelines/coreclr/templates/upload-jits-steps.yml diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 6f98fba1029f1..cdcadcbc04802 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -32,6 +32,7 @@ parameters: preBuildSteps: [] enableRichCodeNavigation: false richCodeNavigationLanguage: 'csharp' + disableComponentGovernance: false jobs: - template: /eng/common/templates/job/job.yml @@ -55,6 +56,8 @@ jobs: # Component governance does not work on musl machines ${{ if eq(parameters.osSubGroup, '_musl') }}: disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} workspace: clean: all diff --git a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml index e7759778d8bd0..9815e89290c7e 100644 --- a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml +++ b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -4,6 +4,7 @@ parameters: runtimeFlavor: 'coreclr' runtimeVariant: '' liveLibrariesBuildConfig: '' + testGroup: 'innerloop' variables: - name: binTestsPath diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index b8964485b9030..ce44dbe6341f7 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -150,11 +150,10 @@ jobs: value: 'run' - name: CollectionName value: 'coreclr_tests' + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' - name: MchFilesLocation value: '$(Build.SourcesDirectory)\artifacts\helixresults\' - name: MergedMchFileLocation @@ -162,10 +161,6 @@ jobs: - name: SpmiLogsLocation value: '$(Build.SourcesDirectory)\artifacts\spmi_logs\' - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'python3' - - name: PipScript - value: 'pip3' - name: MchFilesLocation value: '$(Build.SourcesDirectory)/artifacts/helixresults/' - name: MergedMchFileLocation @@ -648,6 +643,9 @@ jobs: displayName: Create SuperPMI directories condition: always() + - script: $(PythonSetupScript) + displayName: Enable python venv + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch displayName: 'Merge $(CollectionName)-$(CollectionType) SuperPMI collections' condition: always() diff --git a/eng/pipelines/coreclr/jitrollingbuild.yml b/eng/pipelines/coreclr/jitrollingbuild.yml index b2d2b86f2a1c2..35f06788b49a9 100644 --- a/eng/pipelines/coreclr/jitrollingbuild.yml +++ b/eng/pipelines/coreclr/jitrollingbuild.yml @@ -22,10 +22,9 @@ extends: stages: - stage: Build jobs: - - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + jobTemplate: /eng/pipelines/common/platform-matrix.yml buildConfig: checked platforms: - osx_arm64 @@ -38,10 +37,15 @@ extends: - windows_arm64 jobParameters: disableComponentGovernance: true # Not a shipping artifact + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/upload-jits-steps.yml + extraVariablesTemplates: + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + jobTemplate: /eng/pipelines/common/platform-matrix.yml buildConfig: release platforms: - osx_arm64 @@ -54,3 +58,8 @@ extends: - windows_arm64 jobParameters: disableComponentGovernance: true # Not a shipping artifact + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/upload-jits-steps.yml + extraVariablesTemplates: + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml diff --git a/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml b/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml index a356acb4fd9cf..bb979a9cc8f29 100644 --- a/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml +++ b/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml @@ -17,26 +17,46 @@ extends: stages: - stage: Build jobs: - - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - windows_x64 - windows_x86 jobParameters: - uploadAs: 'pipelineArtifacts' + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType) + displayName: JIT and SuperPMI Assets - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - windows_x64 - windows_x86 + - linux_x64 jobParameters: - uploadAs: 'pipelineArtifacts' + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: ReleaseJIT_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: JIT and SuperPMI Assets - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/coreclr/superpmi-diffs.yml b/eng/pipelines/coreclr/superpmi-diffs.yml index 423f1e8b1dd1a..05862e4f18ef7 100644 --- a/eng/pipelines/coreclr/superpmi-diffs.yml +++ b/eng/pipelines/coreclr/superpmi-diffs.yml @@ -45,25 +45,45 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - windows_x64 - windows_x86 jobParameters: - uploadAs: 'pipelineArtifacts' + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType) + displayName: JIT and SuperPMI Assets condition: not(eq(dependencies.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true)) - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - windows_x64 - windows_x86 - linux_x64 jobParameters: - uploadAs: 'pipelineArtifacts' + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: ReleaseJIT_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: JIT and SuperPMI Assets condition: not(eq(dependencies.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true)) - template: /eng/pipelines/common/platform-matrix.yml diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 88e575b37d72a..1e0a377ecfb86 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -35,13 +35,23 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - windows_x64 - windows_x86 jobParameters: - uploadAs: 'pipelineArtifacts' + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType) + displayName: JIT and SuperPMI Assets condition: not(eq(dependencies.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true)) - template: /eng/pipelines/common/platform-matrix.yml diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml deleted file mode 100644 index 324fd3a3fbde0..0000000000000 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ /dev/null @@ -1,141 +0,0 @@ -parameters: - archType: '' - buildConfig: '' - container: '' - crossBuild: false - osGroup: '' - osSubgroup: '' - condition: true - pool: '' - timeoutInMinutes: '' - variables: {} - dependOnEvaluatePaths: false - disableComponentGovernance: false - uploadAs: 'azureBlob' - -### Product build -jobs: -- template: xplat-pipeline-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - condition: ${{ parameters.condition }} - helixType: 'build/product/' - enableMicrobuild: true - pool: ${{ parameters.pool }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - - # Compute job name from template parameters - name: ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('CoreCLR JIT Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - # Run all steps in the container. - # Note that the containers are defined in platform-matrix.yml - container: ${{ parameters.container }} - - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - crossBuild: ${{ parameters.crossBuild }} - - gatherAssetManifests: true - - variables: - - name: osGroup - value: ${{ parameters.osGroup }} - - name: osSubgroup - value: ${{ parameters.osSubgroup }} - - - name: publishLogsArtifactPrefix - value: 'BuildLogs_CoreCLR_JIT' - - name: uploadAs - value: ${{ parameters.uploadAs }} - - - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonSetupScript - value: 'py -3 -m venv $(Build.SourcesDirectory)\venv' - - name: PythonScript - value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe' - - name: PipScript - value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe -m pip' - - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonSetupScript - value: 'python3 -m venv $(Build.SourcesDirectory)/venv' - - name: PythonScript - value: '$(Build.SourcesDirectory)/venv/bin/python3' - - name: PipScript - value: '$(Build.SourcesDirectory)/venv/bin/pip3' - - - ${{ parameters.variables }} - - steps: - - # Install native dependencies - # Linux builds use docker images with dependencies preinstalled, - # and FreeBSD builds use a build agent with dependencies - # preinstalled, so we only need this step for OSX and Windows. - - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup) - displayName: Install native dependencies (OSX) - - # Install internal tools on official builds - # Since our internal tools are behind an authenticated feed, - # we need to use the DotNetCli AzDO task to restore from the feed using a service connection. - # We can't do this from within the build, so we need to do this as a separate step. - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/pipelines/common/restore-internal-tools.yml - - # Build/Generate native prerequisites - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.nativeprereqs $(crossArg) -arch $(archType) -c $(buildConfig) -ci /bl:$(Build.SourcesDirectory)artifacts/log/$(buildConfig)/CoreCLRNativePrereqs.binlog - displayName: Build and generate native prerequisites - - # Build CoreCLR JIT - - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) -ci -component alljits -component spmi - displayName: Build CoreCLR JIT - - ${{ if eq(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -component alljits -component spmi - displayName: Build CoreCLR JIT - - - ${{ if eq(parameters.uploadAs, 'azureBlob') }}: - # Add authenticated pip feed - - task: PipAuthenticate@1 - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: public/dotnet-public-pypi - onlyAddExtraIndex: false - - - script: $(PythonSetupScript) - displayName: Enable python venv - - # Ensure the Python azure-storage-blob package is installed before doing the upload. - - script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall - displayName: Upgrade Pip to latest and install azure-storage-blob Python package - - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(buildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion) --use_latest_jit_change - displayName: Upload JIT to Azure Storage - env: - CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline - - - ${{ if eq(parameters.uploadAs, 'pipelineArtifacts') }}: - # Publish product output directory for consumption by tests. - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(buildProductRootFolderPath) - includeRootFolder: false - archiveType: $(archiveType) - tarCompression: $(tarCompression) - archiveExtension: $(archiveExtension) - artifactName: $(buildProductArtifactName) - displayName: 'product build' - - # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: '$(publishLogsArtifactPrefix)_Attempt$(System.JobAttempt)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - continueOnError: true - condition: always() diff --git a/eng/pipelines/coreclr/templates/format-job.yml b/eng/pipelines/coreclr/templates/format-job.yml index 88e5184db032c..bc749d4f84cdf 100644 --- a/eng/pipelines/coreclr/templates/format-job.yml +++ b/eng/pipelines/coreclr/templates/format-job.yml @@ -28,17 +28,12 @@ jobs: helixType: 'format' pool: ${{ parameters.pool }} variables: - + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} - - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'py -3' - - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'python3' - - ${{ if eq(parameters.osGroup, 'linux') }}: - name: LinuxCrossArg value: '--cross' @@ -59,6 +54,9 @@ jobs: includePreviewVersions: true installationPath: $(Agent.ToolsDirectory)/dotnet + - script: $(PythonSetupScript) + displayName: Enable python venv + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitformat.py -r $(Build.SourcesDirectory) -o $(osGroup) -a $(archType) $(LinuxCrossArg) displayName: Run jitformat.py diff --git a/eng/pipelines/coreclr/templates/jit-exploratory-steps.yml b/eng/pipelines/coreclr/templates/jit-exploratory-steps.yml index 7cf7757653a18..b26906ded6ad0 100644 --- a/eng/pipelines/coreclr/templates/jit-exploratory-steps.yml +++ b/eng/pipelines/coreclr/templates/jit-exploratory-steps.yml @@ -7,6 +7,9 @@ parameters: toolName: '' # required -- which tool to use: Antigen or Fuzzlyn? steps: +- script: $(PythonSetupScript) + displayName: Enable python venv + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/fuzzer_setup.py -tool_name $(toolName) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) displayName: ${{ format('{0} setup ({1}-{2})', parameters.toolName, parameters.osGroup, parameters.archType) }} diff --git a/eng/pipelines/coreclr/templates/jit-exploratory-variables.yml b/eng/pipelines/coreclr/templates/jit-exploratory-variables.yml index 7ed2596fe87cd..9af139026c5d9 100644 --- a/eng/pipelines/coreclr/templates/jit-exploratory-variables.yml +++ b/eng/pipelines/coreclr/templates/jit-exploratory-variables.yml @@ -6,6 +6,9 @@ parameters: toolName: '' # required -- which tool to use: Antigen or Fuzzlyn? variables: +- template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} - ${{ if in(variables['Build.Reason'], 'Schedule') }}: - name: RunReason @@ -14,10 +17,6 @@ variables: - name: RunReason value: 'PR' - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' - name: Core_Root_Dir value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' - name: HelixResults @@ -30,10 +29,6 @@ variables: value: '$(Build.SourcesDirectory)\artifacts\fuzzlyn_logs\' - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'python3' - - name: PipScript - value: 'pip3' - name: Core_Root_Dir value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Tests/Core_Root' - name: HelixResults diff --git a/eng/pipelines/coreclr/templates/jit-python-variables.yml b/eng/pipelines/coreclr/templates/jit-python-variables.yml new file mode 100644 index 0000000000000..c59c219a29ec2 --- /dev/null +++ b/eng/pipelines/coreclr/templates/jit-python-variables.yml @@ -0,0 +1,20 @@ +parameters: + osGroup: '' + +variables: +- ${{ if eq(parameters.osGroup, 'windows') }}: + - name: PythonSetupScript + value: 'py -3 -m venv $(Build.SourcesDirectory)\venv' + - name: PythonScript + value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe' + - name: PipScript + value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe -m pip' +- ${{ if ne(parameters.osGroup, 'windows') }}: + - name: PythonSetupScript + value: 'python3 -m venv $(Build.SourcesDirectory)/venv' + - name: PythonScript + value: '$(Build.SourcesDirectory)/venv/bin/python3' + - name: PipScript + value: '$(Build.SourcesDirectory)/venv/bin/pip3' + + - ${{ parameters.variables }} diff --git a/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml index 155d82205938c..791a71c734838 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml @@ -38,11 +38,10 @@ jobs: displayName: '${{ parameters.jobName }}' variables: + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' - name: SpmiCollectionLocation value: '$(Build.SourcesDirectory)\artifacts\spmi\' - name: SpmiLogsLocation @@ -65,6 +64,9 @@ jobs: mkdir -p $(SpmiCollectionLocation) displayName: Create directory for SPMI collection + - script: $(PythonSetupScript) + displayName: Enable python venv + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_asmdiffs_checked_release_setup.py -source_directory $(Build.SourcesDirectory) -checked_directory $(buildProductRootFolderPath) -release_directory $(releaseProductRootFolderPath) -arch $(archType) displayName: ${{ format('SuperPMI asmdiffs checked release setup ({0} {1})', parameters.osGroup, parameters.archType) }} diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index d83f6e7e089da..dad4986181c85 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -54,13 +54,11 @@ jobs: - CollectionType: ${{ parameters.collectionType }} - CollectionName: ${{ parameters.collectionName }} + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} + - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonSetupScript - value: 'py -3 -m venv $(Build.SourcesDirectory)\venv' - - name: PythonScript - value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe' - - name: PipScript - value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe -m pip' - name: Core_Root_Dir value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' - name: MchFilesLocation @@ -72,12 +70,6 @@ jobs: - name: PayloadLocation value: '$(Build.SourcesDirectory)\payload' - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonSetupScript - value: 'python3 -m venv $(Build.SourcesDirectory)/venv' - - name: PythonScript - value: '$(Build.SourcesDirectory)/venv/bin/python3' - - name: PipScript - value: '$(Build.SourcesDirectory)/venv/bin/pip3' - name: Core_Root_Dir value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Tests/Core_Root' - name: MchFilesLocation diff --git a/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml index da61351842a58..2e00e66d7f9c9 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml @@ -47,11 +47,11 @@ jobs: - name: diffType value: ${{ parameters.diffType }} + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} + - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' - name: SpmiCollectionLocation value: '$(Build.SourcesDirectory)\artifacts\spmi\' - name: SpmiLogsLocation @@ -61,10 +61,6 @@ jobs: - name: HelixResultLocation value: '$(Build.SourcesDirectory)\artifacts\helixresults\' - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'python3' - - name: PipScript - value: 'pip3' - name: SpmiCollectionLocation value: '$(Build.SourcesDirectory)/artifacts/spmi/' - name: SpmiLogsLocation @@ -94,6 +90,9 @@ jobs: steps: - ${{ parameters.steps }} + - script: $(PythonSetupScript) + displayName: Enable python venv + - ${{ if ne(parameters.osGroup, 'windows') }}: - script: | mkdir -p $(SpmiCollectionLocation) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml index 3083a080c230b..25fabb2f7670e 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml @@ -41,10 +41,10 @@ jobs: - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} + - name: SpmiCollectionLocation value: '$(Build.SourcesDirectory)\artifacts\spmi\' - name: SpmiLogsLocation @@ -60,6 +60,9 @@ jobs: steps: - ${{ parameters.steps }} + - script: $(PythonSetupScript) + displayName: Enable python venv + - script: | mkdir $(SpmiCollectionLocation) mkdir $(SpmiLogsLocation) diff --git a/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml b/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml index 659483c9bc302..8681076ae5706 100644 --- a/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml @@ -24,8 +24,8 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} helixQueues: ${{ parameters.helixQueues }} dependsOn: - - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, 'checked') }} - - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, 'release') }} + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_checked_' + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_release_' variables: - ${{ each variable in parameters.variables }}: @@ -33,23 +33,22 @@ jobs: - name: releaseProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).Release' - - name: releaseProductArtifactName - value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_release' steps: - # Download jit checked builds - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + artifactFileName: 'CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType)' displayName: 'JIT checked build' + cleanupUnpackFolder: false - #Download jit release builds + # Download jit release builds - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(releaseProductRootFolderPath) - artifactFileName: '$(releaseProductArtifactName)$(archiveExtension)' - artifactName: '$(releaseProductArtifactName)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + artifactFileName: 'ReleaseJIT_$(osGroup)_$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'ReleaseJIT_$(osGroup)_$(osSubgroup)_$(archType)' displayName: 'JIT release build' + cleanupUnpackFolder: false diff --git a/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml b/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml index 2375089e250b9..762d39b16148f 100644 --- a/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml @@ -33,9 +33,9 @@ jobs: diffJitOptions: ${{ parameters.diffJitOptions }} dependsOn: - ${{ if in(parameters.diffType, 'asmdiffs', 'all') }}: - - ${{ format('coreclr_jit_build_{0}{1}_{2}_checked', parameters.osGroup, parameters.osSubgroup, parameters.archType) }} + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_checked_' - ${{ if in(parameters.diffType, 'tpdiff', 'all') }}: - - ${{ format('coreclr_jit_build_{0}{1}_{2}_release', parameters.osGroup, parameters.osSubgroup, parameters.archType) }} + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_release_' variables: - ${{ each variable in parameters.variables }}: @@ -48,8 +48,6 @@ jobs: - ${{ if ne(parameters.osGroup, 'windows') }}: - name: releaseProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).Release' - - name: releaseProductArtifactName - value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_release' steps: @@ -57,16 +55,18 @@ jobs: # Download jit checked builds - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + artifactFileName: 'CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType)' displayName: 'JIT checked build' + cleanupUnpackFolder: false - ${{ if in(parameters.diffType, 'tpdiff', 'all') }}: # Download jit release builds - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(releaseProductRootFolderPath) - artifactFileName: '$(releaseProductArtifactName)$(archiveExtension)' - artifactName: '$(releaseProductArtifactName)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + artifactFileName: 'ReleaseJIT_$(osGroup)_$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'ReleaseJIT_$(osGroup)_$(osSubgroup)_$(archType)' displayName: 'JIT release build' + cleanupUnpackFolder: false diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 979f95ce685d8..7296c2f4f894c 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -35,7 +35,8 @@ jobs: # Download jit builds - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' - displayName: 'JIT product build' \ No newline at end of file + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + artifactFileName: 'CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType)' + displayName: 'JIT checked build' + cleanupUnpackFolder: false \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/upload-jits-steps.yml b/eng/pipelines/coreclr/templates/upload-jits-steps.yml new file mode 100644 index 0000000000000..912732b90b847 --- /dev/null +++ b/eng/pipelines/coreclr/templates/upload-jits-steps.yml @@ -0,0 +1,19 @@ +steps: +# Add authenticated pip feed +- task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: public/dotnet-public-pypi + onlyAddExtraIndex: false + +- script: $(PythonSetupScript) + displayName: Enable python venv + +# Ensure the Python azure-storage-blob package is installed before doing the upload. +- script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall + displayName: Upgrade Pip to latest and install azure-storage-blob Python package + +- script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(_BuildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion) --use_latest_jit_change + displayName: Upload JIT to Azure Storage + env: + CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 78f2d73e49f6f..8e0a94d0a608d 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -81,6 +81,9 @@ jobs: - _archiveTestsParameter: /p:ArchiveTests=true - ${{ if eq(parameters.SuperPmiCollect, true) }}: + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} - template: /eng/pipelines/libraries/superpmi-collect-variables.yml parameters: buildConfig: ${{ parameters.buildConfig }} @@ -258,5 +261,6 @@ jobs: MchFilesLocation: $(MchFilesLocation) SpmiLogsLocation: $(SpmiLogsLocation) SuperPmiMcsPath: $(SuperPmiMcsPath) + PythonSetupScript: $(PythonSetupScript) PythonScript: $(PythonScript) PipScript: $(PipScript) diff --git a/eng/pipelines/libraries/superpmi-collect-variables.yml b/eng/pipelines/libraries/superpmi-collect-variables.yml index 509bd60942450..4cfde9ea0a0b7 100644 --- a/eng/pipelines/libraries/superpmi-collect-variables.yml +++ b/eng/pipelines/libraries/superpmi-collect-variables.yml @@ -23,10 +23,6 @@ variables: - name: SuperPmiMcsPath value: $(_runtimeX64DownloadPath) - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' - name: MchFilesLocation value: '$(Build.SourcesDirectory)\artifacts\helixresults\' - name: MergedMchFileLocation @@ -34,10 +30,6 @@ variables: - name: SpmiLogsLocation value: '$(Build.SourcesDirectory)\artifacts\spmi_logs\' - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'python3' - - name: PipScript - value: 'pip3' - name: MchFilesLocation value: '$(Build.SourcesDirectory)/artifacts/helixresults/' - name: MergedMchFileLocation diff --git a/eng/pipelines/libraries/superpmi-postprocess-step.yml b/eng/pipelines/libraries/superpmi-postprocess-step.yml index ef29b332c7517..e4c47fcc2092a 100644 --- a/eng/pipelines/libraries/superpmi-postprocess-step.yml +++ b/eng/pipelines/libraries/superpmi-postprocess-step.yml @@ -18,6 +18,7 @@ parameters: MchFilesLocation: '' SpmiLogsLocation: '' SuperPmiMcsPath: '' + PythonSetupScript: '' PythonScript: '' PipScript: '' @@ -35,7 +36,10 @@ steps: mkdir ${{ parameters.MergedMchFileLocation }} mkdir ${{ parameters.SpmiLogsLocation }} displayName: 'Create SuperPMI directories' - condition: always() + condition: always() + + - script: ${{ parameters.PythonSetupScript }} + displayName: Enable python venv - script: ${{ parameters.PythonScript }} $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch -log_level DEBUG -pattern ${{ parameters.MchFilesLocation }}${{ parameters.SuperPmiCollectionName }}.${{ parameters.SuperPmiCollectionType }}*.mch -output_mch_path ${{ parameters.MergedMchFileLocation }}${{ parameters.SuperPmiCollectionName }}.${{ parameters.SuperPmiCollectionType }}.${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}.mch -core_root ${{ parameters.SuperPmiMcsPath }} displayName: 'Merge ${{ parameters.SuperPmiCollectionName }}-${{ parameters.SuperPmiCollectionType }} SuperPMI collections' From cc647ae60743502e47ef52630cd0443b67e05efa Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 27 Feb 2024 16:11:36 -0800 Subject: [PATCH 26/79] Move runtime-extra-platforms-other.yml to exclusively global-build-job based builds. --- .../runtime-extra-platforms-other.yml | 148 ++++-------------- 1 file changed, 34 insertions(+), 114 deletions(-) diff --git a/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml b/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml index 10b3e9baca01f..4b16485873a83 100644 --- a/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml +++ b/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml @@ -7,50 +7,13 @@ parameters: jobs: -# -# Build CoreCLR release -# Always as they are needed by Installer and we always build and test the Installer. -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: release - platforms: - - linux_x64 - - linux_arm - - linux_arm64 - - linux_musl_x64 - - osx_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - jobParameters: - testGroup: innerloop - -# -# Build libraries using live CoreLib -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Release - platforms: - - linux_x64 - - linux_arm - - linux_arm64 - - linux_musl_x64 - - osx_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - # # Libraries Release Test Execution against a release coreclr runtime # Only when the PR contains a libraries change # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: Release platforms: - linux_x64 @@ -63,10 +26,14 @@ jobs: - windows_arm64 helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml jobParameters: - isOfficialBuild: false - isExtraPlatforms: ${{ parameters.isExtraPlatformsBuild }} - testScope: innerloop - liveRuntimeBuildConfig: release + buildArgs: -s tools+clr+libs+libs.tests -c $(_BuildConfig) -testscope innerloop /p:ArchiveTests=true + nameSuffix: Libraries_Release_CoreCLR + timeoutInMinutes: 150 + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Libraries_Release_CoreCLR condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), @@ -138,67 +105,6 @@ jobs: eq(variables['monoContainsChange'], true), eq(variables['isRollingBuild'], true)) -# -# Build the whole product using Mono and run runtime tests -# Build Mono release -# Only when libraries, mono, or the runtime tests changed -# Currently only these architectures are needed for the runtime tests. -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono - buildConfig: release - platforms: - - linux_arm64 - jobParameters: - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(variables['isRollingBuild'], true)) - -# -# Mono Test builds with CoreCLR runtime tests using live libraries debug build -# Only when Mono is changed -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: innerloop - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) - -# -# Mono CoreCLR runtime Test executions using live libraries in jit mode -# Only when Mono is changed -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - linux_arm64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: innerloop - liveLibrariesBuildConfig: Release - liveRuntimeBuildConfig: release - runtimeVariant: minijit - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) - # # Mono CoreCLR runtime Test executions using live libraries and LLVM Full AOT # Only when Mono is changed @@ -241,28 +147,42 @@ jobs: # # Mono CoreCLR runtime Test executions using live libraries in interpreter mode # Only when Mono is changed + - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: release + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + buildConfig: Release runtimeFlavor: mono platforms: - - linux_arm64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + - linux_arm64 + - osx_arm64 + variables: + - name: timeoutPerTestInMinutes + value: 60 + - name: timeoutPerTestCollectionInMinutes + value: 180 jobParameters: testGroup: innerloop - liveLibrariesBuildConfig: Release - liveRuntimeBuildConfig: release + nameSuffix: AllSubsets_Mono_Interpreter_RuntimeTests runtimeVariant: monointerpreter + buildArgs: -s mono+libs+clr.hosts+clr.iltools -c Release + timeoutInMinutes: 180 condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), eq(variables['isRollingBuild'], true)) + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_Release + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/test-variables.yml # -# Mono CoreCLR runtime Test executions using live libraries in interpreter mode +# Mono CoreCLR runtime Test executions using live libraries in JIT mode # Only when Mono is changed - template: /eng/pipelines/common/platform-matrix.yml @@ -272,7 +192,7 @@ jobs: buildConfig: Release runtimeFlavor: mono platforms: - - osx_arm64 + - linux_arm64 variables: - name: timeoutPerTestInMinutes value: 60 @@ -280,8 +200,8 @@ jobs: value: 180 jobParameters: testGroup: innerloop - nameSuffix: AllSubsets_Mono_Interpreter_RuntimeTests - runtimeVariant: monointerpreter + nameSuffix: AllSubsets_Mono_Minijit_RuntimeTests + runtimeVariant: minijit buildArgs: -s mono+libs+clr.hosts+clr.iltools -c Release timeoutInMinutes: 180 condition: >- From f98d6695c98418131b5ad5f910aaeca09e5203ad Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 27 Feb 2024 16:33:26 -0800 Subject: [PATCH 27/79] Convert crossgen2 outerloop job to use the global-build-job approach --- eng/pipelines/coreclr/crossgen2-outerloop.yml | 81 +++++++++++-------- .../crossgen2-comparison-build-job.yml | 20 +---- .../templates/crossgen2-comparison-job.yml | 26 +----- 3 files changed, 54 insertions(+), 73 deletions(-) diff --git a/eng/pipelines/coreclr/crossgen2-outerloop.yml b/eng/pipelines/coreclr/crossgen2-outerloop.yml index 978143090b427..bb3dd3c2070ee 100644 --- a/eng/pipelines/coreclr/crossgen2-outerloop.yml +++ b/eng/pipelines/coreclr/crossgen2-outerloop.yml @@ -20,41 +20,10 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_arm - - linux_x64 - - linux_arm64 - - osx_arm64 - - osx_x64 - - windows_x86 - - windows_x64 - - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: outerloop - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: Release - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - osx_arm64 - - windows_x86 - - windows_x64 - jobParameters: - testGroup: outerloop - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Release - platforms: - - linux_arm - linux_arm64 - linux_x64 - osx_arm64 @@ -62,10 +31,23 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - isOfficialBuild: false - liveRuntimeBuildConfig: Release + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Checked_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -95,6 +77,7 @@ extends: compositeBuildMode: true displayNameArgs: R2R_Composite liveLibrariesBuildConfig: Release + unifiedArtifactsName: Checked_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) # Outerloop testing in non-composite mode - template: /eng/pipelines/common/platform-matrix.yml @@ -116,6 +99,34 @@ extends: readyToRun: true displayNameArgs: R2R liveLibrariesBuildConfig: Release + unifiedArtifactsName: Checked_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + + # Build release CoreCLR for Crossgen2 baseline generation + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: + - linux_arm + - linux_arm64 + - linux_x64 + - osx_arm64 + - osx_x64 + - windows_x86 + - windows_x64 + - windows_arm64 + jobParameters: + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Release_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets # Build Crossgen2 baselines # These are the various crossgen2 targets that are supported, and cover all major diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml index 3ca668998b133..a75dba476d6a6 100644 --- a/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml @@ -82,9 +82,7 @@ jobs: # Test job depends on the corresponding build job dependsOn: - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_ # Run all steps in the container. # Note that the containers are defined in platform-matrix.yml @@ -96,21 +94,11 @@ jobs: # Download product build - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: 'Release_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactName: 'Release_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)' displayName: 'product build' - # Optionally download live-built libraries - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir) - cleanUnpackFolder: false - artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' - artifactName: '$(librariesBuildArtifactName)' - displayName: 'live-built libraries' - # Populate Core_Root - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) $(crossArg) generatelayoutonly displayName: Populate Core_Root diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml index 4faa6c501cfe1..71f206beb929d 100644 --- a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml @@ -12,12 +12,6 @@ parameters: pool: '' targetarch: '' targetos: '' - - # When set to a non-empty value (Debug / Release), it determines libraries - # build configuration to use for the tests. Setting this property implies - # a dependency of this job on the appropriate libraries build and is used - # to construct the name of the Azure artifact representing libraries build - # to use for building the tests. liveLibrariesBuildConfig: '' ### Crossgen-comparison job @@ -84,9 +78,7 @@ jobs: # Test job depends on the corresponding build job dependsOn: - ${{ format('test_crossgen2_comparison_build_{0}_{1}_Release', parameters.targetos, parameters.targetarch)}} - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_ # Run all steps in the container. # Note that the containers are defined in platform-matrix.yml @@ -98,21 +90,11 @@ jobs: # Download product build - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: 'Release_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactName: 'Release_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)' displayName: 'product build' - # Optionally download live-built libraries - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir) - cleanUnpackFolder: false - artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' - artifactName: '$(librariesBuildArtifactName)' - displayName: 'live-built libraries' - # Populate Core_Root - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) $(crossArg) generatelayoutonly displayName: Populate Core_Root From d032532c7362cb280fa52b0b694b44bfd44dddfd Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 27 Feb 2024 17:05:16 -0800 Subject: [PATCH 28/79] Convert libraries stress pipelines to use the global build job templates with helix execution in-job --- .../coreclr/libraries-gcstress-extra.yml | 37 ++++---- .../libraries-gcstress0x3-gcstress0xc.yml | 40 ++++---- .../coreclr/libraries-jitstress-random.yml | 40 +++----- eng/pipelines/coreclr/libraries-jitstress.yml | 46 ++++------ .../libraries-jitstress2-jitstressregs.yml | 47 ++++------ .../coreclr/libraries-jitstressregs.yml | 47 ++++------ eng/pipelines/coreclr/libraries-pgo.yml | 65 ++++++++----- eng/pipelines/libraries/run-test-job.yml | 91 ------------------- 8 files changed, 149 insertions(+), 264 deletions(-) diff --git a/eng/pipelines/coreclr/libraries-gcstress-extra.yml b/eng/pipelines/coreclr/libraries-gcstress-extra.yml index 5720cddd6ea17..c6bc819be20eb 100644 --- a/eng/pipelines/coreclr/libraries-gcstress-extra.yml +++ b/eng/pipelines/coreclr/libraries-gcstress-extra.yml @@ -19,30 +19,27 @@ extends: - stage: Build jobs: - # - # Build CoreCLR checked and libraries Release - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked - platformGroup: gcstress - - # - # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platformGroup: gcstress helixQueueGroup: libraries helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + platformGroup: gcstress jobParameters: # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. - timeoutInMinutes: 600 - testScope: innerloop - liveRuntimeBuildConfig: checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - coreclrTestGroup: gcstress-extra + timeoutInMinutes: 660 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + - heapverify1 + - gcstress0xc_disabler2r + - gcstress0xc_disabler2r_jitstress2 + - gcstress0xc_disabler2r_heapverify1 + - gcstress0xc_jitstress1 + - gcstress0xc_jitstress2 + - gcstress0xc_jitminopts_heapverify1 diff --git a/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml b/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml index ca04549c65e34..90f9803f74517 100644 --- a/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml +++ b/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml @@ -12,6 +12,12 @@ trigger: none variables: - template: /eng/pipelines/common/variables.yml +extends: + template: /eng/pipelines/common/templates/pipeline-with-resources.yml + parameters: + stages: + - stage: Build + jobs: extends: template: /eng/pipelines/common/templates/pipeline-with-resources.yml parameters: @@ -19,30 +25,24 @@ extends: - stage: Build jobs: - # - # Build CoreCLR checked and libraries Release - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked - platformGroup: gcstress - - # - # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platformGroup: gcstress helixQueueGroup: libraries helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + platformGroup: gcstress jobParameters: # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. - timeoutInMinutes: 600 - testScope: innerloop - liveRuntimeBuildConfig: checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - coreclrTestGroup: gcstress0x3-gcstress0xc + timeoutInMinutes: 660 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + # Disable gcstress0x3 for now; it causes lots of test timeouts. Investigate this after + # gcstress0xc runs are clean. Tracking issue: https://github.com/dotnet/runtime/issues/38903. + # - gcstress0x3 + - gcstress0xc diff --git a/eng/pipelines/coreclr/libraries-jitstress-random.yml b/eng/pipelines/coreclr/libraries-jitstress-random.yml index 859bff6a949b0..b9ab14fc30804 100644 --- a/eng/pipelines/coreclr/libraries-jitstress-random.yml +++ b/eng/pipelines/coreclr/libraries-jitstress-random.yml @@ -18,13 +18,12 @@ extends: - stage: Build jobs: - # - # Build CoreCLR checked and libraries Release - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml platforms: - linux_x64 - linux_arm @@ -32,28 +31,15 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - - # - # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - - windows_x86 - helixQueueGroup: libraries - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. - timeoutInMinutes: 300 - testScope: innerloop - liveRuntimeBuildConfig: checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - coreclrTestGroup: jitstress-random + timeoutInMinutes: 360 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + - jitstress_random_1 + - jitstress_random_2 diff --git a/eng/pipelines/coreclr/libraries-jitstress.yml b/eng/pipelines/coreclr/libraries-jitstress.yml index ba921b4f1fa65..501f233f47137 100644 --- a/eng/pipelines/coreclr/libraries-jitstress.yml +++ b/eng/pipelines/coreclr/libraries-jitstress.yml @@ -18,13 +18,12 @@ extends: - stage: Build jobs: - # - # Build CoreCLR checked and libraries Release - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml platforms: - linux_x64 - linux_arm @@ -32,28 +31,21 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - - # - # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - - windows_x86 - helixQueueGroup: libraries - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. - timeoutInMinutes: 300 - testScope: innerloop - liveRuntimeBuildConfig: checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - coreclrTestGroup: jitstress + timeoutInMinutes: 360 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + - no_tiered_compilation + - jitminopts + - jitstress1 + - jitstress1_tiered + - jitstress2 + - jitstress2_tiered + - disabler2r + - tailcallstress diff --git a/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml b/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml index 6d8145098125f..9c50b8d5f4588 100644 --- a/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml +++ b/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml @@ -18,13 +18,12 @@ extends: - stage: Build jobs: - # - # Build CoreCLR checked and libraries Release - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml platforms: - linux_x64 - linux_arm @@ -32,28 +31,22 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - - # - # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - - windows_x86 - helixQueueGroup: libraries - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. - timeoutInMinutes: 300 - testScope: innerloop - liveRuntimeBuildConfig: checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - coreclrTestGroup: jitstress2-jitstressregs + timeoutInMinutes: 360 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + - jitstress2_jitstressregs1 + - jitstress2_jitstressregs2 + - jitstress2_jitstressregs3 + - jitstress2_jitstressregs4 + - jitstress2_jitstressregs8 + - jitstress2_jitstressregs0x10 + - jitstress2_jitstressregs0x80 + - jitstress2_jitstressregs0x1000 + - jitstress2_jitstressregs0x2000 \ No newline at end of file diff --git a/eng/pipelines/coreclr/libraries-jitstressregs.yml b/eng/pipelines/coreclr/libraries-jitstressregs.yml index b4ad87ff13579..5ab04e9a27add 100644 --- a/eng/pipelines/coreclr/libraries-jitstressregs.yml +++ b/eng/pipelines/coreclr/libraries-jitstressregs.yml @@ -18,13 +18,12 @@ extends: - stage: Build jobs: - # - # Build CoreCLR checked and libraries Release - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml platforms: - linux_x64 - linux_arm @@ -32,28 +31,22 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - - # - # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - - windows_x86 - helixQueueGroup: libraries - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. - timeoutInMinutes: 300 - testScope: innerloop - liveRuntimeBuildConfig: checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - coreclrTestGroup: jitstressregs + timeoutInMinutes: 360 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + - jitstressregs1 + - jitstressregs2 + - jitstressregs3 + - jitstressregs4 + - jitstressregs8 + - jitstressregs0x10 + - jitstressregs0x80 + - jitstressregs0x1000 + - jitstressregs0x2000 diff --git a/eng/pipelines/coreclr/libraries-pgo.yml b/eng/pipelines/coreclr/libraries-pgo.yml index addaf68306559..a7766f2fb881c 100644 --- a/eng/pipelines/coreclr/libraries-pgo.yml +++ b/eng/pipelines/coreclr/libraries-pgo.yml @@ -18,41 +18,56 @@ extends: - stage: Build jobs: - # - # Build CoreCLR checked and libraries Release - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml platforms: - - linux_x64 - - linux_arm - - linux_arm64 - - windows_x86 - - windows_x64 - windows_arm64 + jobParameters: + # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. + timeoutInMinutes: 660 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + - defaultpgo - # - # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml platforms: + - linux_x64 - linux_arm - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - windows_x86 - helixQueueGroup: libraries - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + - windows_x64 jobParameters: - timeoutInMinutes: 600 - testScope: innerloop - liveRuntimeBuildConfig: checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - coreclrTestGroup: pgo + # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. + timeoutInMinutes: 660 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + - defaultpgo + - fullpgo + - fullpgo_methodprofiling + - fullpgo_random_gdv + - fullpgo_random_gdv_methodprofiling_only + - fullpgo_random_gdv_edge + - jitosr_stress + - jitosr_stress_random + - syntheticpgo + - syntheticpgo_blend + - jitcrossblocklocalassertionprop diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 8e0a94d0a608d..b8331733b688e 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -149,103 +149,12 @@ jobs: SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} - # coreclrTestGroup: The following mappings of 'coreclrTestGroup' to 'scenarios' is copied from - # eng/pipelines/common/templates/runtimes/run-test-job.yml (with 'testGroup' replaced by 'coreclrTestGroup' - # for clarity), and should remain in sync. This is only a subset; only the testGroups that are - # used to test the libraries have been added here. More could be added if we decided to test the - # libraries with more stress modes. The scenario tags are interpreted by - # src\tests\Common\testenvironment.proj. - # - # The one difference here compared to eng/pipelines/common/templates/runtimes/run-test-job.yml is - # that 'jitstress' contains 'no_tiered_compilation'. The 'normal' (default) test mode - # is run in a regular CI job, so there is no need to duplicate it here. So, add 'no_tiered_compilation' - # to the 'jitstress' job instead of adding a new job just for 'no_tiered_compilation'. - - # src/libraries/sendtohelix.proj processes one scenario per parallel MSBuild invocation. Each invocation only - # creates Helix work items and them waits for their completion on the remote Helix machines, so is not - # computationally intensive. We want Helix to be provided with all the possible work items in up front, - # so can do as much work in parallel as possible. Thus, increase the amount of allowed MSBuild parallelism - # to at least the maximum number of scenarios to be processed in a coreclrTestGroup. If there is no - # coreclrTestGroup then there is only one scenario (the default scenario), so don't change the MSBuild argument. - ${{ if ne(parameters.coreclrTestGroup, '') }}: - extraHelixArguments: $(_extraHelixArguments) /maxcpucount:10 - ${{ else }}: - extraHelixArguments: $(_extraHelixArguments) - ${{ if in(parameters.coreclrTestGroup, 'superpmi_collection') }}: scenarios: - normal ${{ if in(parameters.coreclrTestGroup, 'superpmi_collection_no_tiered_compilation') }}: scenarios: - no_tiered_compilation - ${{ if in(parameters.coreclrTestGroup, 'jitstress') }}: - scenarios: - - no_tiered_compilation - - jitminopts - - jitstress1 - - jitstress1_tiered - - jitstress2 - - jitstress2_tiered - - disabler2r - - tailcallstress - ${{ if in(parameters.coreclrTestGroup, 'jitstress-random') }}: - scenarios: - - jitstress_random_1 - - jitstress_random_2 - ${{ if in(parameters.coreclrTestGroup, 'jitstressregs' ) }}: - scenarios: - - jitstressregs1 - - jitstressregs2 - - jitstressregs3 - - jitstressregs4 - - jitstressregs8 - - jitstressregs0x10 - - jitstressregs0x80 - - jitstressregs0x1000 - - jitstressregs0x2000 - ${{ if in(parameters.coreclrTestGroup, 'jitstress2-jitstressregs') }}: - scenarios: - - jitstress2_jitstressregs1 - - jitstress2_jitstressregs2 - - jitstress2_jitstressregs3 - - jitstress2_jitstressregs4 - - jitstress2_jitstressregs8 - - jitstress2_jitstressregs0x10 - - jitstress2_jitstressregs0x80 - - jitstress2_jitstressregs0x1000 - - jitstress2_jitstressregs0x2000 - ${{ if in(parameters.coreclrTestGroup, 'gcstress0x3-gcstress0xc') }}: - scenarios: - # Disable gcstress0x3 for now; it causes lots of test timeouts. Investigate this after - # gcstress0xc runs are clean. Tracking issue: https://github.com/dotnet/runtime/issues/38903. - # - gcstress0x3 - - gcstress0xc - ${{ if in(parameters.coreclrTestGroup, 'gcstress-extra') }}: - scenarios: - - heapverify1 - - gcstress0xc_disabler2r - - gcstress0xc_disabler2r_jitstress2 - - gcstress0xc_disabler2r_heapverify1 - - gcstress0xc_jitstress1 - - gcstress0xc_jitstress2 - - gcstress0xc_jitminopts_heapverify1 - ${{ if in(parameters.coreclrTestGroup, 'pgo') }}: - ${{ if and(eq(parameters.osGroup, 'windows'), eq(parameters.archType, 'arm64')) }}: - scenarios: - - defaultpgo - ${{ else }}: - scenarios: - - defaultpgo - - fullpgo - - fullpgo_methodprofiling - - fullpgo_random_gdv - - fullpgo_random_gdv_methodprofiling_only - - fullpgo_random_gdv_edge - - jitosr_stress - - jitosr_stress_random - - syntheticpgo - - syntheticpgo_blend - - jitcrossblocklocalassertionprop - ${{ if eq(parameters.SuperPmiCollect, true) }}: - template: /eng/pipelines/libraries/superpmi-postprocess-step.yml From a6729e4a5d65342dd560f12985e240e8ce09a72b Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 28 Feb 2024 10:11:12 -0800 Subject: [PATCH 29/79] Convert runtime-llvm and runtimelab pipelines to use the global-build-job templates --- eng/pipelines/runtime-llvm.yml | 74 +++-- eng/pipelines/runtimelab.yml | 254 +++++++----------- .../runtimelab-post-build-steps.yml | 76 ------ 3 files changed, 121 insertions(+), 283 deletions(-) delete mode 100644 eng/pipelines/runtimelab/runtimelab-post-build-steps.yml diff --git a/eng/pipelines/runtime-llvm.yml b/eng/pipelines/runtime-llvm.yml index 9d358e5f79308..15be109bc9acd 100644 --- a/eng/pipelines/runtime-llvm.yml +++ b/eng/pipelines/runtime-llvm.yml @@ -146,73 +146,59 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isRollingBuild'], true)) - # - # Build Mono release with LLVM AOT - # Only when mono, or the runtime tests changed - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono - buildConfig: release - platforms: - - linux_x64 - - linux_arm64 - jobParameters: - runtimeVariant: llvmaot - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # Mono CoreCLR runtime Test executions using live libraries and LLVM AOT - # Only when Mono is changed - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml buildConfig: release runtimeFlavor: mono platforms: - - linux_x64_llvmaot + - linux_x64 # Disabled pending outcome of https://github.com/dotnet/runtime/issues/60234 investigation #- linux_arm64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testGroup: innerloop - liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - liveRuntimeBuildConfig: release - runtimeVariant: llvmaot + nameSuffix: AllSubsets_Mono_LLVMAOT_RuntimeTests + buildArgs: -s mono+clr.iltools+clr.hosts+libs+host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} + /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_Release + llvmAotStepContainer: linux_x64_llvmaot + runtimeVariant: llvmaot condition: >- or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isRollingBuild'], true)) - # - # Mono CoreCLR runtime Test executions using live libraries and LLVM Full AOT - # Only when Mono is changed - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml buildConfig: release runtimeFlavor: mono platforms: - - linux_x64_llvmaot + - linux_x64 - linux_arm64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testGroup: innerloop - liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - liveRuntimeBuildConfig: release - runtimeVariant: llvmfullaot + nameSuffix: AllSubsets_Mono_LLVMFullAOT_RuntimeTests + buildArgs: -s mono+clr.iltools+clr.hosts+libs+host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} + /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_Release + llvmAotStepContainer: linux_x64_llvmaot + runtimeVariant: llvmfullaot condition: >- or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isRollingBuild'], true)) diff --git a/eng/pipelines/runtimelab.yml b/eng/pipelines/runtimelab.yml index 7c34126757d73..b1b65f5343de9 100644 --- a/eng/pipelines/runtimelab.yml +++ b/eng/pipelines/runtimelab.yml @@ -49,175 +49,103 @@ extends: stages: - stage: Build jobs: + - ${{ if ne(variables.isOfficialBuild, true) }}: + # + # Build the whole product with Checked CoreCLR and run runtime tests + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + buildConfig: checked + platforms: + - linux_x64 + - windows_x64 + jobParameters: + timeoutInMinutes: 200 + buildArgs: -s clr+libs+hosts+packs -c debug -rc $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) - # - # Build with Debug config and Checked runtimeConfiguration - # - - ${{ if ne(variables.isOfficialBuild, true) }}: - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: Checked - platforms: - - linux_x64 - - windows_x64 - jobParameters: - timeoutInMinutes: 100 - testGroup: innerloop - buildArgs: -s clr+libs+host+packs -c debug -runtimeConfiguration Checked - postBuildSteps: - - template: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml - parameters: - uploadRuntimeTests: true + # + # Build the whole product with Release CoreCLR and run libraries tests + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: release + platforms: + - linux_x64 + - windows_x64 + jobParameters: + timeoutInMinutes: 180 + buildArgs: -s clr+libs+libs.tests+hosts+packs -c $(_BuildConfig) /p:ArchiveTests=true + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Libraries_$(_BuildConfig) - # - # Build with Release config and Release runtimeConfiguration - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: Release - platforms: - - linux_x64 - - windows_x64 - jobParameters: - timeoutInMinutes: 100 - isOfficialBuild: ${{ variables.isOfficialBuild }} - testGroup: innerloop - postBuildSteps: - - template: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml - parameters: - uploadLibrariesTests: ${{ eq(variables.isOfficialBuild, false) }} - uploadIntermediateArtifacts: false - ${{ if eq(variables.isOfficialBuild, false) }}: - buildArgs: -s clr+libs+libs.tests+host+packs -c $(_BuildConfig) /p:ArchiveTests=true - ${{ if eq(variables.isOfficialBuild, true) }}: - buildArgs: -s clr+libs -c $(_BuildConfig) + # + # Build and test libraries AllConfigurations + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + platforms: + - windows_x64 + jobParameters: + buildArgs: -test -s tools+libs+libs.tests -allConfigurations -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + nameSuffix: Libraries_AllConfigurations + timeoutInMinutes: 150 - # - # Build with Release allConfigurations to produce packages - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: Release - platforms: - - windows_x64 - jobParameters: - isOfficialBuild: ${{ variables.isOfficialBuild }} - testGroup: innerloop - nameSuffix: AllConfigurations - buildArgs: -s libs -c $(_BuildConfig) -allConfigurations - ${{ if eq(variables.isOfficialBuild, true) }}: - postBuildSteps: - - template: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml + - ${{ else }}: + # + # Build the whole product with Release CoreCLR + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: release + platforms: + - linux_x64 + - windows_x64 + jobParameters: + isOfficialBuild: true + timeoutInMinutes: 180 + buildArgs: -s clr+libs+hosts+packs -c $(_BuildConfig) + postBuildSteps: + # Upload the results. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml parameters: - uploadIntermediateArtifacts: true - isOfficialBuild: true - librariesBinArtifactName: libraries_bin_official_allconfigurations - - # Installer official builds need to build installers and need the libraries all configurations build - - ${{ if eq(variables.isOfficialBuild, true) }}: - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/installer/jobs/build-job.yml - jobParameters: - liveRuntimeBuildConfig: Release - liveLibrariesBuildConfig: Release - isOfficialBuild: ${{ variables.isOfficialBuild }} - useOfficialAllConfigurations: true - dependsOnGlobalBuild: true - platforms: - - linux_x64 - - windows_x64 - - - ${{ if ne(variables.isOfficialBuild, true) }}: - # - # CoreCLR Test builds using live libraries release build - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: Checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: innerloop - dependsOn: - - build_linux_x64_Checked_ - - build_linux_x64_Release_ - - # - # CoreCLR Test executions using live libraries - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: Checked - platforms: - - linux_x64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: innerloop - liveLibrariesBuildConfig: Release - dependsOn: - - coreclr_common_test_build_p0_AnyOS_AnyCPU_Checked - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: Checked - platforms: - - windows_x64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: innerloop - liveLibrariesBuildConfig: Release - dependsOn: - - coreclr_common_test_build_p0_AnyOS_AnyCPU_Checked - - build_windows_x64_Checked_ - - build_windows_x64_Release_ - - # - # Libraries Release Test Execution against a release coreclr runtime - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - linux_x64 - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - jobParameters: - isRollingBuild: false - isOfficialBuild: false - testScope: innerloop - liveRuntimeBuildConfig: Release - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - dependsOn: - - build_linux_x64_Release_ + name: $(osGroup)$(osSubgroup)_$(archType) - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - windows_x64 - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - jobParameters: - isRollingBuild: false - isOfficialBuild: false - testScope: innerloop - liveRuntimeBuildConfig: Release - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - dependsOn: - - build_windows_x64_Release_ + # + # Build libraries AllConfigurations for packages + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: Release + platforms: + - windows_x64 + jobParameters: + buildArgs: -s tools+libs -allConfigurations -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + nameSuffix: Libraries_AllConfigurations + isOfficialBuild: true + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: Libraries_AllConfigurations + timeoutInMinutes: 95 - ${{ if eq(variables.isOfficialBuild, true) }}: - template: /eng/pipelines/official/stages/publish.yml parameters: - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: true \ No newline at end of file diff --git a/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml b/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml deleted file mode 100644 index 97c007ba4b307..0000000000000 --- a/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml +++ /dev/null @@ -1,76 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osSubgroup: '' - nameSuffix: '' - platform: '' - runtimeVariant: '' - librariesBinArtifactName: '' - isOfficialBuild: false - uploadLibrariesTests: false - uploadRuntimeTests: false - uploadIntermediateArtifacts: false - -steps: - # Build coreclr native test output - - ${{ if eq(parameters.uploadRuntimeTests, true) }}: - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(buildConfigUpper) ${{ parameters.archType }} - displayName: Build native test components - - # Copy all build output into artifacts staging directory - - template: /eng/pipelines/libraries/prepare-for-bin-publish.yml - parameters: - isOfficialBuild: ${{ parameters.isOfficialBuild }} - - # Zip CoreCLR Build Output - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper) - archiveType: $(archiveType) - tarCompression: $(tarCompression) - includeRootFolder: false - archiveExtension: $(archiveExtension) - artifactName: CoreCLRProduct_${{ parameters.runtimeVariant }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }} - displayName: 'CoreCLR product build' - - # Zip Test Build - - ${{ if eq(parameters.uploadLibrariesTests, true) }}: - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/helix - includeRootFolder: true - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: libraries_test_assets_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }} - displayName: Test Assets - - # Zip runtime native assets for use by Tests - - ${{ if eq(parameters.uploadRuntimeTests, true) }}: - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/tests/coreclr/obj/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper) - includeRootFolder: false - archiveType: $(archiveType) - tarCompression: $(tarCompression) - archiveExtension: $(archiveExtension) - artifactName: CoreCLRNativeTestArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }} - displayName: 'native test components' - - # Zip Libraries Build Output - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.ArtifactStagingDirectory)/artifacts - archiveType: $(archiveType) - tarCompression: $(tarCompression) - includeRootFolder: false - archiveExtension: $(archiveExtension) - artifactName: ${{ coalesce(parameters.librariesBinArtifactName, format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig)) }} - displayName: Build Assets - - # Upload unsigned artifacts - - ${{ if eq(parameters.uploadIntermediateArtifacts, true) }}: - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: ${{ parameters.platform }}${{ parameters.nameSuffix }} From b44a4534862c290e82e6018cb06234839b47aa07 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 28 Feb 2024 11:03:55 -0800 Subject: [PATCH 30/79] Convert superpmi-collect.yml to use global-build-job to build the product assets. --- .../common/templates/global-build-step.yml | 3 + .../templates/runtimes/run-test-job.yml | 3 - eng/pipelines/coreclr/superpmi-collect.yml | 117 ++++++++++++++++-- .../templates/superpmi-collect-job.yml | 60 +++------ eng/pipelines/libraries/base-job.yml | 3 +- eng/pipelines/libraries/run-test-job.yml | 103 +++++++-------- 6 files changed, 181 insertions(+), 108 deletions(-) diff --git a/eng/pipelines/common/templates/global-build-step.yml b/eng/pipelines/common/templates/global-build-step.yml index 7f38a9fd11840..172d4d3127be0 100644 --- a/eng/pipelines/common/templates/global-build-step.yml +++ b/eng/pipelines/common/templates/global-build-step.yml @@ -4,9 +4,12 @@ parameters: shouldContinueOnError: false archParameter: $(_archParameter) displayName: Build product + container: '' steps: - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) $(crossArg) ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) displayName: ${{ parameters.displayName }} ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}: continueOnError: ${{ parameters.shouldContinueOnError }} + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index ce44dbe6341f7..149a9e714589f 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -77,9 +77,6 @@ jobs: - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - # SuperPMI collection needs to run mcs.exe on the AzDO machine. Assume that's an x64 machine, and download an x64 product build if needed. - - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}: - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', '', parameters.osGroup, parameters.osSubgroup, 'x64', parameters.buildConfig) }} # Compute job name from template parameters ${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 1a9f70c46d51a..6a7ceadf3cd56 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -35,32 +35,116 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - windows_x64 + - linux_x64 + jobParameters: + testGroup: outerloop + buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -lc Release /p:ArchiveTests=true + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: LibrariesTestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + extraVariablesTemplates: + - template: /eng/pipelines/coreclr/templates/native-test-assets-variables.yml + parameters: + testGroup: outerloop + disableComponentGovernance: true # No shipping artifacts produced by this pipeline + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + platforms: - windows_x86 - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 + - osx_arm64 jobParameters: testGroup: outerloop + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + # Build CLR assets for x64 as well as the target as we need an x64 mcs + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.spmi -c $(_BuildConfig) + archParameter: -arch x64 + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: LibrariesTestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + extraVariablesTemplates: + - template: /eng/pipelines/coreclr/templates/native-test-assets-variables.yml + parameters: + testGroup: outerloop disableComponentGovernance: true # No shipping artifacts produced by this pipeline - # superpmi-collect-job that targets macOS/arm64 depends on coreclr binaries produced by the macOS/x64 job - # We don't collect osx-x64 (it's essentially the same as linux-x64). If we did, we'd add osx_x64 in the - # build-coreclr-and-libraries-job.yml above, and remove this. - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - - osx_x64 + - linux_arm + - linux_arm64 jobParameters: testGroup: outerloop + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + # Build CLR assets for x64 as well as the target as we need an x64 mcs + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.spmi -c $(_BuildConfig) + archParameter: -arch x64 + container: linux_x64 + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: LibrariesTestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + extraVariablesTemplates: + - template: /eng/pipelines/coreclr/templates/native-test-assets-variables.yml + parameters: + testGroup: outerloop disableComponentGovernance: true # No shipping artifacts produced by this pipeline - template: /eng/pipelines/common/platform-matrix.yml @@ -213,6 +297,7 @@ extends: testGroup: outerloop liveLibrariesBuildConfig: Release SuperPmiCollect: true + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -254,9 +339,12 @@ extends: liveRuntimeBuildConfig: checked dependsOnTestBuildConfiguration: Release dependsOnTestArchitecture: x64 - coreclrTestGroup: superpmi_collection + scenarios: + - normal SuperPmiCollect: true SuperPmiCollectionName: libraries_tests + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + helixArtifactsName: LibrariesTestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) # # Collection of libraries test run: no_tiered_compilation @@ -281,6 +369,9 @@ extends: liveRuntimeBuildConfig: checked dependsOnTestBuildConfiguration: Release dependsOnTestArchitecture: x64 - coreclrTestGroup: superpmi_collection_no_tiered_compilation + scenarios: + - no_tiered_compilation SuperPmiCollect: true SuperPmiCollectionName: libraries_tests_no_tiered_compilation + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + helixArtifactsName: LibrariesTestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml index b77ec27e6e5ce..c66d2ad990f58 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml @@ -42,33 +42,30 @@ jobs: steps: # Extra steps that will be passed to the superpmi template and run before sending the job to helix (all of which is done in the template) - - # Optionally download live-built libraries - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir) - cleanUnpackFolder: false - artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' - artifactName: '$(librariesBuildArtifactName)' - displayName: 'live-built libraries' - - # Download coreclr + + # Download runtime artifacts - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' - displayName: 'Coreclr product build' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: 'Runtime build artifacts' - # Download x64 coreclr if running on non-x64 configuration - - ${{ if ne(parameters.archType, 'x64') }}: + # Unzip individual test projects + - ${{ if eq(parameters.collectionName, 'libraries_tests') }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper)' - artifactFileName: 'CoreCLRProduct__$(osGroup)$(osSubgroup)_x64_$(buildConfig)$(archiveExtension)' - artifactName: 'CoreCLRProduct__$(osGroup)$(osSubgroup)_x64_$(buildConfig)' - displayName: 'Coreclr product build (x64)' + unpackFolder: '$(Build.SourcesDirectory)/artifacts/helix/' + artifactFileName: 'LibrariesTestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactName: LibrariesTestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: 'Libraries test artifacts' + - task: ExtractFiles@1 + displayName: 'Unzip libraries tests' + inputs: + archiveFilePatterns: '$(Build.SourcesDirectory)/artifacts/helix/**/*.zip' + destinationFolder: '$(Build.SourcesDirectory)/artifacts/tests/libraries/$(osGroup).$(archType).$(buildConfigUpper)' + cleanDestinationFolder: true + overwriteExistingFiles: true # Download and unzip managed test artifacts - ${{ if eq(parameters.collectionName, 'coreclr_tests') }}: @@ -79,25 +76,6 @@ jobs: artifactName: '$(managedGenericTestArtifactName)' displayName: 'generic managed test artifacts' - # Download and unzip libraries test artifacts - - ${{ if eq(parameters.collectionName, 'libraries_tests') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: '$(Build.SourcesDirectory)/artifacts/tests/libraries_zipped/$(osGroup).$(archType).$(buildConfigUpper)' - artifactFileName: 'libraries_test_assets_${{ parameters.osGroup }}_$(archType)_Release$(archiveExtension)' - artifactName: ${{ format('libraries_test_assets_{0}_$(archType)_Release', parameters.osGroup) }} - displayName: 'generic libraries test artifacts' - - # Unzip individual test projects - - ${{ if eq(parameters.collectionName, 'libraries_tests') }}: - - task: ExtractFiles@1 - displayName: 'Unzip Tests.zip' - inputs: - archiveFilePatterns: '$(Build.SourcesDirectory)/artifacts/tests/libraries_zipped/$(osGroup).$(archType).$(buildConfigUpper)/**/*.zip' - destinationFolder: '$(Build.SourcesDirectory)/artifacts/tests/libraries/$(osGroup).$(archType).$(buildConfigUpper)' - cleanDestinationFolder: true - overwriteExistingFiles: true - # Create Core_Root - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) displayName: Create Core_Root diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index f41dd67d02a42..5b381c958bde7 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -23,6 +23,7 @@ parameters: pool: '' runTests: false SuperPmiCollect: false + unifiedArtifactsName: '' jobs: - template: /eng/common/templates/job/job.yml @@ -117,7 +118,7 @@ jobs: clean: true fetchDepth: $(checkoutFetchDepth) - - ${{ if and(ne(parameters.liveRuntimeBuildConfig, ''), eq(parameters.runTests, true)) }}: + - ${{ if and(eq(parameters.unifiedArtifactsName, ''), ne(parameters.liveRuntimeBuildConfig, ''), eq(parameters.runTests, true)) }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(_runtimeDownloadPath) diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index b8331733b688e..483974c45d256 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -19,14 +19,13 @@ parameters: condition: true shouldContinueOnError: false variables: {} - # coreclrTestGroup: if empty, then a normal, default test run is created. If set, it indicates a set of - # stress modes that each test will be run with. This is the same usage as 'testGroup' in - # eng/pipelines/common/templates/runtimes/run-test-job.yml. - coreclrTestGroup: '' + scenarios: [] SuperPmiCollect: false SuperPmiCollectionType: 'run' SuperPmiCollectionName: 'libraries_tests' dependsOn: [] + unifiedArtifactsName: '' + helixArtifactsName: '' jobs: - template: /eng/pipelines/libraries/base-job.yml @@ -46,6 +45,7 @@ jobs: condition: ${{ parameters.condition }} testScope: ${{ parameters.testScope }} SuperPmiCollect: ${{ parameters.SuperPmiCollect }} + unifiedArtifactsName: ${{ parameters.unifiedArtifactsName }} runTests: true ${{ if eq(parameters.SuperPmiCollect, true) }}: displayName: ${{ format('SuperPMI collection {0} {1} {2}', parameters.SuperPmiCollectionName, parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }} @@ -68,12 +68,12 @@ jobs: - ${{ if ne(parameters.dependsOn[0], '') }}: - ${{ parameters.dependsOn }} - ${{ if eq(parameters.dependsOn[0], '') }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - # SuperPMI collection needs to run mcs.exe on the AzDO machine. Assume that's an x64 machine, and download an x64 product build if needed. - - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}: - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', 'coreclr', '', parameters.osGroup, parameters.osSubgroup, 'x64', parameters.liveRuntimeBuildConfig) }} + - ${{ if ne(parameters.unifiedArtifactsName, '')}}: + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_' + - ${{ else }}: + - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: + - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} variables: @@ -94,43 +94,50 @@ jobs: - ${{ parameters.variables }} steps: - - # SuperPMI collection: Download x64 coreclr if running on non-x64 configuration (needed for mcs.exe on AzDO machine; see `SuperPmiMcsPath`. - - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(_runtimeX64DownloadPath) - artifactFileName: 'CoreCLRProduct__${{ parameters.osGroup }}${{ parameters.osSubgroup }}_x64_${{ parameters.liveRuntimeBuildConfig }}$(archiveExtension)' - artifactName: 'CoreCLRProduct__${{ parameters.osGroup }}${{ parameters.osSubgroup }}_x64_${{ parameters.liveRuntimeBuildConfig }}' - displayName: 'CoreCLR product build (x64)' - - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - displayName: Build Assets - cleanUnpackFolder: false - artifactName: $(librariesBuildArtifactName) - artifactFileName: $(librariesBuildArtifactName)$(archiveExtension) - unpackFolder: $(Build.SourcesDirectory)/artifacts - - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - displayName: Test Assets - cleanUnpackFolder: false - artifactName: $(librariesTestsArtifactName) - artifactFileName: $(librariesTestsArtifactName)$(archiveExtension) - unpackFolder: $(Build.SourcesDirectory)/artifacts - - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} displayName: Install Build Dependencies - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - script: $(_buildScript) - -subset host.native+libs.pretest - $(_buildArguments) - /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideRuntimeFromLiveDrop.binlog - displayName: Prepare TestHost with runtime $(runtimeFlavorName) + - ${{ if ne(parameters.unifiedArtifactsName, '') }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: '${{ parameters.unifiedArtifactsName }}$(archiveExtension)' + artifactName: '${{ parameters.unifiedArtifactsName }}' + displayName: 'unified artifacts' + + - ${{ else }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + displayName: Build Assets + cleanUnpackFolder: false + artifactName: $(librariesBuildArtifactName) + artifactFileName: $(librariesBuildArtifactName)$(archiveExtension) + unpackFolder: $(Build.SourcesDirectory)/artifacts + + - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: + - script: $(_buildScript) + -subset host.native+libs.pretest + $(_buildArguments) + /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideRuntimeFromLiveDrop.binlog + displayName: Prepare TestHost with runtime $(runtimeFlavorName) + + - ${{ if ne(parameters.helixArtifactsName, '') }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + displayName: Test Assets + artifactName: ${{ parameters.helixArtifactsName }} + artifactFileName: '${{ parameters.helixArtifactsName }}$(archiveExtension)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/helix + - ${{ else }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + displayName: Test Assets + cleanUnpackFolder: false + artifactName: $(librariesTestsArtifactName) + artifactFileName: $(librariesTestsArtifactName)$(archiveExtension) + unpackFolder: $(Build.SourcesDirectory)/artifacts - template: /eng/pipelines/libraries/helix.yml parameters: @@ -148,13 +155,9 @@ jobs: SuperPmiCollect: ${{ parameters.SuperPmiCollect }} SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} - - ${{ if in(parameters.coreclrTestGroup, 'superpmi_collection') }}: - scenarios: - - normal - ${{ if in(parameters.coreclrTestGroup, 'superpmi_collection_no_tiered_compilation') }}: - scenarios: - - no_tiered_compilation + + ${{ if ne(parameters.scenarios[0], '') }}: + scenarios: ${{ parameters.scenarios }} - ${{ if eq(parameters.SuperPmiCollect, true) }}: - template: /eng/pipelines/libraries/superpmi-postprocess-step.yml From 2550485068bfdb17ff0b12c59611026c330ac2b1 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 28 Feb 2024 11:54:24 -0800 Subject: [PATCH 31/79] Convert the perf jobs to use global build everywhere. --- .../build-coreclr-and-libraries-job.yml | 49 ------------- eng/pipelines/coreclr/perf-non-wasm-jobs.yml | 34 ++++++++-- eng/pipelines/coreclr/perf-wasm-jobs.yml | 5 -- eng/pipelines/coreclr/perf_slow.yml | 44 ++++++++---- eng/pipelines/coreclr/templates/perf-job.yml | 68 ++++++------------- eng/pipelines/libraries/base-job.yml | 3 - 6 files changed, 80 insertions(+), 123 deletions(-) delete mode 100644 eng/pipelines/common/build-coreclr-and-libraries-job.yml diff --git a/eng/pipelines/common/build-coreclr-and-libraries-job.yml b/eng/pipelines/common/build-coreclr-and-libraries-job.yml deleted file mode 100644 index 147a346b9bf4e..0000000000000 --- a/eng/pipelines/common/build-coreclr-and-libraries-job.yml +++ /dev/null @@ -1,49 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osSubgroup: '' - targetRid: '' - container: '' - testGroup: '' - crossBuild: false - timeoutInMinutes: '' - variables: {} - pool: '' - platform: '' - disableComponentGovernance: false - -jobs: -- template: /eng/pipelines/coreclr/templates/build-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - targetRid: ${{ parameters.targetRid }} - container: ${{ parameters.container }} - testGroup: ${{ parameters.testGroup }} - crossBuild: ${{ parameters.crossBuild }} - timeoutInminutes: ${{ parameters.timeoutInMinutes }} - variables: ${{ parameters.variables }} - pool: ${{ parameters.pool }} - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - -- template: /eng/pipelines/libraries/build-job.yml - parameters: - ${{ if eq(parameters.buildConfig, 'debug') }}: - buildConfig: Debug - ${{ if ne(parameters.buildConfig, 'debug') }}: - buildConfig: Release - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - container: ${{ parameters.container }} - testGroup: ${{ parameters.testGroup }} - crossBuild: ${{ parameters.crossBuild }} - timeoutInminutes: ${{ parameters.timeoutInMinutes }} - variables: ${{ parameters.variables }} - pool: ${{ parameters.pool }} - liveRuntimeBuildConfig: ${{ parameters.buildConfig }} - platform: ${{ parameters.platform }} - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} diff --git a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml index 547769e082217..ab60b9392db25 100644 --- a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml +++ b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml @@ -5,7 +5,7 @@ jobs: # build coreclr and libraries - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - linux_x64 @@ -13,7 +13,19 @@ jobs: - windows_x86 - linux_musl_x64 jobParameters: - testGroup: perf + nameSuffix: coreclr + buildArgs: -s clr+libs+host+packs -c $(_BuildConfig) + isOfficialBuild: false + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr + displayName: Build Assets # build mono for AOT - template: /eng/pipelines/common/platform-matrix.yml @@ -64,11 +76,25 @@ jobs: # build mono - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release + runtimeFlavor: mono platforms: - linux_x64 + jobParameters: + nameSuffix: mono + buildArgs: -s mono+clr.iltools+clr.hosts+libs+host+packs -c $(_BuildConfig) + isOfficialBuild: false + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_mono + displayName: Build Assets # Build and run iOS Mono and NativeAOT scenarios - template: /eng/pipelines/coreclr/templates/build-and-run-perf-ios-scenarios.yml diff --git a/eng/pipelines/coreclr/perf-wasm-jobs.yml b/eng/pipelines/coreclr/perf-wasm-jobs.yml index 9b6fa299752b1..3f33e90cbc8f1 100644 --- a/eng/pipelines/coreclr/perf-wasm-jobs.yml +++ b/eng/pipelines/coreclr/perf-wasm-jobs.yml @@ -67,7 +67,6 @@ jobs: jobparameters: testgroup: perf livelibrariesbuildconfig: Release - skipLiveLibrariesDownload: true runtimetype: wasm codegentype: 'aot' projectfile: microbenchmarks.proj @@ -109,7 +108,6 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - skipLiveLibrariesDownload: true runtimeType: wasm codeGenType: 'wasm' projectFile: microbenchmarks.proj @@ -138,7 +136,6 @@ jobs: jobparameters: testgroup: perf livelibrariesbuildconfig: Release - skipLiveLibrariesDownload: true runtimetype: wasm codegentype: 'aot' projectfile: microbenchmarks.proj @@ -165,7 +162,6 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - skipLiveLibrariesDownload: true runtimeType: wasm projectFile: blazor_perf.proj runKind: blazor_scenarios @@ -188,7 +184,6 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - skipLiveLibrariesDownload: true runtimeType: wasm projectFile: blazor_perf.proj runKind: blazor_scenarios diff --git a/eng/pipelines/coreclr/perf_slow.yml b/eng/pipelines/coreclr/perf_slow.yml index 49afdad10f577..409f8d4db8c41 100644 --- a/eng/pipelines/coreclr/perf_slow.yml +++ b/eng/pipelines/coreclr/perf_slow.yml @@ -36,24 +36,28 @@ extends: - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')) }}: - # build mono - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono - buildConfig: release - platforms: - - linux_arm64 - - # build coreclr and libraries - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release + runtimeFlavor: mono + runtimeVariant: monointerpreter platforms: - linux_arm64 jobParameters: - testGroup: perf + nameSuffix: mono + buildArgs: -s mono+clr.iltools+clr.hosts+libs+host+packs -c $(_BuildConfig) + isOfficialBuild: false + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_mono + displayName: Build Assets # run arm64 interpreter jobs for mono - template: /eng/pipelines/common/platform-matrix.yml @@ -79,7 +83,7 @@ extends: # build coreclr and libraries - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - linux_x64 @@ -87,7 +91,19 @@ extends: - linux_arm64 - windows_arm64 jobParameters: - testGroup: perf + nameSuffix: coreclr + buildArgs: -s clr+libs+host+packs -c $(_BuildConfig) + isOfficialBuild: false + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr + displayName: Build Assets - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 450cd2799fa0c..eeda29812d569 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -24,7 +24,6 @@ parameters: iOSLlvmBuild: 'False' iOSStripSymbols: 'False' hybridGlobalization: 'False' - skipLiveLibrariesDownload: false collectHelixLogsScript: '' timeoutInMinutes: 320 compare: false @@ -79,11 +78,9 @@ jobs: ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: dependsOn: - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono', 'iOSNativeAOT', 'wasm')) }}: - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if and(ne(parameters.liveLibrariesBuildConfig, ''), eq(parameters.skipLiveLibrariesDownload, 'false')) }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, 'coreclr') }} - ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}: - - ${{ format('mono_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, 'mono') }} - ${{ if eq(parameters.runtimeType, 'wasm')}}: - ${{ format('build_{0}{1}_{2}_{3}_{4}_{5}', 'browser', '', 'wasm', 'linux', parameters.buildConfig, parameters.runtimeType) }} - ${{ if and(eq(parameters.codeGenType, 'AOT'), ne(parameters.runtimeType, 'wasm'))}}: @@ -150,35 +147,6 @@ jobs: $(Build.SourcesDirectory)/eng/common/msbuild.sh $(Build.SourcesDirectory)/eng/testing/performance/add_properties_to_pipeline.proj /t:SetVariables displayName: Add Properties To Pipeline Env - # Optionally download live-built libraries - - ${{ if and(ne(parameters.liveLibrariesBuildConfig, ''), eq(parameters.skipLiveLibrariesDownload, 'false')) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir) - cleanUnpackFolder: false - artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' - artifactName: '$(librariesBuildArtifactName)' - displayName: 'live-built libraries' - - # Download coreclr - - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono', 'iOSNativeAOT', 'wasm')) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' - displayName: 'Coreclr product build' - - # Download mono - - ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) - cleanUnpackFolder: false - artifactFileName: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)_$(archType)_$(buildConfig)$(archiveExtension)' - artifactName: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)_$(archType)_$(buildConfig)' - displayName: 'Mono runtime' - # Download wasm - ${{ if eq(parameters.runtimeType, 'wasm') }}: - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: @@ -206,9 +174,15 @@ jobs: find $(librariesDownloadDir)/bin/wasm -type d && find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \; displayName: "Create wasm directory (Linux)" + - ${{ elseif eq(parameters.codeGenType, 'AOT') }}: + # Download mono AOT + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr' + displayName: 'Runtime artifacts' - # Download mono AOT - - ${{ if and(eq(parameters.codeGenType, 'AOT'), ne(parameters.runtimeType, 'wasm')) }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/LinuxMonoAOT @@ -218,9 +192,16 @@ jobs: - script: "mkdir -p $(librariesDownloadDir)/bin/aot;mkdir -p $(librariesDownloadDir)/bin/aot/pack;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/mono/linux.${{ parameters.archType }}.Release/cross/linux-${{ parameters.archType }}/* $(librariesDownloadDir)/bin/aot;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/microsoft.netcore.app.runtime.linux-${{ parameters.archType }}/Release/* $(librariesDownloadDir)/bin/aot/pack" displayName: "Create aot directory (Linux)" + - ${{ elseif in(parameters.runtimeType, 'coreclr', 'mono') }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_${{ parameters.runtimeType }}' + displayName: 'Runtime artifacts' - # Download artifacts for Android Testing - - ${{ if eq(parameters.runtimeType, 'AndroidMono')}}: + - ${{ elseif eq(parameters.runtimeType, 'AndroidMono')}}: + # Download artifacts for Android Testing - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/androidHelloWorld @@ -237,7 +218,7 @@ jobs: displayName: 'Mono Android BDN Apk' # Download iOSMono and Native AOT tests - - ${{ if or(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.runtimeType, 'iOSNativeAOT')) }}: + - ${{ elseif or(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.runtimeType, 'iOSNativeAOT')) }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/iosHelloWorld @@ -286,12 +267,3 @@ jobs: - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) $(_crossBuildPropertyArg) displayName: Create Core_Root condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono'), ne('${{ parameters.runtimeType }}', 'wasm'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) - - # Copy the runtime directory into the testhost folder to include OOBs. - - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\runtime\\${{parameters.framework}}-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\${{parameters.framework}}-$(osGroup)-$(buildConfigUpper)-$(archType)\\shared\\Microsoft.NETCore.App\\$(productVersion) /E /I /Y;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\${{parameters.framework}}-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y;copy $(Build.SourcesDirectory)\\artifacts\\bin\\coreclr\\$(osGroup).$(archType).$(buildConfigUpper)\\corerun.exe $(Build.SourcesDirectory)\\.dotnet-mono\\shared\\Microsoft.NETCore.App\\$(productVersion)\\corerun.exe" - displayName: "Create mono dotnet (Windows)" - condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), eq(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) - - - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)/artifacts/bin/runtime/${{parameters.framework}}-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/artifacts/bin/testhost/${{parameters.framework}}-$(osGroup)-$(buildConfigUpper)-$(archType)/shared/Microsoft.NETCore.App/$(productVersion) -rf;cp $(Build.SourcesDirectory)/artifacts/bin/testhost/${{parameters.framework}}-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono -r;cp $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)/corerun $(Build.SourcesDirectory)/.dotnet-mono/shared/Microsoft.NETCore.App/$(productVersion)/corerun" - displayName: "Create mono dotnet (Linux)" - condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 5b381c958bde7..b9af7877ce719 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -100,9 +100,6 @@ jobs: - _buildArguments: $(_runtimeConfigurationArg) -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_testModeArg) $(_testScopeArg) $(_osArg) $(_msbuildCommonParameters) $(_runtimeArtifactsPathArg) $(_crossBuildPropertyArg) - ${{ parameters.variables }} - # we need to override this value to support build-coreclr-and-libraries-job.yml - - _BuildConfig: ${{ parameters.buildConfig }} - dependsOn: - ${{ if eq(parameters.dependOnEvaluatePaths, true) }}: - evaluate_paths From efafcc289f3f9ff11b07845d6ed1d5556559a858 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 28 Feb 2024 11:59:28 -0800 Subject: [PATCH 32/79] Fix archive file name --- eng/pipelines/coreclr/templates/perf-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index eeda29812d569..922d630fd91df 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -179,7 +179,7 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - artifactFileName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactFileName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr$(archiveExtension)' artifactName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr' displayName: 'Runtime artifacts' @@ -196,7 +196,7 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - artifactFileName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactFileName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_${{ parameters.runtimeType }}$(archiveExtension)' artifactName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_${{ parameters.runtimeType }}' displayName: 'Runtime artifacts' From 401ce992d0554288805d39378e488d8085956368 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 28 Feb 2024 12:12:58 -0800 Subject: [PATCH 33/79] move coreclr outerloop pipeline to use global-build-job --- eng/pipelines/coreclr/ci.yml | 61 ++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml index 1b7b180a85f1f..d3a3a4736b754 100644 --- a/eng/pipelines/coreclr/ci.yml +++ b/eng/pipelines/coreclr/ci.yml @@ -44,7 +44,7 @@ extends: # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + jobTemplate: /eng/pipelines/common/templates/global-build-job.yml buildConfig: debug platforms: - linux_arm @@ -56,29 +56,20 @@ extends: - osx_x64 - windows_arm64 jobParameters: - testGroup: outerloop - - # - # Checked builds - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: checked - platformGroup: all - platforms: - # It is too early to include osx_arm64 in platform group all - # Adding it here will enable it also - - osx_arm64 - jobParameters: - testGroup: outerloop + buildArgs: -s clr -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop # # Release builds # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + jobTemplate: /eng/pipelines/common/templates/global-build-job.yml buildConfig: release platforms: - linux_arm @@ -88,23 +79,43 @@ extends: - osx_x64 - windows_x86 jobParameters: - testGroup: outerloop + buildArgs: -s clr -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop # - # Release library builds + # Checked builds # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Release + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked platformGroup: all platforms: # It is too early to include osx_arm64 in platform group all # Adding it here will enable it also - osx_arm64 jobParameters: - isOfficialBuild: false - liveRuntimeBuildConfig: checked + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop # # Checked test builds @@ -134,6 +145,7 @@ extends: jobParameters: testGroup: outerloop liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) # # Checked R2R test runs @@ -158,6 +170,7 @@ extends: readyToRun: true displayNameArgs: R2R_CG2 liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) # # Formatting From f3271b325c4a74bbcf3eef9bffdbe369f5e296dc Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 28 Feb 2024 16:31:09 -0800 Subject: [PATCH 34/79] Update runtime pipeline to use the global build jobs --- .../templates/runtimes/run-test-job.yml | 6 +- eng/pipelines/installer/jobs/build-job.yml | 172 +--- .../jobs/steps/upload-job-artifacts.yml | 68 +- eng/pipelines/libraries/run-test-job.yml | 4 +- eng/pipelines/runtime.yml | 868 +++++++++++------- 5 files changed, 567 insertions(+), 551 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 149a9e714589f..c5542ad0ecfab 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -9,7 +9,6 @@ parameters: readyToRun: false hotColdSplitting: false liveLibrariesBuildConfig: '' - unifiedArtifactsName: '' compositeBuildMode: false useCodeFlowEnforcement: '' helixQueues: '' @@ -25,6 +24,9 @@ parameters: dependsOn: [] dependOnEvaluatePaths: false SuperPmiCollect: false + unifiedArtifactsName: '' + unifiedBuildNameSuffix: '' + unifiedBuildConfigOverride: '' ### Test run job @@ -62,7 +64,7 @@ jobs: - ${{ if notIn(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: - '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}' - ${{ if ne(parameters.unifiedArtifactsName, '')}}: - - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_' + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ coalesce(parameters.unifiedBuildConfigOverride, parameters.buildConfig) }}_${{ parameters.unifiedBuildNameSuffix }}' - ${{ else }}: - ${{ if or( eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter'), eq(parameters.runtimeVariant, 'llvmaot'), eq(parameters.runtimeVariant, 'llvmfullaot')) }}: # This is needed for creating a CORE_ROOT in the current design. diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml index bd68b5f81e28f..a0b2e371bed95 100644 --- a/eng/pipelines/installer/jobs/build-job.yml +++ b/eng/pipelines/installer/jobs/build-job.yml @@ -18,7 +18,6 @@ parameters: displayName: '' runtimeVariant: '' pool: '' - runOnlyIfDependenciesSucceeded: false # The target names here should match container names in the resources section in our pipelines, like runtime.yml packageDistroList: @@ -35,6 +34,9 @@ parameters: liveRuntimeBuildConfig: '' liveLibrariesBuildConfig: '' runtimeFlavor: 'coreclr' + unifiedArtifactsName: '' + unifiedBuildNameSuffix: '' + unifiedBuildConfigOverride: '' ### Product build jobs: @@ -50,16 +52,11 @@ jobs: enableMicrobuild: true pool: ${{ parameters.pool }} - ${{ if eq(parameters.runOnlyIfDependenciesSucceeded, true) }}: - condition: and(succeeded(), ${{ parameters.condition }}) - ${{ if ne(parameters.runOnlyIfDependenciesSucceeded, true) }}: - condition: ${{ parameters.condition }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} # Compute job name from template parameters - name: ${{ format('installer_{0}_{1}_{2}_{3}_', parameters.runtimeFlavor, parameters.runtimeVariant, coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} - displayName: ${{ format('Installer Build and Test {0} {1} {2} {3}', parameters.runtimeFlavor, parameters.runtimeVariant, coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} + name: ${{ format('installer_{0}_{1}', coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} + displayName: ${{ format('Installer Build and Test {0} {1}', coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} # Run all steps in the container. # Note that the containers are defined in platform-matrix.yml @@ -79,41 +76,9 @@ jobs: - ${{ each variable in parameters.variables }}: - ${{ variable }} - - name: OfficialBuildArg - value: '' - - # Explicitly enable tests for linux even though it is a cross build using mariner - # They still work in this configuration and until they run on Helix, it is our only - # linux test coverage in the installer pipeline - - name: SkipTests - value: ${{ or( - not(in(parameters.archType, 'x64', 'x86')), - eq(parameters.runtimeFlavor, 'mono'), - eq(parameters.isOfficialBuild, true), - and( - eq(parameters.crossBuild, true), - not(and( - eq(parameters.osGroup, 'linux'), - eq(parameters.osSubgroup, '')) - ))) }} - - name: BuildAction value: -test - - ${{ if eq(or(not(in(parameters.archType, 'x64', 'x86')), eq(parameters.runtimeFlavor, 'mono'), eq(parameters.isOfficialBuild, true), and(eq(parameters.crossBuild, true), not(and(eq(parameters.osGroup, 'linux'), eq(parameters.osSubgroup, ''))))), true) }}: - - name: BuildAction - value: '' - - - name: SignType - value: test - - # Set up non-PR build from internal project - - ${{ if eq(parameters.isOfficialBuild, true) }}: - - name: SignType - value: $[ coalesce(variables.OfficialSignType, 'real') ] - - name: OfficialBuildArg - value: /p:OfficialBuildId=$(Build.BuildNumber) - - name: buildCommandSourcesDirectory value: '$(Build.SourcesDirectory)/' @@ -127,13 +92,7 @@ jobs: value: >- /p:TargetArchitecture=${{ parameters.archType }} /p:PortableBuild=true - /p:SkipTests=$(SkipTests) /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} - $(OfficialBuildArg) - - name: MsbuildSigningArguments - value: >- - /p:CertificateId=400 - /p:DotNetSignType=$(SignType) - name: TargetArchitecture value: ${{ parameters.archType }} @@ -142,16 +101,13 @@ jobs: build.cmd -subset host+packs -ci $(BuildAction) -configuration $(_BuildConfig) - $(LiveOverridePathArgs) $(CommonMSBuildArgs) - $(MsbuildSigningArguments) - - ${{ if eq(parameters.osGroup, 'osx') }}: + - ${{ elseif eq(parameters.osGroup, 'osx') }}: - name: CommonMSBuildArgs value: >- /p:PortableBuild=true - /p:SkipTests=$(SkipTests) /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /p:TargetArchitecture=${{ parameters.archType }} /p:CrossBuild=${{ parameters.crossBuild }} @@ -162,37 +118,15 @@ jobs: $(BuildAction) -configuration $(_BuildConfig) -arch ${{ parameters.archType }} - $(LiveOverridePathArgs) - $(CommonMSBuildArgs) - $(OfficialBuildArg) - - - ${{ if in(parameters.osGroup, 'ios', 'tvos', 'android', 'browser') }}: - - - name: CommonMSBuildArgs - value: >- - /p:PortableBuild=true - /p:SkipTests=$(SkipTests) - - - name: BaseJobBuildCommand - value: >- - $(Build.SourcesDirectory)/build.sh -subset packs -ci - $(BuildAction) - -configuration $(_BuildConfig) - -os ${{ parameters.osGroup }} - -arch ${{ parameters.archType }} - /p:StripSymbols=true - $(LiveOverridePathArgs) $(CommonMSBuildArgs) - $(OfficialBuildArg) - - ${{ if in(parameters.osGroup, 'linux', 'freebsd') }}: + - ${{ elseif in(parameters.osGroup, 'linux', 'freebsd') }}: - name: CommonMSBuildArgs value: >- /p:Configuration=$(_BuildConfig) /p:TargetOS=${{ parameters.osGroup }} /p:TargetArchitecture=${{ parameters.archType }} /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} - $(OfficialBuildArg) - name: _PortableBuild value: ${{ eq(parameters.osSubgroup, '') }} @@ -212,80 +146,16 @@ jobs: $(ArchArg) /p:CrossBuild=${{ parameters.crossBuild }} /p:PortableBuild=$(_PortableBuild) - /p:SkipTests=$(SkipTests) - $(LiveOverridePathArgs) $(CommonMSBuildArgs) - name: installersSubsetArg value: --subset packs.installers - ### - ### Common Live build override variable setup - ### - - - name: LiveOverridePathArgs - value: >- - $(RuntimeArtifactsArgs) - $(LibrariesConfigurationArg) - - - name: RuntimeArtifactsArgs - value: '' - - name: LibrariesConfigurationArg - value: '' - - - name: RuntimeDownloadPath - value: '' - - name: LibrariesDownloadPath - value: '' - - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - name: liveRuntimeLegName - value: ${{ format('{0}{1}_{2}_{3}', - parameters.osGroup, - parameters.osSubgroup, - parameters.archType, - parameters.liveRuntimeBuildConfig) }} - - name: RuntimeDownloadPath - value: 'artifacts/transport/${{ parameters.runtimeFlavor }}' - - name: RuntimeArtifactsArgs - value: >- - /p:RuntimeArtifactsPath=$(buildCommandSourcesDirectory)$(RuntimeDownloadPath) - /p:RuntimeConfiguration=${{ parameters.liveRuntimeBuildConfig }} - - name: RuntimeArtifactName - value: $(runtimeFlavorName)Product_${{ parameters.runtimeVariant }}_$(liveRuntimeLegName) - - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - name: liveLibrariesLegName - value: ${{ format('{0}{1}_{2}_{3}', - parameters.osGroup, - parameters.osSubgroup, - parameters.archType, - parameters.liveLibrariesBuildConfig) }} - - name: LibrariesDownloadPath - value: 'artifacts' - - name: LibrariesArtifactName - value: libraries_bin_$(liveLibrariesLegName) - - name: LibrariesConfigurationArg - value: ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' - dependsOn: - ${{ if eq(parameters.dependOnEvaluatePaths, true) }}: - evaluate_paths + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ coalesce(parameters.unifiedBuildConfigOverride, parameters.buildConfig) }}_${{ parameters.unifiedBuildNameSuffix }}' - ${{ parameters.dependsOn }} - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', - parameters.runtimeFlavor, - parameters.runtimeVariant, - parameters.osGroup, - parameters.osSubgroup, - parameters.archType, - parameters.liveRuntimeBuildConfig) }} - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - libraries_build_${{ format('{0}{1}_{2}_{3}', - parameters.osGroup, - parameters.osSubgroup, - parameters.archType, - parameters.liveLibrariesBuildConfig) }} steps: - ${{ if eq(parameters.osGroup, 'windows') }}: - template: /eng/pipelines/common/templates/disable-vsupdate-or-failfast.yml @@ -311,22 +181,12 @@ jobs: env: Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(Build.SourcesDirectory)/$(RuntimeDownloadPath) - artifactFileName: '$(RuntimeArtifactName)$(archiveExtension)' - artifactName: '$(RuntimeArtifactName)' - displayName: '$(runtimeFlavorName) artifacts' - - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(Build.SourcesDirectory)/$(LibrariesDownloadPath) - artifactFileName: '$(LibrariesArtifactName)$(archiveExtension)' - artifactName: '$(LibrariesArtifactName)' - displayName: 'Libraries artifacts' - cleanUnpackFolder: false + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: '${{ parameters.unifiedArtifactsName }}$(archiveExtension)' + artifactName: '${{ parameters.unifiedArtifactsName }}' + displayName: 'unified artifacts' - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} @@ -339,7 +199,7 @@ jobs: - script: $(BaseJobBuildCommand) displayName: Build - continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} + continueOnError: ${{ eq(parameters.shouldContinueOnError, true) }} - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log @@ -373,8 +233,6 @@ jobs: name: ${{ coalesce(parameters.name, parameters.platform) }} runtimeFlavor: ${{ parameters.runtimeFlavor }} runtimeVariant: ${{ parameters.runtimeVariant }} - isOfficialBuild: ${{ eq(parameters.isOfficialBuild, true) }} - skipTests: ${{ eq(variables.SkipTests, true) }} - ${{ if ne(parameters.osGroup, 'windows') }}: - script: set -x && df -h diff --git a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml b/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml index 4012b9a4fa34c..8cdf6e8fb7778 100644 --- a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml +++ b/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml @@ -1,16 +1,7 @@ parameters: name: '' - runtimeFlavor: 'coreclr' - runtimeVariant: '' - isOfficialBuild: false steps: -# Upload build artifacts (packages) to pipeline only if official, to save storage space. -- ${{ if eq(parameters.isOfficialBuild, true) }}: - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: ${{ parameters.name }} - - task: PublishTestResults@2 displayName: Publish Test Results inputs: @@ -18,41 +9,40 @@ steps: testResultsFiles: '*.trx' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' mergeTestResults: true - testRunTitle: Installer-${{ parameters.runtimeFlavor }}-${{ parameters.name }}-$(_BuildConfig) + testRunTitle: Installer-${{ parameters.name }}-$(_BuildConfig) continueOnError: true - condition: eq(variables.SkipTests, false) + condition: always() # Upload binaries and symbols on failure to allow debugging issues -- ${{ if eq(parameters.skipTests, false) }}: - - task: CopyFiles@2 - displayName: Prepare binaries to publish - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/bin' - Contents: | - */corehost/** - */corehost_test/** - TargetFolder: '$(Build.StagingDirectory)/Binaries' - continueOnError: true - condition: failed() +- task: CopyFiles@2 + displayName: Prepare binaries to publish + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/bin' + Contents: | + */corehost/** + */corehost_test/** + TargetFolder: '$(Build.StagingDirectory)/Binaries' + continueOnError: true + condition: failed() - - task: ArchiveFiles@2 - displayName: Zip binaries - inputs: - rootFolderOrFile: '$(Build.StagingDirectory)/Binaries' - archiveFile: '$(Build.StagingDirectory)/corehost-bin-${{ parameters.name }}-$(_BuildConfig)$(archiveExtension)' - archiveType: $(archiveType) - tarCompression: $(tarCompression) - includeRootFolder: false - continueOnError: true - condition: failed() +- task: ArchiveFiles@2 + displayName: Zip binaries + inputs: + rootFolderOrFile: '$(Build.StagingDirectory)/Binaries' + archiveFile: '$(Build.StagingDirectory)/corehost-bin-${{ parameters.name }}-$(_BuildConfig)$(archiveExtension)' + archiveType: $(archiveType) + tarCompression: $(tarCompression) + includeRootFolder: false + continueOnError: true + condition: failed() - - task: PublishBuildArtifacts@1 - displayName: Publish binaries - inputs: - pathtoPublish: '$(Build.StagingDirectory)/corehost-bin-${{ parameters.name }}-$(_BuildConfig)$(archiveExtension)' - artifactName: Installer-Binaries-${{ parameters.runtimeFlavor }}-${{ parameters.runtimeVariant }}-${{ parameters.name }}-$(_BuildConfig) - continueOnError: true - condition: failed() +- task: PublishBuildArtifacts@1 + displayName: Publish binaries + inputs: + pathtoPublish: '$(Build.StagingDirectory)/corehost-bin-${{ parameters.name }}-$(_BuildConfig)$(archiveExtension)' + artifactName: Installer-Binaries-${{ parameters.name }}-$(_BuildConfig) + continueOnError: true + condition: failed() - task: CopyFiles@2 displayName: Prepare BuildLogs staging directory diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 483974c45d256..9dc004ab2b349 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -26,6 +26,8 @@ parameters: dependsOn: [] unifiedArtifactsName: '' helixArtifactsName: '' + unifiedBuildNameSuffix: '' + unifiedBuildConfigOverride: '' jobs: - template: /eng/pipelines/libraries/base-job.yml @@ -69,7 +71,7 @@ jobs: - ${{ parameters.dependsOn }} - ${{ if eq(parameters.dependsOn[0], '') }}: - ${{ if ne(parameters.unifiedArtifactsName, '')}}: - - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_' + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ coalesce(parameters.unifiedBuildConfigOverride, parameters.buildConfig) }}_${{ parameters.unifiedBuildNameSuffix }}' - ${{ else }}: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 2f43741e572d5..f559ad50d56c5 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -67,71 +67,138 @@ extends: - template: /eng/pipelines/common/evaluate-default-paths.yml # - # Build CoreCLR checked - # Only when CoreCLR is changed + # Build CoreCLR verticals where we don't run host tests # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: checked + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - - linux_x64 - - linux_arm - - linux_arm64 - linux_musl_arm - linux_musl_arm64 - - linux_musl_x64 - - osx_arm64 - - windows_x86 - - windows_x64 - windows_arm64 + - linux_arm jobParameters: - testGroup: innerloop + nameSuffix: AllSubsets_CoreCLR_ReleaseRuntimeLibs + buildArgs: -s clr+libs+host+packs -rc Release -lc Release -c $(_BuildConfig) + timeoutInMinutes: 120 condition: >- or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + platforms: + - osx_arm64 + - linux_arm64 + - linux_musl_x64 + jobParameters: + nameSuffix: AllSubsets_CoreCLR + buildArgs: -s clr+libs+host+packs -rc Release -c Release -lc $(_BuildConfig) + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) # - # Build the whole product using GNU compiler toolchain - # When CoreCLR, Mono, Libraries, Installer and src/tests are changed + # Build CoreCLR and Libraries with Libraries tests + # For running libraries tests and installer tests # - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: checked + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - - gcc_linux_x64 + - linux_x64 + - linux_musl_x64 + - osx_x64 + - windows_x64 jobParameters: - testGroup: innerloop - nameSuffix: Native_GCC - buildArgs: -s clr.native+libs.native+mono+host.native -c $(_BuildConfig) -gcc + nameSuffix: CoreCLR_Libraries + buildArgs: -s clr+libs+libs.tests -c Release -lc $(_BuildConfig) /p:ArchiveTests=true + timeoutInMinutes: 120 postBuildSteps: - - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests.yml + - template: /eng/pipelines/common/upload-artifact-step.yml parameters: - testBuildArgs: skipmanaged skipgeneratelayout skiprestorepackages -gcc + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreCLR_Libraries_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) condition: >- or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), eq(variables['isRollingBuild'], true)) + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: + - windows_x86 + jobParameters: + nameSuffix: CoreCLR_Libraries + buildArgs: -s clr+libs+libs.tests -c Release $(_BuildConfig) /p:ArchiveTests=true + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreCLR_Libraries_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(variables['isRollingBuild'], true)) + # - # Build CoreCLR osx_x64 checked - # Only when CoreCLR or Libraries is changed + # Build CoreCLR and Libraries with the respective tests + # for the test configurations we run. # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: checked + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: + - linux_arm64 - osx_x64 jobParameters: - testGroup: innerloop + nameSuffix: Libraries_CheckedCoreCLR + buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:ArchiveTests=true + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: innerloop condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), @@ -139,33 +206,195 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), eq(variables['isRollingBuild'], true)) - # - # Build CoreCLR release - # Always as they are needed by Installer and we always build and test the Installer. - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: release + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - - osx_arm64 - - osx_x64 - linux_x64 - - linux_arm - - linux_arm64 + - windows_x64 + jobParameters: + nameSuffix: Libraries_CheckedCoreCLR + buildArgs: -s clr+libs -c $(_BuildConfig) -rc Checked + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: innerloop + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + platforms: - linux_musl_x64 + - windows_x86 + jobParameters: + nameSuffix: Libraries_CheckedCoreCLR + buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:ArchiveTests=true + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + platforms: + - osx_arm64 + jobParameters: + nameSuffix: Libraries_CheckedCoreCLR + buildArgs: -s clr+libs -c $(_BuildConfig) -rc Checked + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: innerloop + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr_AppleSilicon.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + platforms: + - linux_arm + - windows_arm64 + - windows_x86 + jobParameters: + nameSuffix: CoreClr_ReleaseLibraries + buildArgs: -s clr+libs -c $(_BuildConfig) -rc Checked + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreClr_ReleaseLibraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: innerloop + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + platforms: + - linux_x64 - linux_musl_arm - linux_musl_arm64 - windows_x64 - - windows_x86 - - windows_arm64 + jobParameters: + nameSuffix: CoreClr_ReleaseLibraries + buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreClr_ReleaseLibraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + # + # Build the whole product using GNU compiler toolchain + # When CoreCLR, Mono, Libraries, Installer and src/tests are changed + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + platforms: + - gcc_linux_x64 jobParameters: testGroup: innerloop - # Mono/runtimetests also need this, but skip for wasm - condition: + nameSuffix: Native_GCC + buildArgs: -s clr.native+libs.native+mono+host.native -c $(_BuildConfig) -gcc + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests.yml + parameters: + testBuildArgs: skipmanaged skipgeneratelayout skiprestorepackages -gcc + condition: >- or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), eq(variables['isRollingBuild'], true)) @@ -932,7 +1161,7 @@ extends: # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml runtimeFlavor: mono buildConfig: debug platforms: @@ -945,6 +1174,8 @@ extends: - windows_x86 # - windows_arm64 jobParameters: + buildSuffix: Mono_Runtime + buildArgs: -s mono -c $(_BuildConfig) condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), @@ -952,11 +1183,10 @@ extends: # # Build Mono release AOT cross-compilers - # Only when mono changed # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml runtimeFlavor: mono buildConfig: release platforms: @@ -964,9 +1194,37 @@ extends: - linux_musl_x64 - linux_arm64 - linux_musl_arm64 + jobParameters: + buildArgs: -s mono+packs -c $(_BuildConfig) + /p:MonoCrossAOTTargetOS=android+browser+wasi /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true + nameSuffix: CrossAOT_Mono + runtimeVariant: crossaot + dependsOn: + - mono_android_offsets + - mono_browser_offsets + - mono_wasi_offsets + monoCrossAOTTargetOS: + - android + - browser + - wasi + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + runtimeFlavor: mono + buildConfig: release + platforms: - windows_arm64 - windows_x64 jobParameters: + buildArgs: -s mono+packs -c $(_BuildConfig) + /p:MonoCrossAOTTargetOS=android+browser+wasi /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true + nameSuffix: CrossAOT_Mono runtimeVariant: crossaot dependsOn: - mono_android_offsets @@ -981,16 +1239,23 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isRollingBuild'], true)) + # + # Build Mono release AOT cross-compilers + # Only when mono changed + # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml runtimeFlavor: mono buildConfig: release platforms: - osx_x64 - osx_arm64 jobParameters: + buildArgs: -s mono+packs -c $(_BuildConfig) + /p:MonoCrossAOTTargetOS=android+browser+wasi+tvos+ios+maccatalyst /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true + nameSuffix: CrossAOT_Mono runtimeVariant: crossaot dependsOn: - mono_android_offsets @@ -1013,182 +1278,40 @@ extends: eq(variables['isRollingBuild'], true)) # - # Build Mono release - # Only when libraries or mono changed + # Build and test libraries for .NET Framework # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono - buildConfig: release + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: Release platforms: - - linux_x64 - # - linux_musl_arm64 - - windows_x64 - windows_x86 - # - windows_arm64 + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml jobParameters: + framework: net48 + buildArgs: -s tools+libs+libs.tests -framework net48 -c $(_BuildConfig) -testscope innerloop /p:ArchiveTests=true + nameSuffix: Libraries_NET48 + timeoutInMinutes: 150 + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: NET48_$(_BuildConfig) + extraHelixArguments: /p:BuildTargetFramework=net48 condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) # - # Build Mono release - # Only when libraries, mono, or the runtime tests changed - # Currently only these architectures are needed for the runtime tests. + # Build and test libraries AllConfigurations + # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono - buildConfig: release + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - - osx_x64 - - linux_arm64 - jobParameters: - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # Build Mono release with LLVM AOT - # Only when mono, or the runtime tests changed - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono - buildConfig: release - platforms: - - linux_x64 - - linux_arm64 - jobParameters: - runtimeVariant: llvmaot - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # Build libraries using live CoreLib - # These set of libraries are built always no matter what changed - # The reason for that is because Corelib and Installer needs it and - # These are part of the test matrix for Libraries changes. - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Release - platforms: - - linux_arm - - linux_musl_arm - - linux_musl_arm64 - - windows_arm64 - - windows_x86 - jobParameters: - condition: - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - platforms: - - linux_arm64 - - linux_musl_x64 - - linux_x64 - - osx_arm64 - - osx_x64 - - windows_x64 - jobParameters: - testScope: innerloop - condition: - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # Libraries debug build that only runs when coreclr is changed - # Only do this on PR builds since we use the Release builds for these test runs in CI - # and those are already built above - # - - ${{ if eq(variables['isRollingBuild'], false) }}: - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Debug - platforms: - - windows_x86 - jobParameters: - condition: >- - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true) - - # - # Libraries release build that only runs when coreclr is changed in PRs - # We need these for checked coreclr + release libraries tests runs. - # - - ${{ if eq(variables['isRollingBuild'], false) }}: - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Release - platforms: - - linux_x64 - - windows_x64 - jobParameters: - condition: >- - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true) - - # - # Build and test libraries for .NET Framework - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: Release - platforms: - - windows_x86 - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - jobParameters: - framework: net48 - buildArgs: -s tools+libs+libs.tests -framework net48 -c $(_BuildConfig) -testscope innerloop /p:ArchiveTests=true - nameSuffix: Libraries_NET48 - timeoutInMinutes: 150 - postBuildSteps: - - template: /eng/pipelines/libraries/helix.yml - parameters: - creator: dotnet-bot - testRunNamePrefixSuffix: NET48_$(_BuildConfig) - extraHelixArguments: /p:BuildTargetFramework=net48 - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # Build and test libraries AllConfigurations - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - platforms: - - windows_x64 + - windows_x64 jobParameters: buildArgs: -test -s tools+libs+libs.tests -allConfigurations -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true nameSuffix: Libraries_AllConfigurations @@ -1217,7 +1340,9 @@ extends: jobParameters: liveRuntimeBuildConfig: release liveLibrariesBuildConfig: Release - runOnlyIfDependenciesSucceeded: true + unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_Release + unifiedBuildConfigOverride: release + unifiedBuildNameSuffix: CoreCLR_Libraries condition: or( eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), @@ -1237,126 +1362,50 @@ extends: jobParameters: liveRuntimeBuildConfig: release liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - runOnlyIfDependenciesSucceeded: true + unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} + unifiedBuildNameSuffix: CoreCLR_Libraries condition: or( eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) # - # CoreCLR Test builds using live libraries release build - # Only when CoreCLR is changed - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: innerloop - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # CoreCLR Test executions using live libraries - # Only when CoreCLR is changed + # Build the whole product using Mono and run runtime tests # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_arm - - windows_x86 - - windows_arm64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: innerloop - liveLibrariesBuildConfig: Release - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_x64 - - linux_x64 - - linux_arm64 - - windows_x64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: innerloop - liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - helixQueueGroup: pr + jobTemplate: /eng/pipelines/common/global-build-job.yml helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: innerloop - liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr_AppleSilicon.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # Mono Test builds with CoreCLR runtime tests using live libraries debug build - # Only when Mono is changed - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: innerloop - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: release + buildConfig: Release runtimeFlavor: mono platforms: - - windows_x64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + - osx_x64 + - linux_arm64 + variables: + - name: timeoutPerTestInMinutes + value: 60 + - name: timeoutPerTestCollectionInMinutes + value: 180 jobParameters: testGroup: innerloop - liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - liveRuntimeBuildConfig: release + nameSuffix: AllSubsets_Mono_Minijit_RuntimeTests runtimeVariant: minijit + buildArgs: -s mono+libs+clr.hosts+clr.iltools -c Release + timeoutInMinutes: 180 condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_Release + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/test-variables.yml # # Build the whole product using Mono and run runtime tests @@ -1368,8 +1417,7 @@ extends: buildConfig: Release runtimeFlavor: mono platforms: - - osx_x64 - - linux_arm64 + - windows_x64 variables: - name: timeoutPerTestInMinutes value: 60 @@ -1379,7 +1427,7 @@ extends: testGroup: innerloop nameSuffix: AllSubsets_Mono_Minijit_RuntimeTests runtimeVariant: minijit - buildArgs: -s mono+libs+clr.hosts+clr.iltools -c Release + buildArgs: -s mono+libs+clr.hosts+clr.iltools -c Release -lc ${{ variables.debugOnPrReleaseOnRolling }} timeoutInMinutes: 180 condition: >- or( @@ -1471,55 +1519,89 @@ extends: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml # - # Libraries Release Test Execution against a release mono runtime. - # Only when libraries or mono changed + # CoreCLR Test builds using live libraries release build + # Only when CoreCLR is changed # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - runtimeFlavor: mono - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml + buildConfig: checked platforms: - # - windows_x64 - - osx_x64 - - linux_arm64 - - linux_x64 - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - isOfficialBuild: false - runtimeDisplayName: mono - testScope: innerloop - liveRuntimeBuildConfig: release + testGroup: innerloop condition: >- or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), eq(variables['isRollingBuild'], true)) # - # Libraries Release Test Execution against a release mono interpreter runtime. - # Only when libraries or mono changed + # CoreCLR Test executions using live libraries + # Only when CoreCLR is changed # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - runtimeFlavor: mono - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + buildConfig: checked platforms: - # - windows_x64 - #- osx_x64 + - linux_arm + - windows_x86 + - windows_arm64 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: innerloop + liveLibrariesBuildConfig: Release + unifiedArtifactsName: CoreClr_ReleaseLibraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedBuildNameSuffix: CoreClr_ReleaseLibraries + unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + buildConfig: checked + platforms: + - osx_x64 - linux_x64 - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + - linux_arm64 + - windows_x64 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: - isOfficialBuild: false - interpreter: true - runtimeDisplayName: mono_interpreter - testScope: innerloop - liveRuntimeBuildConfig: release + testGroup: innerloop + liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedBuildNameSuffix: Libraries_CheckedCoreCLR + unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} condition: >- or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + buildConfig: checked + platforms: + - osx_arm64 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: innerloop + liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedBuildNameSuffix: Libraries_CheckedCoreCLR + unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr_AppleSilicon.containsChange'], true), eq(variables['isRollingBuild'], true)) # @@ -1537,6 +1619,10 @@ extends: isOfficialBuild: false testScope: innerloop liveRuntimeBuildConfig: release + unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + helixArtifactsName: CoreCLR_Libraries_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedBuildNameSuffix: CoreCLR_Libraries + unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), @@ -1560,11 +1646,14 @@ extends: isOfficialBuild: false testScope: innerloop liveRuntimeBuildConfig: release + unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + helixArtifactsName: CoreCLR_Libraries_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedBuildNameSuffix: CoreCLR_Libraries + unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(variables['isRollingBuild'], true)) - # The next three jobs run checked coreclr + libraries tests. # The matrix looks like the following, where the right columns specify which configurations # the libraries tests are built in. @@ -1597,6 +1686,9 @@ extends: jobParameters: testScope: innerloop liveRuntimeBuildConfig: checked + unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + helixArtifactsName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedBuildNameSuffix: Libraries_CheckedCoreCLR condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), @@ -1620,6 +1712,10 @@ extends: jobParameters: testScope: innerloop liveRuntimeBuildConfig: checked + unifiedArtifactsName: CoreCLR_ReleaseLibraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_checked + helixArtifactsName: CoreCLR_ReleaseLibraries_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_checked + unifiedBuildNameSuffix: CoreCLR_ReleaseLibraries + unifiedBuildConfigOverride: checked condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), @@ -1636,12 +1732,80 @@ extends: jobParameters: testScope: innerloop liveRuntimeBuildConfig: checked + unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + helixArtifactsName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedBuildNameSuffix: Libraries_CheckedCoreCLR condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(variables['isRollingBuild'], true)) + # + # Build and test Mono Interpreter with the libraries testss + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + runtimeFlavor: mono + runtimeVariant: monointerpreter + platforms: + - linux_x64 + #- osx_x64 + #- windows_x64 + jobParameters: + testGroup: innerloop + nameSuffix: Mono_Interpreter_LibrariesTests + isExtraPlatforms: ${{ parameters.isExtraPlatformsBuild }} + buildArgs: -s mono+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true + timeoutInMinutes: 480 + # extra steps, run tests + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_Interpreter_$(_BuildConfig) + interpreter: true + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + # + # Build and test Mono Minijit with the libraries testss + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + runtimeFlavor: mono + platforms: + - linux_arm64 + - linux_x64 + - osx_x64 + #- windows_x64 + jobParameters: + testGroup: innerloop + nameSuffix: Mono_MiniJIT_LibrariesTests + isExtraPlatforms: ${{ parameters.isExtraPlatformsBuild }} + buildArgs: -s mono+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true + timeoutInMinutes: 480 + # extra steps, run tests + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_Minijit_$(_BuildConfig) + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) + # # Sourcebuild legs # We have 3 important legs for source-build: From a62f944b417998883fb756da728be98d9e2edcd0 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 28 Feb 2024 16:31:52 -0800 Subject: [PATCH 35/79] Make artifact names consistent with how we format names throughout the YAML scripts --- eng/pipelines/coreclr/ci.yml | 6 +-- eng/pipelines/coreclr/clrinterpreter.yml | 4 +- eng/pipelines/coreclr/crossgen2-composite.yml | 4 +- eng/pipelines/coreclr/crossgen2-gcstress.yml | 4 +- eng/pipelines/coreclr/crossgen2-outerloop.yml | 8 +-- eng/pipelines/coreclr/crossgen2.yml | 6 +-- eng/pipelines/coreclr/gc-longrunning.yml | 4 +- eng/pipelines/coreclr/gc-simulator.yml | 4 +- eng/pipelines/coreclr/gc-standalone.yml | 6 +-- eng/pipelines/coreclr/gcstress-extra.yml | 4 +- .../coreclr/gcstress0x3-gcstress0xc.yml | 4 +- eng/pipelines/coreclr/perf-non-wasm-jobs.yml | 4 +- eng/pipelines/coreclr/perf_slow.yml | 4 +- eng/pipelines/coreclr/r2r-extra.yml | 4 +- eng/pipelines/coreclr/r2r.yml | 4 +- eng/pipelines/coreclr/release-tests.yml | 6 +-- eng/pipelines/coreclr/runincontext.yml | 4 +- .../superpmi-asmdiffs-checked-release.yml | 4 +- eng/pipelines/coreclr/superpmi-collect.yml | 22 ++++---- eng/pipelines/coreclr/superpmi-diffs.yml | 4 +- eng/pipelines/coreclr/superpmi-replay.yml | 2 +- .../crossgen2-comparison-build-job.yml | 4 +- .../templates/crossgen2-comparison-job.yml | 4 +- .../templates/jit-outerloop-pipeline.yml | 4 +- eng/pipelines/coreclr/templates/perf-job.yml | 8 +-- .../superpmi-asmdiffs-checked-release-job.yml | 8 +-- .../templates/superpmi-collect-job.yml | 8 +-- .../coreclr/templates/superpmi-diffs-job.yml | 8 +-- .../coreclr/templates/superpmi-replay-job.yml | 4 +- eng/pipelines/coreclr/tieringtest.yml | 4 +- eng/pipelines/runtime-cet.yml | 4 +- eng/pipelines/runtime.yml | 52 +++++++++---------- 32 files changed, 110 insertions(+), 110 deletions(-) diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml index d3a3a4736b754..8ce9cf6a4deed 100644 --- a/eng/pipelines/coreclr/ci.yml +++ b/eng/pipelines/coreclr/ci.yml @@ -110,7 +110,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -145,7 +145,7 @@ extends: jobParameters: testGroup: outerloop liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) # # Checked R2R test runs @@ -170,7 +170,7 @@ extends: readyToRun: true displayNameArgs: R2R_CG2 liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) # # Formatting diff --git a/eng/pipelines/coreclr/clrinterpreter.yml b/eng/pipelines/coreclr/clrinterpreter.yml index 06d8863fbb5a1..288e297c6f905 100644 --- a/eng/pipelines/coreclr/clrinterpreter.yml +++ b/eng/pipelines/coreclr/clrinterpreter.yml @@ -42,7 +42,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: CoreCLRInterpreterBuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: CoreCLRInterpreterBuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -77,4 +77,4 @@ extends: dependsOn: - build_$(osGroup)$(osSubgroup)_$(archType)_checked_ - coreclr_common_test_build_pri0_AnyOS_AnyCPU_checked - unifiedArtifactsName: CoreCLRInterpreterBuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: CoreCLRInterpreterBuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/crossgen2-composite.yml b/eng/pipelines/coreclr/crossgen2-composite.yml index 328f394d39419..0dbe78f5907f3 100644 --- a/eng/pipelines/coreclr/crossgen2-composite.yml +++ b/eng/pipelines/coreclr/crossgen2-composite.yml @@ -42,7 +42,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -78,4 +78,4 @@ extends: compositeBuildMode: true displayNameArgs: Composite liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/crossgen2-gcstress.yml b/eng/pipelines/coreclr/crossgen2-gcstress.yml index b43c129db79e3..a509018edc0b1 100644 --- a/eng/pipelines/coreclr/crossgen2-gcstress.yml +++ b/eng/pipelines/coreclr/crossgen2-gcstress.yml @@ -41,7 +41,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -76,4 +76,4 @@ extends: compositeBuildMode: true displayNameArgs: Composite liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/crossgen2-outerloop.yml b/eng/pipelines/coreclr/crossgen2-outerloop.yml index bb3dd3c2070ee..833513ea532e2 100644 --- a/eng/pipelines/coreclr/crossgen2-outerloop.yml +++ b/eng/pipelines/coreclr/crossgen2-outerloop.yml @@ -42,7 +42,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: Checked_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: Checked_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -77,7 +77,7 @@ extends: compositeBuildMode: true displayNameArgs: R2R_Composite liveLibrariesBuildConfig: Release - unifiedArtifactsName: Checked_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: Checked_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) # Outerloop testing in non-composite mode - template: /eng/pipelines/common/platform-matrix.yml @@ -99,7 +99,7 @@ extends: readyToRun: true displayNameArgs: R2R liveLibrariesBuildConfig: Release - unifiedArtifactsName: Checked_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: Checked_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) # Build release CoreCLR for Crossgen2 baseline generation - template: /eng/pipelines/common/platform-matrix.yml @@ -125,7 +125,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: Release_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: Release_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets # Build Crossgen2 baselines diff --git a/eng/pipelines/coreclr/crossgen2.yml b/eng/pipelines/coreclr/crossgen2.yml index e2bdb4781ddd0..a52fbae3a091b 100644 --- a/eng/pipelines/coreclr/crossgen2.yml +++ b/eng/pipelines/coreclr/crossgen2.yml @@ -40,7 +40,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -73,7 +73,7 @@ extends: readyToRun: true displayNameArgs: R2R_CG2 liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -115,4 +115,4 @@ extends: hotColdSplitting: true displayNameArgs: R2R_CG2_HotColdSplitting liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/gc-longrunning.yml b/eng/pipelines/coreclr/gc-longrunning.yml index 40aeab2c3e19b..0dbd66cd9ea61 100644 --- a/eng/pipelines/coreclr/gc-longrunning.yml +++ b/eng/pipelines/coreclr/gc-longrunning.yml @@ -39,7 +39,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -69,4 +69,4 @@ extends: jobParameters: testGroup: gc-longrunning liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/gc-simulator.yml b/eng/pipelines/coreclr/gc-simulator.yml index dd7f08c353fa3..a0bda3d462105 100644 --- a/eng/pipelines/coreclr/gc-simulator.yml +++ b/eng/pipelines/coreclr/gc-simulator.yml @@ -40,7 +40,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -71,4 +71,4 @@ extends: jobParameters: testGroup: gc-simulator liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/gc-standalone.yml b/eng/pipelines/coreclr/gc-standalone.yml index 9da4a15957041..6089ac89178e8 100644 --- a/eng/pipelines/coreclr/gc-standalone.yml +++ b/eng/pipelines/coreclr/gc-standalone.yml @@ -38,7 +38,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -69,7 +69,7 @@ extends: testGroup: gc-standalone displayNameArgs: GCStandAlone liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -86,4 +86,4 @@ extends: testGroup: gc-standalone-server displayNameArgs: GCStandAloneServer liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/gcstress-extra.yml b/eng/pipelines/coreclr/gcstress-extra.yml index 7076bb9a8795b..d981efa133d1a 100644 --- a/eng/pipelines/coreclr/gcstress-extra.yml +++ b/eng/pipelines/coreclr/gcstress-extra.yml @@ -38,7 +38,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -68,4 +68,4 @@ extends: jobParameters: testGroup: gcstress-extra liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml b/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml index 5e6bbe82cc524..5376898b7bf32 100644 --- a/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml +++ b/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml @@ -38,7 +38,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -68,4 +68,4 @@ extends: jobParameters: testGroup: gcstress0x3-gcstress0xc liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml index ab60b9392db25..7de5a003edff2 100644 --- a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml +++ b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml @@ -24,7 +24,7 @@ jobs: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr displayName: Build Assets # build mono for AOT @@ -93,7 +93,7 @@ jobs: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_mono + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_mono displayName: Build Assets # Build and run iOS Mono and NativeAOT scenarios diff --git a/eng/pipelines/coreclr/perf_slow.yml b/eng/pipelines/coreclr/perf_slow.yml index 409f8d4db8c41..92bd74ab5c513 100644 --- a/eng/pipelines/coreclr/perf_slow.yml +++ b/eng/pipelines/coreclr/perf_slow.yml @@ -56,7 +56,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_mono + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_mono displayName: Build Assets # run arm64 interpreter jobs for mono @@ -102,7 +102,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr displayName: Build Assets - template: /eng/pipelines/common/platform-matrix.yml diff --git a/eng/pipelines/coreclr/r2r-extra.yml b/eng/pipelines/coreclr/r2r-extra.yml index f538edfcde62f..71324e3224f38 100644 --- a/eng/pipelines/coreclr/r2r-extra.yml +++ b/eng/pipelines/coreclr/r2r-extra.yml @@ -38,7 +38,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -70,4 +70,4 @@ extends: readyToRun: true displayNameArgs: R2R liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/r2r.yml b/eng/pipelines/coreclr/r2r.yml index f3d1f1063ba59..977580703f71d 100644 --- a/eng/pipelines/coreclr/r2r.yml +++ b/eng/pipelines/coreclr/r2r.yml @@ -41,7 +41,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -76,4 +76,4 @@ extends: readyToRun: true displayNameArgs: R2R liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/release-tests.yml b/eng/pipelines/coreclr/release-tests.yml index 9a21c4664b887..d25c3069b7cf6 100644 --- a/eng/pipelines/coreclr/release-tests.yml +++ b/eng/pipelines/coreclr/release-tests.yml @@ -41,7 +41,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -77,7 +77,7 @@ extends: jobParameters: testGroup: outerloop liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) # # Release R2R test runs @@ -94,5 +94,5 @@ extends: liveLibrariesBuildConfig: Release readyToRun: true displayNameArgs: R2R - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/runincontext.yml b/eng/pipelines/coreclr/runincontext.yml index acc6d76c48758..6ade58de8fe5a 100644 --- a/eng/pipelines/coreclr/runincontext.yml +++ b/eng/pipelines/coreclr/runincontext.yml @@ -37,7 +37,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -68,4 +68,4 @@ extends: runInUnloadableContext: true displayNameArgs: RunInContext liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml b/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml index bb979a9cc8f29..e5dcc6800e1b2 100644 --- a/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml +++ b/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml @@ -34,7 +34,7 @@ extends: archiveType: $(archiveType) tarCompression: $(tarCompression) archiveExtension: $(archiveExtension) - artifactName: CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType) + artifactName: CheckedJIT_$(osGroup)$(osSubgroup)_$(archType) displayName: JIT and SuperPMI Assets - template: /eng/pipelines/common/platform-matrix.yml @@ -55,7 +55,7 @@ extends: archiveType: $(archiveType) tarCompression: $(tarCompression) archiveExtension: $(archiveExtension) - artifactName: ReleaseJIT_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: JIT and SuperPMI Assets - template: /eng/pipelines/common/platform-matrix.yml diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 6a7ceadf3cd56..7f3bf2343766c 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -52,7 +52,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - template: /eng/pipelines/common/upload-artifact-step.yml parameters: rootFolder: $(Build.SourcesDirectory)/artifacts/helix @@ -60,7 +60,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: LibrariesTestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) extraVariablesTemplates: - template: /eng/pipelines/coreclr/templates/native-test-assets-variables.yml parameters: @@ -92,7 +92,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - template: /eng/pipelines/common/upload-artifact-step.yml parameters: rootFolder: $(Build.SourcesDirectory)/artifacts/helix @@ -100,7 +100,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: LibrariesTestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) extraVariablesTemplates: - template: /eng/pipelines/coreclr/templates/native-test-assets-variables.yml parameters: @@ -132,7 +132,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - template: /eng/pipelines/common/upload-artifact-step.yml parameters: rootFolder: $(Build.SourcesDirectory)/artifacts/helix @@ -140,7 +140,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: LibrariesTestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) extraVariablesTemplates: - template: /eng/pipelines/coreclr/templates/native-test-assets-variables.yml parameters: @@ -297,7 +297,7 @@ extends: testGroup: outerloop liveLibrariesBuildConfig: Release SuperPmiCollect: true - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -343,8 +343,8 @@ extends: - normal SuperPmiCollect: true SuperPmiCollectionName: libraries_tests - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) - helixArtifactsName: LibrariesTestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) # # Collection of libraries test run: no_tiered_compilation @@ -373,5 +373,5 @@ extends: - no_tiered_compilation SuperPmiCollect: true SuperPmiCollectionName: libraries_tests_no_tiered_compilation - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) - helixArtifactsName: LibrariesTestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/superpmi-diffs.yml b/eng/pipelines/coreclr/superpmi-diffs.yml index 05862e4f18ef7..d97452f841c71 100644 --- a/eng/pipelines/coreclr/superpmi-diffs.yml +++ b/eng/pipelines/coreclr/superpmi-diffs.yml @@ -60,7 +60,7 @@ extends: archiveType: $(archiveType) tarCompression: $(tarCompression) archiveExtension: $(archiveExtension) - artifactName: CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType) + artifactName: CheckedJIT_$(osGroup)$(osSubgroup)_$(archType) displayName: JIT and SuperPMI Assets condition: not(eq(dependencies.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true)) @@ -82,7 +82,7 @@ extends: archiveType: $(archiveType) tarCompression: $(tarCompression) archiveExtension: $(archiveExtension) - artifactName: ReleaseJIT_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: JIT and SuperPMI Assets condition: not(eq(dependencies.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true)) diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 1e0a377ecfb86..638f9840b5391 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -50,7 +50,7 @@ extends: archiveType: $(archiveType) tarCompression: $(tarCompression) archiveExtension: $(archiveExtension) - artifactName: CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType) + artifactName: CheckedJIT_$(osGroup)$(osSubgroup)_$(archType) displayName: JIT and SuperPMI Assets condition: not(eq(dependencies.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true)) diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml index a75dba476d6a6..ea328a9c543ec 100644 --- a/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml @@ -95,8 +95,8 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - artifactFileName: 'Release_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' - artifactName: 'Release_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)' + artifactFileName: 'Release_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactName: 'Release_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)' displayName: 'product build' # Populate Core_Root diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml index 71f206beb929d..85a303c6e9dd6 100644 --- a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml @@ -91,8 +91,8 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - artifactFileName: 'Release_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' - artifactName: 'Release_CoreCLR_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)' + artifactFileName: 'Release_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactName: 'Release_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)' displayName: 'product build' # Populate Core_Root diff --git a/eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml b/eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml index 86965e2f310b7..4d76aa00e28ca 100644 --- a/eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml +++ b/eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml @@ -28,7 +28,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -62,4 +62,4 @@ extends: jobParameters: testGroup: ${{ parameters.testGroup }} liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 922d630fd91df..d1d7960c1788b 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -179,8 +179,8 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - artifactFileName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr$(archiveExtension)' - artifactName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr' + artifactFileName: 'BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr$(archiveExtension)' + artifactName: 'BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr' displayName: 'Runtime artifacts' - template: /eng/pipelines/common/download-artifact-step.yml @@ -196,8 +196,8 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - artifactFileName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_${{ parameters.runtimeType }}$(archiveExtension)' - artifactName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)_${{ parameters.runtimeType }}' + artifactFileName: 'BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_${{ parameters.runtimeType }}$(archiveExtension)' + artifactName: 'BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_${{ parameters.runtimeType }}' displayName: 'Runtime artifacts' - ${{ elseif eq(parameters.runtimeType, 'AndroidMono')}}: diff --git a/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml b/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml index 8681076ae5706..3ef86c53ca91e 100644 --- a/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml @@ -39,8 +39,8 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr - artifactFileName: 'CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType)$(archiveExtension)' - artifactName: 'CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType)' + artifactFileName: 'CheckedJIT_$(osGroup)$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'CheckedJIT_$(osGroup)$(osSubgroup)_$(archType)' displayName: 'JIT checked build' cleanupUnpackFolder: false @@ -48,7 +48,7 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr - artifactFileName: 'ReleaseJIT_$(osGroup)_$(osSubgroup)_$(archType)$(archiveExtension)' - artifactName: 'ReleaseJIT_$(osGroup)_$(osSubgroup)_$(archType)' + artifactFileName: 'ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType)' displayName: 'JIT release build' cleanupUnpackFolder: false diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml index c66d2ad990f58..df59120dd779e 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml @@ -47,8 +47,8 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - artifactFileName: 'BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactFileName: 'BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: 'Runtime build artifacts' # Unzip individual test projects @@ -56,8 +56,8 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: '$(Build.SourcesDirectory)/artifacts/helix/' - artifactFileName: 'LibrariesTestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' - artifactName: LibrariesTestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactFileName: 'LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: 'Libraries test artifacts' - task: ExtractFiles@1 displayName: 'Unzip libraries tests' diff --git a/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml b/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml index 762d39b16148f..6bab325a0b21e 100644 --- a/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml @@ -56,8 +56,8 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr - artifactFileName: 'CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType)$(archiveExtension)' - artifactName: 'CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType)' + artifactFileName: 'CheckedJIT_$(osGroup)$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'CheckedJIT_$(osGroup)$(osSubgroup)_$(archType)' displayName: 'JIT checked build' cleanupUnpackFolder: false @@ -66,7 +66,7 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr - artifactFileName: 'ReleaseJIT_$(osGroup)_$(osSubgroup)_$(archType)$(archiveExtension)' - artifactName: 'ReleaseJIT_$(osGroup)_$(osSubgroup)_$(archType)' + artifactFileName: 'ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType)' displayName: 'JIT release build' cleanupUnpackFolder: false diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 7296c2f4f894c..1c234e2984ea9 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -36,7 +36,7 @@ jobs: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr - artifactFileName: 'CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType)$(archiveExtension)' - artifactName: 'CheckedJIT_$(osGroup)_$(osSubgroup)_$(archType)' + artifactFileName: 'CheckedJIT_$(osGroup)$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'CheckedJIT_$(osGroup)$(osSubgroup)_$(archType)' displayName: 'JIT checked build' cleanupUnpackFolder: false \ No newline at end of file diff --git a/eng/pipelines/coreclr/tieringtest.yml b/eng/pipelines/coreclr/tieringtest.yml index dd426c37d4c33..2ce790dcc26f9 100644 --- a/eng/pipelines/coreclr/tieringtest.yml +++ b/eng/pipelines/coreclr/tieringtest.yml @@ -37,7 +37,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -68,4 +68,4 @@ extends: tieringTest: true displayNameArgs: TieringTest liveLibrariesBuildConfig: Release - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/runtime-cet.yml b/eng/pipelines/runtime-cet.yml index 120f7ef031344..459905bd89ee6 100644 --- a/eng/pipelines/runtime-cet.yml +++ b/eng/pipelines/runtime-cet.yml @@ -57,7 +57,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -92,4 +92,4 @@ extends: testGroup: innerloop liveLibrariesBuildConfig: release useCodeFlowEnforcement: true - unifiedArtifactsName: BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index f559ad50d56c5..15ab41581142f 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -129,7 +129,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: CoreCLR_Libraries_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: CoreCLR_Libraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), @@ -154,7 +154,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: CoreCLR_Libraries_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: CoreCLR_Libraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), @@ -185,7 +185,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets - template: /eng/pipelines/common/upload-artifact-step.yml parameters: @@ -194,7 +194,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml parameters: @@ -226,7 +226,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -257,7 +257,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets - template: /eng/pipelines/common/upload-artifact-step.yml parameters: @@ -266,7 +266,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), @@ -291,7 +291,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -323,7 +323,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: CoreClr_ReleaseLibraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: CoreClr_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -356,7 +356,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: CoreClr_ReleaseLibraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: CoreClr_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets - template: /eng/pipelines/common/upload-artifact-step.yml parameters: @@ -365,7 +365,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), @@ -1340,7 +1340,7 @@ extends: jobParameters: liveRuntimeBuildConfig: release liveLibrariesBuildConfig: Release - unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_Release + unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release unifiedBuildConfigOverride: release unifiedBuildNameSuffix: CoreCLR_Libraries condition: @@ -1362,7 +1362,7 @@ extends: jobParameters: liveRuntimeBuildConfig: release liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} unifiedBuildNameSuffix: CoreCLR_Libraries condition: @@ -1553,7 +1553,7 @@ extends: jobParameters: testGroup: innerloop liveLibrariesBuildConfig: Release - unifiedArtifactsName: CoreClr_ReleaseLibraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedArtifactsName: CoreClr_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) unifiedBuildNameSuffix: CoreClr_ReleaseLibraries unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} condition: >- @@ -1576,7 +1576,7 @@ extends: jobParameters: testGroup: innerloop liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) unifiedBuildNameSuffix: Libraries_CheckedCoreCLR unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} condition: >- @@ -1596,7 +1596,7 @@ extends: jobParameters: testGroup: innerloop liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) unifiedBuildNameSuffix: Libraries_CheckedCoreCLR unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} condition: >- @@ -1619,8 +1619,8 @@ extends: isOfficialBuild: false testScope: innerloop liveRuntimeBuildConfig: release - unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) - helixArtifactsName: CoreCLR_Libraries_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + helixArtifactsName: CoreCLR_Libraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) unifiedBuildNameSuffix: CoreCLR_Libraries unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} condition: >- @@ -1646,8 +1646,8 @@ extends: isOfficialBuild: false testScope: innerloop liveRuntimeBuildConfig: release - unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) - helixArtifactsName: CoreCLR_Libraries_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + helixArtifactsName: CoreCLR_Libraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) unifiedBuildNameSuffix: CoreCLR_Libraries unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} condition: >- @@ -1686,8 +1686,8 @@ extends: jobParameters: testScope: innerloop liveRuntimeBuildConfig: checked - unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) - helixArtifactsName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + helixArtifactsName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) unifiedBuildNameSuffix: Libraries_CheckedCoreCLR condition: >- or( @@ -1712,8 +1712,8 @@ extends: jobParameters: testScope: innerloop liveRuntimeBuildConfig: checked - unifiedArtifactsName: CoreCLR_ReleaseLibraries_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_checked - helixArtifactsName: CoreCLR_ReleaseLibraries_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_checked + unifiedArtifactsName: CoreCLR_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_checked + helixArtifactsName: CoreCLR_ReleaseLibraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_checked unifiedBuildNameSuffix: CoreCLR_ReleaseLibraries unifiedBuildConfigOverride: checked condition: >- @@ -1732,8 +1732,8 @@ extends: jobParameters: testScope: innerloop liveRuntimeBuildConfig: checked - unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) - helixArtifactsName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)_$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + helixArtifactsName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) unifiedBuildNameSuffix: Libraries_CheckedCoreCLR condition: >- or( From 019ddec2c7933bab96159f646364eef95641c64e Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 28 Feb 2024 16:52:55 -0800 Subject: [PATCH 36/79] Remove now-unused YAML --- .../runtimes/native-test-assets-variables.yml | 30 +-- .../templates/runtimes/run-test-job.yml | 42 +--- eng/pipelines/coreclr/templates/build-job.yml | 183 ----------------- eng/pipelines/libraries/base-job.yml | 144 -------------- eng/pipelines/libraries/build-job.yml | 98 ---------- eng/pipelines/libraries/run-test-job.yml | 122 ++++++------ eng/pipelines/mono/templates/build-job.yml | 184 ------------------ .../mono/templates/generate-offsets.yml | 2 +- .../mono/templates/xplat-pipeline-job.yml | 65 ------- 9 files changed, 67 insertions(+), 803 deletions(-) delete mode 100644 eng/pipelines/coreclr/templates/build-job.yml delete mode 100644 eng/pipelines/libraries/base-job.yml delete mode 100644 eng/pipelines/libraries/build-job.yml delete mode 100644 eng/pipelines/mono/templates/build-job.yml delete mode 100644 eng/pipelines/mono/templates/xplat-pipeline-job.yml diff --git a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml index 9815e89290c7e..9b84882f17f45 100644 --- a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml +++ b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -11,34 +11,8 @@ variables: value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr' # Build product defines what we are trying to build, either coreclr or mono - - ${{ if ne(parameters.runtimeFlavor, 'mono') }}: - - name: buildProductArtifactName - value: 'CoreCLRProduct___$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: buildProductRootFolderPath - value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' - - ${{ else }}: - - name: buildProductArtifactName - value: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - # minijit and monointerpreter do not use separate product builds. - - ${{ if or(eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter')) }}: - - name : buildProductArtifactName - value : 'MonoProduct__$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - ${{ if eq(parameters.runtimeVariant, 'llvmfullaot') }}: - - name : buildProductArtifactName - value : 'MonoProduct_llvmaot_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: buildProductRootFolderPath - value: '$(Build.SourcesDirectory)/artifacts/bin/mono/$(osGroup).$(archType).$(buildConfigUpper)' - - # We need this because both mono and coreclr build currently depends on CoreClr - - name: coreClrProductArtifactName - value: 'CoreCLRProduct___$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: coreClrProductRootFolderPath - value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' + - name: buildProductRootFolderPath + value: '$(Build.SourcesDirectory)/artifacts/bin/${{ parameters.runtimeFlavor }}/$(osGroup).$(archType).$(buildConfigUpper)' - ${{ if ne(parameters.runtimeFlavor, 'mono') }}: - name: managedGenericTestArtifactName diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index c5542ad0ecfab..07d580cb96a95 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -204,42 +204,12 @@ jobs: steps: - - ${{ if ne(parameters.unifiedArtifactsName, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - artifactFileName: '${{ parameters.unifiedArtifactsName }}$(archiveExtension)' - artifactName: '${{ parameters.unifiedArtifactsName }}' - displayName: 'unified artifacts' - - - ${{ else }}: - - # Optionally download live-built libraries - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir) - cleanUnpackFolder: false - artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' - artifactName: '$(librariesBuildArtifactName)' - displayName: 'live-built libraries' - - # Download product binaries directory - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' - displayName: 'product build' - - - ${{ if eq(parameters.runtimeFlavor, 'mono') }}: - # We need to explicitly download CoreCLR for Mono - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(coreClrProductRootFolderPath) - artifactFileName: '$(coreClrProductArtifactName)$(archiveExtension)' - artifactName: '$(coreClrProductArtifactName)' - displayName: 'CoreCLR product download for Mono' + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: '${{ parameters.unifiedArtifactsName }}$(archiveExtension)' + artifactName: '${{ parameters.unifiedArtifactsName }}' + displayName: 'unified artifacts' # Download and unzip the Microsoft.NET.Sdk.IL package needed for traversing # ilproj test projects during copynativeonly. diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml deleted file mode 100644 index f1a2791862f1e..0000000000000 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ /dev/null @@ -1,183 +0,0 @@ -parameters: - archType: '' - buildConfig: '' - condition: true - container: '' - crossBuild: false - dependOnEvaluatePaths: false - disableComponentGovernance: false - isOfficialBuild: false - osGroup: '' - osSubgroup: '' - platform: '' - pool: '' - runtimeVariant: '' - testGroup: '' - timeoutInMinutes: '' - variables: {} - -### Product build -jobs: -- template: xplat-pipeline-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - runtimeVariant: ${{ parameters.runtimeVariant }} - testGroup: ${{ parameters.testGroup }} - helixType: 'build/product/' - pool: ${{ parameters.pool }} - condition: ${{ parameters.condition }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - - # Compute job name from template parameters - name: ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', - parameters.runtimeVariant, - parameters.osGroup, - parameters.osSubgroup, - parameters.archType, - parameters.buildConfig) }} - displayName: ${{ format('CoreCLR {0} Product Build {1}{2} {3} {4}', - parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, - parameters.archType, - parameters.buildConfig) }} - - # Run all steps in the container. - # Note that the containers are defined in platform-matrix.yml - container: ${{ parameters.container }} - - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - crossBuild: ${{ parameters.crossBuild }} - - gatherAssetManifests: true - variables: - - name: osGroup - value: ${{ parameters.osGroup }} - - name: osSubgroup - value: ${{ parameters.osSubgroup }} - - name: publishLogsArtifactPrefix - value: 'BuildLogs_CoreCLR' - - name: officialBuildIdArg - value: '' - - ${{ if eq(parameters.isOfficialBuild, true) }}: - - name: officialBuildIdArg - value: '/p:OfficialBuildId=$(Build.BuildNumber)' - - name: enforcePgoArg - value: '' - # The EnforcePGO script is only supported on Windows and is not supported on arm64. - - ${{ if and(eq(parameters.buildConfig, 'Release'), and(eq(parameters.osGroup, 'windows'), ne(parameters.archType, 'arm64'))) }}: - - name: enforcePgoArg - value: '-enforcepgo' - - - name: clrRuntimeComponentsBuildArg - value: '' - - ${{ if ne(parameters.testGroup, 'innerloop') }}: - - name: clrRuntimeComponentsBuildArg - value: '-component runtime -component alljits -component nativeaot -component spmi ' - - # Set a default empty argument for the pgo path. - # This will be set during the 'native prerequisites' step if PGO optimization is enabled. - - name: CoreClrPgoDataArg - value: '' - - - name: nativeSymbols - value: '' - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - name: nativeSymbols - value: '--keepnativesymbols' - - - ${{ parameters.variables }} - - steps: - # Install native dependencies - # Linux builds use docker images with dependencies preinstalled, - # and FreeBSD builds use a build agent with dependencies - # preinstalled, so we only need this step for OSX and Windows. - - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup) - displayName: Install native dependencies - - # Install internal tools on official builds - # Since our internal tools are behind an authenticated feed, - # we need to use the DotNetCli AzDO task to restore from the feed using a service connection. - # We can't do this from within the build, so we need to do this as a separate step. - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/pipelines/common/restore-internal-tools.yml - - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - ${{ if ne(parameters.osGroup, 'windows') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if eq(parameters.osGroup, 'windows') }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage before Build - - # Build/Generate native prerequisites - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.nativeprereqs $(crossArg) -arch $(archType) -c $(buildConfig) $(officialBuildIdArg) -ci /bl:$(Build.SourcesDirectory)artifacts/log/$(buildConfig)/CoreCLRNativePrereqs.binlog - displayName: Build and generate native prerequisites - - # Build CoreCLR Runtime - - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) -ci $(clrRuntimeComponentsBuildArg) $(officialBuildIdArg) $(CoreClrPgoDataArg) $(nativeSymbols) - displayName: Build CoreCLR Runtime - - ${{ if eq(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(officialBuildIdArg) $(CoreClrPgoDataArg) - displayName: Build CoreCLR Runtime - - - ${{ if or(eq(parameters.crossBuild, 'true'), ne(parameters.archType, 'x64')) }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -hostarch x64 -ci -component crosscomponents -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" $(officialBuildIdArg) - displayName: Build CoreCLR Cross-Arch Tools (Tools that run on x64 targeting x86) - - - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage after Build - - # Build CoreCLR Managed Components - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages $(crossArg) -arch $(archType) -c $(buildConfig) $(officialBuildIdArg) -ci - displayName: Build managed product components and packages - - # Publish product output directory for consumption by tests. - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(buildProductRootFolderPath) - includeRootFolder: false - archiveType: $(archiveType) - tarCompression: $(tarCompression) - archiveExtension: $(archiveExtension) - artifactName: $(buildProductArtifactName) - displayName: 'product build' - - - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: - - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log - displayName: Collect vslogs on exit - condition: always() - - # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: '$(publishLogsArtifactPrefix)_Attempt$(System.JobAttempt)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - continueOnError: true - condition: always() diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml deleted file mode 100644 index b9af7877ce719..0000000000000 --- a/eng/pipelines/libraries/base-job.yml +++ /dev/null @@ -1,144 +0,0 @@ -parameters: - buildConfig: '' - osGroup: '' - archType: '' - osSubgroup: '' - crossBuild: false - framework: 'net9.0' - isSourceBuild: false - liveRuntimeBuildConfig: '' - runtimeFlavor: 'coreclr' - timeoutInMinutes: 150 - condition: true - container: '' - steps: [] - dependsOn: [] - dependOnEvaluatePaths: false - disableComponentGovernance: false - variables: {} - name: '' - displayName: '' - testDisplayName: '' - testScope: '' - pool: '' - runTests: false - SuperPmiCollect: false - unifiedArtifactsName: '' - -jobs: - - template: /eng/common/templates/job/job.yml - parameters: - displayName: ${{ format('Libraries {0} {1}{2} {3} {4}', parameters.displayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - name: ${{ format('libraries_{0}_{1}{2}_{3}_{4}', parameters.name, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - enableTelemetry: ${{ parameters.isOfficialBuild }} # TODO: figure out if it's needed - container: ${{ parameters.container }} - condition: and(succeeded(), ${{ parameters.condition }}) - helixRepo: dotnet/runtime - pool: ${{ parameters.pool }} - - # Disable component governance if requested or on musl machines where it does not work well - ${{ if or(eq(parameters.disableComponentGovernance, true), eq(parameters.osSubGroup, '_musl')) }}: - disableComponentGovernance: true - - variables: - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNet-HelixApi-Access - - group: AzureDevOps-Artifact-Feeds-Pats - - - _buildScriptFileName: build - - - _msbuildCommonParameters: '' - # rename this variable, due to collision with build-native.proj - - _osArg: '' - - _finalFrameworkArg: '' - - _testModeArg: '' - - _buildScript: $(_buildScriptFileName)$(scriptExt) - - _testScopeArg: '' - - _extraHelixArguments: '' - - _crossBuildPropertyArg: '' - - _testRunNamePrefixSuffix: '' - - - librariesBuildArtifactName: ${{ format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - - ${{ if ne(parameters.testScope, '') }}: - - _testScopeArg: -testscope ${{ parameters.testScope }} - - - ${{ if eq(parameters.crossBuild, true) }}: - - _crossBuildPropertyArg: /p:CrossBuild=true - - # force a value for OS when cross-building - - ${{ if in(parameters.osGroup, 'browser', 'ios', 'tvos', 'android', 'freebsd') }}: - - _osArg: -os ${{ parameters.osGroup }} - - - ${{ if ne(parameters.framework, '') }}: - - _finalFrameworkArg: -framework ${{ parameters.framework }} - - _extraHelixArguments: /p:BuildTargetFramework=${{ parameters.framework }} - - - ${{ if eq(parameters.isOfficialBuild, true) }}: - - _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber) - - - _runtimeArtifactName: '' - - _runtimeDownloadPath: '' - - _runtimeArtifactsPathArg: '' - - _runtimeConfigurationArg: '' - - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}' - - _runtimeConfigurationArg: -rc ${{ parameters.liveRuntimeBuildConfig }} - - ${{ if eq(parameters.runTests, true) }}: - - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.runtimeVariant}}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' - - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)' - - ${{ if eq(parameters.testDisplayName, '') }}: - - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }} - - ${{ if ne(parameters.testDisplayName, '') }}: - - _testRunNamePrefixSuffix: ${{ parameters.testDisplayName }} - - - ${{ if ne(parameters.osGroup, 'windows') }}: - - _buildScript: ./$(_buildScriptFileName)$(scriptExt) - - - _buildArguments: $(_runtimeConfigurationArg) -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_testModeArg) $(_testScopeArg) $(_osArg) $(_msbuildCommonParameters) $(_runtimeArtifactsPathArg) $(_crossBuildPropertyArg) - - ${{ parameters.variables }} - - dependsOn: - - ${{ if eq(parameters.dependOnEvaluatePaths, true) }}: - - evaluate_paths - - ${{ parameters.dependsOn }} - workspace: - clean: all - - enablePublishBuildArtifacts: true - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - steps: - - checkout: self - clean: true - fetchDepth: $(checkoutFetchDepth) - - - ${{ if and(eq(parameters.unifiedArtifactsName, ''), ne(parameters.liveRuntimeBuildConfig, ''), eq(parameters.runTests, true)) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(_runtimeDownloadPath) - artifactFileName: '$(_runtimeArtifactName)$(archiveExtension)' - artifactName: '$(_runtimeArtifactName)' - displayName: '$(runtimeFlavorName) build drop' - - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - ${{ if ne(parameters.osGroup, 'windows') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if eq(parameters.osGroup, 'windows') }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - - ${{ parameters.steps }} diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml deleted file mode 100644 index b73c8d09f0a6e..0000000000000 --- a/eng/pipelines/libraries/build-job.yml +++ /dev/null @@ -1,98 +0,0 @@ -parameters: - buildConfig: '' - osGroup: '' - osSubgroup: '' - archType: '' - targetRid: '' - crossBuild: false - framework: 'net9.0' - isOfficialBuild: false - runtimeVariant: '' - platform: '' - testScope: '' - - timeoutInMinutes: 150 - container: '' - condition: true - dependOnEvaluatePaths: false - disableComponentGovernance: false - shouldContinueOnError: false - variables: {} - pool: '' - -jobs: - - template: /eng/pipelines/libraries/base-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - archType: ${{ parameters.archType }} - crossBuild: ${{ parameters.crossBuild }} - framework: ${{ parameters.framework }} - isOfficialBuild: ${{ parameters.isOfficialBuild }} - runtimeFlavor: ${{ parameters.runtimeFlavor }} - runTests: false - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - container: ${{ parameters.container }} - condition: ${{ parameters.condition }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - pool: ${{ parameters.pool }} - runtimeVariant: ${{ parameters.runtimeVariant }} - testScope: ${{ parameters.testScope }} - name: build - displayName: 'Build' - - variables: - - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - _subset: tools+libs+libs.tests - - _buildAction: '' - - _additionalBuildArguments: '/p:ArchiveTests=true' - - ${{ parameters.variables }} - - steps: - - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} - displayName: Install Build Dependencies - - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage before Build - - - script: $(_buildScript) - -subset $(_subset) - $(_buildAction) - $(_buildArguments) - $(_additionalBuildArguments) - displayName: Restore and Build Product - - - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage after Build - - - template: /eng/pipelines/libraries/prepare-for-bin-publish.yml - - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.ArtifactStagingDirectory)/artifacts - includeRootFolder: false - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: $(librariesBuildArtifactName) - displayName: Build Assets - - # Upload test assets - # We'll pull them down in another job to send to Helix - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/helix - includeRootFolder: true - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: $(librariesTestsArtifactName) - displayName: Test Assets \ No newline at end of file diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 9dc004ab2b349..e8c4888e8af78 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -4,7 +4,6 @@ parameters: osSubgroup: '' archType: '' targetRid: '' - framework: 'net9.0' isOfficialBuild: false liveRuntimeBuildConfig: '' runtimeFlavor: 'coreclr' @@ -30,43 +29,34 @@ parameters: unifiedBuildConfigOverride: '' jobs: - - template: /eng/pipelines/libraries/base-job.yml + - template: /eng/common/templates/job/job.yml parameters: - buildConfig: ${{ parameters.buildConfig }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - archType: ${{ parameters.archType }} - crossBuild: ${{ parameters.crossBuild }} - framework: ${{ parameters.framework }} - isOfficialBuild: ${{ parameters.isOfficialBuild }} - liveRuntimeBuildConfig: ${{ parameters.liveRuntimeBuildConfig }} - runtimeFlavor: ${{ parameters.runtimeFlavor }} - runtimeVariant: ${{ parameters.runtimeVariant }} + enablePublishBuildArtifacts: true timeoutInMinutes: ${{ parameters.timeoutInMinutes }} container: ${{ parameters.container }} - condition: ${{ parameters.condition }} - testScope: ${{ parameters.testScope }} - SuperPmiCollect: ${{ parameters.SuperPmiCollect }} - unifiedArtifactsName: ${{ parameters.unifiedArtifactsName }} - runTests: true + condition: and(succeeded(), ${{ parameters.condition }}) + helixRepo: dotnet/runtime ${{ if eq(parameters.SuperPmiCollect, true) }}: - displayName: ${{ format('SuperPMI collection {0} {1} {2}', parameters.SuperPmiCollectionName, parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }} - name: ${{ format('spmi_{0}_{1}_{2}', parameters.SuperPmiCollectionName, parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }} + displayName: ${{ format('Libraries SuperPMI collection {0} {1} {2} {3}{4} {5} {6}', parameters.SuperPmiCollectionName, parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + name: ${{ format('spmi_{0}_{1}_{2}_{3}{4}_{5}_{6}', parameters.SuperPmiCollectionName, parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} ${{ else }}: ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - displayName: ${{ format('Test Run {0} {1}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }} - name: ${{ format('test_run_{0}_{1}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }} + displayName: ${{ format('Test Run {0} {1} {2}{3} {4} {5}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + name: ${{ format('test_run_{0}_{1}_{2}{3}_{4}_{5}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} ${{ if eq(parameters.liveRuntimeBuildConfig, '') }}: displayName: 'Test Run' name: test_run - ${{ if eq(parameters.interpreter, 'true') }}: - testDisplayName: ${{ parameters.runtimeFlavor }}_interpreter_${{ parameters.liveRuntimeBuildConfig }} # To run the tests we just send to helix and wait, use ubuntu hosted pools for faster providing and to not back up our build pools pool: ${{ parameters.pool }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} + # Disable component governance if requested or on musl machines where it does not work well + ${{ if or(eq(parameters.disableComponentGovernance, true), eq(parameters.osSubGroup, '_musl')) }}: + disableComponentGovernance: true + dependsOn: + - ${{ if eq(parameters.dependOnEvaluatePaths, true) }}: + - evaluate_paths - ${{ if ne(parameters.dependsOn[0], '') }}: - ${{ parameters.dependsOn }} - ${{ if eq(parameters.dependsOn[0], '') }}: @@ -77,7 +67,13 @@ jobs: - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} + workspace: + clean: all + variables: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - group: DotNet-HelixApi-Access + - group: AzureDevOps-Artifact-Feeds-Pats - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - _archiveTestsParameter: /p:ArchiveTests=true @@ -96,50 +92,45 @@ jobs: - ${{ parameters.variables }} steps: + - checkout: self + clean: true + fetchDepth: $(checkoutFetchDepth) + + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - ${{ if ne(parameters.osGroup, 'windows') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - ${{ if eq(parameters.osGroup, 'windows') }}: + - task: PowerShell@2 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} displayName: Install Build Dependencies - - ${{ if ne(parameters.unifiedArtifactsName, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - artifactFileName: '${{ parameters.unifiedArtifactsName }}$(archiveExtension)' - artifactName: '${{ parameters.unifiedArtifactsName }}' - displayName: 'unified artifacts' + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: '${{ parameters.unifiedArtifactsName }}$(archiveExtension)' + artifactName: '${{ parameters.unifiedArtifactsName }}' + displayName: 'unified artifacts' - - ${{ else }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - displayName: Build Assets - cleanUnpackFolder: false - artifactName: $(librariesBuildArtifactName) - artifactFileName: $(librariesBuildArtifactName)$(archiveExtension) - unpackFolder: $(Build.SourcesDirectory)/artifacts - - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - script: $(_buildScript) - -subset host.native+libs.pretest - $(_buildArguments) - /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideRuntimeFromLiveDrop.binlog - displayName: Prepare TestHost with runtime $(runtimeFlavorName) - - - ${{ if ne(parameters.helixArtifactsName, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - displayName: Test Assets - artifactName: ${{ parameters.helixArtifactsName }} - artifactFileName: '${{ parameters.helixArtifactsName }}$(archiveExtension)' - unpackFolder: $(Build.SourcesDirectory)/artifacts/helix - - ${{ else }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - displayName: Test Assets - cleanUnpackFolder: false - artifactName: $(librariesTestsArtifactName) - artifactFileName: $(librariesTestsArtifactName)$(archiveExtension) - unpackFolder: $(Build.SourcesDirectory)/artifacts + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + displayName: Test Assets + artifactName: ${{ parameters.helixArtifactsName }} + artifactFileName: '${{ parameters.helixArtifactsName }}$(archiveExtension)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/helix - template: /eng/pipelines/libraries/helix.yml parameters: @@ -153,7 +144,10 @@ jobs: interpreter: ${{ parameters.interpreter }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} creator: dotnet-bot - testRunNamePrefixSuffix: $(_testRunNamePrefixSuffix) + ${{ if eq(parameters.interpreter, 'true') }}: + testRunNamePrefixSuffix: ${{ parameters.runtimeFlavor }}_interpreter_${{ parameters.liveRuntimeBuildConfig }} + ${{ else }}: + testRunNamePrefixSuffix: ${{ parameters.runtimeDisplayName }}_${{ parameters.liveRuntimeBuildConfig }} SuperPmiCollect: ${{ parameters.SuperPmiCollect }} SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} diff --git a/eng/pipelines/mono/templates/build-job.yml b/eng/pipelines/mono/templates/build-job.yml deleted file mode 100644 index 86e0813c7c7e5..0000000000000 --- a/eng/pipelines/mono/templates/build-job.yml +++ /dev/null @@ -1,184 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osSubgroup: '' - platform: '' - container: '' - timeoutInMinutes: '' - variables: {} - pool: '' - condition: true - runtimeVariant: '' - isOfficialBuild: false - crossBuild: false - dependsOn: [] - monoCrossAOTTargetOS: [] - dependOnEvaluatePaths: false - -### Product build -jobs: -- template: xplat-pipeline-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - helixType: 'build/product/' - enableMicrobuild: true - pool: ${{ parameters.pool }} - runtimeVariant: ${{ parameters.runtimeVariant }} - crossBuild: ${{ parameters.crossBuild }} - monoCrossAOTTargetOS: ${{ parameters.monoCrossAOTTargetOS }} - condition: ${{ parameters.condition }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - - # Compute job name from template parameters - name: ${{ format('mono_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Mono {0} Product Build {1}{2} {3} {4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - # Run all steps in the container. - # Note that the containers are defined in platform-matrix.yml - container: ${{ parameters.container }} - - dependsOn: ${{ parameters.dependsOn }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - gatherAssetManifests: true - variables: - - name: osGroup - value: ${{ parameters.osGroup }} - - name: osSubgroup - value: ${{ parameters.osSubgroup }} - - name: officialBuildIdArg - value: '' - - name: osOverride - value: '' - - name: aotCrossParameter - value: '' - - name: llvmParameter - value: '' - - name: darwinFrameworks - value: '' - - ${{ if eq(parameters.isOfficialBuild, true) }}: - - name: officialBuildIdArg - value: '/p:OfficialBuildId=$(Build.BuildNumber)' - - ${{ if and(eq(parameters.osSubgroup, '_musl'), eq(parameters.osGroup, 'linux')) }}: - # Set output RID manually: musl isn't properly detected. Make sure to also convert linux to - # lowercase for RID format. (Detection normally converts, but we're preventing it.) - - name: OutputRidArg - value: /p:OutputRID=linux-musl-${{ parameters.archType }} - - name: _PortableBuild - value: true - - ${{ if eq(parameters.osGroup, 'tvos') }}: - - name: osOverride - value: -os tvos - - name: darwinFrameworks - value: /p:BuildDarwinFrameworks=true - - ${{ if eq(parameters.osGroup, 'tvossimulator') }}: - - name: osOverride - value: -os tvossimulator - - name: darwinFrameworks - value: /p:BuildDarwinFrameworks=true - - ${{ if eq(parameters.osGroup, 'ios') }}: - - name: osOverride - value: -os ios - - name: darwinFrameworks - value: /p:BuildDarwinFrameworks=true - - ${{ if eq(parameters.osGroup, 'iossimulator') }}: - - name: osOverride - value: -os iossimulator - - name: darwinFrameworks - value: /p:BuildDarwinFrameworks=true - - ${{ if eq(parameters.osGroup, 'android') }}: - - name: osOverride - value: -os android - - ${{ if eq(parameters.osGroup, 'browser') }}: - - name: archType - value: wasm - - name: osOverride - value: '-os browser' - - ${{ if eq(parameters.osGroup, 'wasi') }}: - - name: archType - value: wasm - - name: osOverride - value: '-os wasi' - - ${{ if eq(parameters.runtimeVariant, 'llvmjit') }}: - - name: llvmParameter - value: /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=false - - ${{ if eq(parameters.runtimeVariant, 'llvmaot') }}: - - name: llvmParameter - value: /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=true - - ${{ if gt(length(parameters.monoCrossAOTTargetOS),0) }}: - - name: aotCrossParameter - value: /p:MonoCrossAOTTargetOS=${{join('+',parameters.monoCrossAOTTargetOS)}} /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true - - ${{ parameters.variables }} - - steps: - - # Install native dependencies - # Linux builds use docker images with dependencies preinstalled, - # and FreeBSD builds use a build agent with dependencies - # preinstalled, so we only need this step for OSX and Windows. - - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup) - displayName: Install native dependencies - - - ${{ each monoCrossAOTTargetOS in parameters.monoCrossAOTTargetOS }}: - - task: DownloadPipelineArtifact@2 - displayName: Download ${{monoCrossAOTTargetOS}} AOT offset files - inputs: - artifact: Mono_Offsets_${{monoCrossAOTTargetOS}} - path: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles' - - - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage before Build - - # Build - - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: ./build$(scriptExt) -subset mono+clr.hosts $(crossArg) -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter) $(darwinFrameworks) - displayName: Build product - - ${{ if eq(parameters.osGroup, 'windows') }}: - - script: build$(scriptExt) -subset mono+clr.hosts -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter) - displayName: Build product - - - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage after Build - - # Publish product output directory for consumption by tests. - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(buildProductRootFolderPath) - includeRootFolder: false - archiveType: $(archiveType) - tarCompression: $(tarCompression) - archiveExtension: $(archiveExtension) - artifactName: $(buildProductArtifactName) - displayName: 'product build' - - # Build packages - - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: ./build$(scriptExt) -subset mono+clr.hosts $(crossArg) -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter) -pack $(OutputRidArg) - displayName: Build nupkg - - ${{ if eq(parameters.osGroup, 'windows') }}: - - script: build$(scriptExt) -subset mono+clr.hosts -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter) -pack $(OutputRidArg) - displayName: Build nupkg - - - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: - - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log - displayName: Collect vslogs on exit - condition: always() - # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'BuildLogs_Attempt$(System.JobAttempt)_Mono_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - continueOnError: true - condition: always() diff --git a/eng/pipelines/mono/templates/generate-offsets.yml b/eng/pipelines/mono/templates/generate-offsets.yml index c68adfe67a9e6..2573b65e1d27f 100644 --- a/eng/pipelines/mono/templates/generate-offsets.yml +++ b/eng/pipelines/mono/templates/generate-offsets.yml @@ -12,7 +12,7 @@ parameters: ### Product build jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} osGroup: ${{ parameters.osGroup }} diff --git a/eng/pipelines/mono/templates/xplat-pipeline-job.yml b/eng/pipelines/mono/templates/xplat-pipeline-job.yml deleted file mode 100644 index 6fac928c278c6..0000000000000 --- a/eng/pipelines/mono/templates/xplat-pipeline-job.yml +++ /dev/null @@ -1,65 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osSubgroup: '' - name: '' - helixType: '(unspecified)' - container: '' - crossBuild: false - liveLibrariesBuildConfig: '' - strategy: '' - pool: '' - runtimeVariant: '' - - # arcade-specific parameters - condition: true - continueOnError: false - dependsOn: '' - dependOnEvaluatePaths: false - displayName: '' - timeoutInMinutes: '' - enableMicrobuild: '' - gatherAssetManifests: false - - variables: {} ## any extra variables to add to the defaults defined below - -jobs: -- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - name: ${{ parameters.name }} - helixType: ${{ parameters.helixType }} - container: ${{ parameters.container }} - crossBuild: ${{ parameters.crossBuild }} - strategy: ${{ parameters.strategy }} - pool: ${{ parameters.pool }} - runtimeVariant: ${{ parameters.runtimeVariant }} - - # arcade-specific parameters - condition: and(succeeded(), ${{ parameters.condition }}) - continueOnError: ${{ parameters.continueOnError }} - dependsOn: ${{ parameters.dependsOn }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - displayName: ${{ parameters.displayName }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - enableMicrobuild: ${{ parameters.enableMicrobuild }} - gatherAssetManifests: ${{ parameters.gatherAssetManifests }} - - variables: - - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml - parameters: - runtimeFlavor: mono - runtimeVariant: ${{ parameters.runtimeVariant }} - liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} - - - name: priorityArg - value: '' - - - ${{ each variable in parameters.variables }}: - - ${{insert}}: ${{ variable }} - - steps: ${{ parameters.steps }} From 469f47a64b455a7da93154f75aed954305314115 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 28 Feb 2024 16:54:23 -0800 Subject: [PATCH 37/79] Remove invalid isExtraPlatformsBuild --- eng/pipelines/runtime.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 15ab41581142f..2d9863428eab7 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -1758,7 +1758,6 @@ extends: jobParameters: testGroup: innerloop nameSuffix: Mono_Interpreter_LibrariesTests - isExtraPlatforms: ${{ parameters.isExtraPlatformsBuild }} buildArgs: -s mono+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true timeoutInMinutes: 480 # extra steps, run tests @@ -1790,8 +1789,7 @@ extends: #- windows_x64 jobParameters: testGroup: innerloop - nameSuffix: Mono_MiniJIT_LibrariesTests - isExtraPlatforms: ${{ parameters.isExtraPlatformsBuild }} + nameSuffix: Mono_MiniJIT_LibrariesTests\ buildArgs: -s mono+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true timeoutInMinutes: 480 # extra steps, run tests From 68d1c63bcb4e229f312d1a524a9d6937a8857af3 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 28 Feb 2024 16:55:04 -0800 Subject: [PATCH 38/79] Fix indentation --- eng/pipelines/coreclr/templates/jit-python-variables.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/jit-python-variables.yml b/eng/pipelines/coreclr/templates/jit-python-variables.yml index c59c219a29ec2..6fd207296eaee 100644 --- a/eng/pipelines/coreclr/templates/jit-python-variables.yml +++ b/eng/pipelines/coreclr/templates/jit-python-variables.yml @@ -17,4 +17,4 @@ variables: - name: PipScript value: '$(Build.SourcesDirectory)/venv/bin/pip3' - - ${{ parameters.variables }} +- ${{ parameters.variables }} From 22dd1e0f7fc6e35d53509d9a17cb2895938dc0db Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 28 Feb 2024 16:56:55 -0800 Subject: [PATCH 39/79] Remove extraneous ability to specify additional variables in variable template --- eng/pipelines/coreclr/templates/jit-python-variables.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/jit-python-variables.yml b/eng/pipelines/coreclr/templates/jit-python-variables.yml index 6fd207296eaee..3ca90600cfb09 100644 --- a/eng/pipelines/coreclr/templates/jit-python-variables.yml +++ b/eng/pipelines/coreclr/templates/jit-python-variables.yml @@ -16,5 +16,3 @@ variables: value: '$(Build.SourcesDirectory)/venv/bin/python3' - name: PipScript value: '$(Build.SourcesDirectory)/venv/bin/pip3' - -- ${{ parameters.variables }} From 7e5db2e782b02242702d94985a39304eb548e285 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 28 Feb 2024 17:00:30 -0800 Subject: [PATCH 40/79] Remove installer builds that are now built in a global build job (which doesn't run tests) --- eng/pipelines/runtime.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 2d9863428eab7..1d2caad38e09a 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -144,7 +144,7 @@ extends: - windows_x86 jobParameters: nameSuffix: CoreCLR_Libraries - buildArgs: -s clr+libs+libs.tests -c Release $(_BuildConfig) /p:ArchiveTests=true + buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true timeoutInMinutes: 120 postBuildSteps: - template: /eng/pipelines/common/upload-artifact-step.yml @@ -1332,11 +1332,7 @@ extends: jobTemplate: /eng/pipelines/installer/jobs/build-job.yml buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - - linux_musl_arm - - linux_musl_arm64 - windows_x86 - - windows_arm64 - - linux_arm jobParameters: liveRuntimeBuildConfig: release liveLibrariesBuildConfig: Release @@ -1353,11 +1349,8 @@ extends: jobTemplate: /eng/pipelines/installer/jobs/build-job.yml buildConfig: Release platforms: - - osx_arm64 - osx_x64 - linux_x64 - - linux_arm64 - - linux_musl_x64 - windows_x64 jobParameters: liveRuntimeBuildConfig: release @@ -1789,7 +1782,7 @@ extends: #- windows_x64 jobParameters: testGroup: innerloop - nameSuffix: Mono_MiniJIT_LibrariesTests\ + nameSuffix: Mono_MiniJIT_LibrariesTests buildArgs: -s mono+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true timeoutInMinutes: 480 # extra steps, run tests From ccc8da70895abf4e204cc94c5a33bc99cad80192 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 28 Feb 2024 17:01:59 -0800 Subject: [PATCH 41/79] Fix configuration match --- eng/pipelines/runtime.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 1d2caad38e09a..a9ddc5241d201 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -1548,7 +1548,6 @@ extends: liveLibrariesBuildConfig: Release unifiedArtifactsName: CoreClr_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) unifiedBuildNameSuffix: CoreClr_ReleaseLibraries - unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), From dd015aa3ca64a8d4106bd4023d43c90e9c54a5da Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 29 Feb 2024 10:52:51 -0800 Subject: [PATCH 42/79] Fix indentation so libraries tests actually run. --- eng/pipelines/libraries/run-test-job.yml | 106 +++++++++++------------ 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index e8c4888e8af78..5adfad97611d9 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -114,61 +114,61 @@ jobs: env: Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} - displayName: Install Build Dependencies - - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(Build.SourcesDirectory)/artifacts/bin - artifactFileName: '${{ parameters.unifiedArtifactsName }}$(archiveExtension)' - artifactName: '${{ parameters.unifiedArtifactsName }}' - displayName: 'unified artifacts' - - - template: /eng/pipelines/common/download-artifact-step.yml + - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: + - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} + displayName: Install Build Dependencies + + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: '${{ parameters.unifiedArtifactsName }}$(archiveExtension)' + artifactName: '${{ parameters.unifiedArtifactsName }}' + displayName: 'unified artifacts' + + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + displayName: Test Assets + artifactName: ${{ parameters.helixArtifactsName }} + artifactFileName: '${{ parameters.helixArtifactsName }}$(archiveExtension)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/helix + + - template: /eng/pipelines/libraries/helix.yml + parameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + osGroup: ${{ parameters.osGroup }} + targetRid: ${{ parameters.targetRid }} + archType: ${{ parameters.archType }} + buildConfig: ${{ parameters.buildConfig }} + helixQueues: ${{ parameters.helixQueues }} + testScope: ${{ parameters.testScope }} + interpreter: ${{ parameters.interpreter }} + shouldContinueOnError: ${{ parameters.shouldContinueOnError }} + creator: dotnet-bot + ${{ if eq(parameters.interpreter, 'true') }}: + testRunNamePrefixSuffix: ${{ parameters.runtimeFlavor }}_interpreter_${{ parameters.liveRuntimeBuildConfig }} + ${{ else }}: + testRunNamePrefixSuffix: ${{ parameters.runtimeDisplayName }}_${{ parameters.liveRuntimeBuildConfig }} + SuperPmiCollect: ${{ parameters.SuperPmiCollect }} + SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} + SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} + + ${{ if ne(parameters.scenarios[0], '') }}: + scenarios: ${{ parameters.scenarios }} + + - ${{ if eq(parameters.SuperPmiCollect, true) }}: + - template: /eng/pipelines/libraries/superpmi-postprocess-step.yml parameters: - displayName: Test Assets - artifactName: ${{ parameters.helixArtifactsName }} - artifactFileName: '${{ parameters.helixArtifactsName }}$(archiveExtension)' - unpackFolder: $(Build.SourcesDirectory)/artifacts/helix - - - template: /eng/pipelines/libraries/helix.yml - parameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} + buildConfig: ${{ parameters.buildConfig }} + buildConfigUpper: $(buildConfigUpper) osGroup: ${{ parameters.osGroup }} - targetRid: ${{ parameters.targetRid }} + osSubgroup: ${{ parameters.osSubgroup }} archType: ${{ parameters.archType }} - buildConfig: ${{ parameters.buildConfig }} - helixQueues: ${{ parameters.helixQueues }} - testScope: ${{ parameters.testScope }} - interpreter: ${{ parameters.interpreter }} - shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - creator: dotnet-bot - ${{ if eq(parameters.interpreter, 'true') }}: - testRunNamePrefixSuffix: ${{ parameters.runtimeFlavor }}_interpreter_${{ parameters.liveRuntimeBuildConfig }} - ${{ else }}: - testRunNamePrefixSuffix: ${{ parameters.runtimeDisplayName }}_${{ parameters.liveRuntimeBuildConfig }} - SuperPmiCollect: ${{ parameters.SuperPmiCollect }} SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} - - ${{ if ne(parameters.scenarios[0], '') }}: - scenarios: ${{ parameters.scenarios }} - - - ${{ if eq(parameters.SuperPmiCollect, true) }}: - - template: /eng/pipelines/libraries/superpmi-postprocess-step.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - buildConfigUpper: $(buildConfigUpper) - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - archType: ${{ parameters.archType }} - SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} - SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} - MergedMchFileLocation: $(MergedMchFileLocation) - MchFilesLocation: $(MchFilesLocation) - SpmiLogsLocation: $(SpmiLogsLocation) - SuperPmiMcsPath: $(SuperPmiMcsPath) - PythonSetupScript: $(PythonSetupScript) - PythonScript: $(PythonScript) - PipScript: $(PipScript) + MergedMchFileLocation: $(MergedMchFileLocation) + MchFilesLocation: $(MchFilesLocation) + SpmiLogsLocation: $(SpmiLogsLocation) + SuperPmiMcsPath: $(SuperPmiMcsPath) + PythonSetupScript: $(PythonSetupScript) + PythonScript: $(PythonScript) + PipScript: $(PipScript) From 6229c012180249c5d7cf8b80e1d16c9606ea783e Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 29 Feb 2024 10:54:39 -0800 Subject: [PATCH 43/79] Upload build assets as well as helix test assets. --- eng/pipelines/runtime.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index a9ddc5241d201..b0a6135b02b2c 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -122,6 +122,15 @@ extends: buildArgs: -s clr+libs+libs.tests -c Release -lc $(_BuildConfig) /p:ArchiveTests=true timeoutInMinutes: 120 postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets - template: /eng/pipelines/common/upload-artifact-step.yml parameters: rootFolder: $(Build.SourcesDirectory)/artifacts/helix @@ -147,6 +156,15 @@ extends: buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true timeoutInMinutes: 120 postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets - template: /eng/pipelines/common/upload-artifact-step.yml parameters: rootFolder: $(Build.SourcesDirectory)/artifacts/helix From f36f0009f02304c4dfec17f9b5dfbe8800976830 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 29 Feb 2024 10:57:22 -0800 Subject: [PATCH 44/79] Fix some config settings --- eng/pipelines/runtime.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index b0a6135b02b2c..d85d8e8ce1db9 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -329,8 +329,8 @@ extends: - windows_arm64 - windows_x86 jobParameters: - nameSuffix: CoreClr_ReleaseLibraries - buildArgs: -s clr+libs -c $(_BuildConfig) -rc Checked + nameSuffix: CoreCLR_ReleaseLibraries + buildArgs: -s clr+libs -rc $(_BuildConfig) -c Release timeoutInMinutes: 120 postBuildSteps: - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml @@ -341,7 +341,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: CoreClr_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: CoreCLR_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -363,7 +363,7 @@ extends: - linux_musl_arm64 - windows_x64 jobParameters: - nameSuffix: CoreClr_ReleaseLibraries + nameSuffix: CoreCLR_ReleaseLibraries buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true timeoutInMinutes: 120 postBuildSteps: @@ -374,7 +374,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: CoreClr_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: CoreCLR_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets - template: /eng/pipelines/common/upload-artifact-step.yml parameters: @@ -1564,8 +1564,8 @@ extends: jobParameters: testGroup: innerloop liveLibrariesBuildConfig: Release - unifiedArtifactsName: CoreClr_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) - unifiedBuildNameSuffix: CoreClr_ReleaseLibraries + unifiedArtifactsName: CoreCLR_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedBuildNameSuffix: CoreCLR_ReleaseLibraries condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), @@ -1586,7 +1586,7 @@ extends: jobParameters: testGroup: innerloop liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) unifiedBuildNameSuffix: Libraries_CheckedCoreCLR unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} condition: >- @@ -1722,8 +1722,8 @@ extends: jobParameters: testScope: innerloop liveRuntimeBuildConfig: checked - unifiedArtifactsName: CoreCLR_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_checked - helixArtifactsName: CoreCLR_ReleaseLibraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_checked + unifiedArtifactsName: CoreCLR_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + helixArtifactsName: CoreCLR_ReleaseLibraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked unifiedBuildNameSuffix: CoreCLR_ReleaseLibraries unifiedBuildConfigOverride: checked condition: >- From 5427f92cb9f56f9012a8001f218c4237fd2bf333 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 29 Feb 2024 12:01:45 -0800 Subject: [PATCH 45/79] Adjust configuration names again --- eng/pipelines/libraries/run-test-job.yml | 4 ++-- eng/pipelines/runtime.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 5adfad97611d9..e56a1519cb74e 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -41,8 +41,8 @@ jobs: name: ${{ format('spmi_{0}_{1}_{2}_{3}{4}_{5}_{6}', parameters.SuperPmiCollectionName, parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} ${{ else }}: ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - displayName: ${{ format('Test Run {0} {1} {2}{3} {4} {5}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - name: ${{ format('test_run_{0}_{1}_{2}{3}_{4}_{5}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Libraries Test Run {0} {1} {2}{3} {4} {5}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + name: ${{ format('libraries_test_run_{0}_{1}_{2}{3}_{4}_{5}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} ${{ if eq(parameters.liveRuntimeBuildConfig, '') }}: displayName: 'Test Run' name: test_run diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index d85d8e8ce1db9..5cfa50ffef4b8 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -129,7 +129,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets - template: /eng/pipelines/common/upload-artifact-step.yml parameters: @@ -163,7 +163,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) displayName: Build Assets - template: /eng/pipelines/common/upload-artifact-step.yml parameters: @@ -1586,7 +1586,7 @@ extends: jobParameters: testGroup: innerloop liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) unifiedBuildNameSuffix: Libraries_CheckedCoreCLR unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} condition: >- From 082b5e7bb186d3a8a297f7ab20e839e858d6403d Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 29 Feb 2024 14:10:57 -0800 Subject: [PATCH 46/79] Fix build command for the installer jobs on mac --- eng/pipelines/installer/jobs/build-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml index a0b2e371bed95..ee0b2c43deda9 100644 --- a/eng/pipelines/installer/jobs/build-job.yml +++ b/eng/pipelines/installer/jobs/build-job.yml @@ -114,7 +114,7 @@ jobs: - name: BaseJobBuildCommand value: >- - $(Build.SourcesDirectory)/build.sh -ci + $(Build.SourcesDirectory)/build.sh -subset host+packs -ci $(BuildAction) -configuration $(_BuildConfig) -arch ${{ parameters.archType }} From 2571c112d7d1649958df32621cd936815987a4ee Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 29 Feb 2024 16:34:45 -0800 Subject: [PATCH 47/79] Fix config overrride for test artifacts --- .../runtimes/native-test-assets-variables.yml | 12 ++++++++++-- eng/pipelines/runtime.yml | 5 ++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml index 9b84882f17f45..bb828a64cae1a 100644 --- a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml +++ b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -5,6 +5,7 @@ parameters: runtimeVariant: '' liveLibrariesBuildConfig: '' testGroup: 'innerloop' + configOverride: '' variables: - name: binTestsPath @@ -24,11 +25,18 @@ variables: - name: managedTestArtifactRootFolderPath value: '$(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper)' + - name: nativeTestArtifactConfig + value: $(_BuildConfig) + + - ${{ if ne(parameters.configOverride, '') }}: + - name: nativeTestArtifactConfig + value: ${{ parameters.configOverride }} + - name: nativeTestArtifactName - value: 'CoreCLRNativeTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)' + value: 'CoreCLRNativeTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(nativeTestArtifactConfig)' - name: nativeTestArtifactRootFolderPath - value: '$(binTestsPath)/obj/$(osGroup).$(archType).$(buildConfigUpper)' + value: '$(binTestsPath)/obj/$(osGroup).$(archType).$(nativeTestArtifactConfig)' - name: microsoftNetSdkIlFolderPath value: '$(Build.SourcesDirectory)/.packages/microsoft.net.sdk.il' diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 5cfa50ffef4b8..9783a26f7694e 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -217,6 +217,7 @@ extends: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml parameters: testGroup: innerloop + configOverride: Checked condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), @@ -250,6 +251,7 @@ extends: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml parameters: testGroup: innerloop + configOverride: Checked condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), @@ -315,6 +317,7 @@ extends: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml parameters: testGroup: innerloop + configOverride: Checked condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr_AppleSilicon.containsChange'], true), @@ -383,7 +386,7 @@ extends: archiveType: $(archiveType) archiveExtension: $(archiveExtension) tarCompression: $(tarCompression) - artifactName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: CoreCLR_ReleaseLibraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), From 1c33a9c7d25e1ecd40d42aac6811538cde79d30a Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 1 Mar 2024 10:06:52 -0800 Subject: [PATCH 48/79] Build the native test components that match the config we want to upload. --- .../coreclr/templates/build-native-test-assets-step.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/build-native-test-assets-step.yml b/eng/pipelines/coreclr/templates/build-native-test-assets-step.yml index 9e65b9875ff31..e8d1fcfe69cc1 100644 --- a/eng/pipelines/coreclr/templates/build-native-test-assets-step.yml +++ b/eng/pipelines/coreclr/templates/build-native-test-assets-step.yml @@ -3,7 +3,7 @@ parameters: compiler: '' steps: - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(_BuildConfig) $(archType) $(crossArg) $(priorityArg) ${{ parameters.compiler }} + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(nativeTestArtifactConfig) $(archType) $(crossArg) $(priorityArg) ${{ parameters.compiler }} displayName: Build native test components - template: /eng/pipelines/common/upload-artifact-step.yml parameters: From c9f3f67bfef74fe3bcb4ca938a43e0faa8bc5db1 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 1 Mar 2024 11:53:41 -0800 Subject: [PATCH 49/79] Fix name suffix --- eng/pipelines/runtime.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 9783a26f7694e..615a0318030fb 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -1195,7 +1195,7 @@ extends: - windows_x86 # - windows_arm64 jobParameters: - buildSuffix: Mono_Runtime + nameSuffix: Mono_Runtime buildArgs: -s mono -c $(_BuildConfig) condition: >- or( From 6fb17faddcb9b44fe6d586217766d36213ff4968 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 4 Mar 2024 09:29:47 -0800 Subject: [PATCH 50/79] Fix test group and pass platforms everywhere necessary in jit-outerloop-pipeline. --- eng/pipelines/coreclr/jit-experimental.yml | 2 +- .../coreclr/templates/jit-outerloop-pipeline.yml | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/coreclr/jit-experimental.yml b/eng/pipelines/coreclr/jit-experimental.yml index 42b2a607ac9f4..56453d894796d 100644 --- a/eng/pipelines/coreclr/jit-experimental.yml +++ b/eng/pipelines/coreclr/jit-experimental.yml @@ -21,4 +21,4 @@ extends: - linux_x64 - windows_x64 - windows_arm64 - testGroup: jitstress-random \ No newline at end of file + testGroup: jit-experimental \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml b/eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml index 4d76aa00e28ca..1e9c70114c561 100644 --- a/eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml +++ b/eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml @@ -48,15 +48,7 @@ extends: parameters: jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 + platforms: ${{ parameters.platforms }} helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: From 3264d2d4cc6ab8431bbee2100ef990900867b55b Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 4 Mar 2024 09:30:30 -0800 Subject: [PATCH 51/79] Add change in JIT subtree to validate JIT pipelines --- src/coreclr/jit/jit.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/jit/jit.h b/src/coreclr/jit/jit.h index 1df8c034d0c1d..b94e73eb6c196 100644 --- a/src/coreclr/jit/jit.h +++ b/src/coreclr/jit/jit.h @@ -6,6 +6,7 @@ #define _JIT_H_ /*****************************************************************************/ + // // clr.sln only defines _DEBUG // The jit uses DEBUG rather than _DEBUG From d83e27469d65d362a67532057e3d1fb6477c7cb1 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 4 Mar 2024 09:33:26 -0800 Subject: [PATCH 52/79] Fix win-x86 libraries test job config --- eng/pipelines/runtime.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index ab1477f56dec4..7c8337a38c4ad 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -1632,10 +1632,9 @@ extends: isOfficialBuild: false testScope: innerloop liveRuntimeBuildConfig: release - unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) - helixArtifactsName: CoreCLR_Libraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + helixArtifactsName: CoreCLR_Libraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) unifiedBuildNameSuffix: CoreCLR_Libraries - unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), From ad00fc159c85233f37fba3771392becd22342400 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 4 Mar 2024 11:45:21 -0800 Subject: [PATCH 53/79] Fix path overrides for cross-config installer builds. --- eng/pipelines/installer/jobs/build-job.yml | 12 ++++++++++++ .../installer/jobs/steps/build-linux-package.yml | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml index ee0b2c43deda9..a59953c64310e 100644 --- a/eng/pipelines/installer/jobs/build-job.yml +++ b/eng/pipelines/installer/jobs/build-job.yml @@ -82,6 +82,12 @@ jobs: - name: buildCommandSourcesDirectory value: '$(Build.SourcesDirectory)/' + - name: librariesBuildConfigArg + value: -lc ${{ coalesce(parameters.liveLibrariesBuildConfig, parameters.buildConfig) }} + + - name: runtimeBuildConfigArg + value: -rc ${{ coalesce(parameters.liveRuntimeBuildConfig, parameters.buildConfig) }} + ### ### Platform-specific variable setup ### @@ -101,6 +107,8 @@ jobs: build.cmd -subset host+packs -ci $(BuildAction) -configuration $(_BuildConfig) + $(librariesBuildConfigArg) + $(runtimeBuildConfigArg) $(CommonMSBuildArgs) - ${{ elseif eq(parameters.osGroup, 'osx') }}: @@ -118,6 +126,8 @@ jobs: $(BuildAction) -configuration $(_BuildConfig) -arch ${{ parameters.archType }} + $(librariesBuildConfigArg) + $(runtimeBuildConfigArg) $(CommonMSBuildArgs) - ${{ elseif in(parameters.osGroup, 'linux', 'freebsd') }}: @@ -143,6 +153,8 @@ jobs: value: >- $(Build.SourcesDirectory)/build.sh -subset host+packs -ci $(BuildAction) + $(librariesBuildConfigArg) + $(runtimeBuildConfigArg) $(ArchArg) /p:CrossBuild=${{ parameters.crossBuild }} /p:PortableBuild=$(_PortableBuild) diff --git a/eng/pipelines/installer/jobs/steps/build-linux-package.yml b/eng/pipelines/installer/jobs/steps/build-linux-package.yml index 102eab770c278..7f8320b55dfe3 100644 --- a/eng/pipelines/installer/jobs/steps/build-linux-package.yml +++ b/eng/pipelines/installer/jobs/steps/build-linux-package.yml @@ -17,7 +17,8 @@ steps: ${{ parameters.subsetArg }} \ ${{ parameters.packagingArgs }} \ $(CommonMSBuildArgs) \ - $(LiveOverridePathArgs) \ + $(librariesBuildConfigArg) \ + $(runtimeBuildConfigArg) \ /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog displayName: Package ${{ parameters.packageStepDescription }} - ${{ parameters.packageType }} target: ${{ parameters.target }} From 6d1c641067e551778e1f05e7019bb6e8ed20ed77 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 4 Mar 2024 11:52:45 -0800 Subject: [PATCH 54/79] Fix config problem with Win-x64 minijit tests --- eng/pipelines/runtime.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 7c8337a38c4ad..e796e0176b356 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -86,7 +86,7 @@ extends: or( eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) - + - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml @@ -178,7 +178,7 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(variables['isRollingBuild'], true)) - + # # Build CoreCLR and Libraries with the respective tests # for the test configurations we run. @@ -1456,6 +1456,8 @@ extends: testRunNamePrefixSuffix: Mono_Release extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml + parameters: + liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} # # Mono CoreCLR runtime Test executions using live libraries in interpreter mode From 483086e07e682b19b3fd029f4c5c5a75abd380f1 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 4 Mar 2024 12:01:17 -0800 Subject: [PATCH 55/79] Flip configuration override to ensure we see the right files. --- eng/pipelines/runtime.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index e796e0176b356..9611a7e3a773c 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -119,7 +119,7 @@ extends: - windows_x64 jobParameters: nameSuffix: CoreCLR_Libraries - buildArgs: -s clr+libs+libs.tests -c Release -lc $(_BuildConfig) /p:ArchiveTests=true + buildArgs: -s clr+libs+libs.tests -rc Release -c $(_BuildConfig) /p:ArchiveTests=true timeoutInMinutes: 120 postBuildSteps: - template: /eng/pipelines/common/upload-artifact-step.yml From 7e00a38796d1dead2f38875914726f8372b61bed Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 4 Mar 2024 12:03:52 -0800 Subject: [PATCH 56/79] Fix superpmi artifacts names. --- eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml | 2 +- eng/pipelines/coreclr/superpmi-diffs.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml b/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml index e5dcc6800e1b2..39020844e68c5 100644 --- a/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml +++ b/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml @@ -55,7 +55,7 @@ extends: archiveType: $(archiveType) tarCompression: $(tarCompression) archiveExtension: $(archiveExtension) - artifactName: ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType) displayName: JIT and SuperPMI Assets - template: /eng/pipelines/common/platform-matrix.yml diff --git a/eng/pipelines/coreclr/superpmi-diffs.yml b/eng/pipelines/coreclr/superpmi-diffs.yml index d97452f841c71..a8c816fc02642 100644 --- a/eng/pipelines/coreclr/superpmi-diffs.yml +++ b/eng/pipelines/coreclr/superpmi-diffs.yml @@ -11,7 +11,7 @@ parameters: # This pipeline only runs on GitHub PRs, not on merges. trigger: none -# Only run on changes to the JIT directory. +# Only run on changes to the JIT directory. pr: branches: include: @@ -82,7 +82,7 @@ extends: archiveType: $(archiveType) tarCompression: $(tarCompression) archiveExtension: $(archiveExtension) - artifactName: ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + artifactName: ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType) displayName: JIT and SuperPMI Assets condition: not(eq(dependencies.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true)) From 9bd50b2478e72d2a385bb1c0f132381fc40b1209 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 7 Mar 2024 17:49:40 -0800 Subject: [PATCH 57/79] Disable PGO optimization on the superpmi and jit builds. Create .dotnet-mono folder from the test host to keep perf harnesses running --- eng/pipelines/coreclr/jitrollingbuild.yml | 2 +- eng/pipelines/coreclr/superpmi-diffs.yml | 2 +- eng/pipelines/coreclr/templates/perf-job.yml | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/jitrollingbuild.yml b/eng/pipelines/coreclr/jitrollingbuild.yml index 35f06788b49a9..61f0d2d05d80e 100644 --- a/eng/pipelines/coreclr/jitrollingbuild.yml +++ b/eng/pipelines/coreclr/jitrollingbuild.yml @@ -58,7 +58,7 @@ extends: - windows_arm64 jobParameters: disableComponentGovernance: true # Not a shipping artifact - buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) -p:NoPgoOptimize=true postBuildSteps: - template: /eng/pipelines/coreclr/templates/upload-jits-steps.yml extraVariablesTemplates: diff --git a/eng/pipelines/coreclr/superpmi-diffs.yml b/eng/pipelines/coreclr/superpmi-diffs.yml index a8c816fc02642..a7d63ef8f0e67 100644 --- a/eng/pipelines/coreclr/superpmi-diffs.yml +++ b/eng/pipelines/coreclr/superpmi-diffs.yml @@ -73,7 +73,7 @@ extends: - windows_x86 - linux_x64 jobParameters: - buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) -p:NoPgoOptimize=true postBuildSteps: - template: /eng/pipelines/common/upload-artifact-step.yml parameters: diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index d1d7960c1788b..e67e622d19b0b 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -267,3 +267,10 @@ jobs: - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) $(_crossBuildPropertyArg) displayName: Create Core_Root condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono'), ne('${{ parameters.runtimeType }}', 'wasm'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) + + - task: CopyFiles@2 + displayName: Create .dotnet-mono folder + inputs: + SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/testhost/${{ parameters.framework }}-$(osGroup)-$(buildConfigUpper)-$(archType) + TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono + condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) From dc2c2606184006dac9c860c5053538cd576603f9 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 7 Mar 2024 17:57:47 -0800 Subject: [PATCH 58/79] Add nopgooptimize for one more job --- eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml b/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml index 39020844e68c5..e0e86db6fc2a2 100644 --- a/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml +++ b/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml @@ -46,7 +46,7 @@ extends: - windows_x86 - linux_x64 jobParameters: - buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) -p:NoPgoOptimize=true postBuildSteps: - template: /eng/pipelines/common/upload-artifact-step.yml parameters: From f379635881d1d2cfab3625908da6a63d457cec9a Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 7 Mar 2024 17:59:11 -0800 Subject: [PATCH 59/79] Copy corerun to .dotnet-mono --- eng/pipelines/coreclr/templates/perf-job.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index e67e622d19b0b..bd7bfbefad86e 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -274,3 +274,11 @@ jobs: SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/testhost/${{ parameters.framework }}-$(osGroup)-$(buildConfigUpper)-$(archType) TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) + + - task: CopyFiles@2 + displayName: Copy corerun to .dotnet-mono + inputs: + SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper) + Contents: 'corerun*' + TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono + condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) From f4b698b9945eff092181aceec6bf609f8d14ba3f Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 7 Mar 2024 18:01:25 -0800 Subject: [PATCH 60/79] Fix job template --- eng/pipelines/coreclr/jitrollingbuild.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/jitrollingbuild.yml b/eng/pipelines/coreclr/jitrollingbuild.yml index 61f0d2d05d80e..a6536f33fed4b 100644 --- a/eng/pipelines/coreclr/jitrollingbuild.yml +++ b/eng/pipelines/coreclr/jitrollingbuild.yml @@ -24,7 +24,7 @@ extends: jobs: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/platform-matrix.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - osx_arm64 @@ -45,7 +45,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/platform-matrix.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - osx_arm64 From be71b347ff95700c07064200f89e90df75a67845 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 7 Mar 2024 18:04:50 -0800 Subject: [PATCH 61/79] Fix template path --- eng/pipelines/coreclr/superpmi-collect.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 7f3bf2343766c..9e236e284dd75 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -62,7 +62,7 @@ extends: tarCompression: $(tarCompression) artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) extraVariablesTemplates: - - template: /eng/pipelines/coreclr/templates/native-test-assets-variables.yml + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml parameters: testGroup: outerloop disableComponentGovernance: true # No shipping artifacts produced by this pipeline @@ -102,7 +102,7 @@ extends: tarCompression: $(tarCompression) artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) extraVariablesTemplates: - - template: /eng/pipelines/coreclr/templates/native-test-assets-variables.yml + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml parameters: testGroup: outerloop disableComponentGovernance: true # No shipping artifacts produced by this pipeline @@ -142,7 +142,7 @@ extends: tarCompression: $(tarCompression) artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) extraVariablesTemplates: - - template: /eng/pipelines/coreclr/templates/native-test-assets-variables.yml + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml parameters: testGroup: outerloop disableComponentGovernance: true # No shipping artifacts produced by this pipeline From bfbe4fc8e5c83494cc8013242bfca1278b406507 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 7 Mar 2024 18:08:00 -0800 Subject: [PATCH 62/79] Use target instead of container --- eng/pipelines/common/templates/global-build-step.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/common/templates/global-build-step.yml b/eng/pipelines/common/templates/global-build-step.yml index 172d4d3127be0..6597ac4feaefc 100644 --- a/eng/pipelines/common/templates/global-build-step.yml +++ b/eng/pipelines/common/templates/global-build-step.yml @@ -12,4 +12,4 @@ steps: ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}: continueOnError: ${{ parameters.shouldContinueOnError }} ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} + target: ${{ parameters.container }} From 37c806e73e3b6326a0e41635437a19de7e8097fe Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 7 Mar 2024 18:12:47 -0800 Subject: [PATCH 63/79] Fix superpmi job dependencies --- eng/pipelines/coreclr/templates/superpmi-collect-job.yml | 9 ++------- eng/pipelines/coreclr/templates/superpmi-replay-job.yml | 4 ++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml index df59120dd779e..6af16c3db8764 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml @@ -29,12 +29,7 @@ jobs: collectionName: ${{ parameters.collectionName }} # Test job depends on the corresponding build job dependsOn: - - ${{ format('coreclr__product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - # Depend on coreclr x64 so we can download it and use mcs.exe from it while publishing non-x64 arch SPMI collection - - ${{ if ne(parameters.archType, 'x64') }}: - - ${{ format('coreclr__product_build_{0}{1}_x64_{2}', parameters.osGroup, parameters.osSubgroup, parameters.buildConfig) }} - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_' - ${{ if eq(parameters.collectionName, 'coreclr_tests') }}: - 'coreclr_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}' @@ -42,7 +37,7 @@ jobs: steps: # Extra steps that will be passed to the superpmi template and run before sending the job to helix (all of which is done in the template) - + # Download runtime artifacts - template: /eng/pipelines/common/download-artifact-step.yml parameters: diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 1c234e2984ea9..59b7ef9ded0c6 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -26,7 +26,7 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} helixQueues: ${{ parameters.helixQueues }} dependsOn: - - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_' variables: ${{ parameters.variables }} @@ -39,4 +39,4 @@ jobs: artifactFileName: 'CheckedJIT_$(osGroup)$(osSubgroup)_$(archType)$(archiveExtension)' artifactName: 'CheckedJIT_$(osGroup)$(osSubgroup)_$(archType)' displayName: 'JIT checked build' - cleanupUnpackFolder: false \ No newline at end of file + cleanupUnpackFolder: false From 8839c53f937609ffb31b58818ea1272f43f2108e Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 7 Mar 2024 18:15:40 -0800 Subject: [PATCH 64/79] Override build config --- eng/pipelines/coreclr/superpmi-collect.yml | 2 ++ eng/pipelines/libraries/run-test-job.yml | 11 +++-------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 9e236e284dd75..a7aa80e8cba6c 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -345,6 +345,7 @@ extends: SuperPmiCollectionName: libraries_tests unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedBuildConfigOverride: checked # # Collection of libraries test run: no_tiered_compilation @@ -375,3 +376,4 @@ extends: SuperPmiCollectionName: libraries_tests_no_tiered_compilation unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedBuildConfigOverride: checked diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index e56a1519cb74e..c92c805365363 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -59,13 +59,8 @@ jobs: - evaluate_paths - ${{ if ne(parameters.dependsOn[0], '') }}: - ${{ parameters.dependsOn }} - - ${{ if eq(parameters.dependsOn[0], '') }}: - - ${{ if ne(parameters.unifiedArtifactsName, '')}}: - - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ coalesce(parameters.unifiedBuildConfigOverride, parameters.buildConfig) }}_${{ parameters.unifiedBuildNameSuffix }}' - - ${{ else }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} + - ${{ else }}: + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ coalesce(parameters.unifiedBuildConfigOverride, parameters.buildConfig) }}_${{ parameters.unifiedBuildNameSuffix }}' workspace: clean: all @@ -151,7 +146,7 @@ jobs: SuperPmiCollect: ${{ parameters.SuperPmiCollect }} SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} - + ${{ if ne(parameters.scenarios[0], '') }}: scenarios: ${{ parameters.scenarios }} From c3febe23b29a0ac41afb2d72bc978a5476e4d374 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 7 Mar 2024 18:30:26 -0800 Subject: [PATCH 65/79] Use /p instead of -p --- eng/pipelines/coreclr/jitrollingbuild.yml | 2 +- eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml | 2 +- eng/pipelines/coreclr/superpmi-diffs.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/coreclr/jitrollingbuild.yml b/eng/pipelines/coreclr/jitrollingbuild.yml index a6536f33fed4b..c99557f4c3cc9 100644 --- a/eng/pipelines/coreclr/jitrollingbuild.yml +++ b/eng/pipelines/coreclr/jitrollingbuild.yml @@ -58,7 +58,7 @@ extends: - windows_arm64 jobParameters: disableComponentGovernance: true # Not a shipping artifact - buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) -p:NoPgoOptimize=true + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) /p:NoPgoOptimize=true postBuildSteps: - template: /eng/pipelines/coreclr/templates/upload-jits-steps.yml extraVariablesTemplates: diff --git a/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml b/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml index e0e86db6fc2a2..4d25196bdc8b1 100644 --- a/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml +++ b/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml @@ -46,7 +46,7 @@ extends: - windows_x86 - linux_x64 jobParameters: - buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) -p:NoPgoOptimize=true + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) /p:NoPgoOptimize=true postBuildSteps: - template: /eng/pipelines/common/upload-artifact-step.yml parameters: diff --git a/eng/pipelines/coreclr/superpmi-diffs.yml b/eng/pipelines/coreclr/superpmi-diffs.yml index a7d63ef8f0e67..1fd327f838c70 100644 --- a/eng/pipelines/coreclr/superpmi-diffs.yml +++ b/eng/pipelines/coreclr/superpmi-diffs.yml @@ -73,7 +73,7 @@ extends: - windows_x86 - linux_x64 jobParameters: - buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) -p:NoPgoOptimize=true + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) /p:NoPgoOptimize=true postBuildSteps: - template: /eng/pipelines/common/upload-artifact-step.yml parameters: From 6918f84323f97ba2a06d45c86f6de21484e69d4d Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 8 Mar 2024 17:05:22 -0800 Subject: [PATCH 66/79] Build libraries tests on all the platforms --- eng/pipelines/coreclr/superpmi-collect.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index a7aa80e8cba6c..293c70f6d34f3 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -77,7 +77,7 @@ extends: - osx_arm64 jobParameters: testGroup: outerloop - buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -lc Release postBuildSteps: # Build CLR assets for x64 as well as the target as we need an x64 mcs - template: /eng/pipelines/common/templates/global-build-step.yml @@ -116,7 +116,7 @@ extends: - linux_arm64 jobParameters: testGroup: outerloop - buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -lc Release postBuildSteps: # Build CLR assets for x64 as well as the target as we need an x64 mcs - template: /eng/pipelines/common/templates/global-build-step.yml From 22820dbf01453a733a0eff00bc1da8ddf7fc94d5 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 11 Mar 2024 11:45:50 -0700 Subject: [PATCH 67/79] Set ArchiveTests --- eng/pipelines/coreclr/superpmi-collect.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 293c70f6d34f3..494689490f64d 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -77,7 +77,7 @@ extends: - osx_arm64 jobParameters: testGroup: outerloop - buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -lc Release + buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -lc Release /p:ArchiveTests=true postBuildSteps: # Build CLR assets for x64 as well as the target as we need an x64 mcs - template: /eng/pipelines/common/templates/global-build-step.yml @@ -116,7 +116,7 @@ extends: - linux_arm64 jobParameters: testGroup: outerloop - buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -lc Release + buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -lc Release /p:ArchiveTests=true postBuildSteps: # Build CLR assets for x64 as well as the target as we need an x64 mcs - template: /eng/pipelines/common/templates/global-build-step.yml From 2932033cb047631fc2289eefdcc6e14873c4e3be Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 11 Mar 2024 13:23:47 -0700 Subject: [PATCH 68/79] More SuperPMI fixes --- .../templates/runtimes/run-test-job.yml | 24 ------------------- eng/pipelines/coreclr/superpmi-collect.yml | 8 +++---- eng/pipelines/libraries/run-test-job.yml | 1 - 3 files changed, 4 insertions(+), 29 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 2d6bae58e08a7..0f0cf346eeca8 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -65,20 +65,6 @@ jobs: - '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}' - ${{ if ne(parameters.unifiedArtifactsName, '')}}: - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ coalesce(parameters.unifiedBuildConfigOverride, parameters.buildConfig) }}_${{ parameters.unifiedBuildNameSuffix }}' - - ${{ else }}: - - ${{ if or( eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter'), eq(parameters.runtimeVariant, 'llvmaot'), eq(parameters.runtimeVariant, 'llvmfullaot')) }}: - # This is needed for creating a CORE_ROOT in the current design. - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', '', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if or( eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter')) }} : - # minijit and mono interpreter runtimevariants do not require any special build of the runtime - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, '', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if not(or(eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter'))) }}: - - ${{ if eq(parameters.runtimeVariant, 'llvmfullaot') }}: - - ${{ format('{0}_llvmaot_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.runtimeVariant, 'llvmfullaot') }}: - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} # Compute job name from template parameters ${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: @@ -236,16 +222,6 @@ jobs: artifactName: '$(nativeTestArtifactName)' displayName: 'native test artifacts' - # SuperPMI collection: Download x64 coreclr if running on non-x64 configuration (needed for mcs.exe) - - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper)' - artifactFileName: 'CoreCLRProduct__$(osGroup)$(osSubgroup)_x64_$(buildConfig)$(archiveExtension)' - artifactName: 'CoreCLRProduct__$(osGroup)$(osSubgroup)_x64_$(buildConfig)' - displayName: 'CoreCLR product build (x64)' - - # Publish native test components to test output folder. Sadly we cannot do this # during product build (so that we could zip up the files in their final test location # and directly unzip them there after download). Unfortunately the logic to copy diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 494689490f64d..b4cf950abc260 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -343,8 +343,8 @@ extends: - normal SuperPmiCollect: true SuperPmiCollectionName: libraries_tests - unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked unifiedBuildConfigOverride: checked # @@ -374,6 +374,6 @@ extends: - no_tiered_compilation SuperPmiCollect: true SuperPmiCollectionName: libraries_tests_no_tiered_compilation - unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked unifiedBuildConfigOverride: checked diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index c92c805365363..581b2ea9c9ede 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -70,7 +70,6 @@ jobs: - group: DotNet-HelixApi-Access - group: AzureDevOps-Artifact-Feeds-Pats - - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - _archiveTestsParameter: /p:ArchiveTests=true - ${{ if eq(parameters.SuperPmiCollect, true) }}: From ec08b1a58464b3611317df9bd11a1820e7898400 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 11 Mar 2024 13:48:27 -0700 Subject: [PATCH 69/79] Update JIT-EE GUID to regenerate collections --- src/coreclr/inc/jiteeversionguid.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index 11675936acfa3..d52c58dde499a 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -43,11 +43,11 @@ typedef const GUID *LPCGUID; #define GUID_DEFINED #endif // !GUID_DEFINED -constexpr GUID JITEEVersionIdentifier = { /* 86eab154-5d93-4fad-bc07-e94fd9268b70 */ - 0x86eab154, - 0x5d93, - 0x4fad, - {0xbc, 0x07, 0xe9, 0x4f, 0xd9, 0x26, 0x8b, 0x70} +constexpr GUID JITEEVersionIdentifier = { /* f3f99424-a3ad-4a01-9ed0-fd6ade3fba84*/ + 0xf3f99424, + 0xa3ad, + 0x4a01, + {0x9e, 0xd0, 0xfd, 0x6a, 0xde, 0x3f, 0xba, 0x84} }; ////////////////////////////////////////////////////////////////////////////////////////////////////////// From 8b40e051238f848262fefd21caf88726beb5dbe5 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 11 Mar 2024 16:30:07 -0700 Subject: [PATCH 70/79] Resolve SuperPMI assets based on the unified artifacts layout (now that we don't have the transport layout any more) --- eng/pipelines/libraries/run-test-job.yml | 2 ++ src/libraries/sendtohelix.proj | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 581b2ea9c9ede..78dd68907c7d6 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -145,6 +145,8 @@ jobs: SuperPmiCollect: ${{ parameters.SuperPmiCollect }} SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} + ${{ if eq(parameters.SuperPmiCollect, true) }}: + extraHelixArguments: /p:RuntimeConfiguration=${{ parameters.liveRuntimeBuildConfig }} ${{ if ne(parameters.scenarios[0], '') }}: scenarios: ${{ parameters.scenarios }} diff --git a/src/libraries/sendtohelix.proj b/src/libraries/sendtohelix.proj index 44a02cd5b828e..803353abaa8a0 100644 --- a/src/libraries/sendtohelix.proj +++ b/src/libraries/sendtohelix.proj @@ -201,7 +201,8 @@ + Condition="'$(SuperPmiCollect)' == 'true'" + DependsOnTargets="ResolveRuntimeFilesFromLocalBuild"> @@ -220,10 +221,10 @@ (superpmi, mcs, superpmi-shim-collector, clrjit), but it's easier to just copy everything. --> - + - + From 30eb6ccd821b73f5b28a684a16c1b78857d79c56 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 12 Mar 2024 11:46:37 -0700 Subject: [PATCH 71/79] Fix configuration, x64 tools lookup, refactor correlation payload into separate directory --- eng/pipelines/coreclr/superpmi-collect.yml | 10 +++++----- eng/pipelines/libraries/superpmi-collect-variables.yml | 8 ++++---- src/libraries/sendtohelix-superpmi-collect.targets | 5 +++++ src/libraries/sendtohelix.proj | 10 +++++++--- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index b4cf950abc260..e784fe45be716 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -42,7 +42,7 @@ extends: - linux_x64 jobParameters: testGroup: outerloop - buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -lc Release /p:ArchiveTests=true + buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true postBuildSteps: - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml - template: /eng/pipelines/common/upload-artifact-step.yml @@ -77,7 +77,7 @@ extends: - osx_arm64 jobParameters: testGroup: outerloop - buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -lc Release /p:ArchiveTests=true + buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true postBuildSteps: # Build CLR assets for x64 as well as the target as we need an x64 mcs - template: /eng/pipelines/common/templates/global-build-step.yml @@ -116,7 +116,7 @@ extends: - linux_arm64 jobParameters: testGroup: outerloop - buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -lc Release /p:ArchiveTests=true + buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true postBuildSteps: # Build CLR assets for x64 as well as the target as we need an x64 mcs - template: /eng/pipelines/common/templates/global-build-step.yml @@ -336,7 +336,7 @@ extends: helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testScope: innerloop - liveRuntimeBuildConfig: checked + liveRuntimeBuildConfig: Checked dependsOnTestBuildConfiguration: Release dependsOnTestArchitecture: x64 scenarios: @@ -367,7 +367,7 @@ extends: helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testScope: innerloop - liveRuntimeBuildConfig: checked + liveRuntimeBuildConfig: Checked dependsOnTestBuildConfiguration: Release dependsOnTestArchitecture: x64 scenarios: diff --git a/eng/pipelines/libraries/superpmi-collect-variables.yml b/eng/pipelines/libraries/superpmi-collect-variables.yml index 4cfde9ea0a0b7..8eda0f8b6e6a5 100644 --- a/eng/pipelines/libraries/superpmi-collect-variables.yml +++ b/eng/pipelines/libraries/superpmi-collect-variables.yml @@ -13,13 +13,13 @@ variables: value: 'Release' - name: _runtimeX64DownloadPath value: '' - # superpmi.py 'merge-mch' needs to be able to find the mcs tool. Point SuperPmiMcsPath at the downloaded CoreCLR binaries. For non-x64 targets, download an x64 - # build and point at that. Pass this to superpmi.py as the '-core_root' argument. It's not actually a "Core_Root" directory, but all it needs is to find mcs. + # superpmi.py 'merge-mch' needs to be able to find the mcs tool. Point SuperPmiMcsPath at the downloaded CoreCLR binaries. For non-x64 targets, we'll also have an x64 mcs tool available. + # so point the non-x64 builds at the x64 artifacts. - name: SuperPmiMcsPath value: $(_runtimeDownloadPath) - ${{ if ne(parameters.archType, 'x64') }}: - name: _runtimeX64DownloadPath - value: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}.x64' + value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/${{ parameters.osGroup }}.x64.$(buildConfigUpper)/' - name: SuperPmiMcsPath value: $(_runtimeX64DownloadPath) - ${{ if eq(parameters.osGroup, 'windows') }}: @@ -35,4 +35,4 @@ variables: - name: MergedMchFileLocation value: '$(Build.SourcesDirectory)/artifacts/spmi_collection/' - name: SpmiLogsLocation - value: '$(Build.SourcesDirectory)/artifacts/spmi_logs/' \ No newline at end of file + value: '$(Build.SourcesDirectory)/artifacts/spmi_logs/' diff --git a/src/libraries/sendtohelix-superpmi-collect.targets b/src/libraries/sendtohelix-superpmi-collect.targets index 5affc58dd36ef..9a48cc9a34e11 100644 --- a/src/libraries/sendtohelix-superpmi-collect.targets +++ b/src/libraries/sendtohelix-superpmi-collect.targets @@ -56,4 +56,9 @@ + + + + + diff --git a/src/libraries/sendtohelix.proj b/src/libraries/sendtohelix.proj index 803353abaa8a0..17aa91b1597bb 100644 --- a/src/libraries/sendtohelix.proj +++ b/src/libraries/sendtohelix.proj @@ -23,7 +23,7 @@ SetStressModes_$(Scenario).sh - + @@ -221,12 +221,16 @@ (superpmi, mcs, superpmi-shim-collector, clrjit), but it's easier to just copy everything. --> - + - + + + Date: Tue, 12 Mar 2024 14:08:29 -0700 Subject: [PATCH 72/79] Fix paths, conditions, item names --- .../coreclr/templates/run-superpmi-collect-job.yml | 3 ++- eng/pipelines/libraries/superpmi-collect-variables.yml | 6 ++---- eng/pipelines/libraries/superpmi-postprocess-step.yml | 5 +++-- src/libraries/sendtohelix-superpmi-collect.targets | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index dad4986181c85..5580169122022 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -89,7 +89,7 @@ jobs: value: '$(Core_Root_Dir)' - ${{ if eq(parameters.collectionName, 'realworld') }}: - name: InputDirectory - value: '$(Core_Root_Dir)' + value: '$(Core_Root_Dir)' - ${{ if eq(parameters.collectionName, 'coreclr_tests') }}: - name: InputDirectory value: '$(managedTestArtifactRootFolderPath)' @@ -110,6 +110,7 @@ jobs: - script: $(PythonSetupScript) displayName: Enable python venv + condition: always() - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} diff --git a/eng/pipelines/libraries/superpmi-collect-variables.yml b/eng/pipelines/libraries/superpmi-collect-variables.yml index 8eda0f8b6e6a5..82ead1c26ca79 100644 --- a/eng/pipelines/libraries/superpmi-collect-variables.yml +++ b/eng/pipelines/libraries/superpmi-collect-variables.yml @@ -16,12 +16,10 @@ variables: # superpmi.py 'merge-mch' needs to be able to find the mcs tool. Point SuperPmiMcsPath at the downloaded CoreCLR binaries. For non-x64 targets, we'll also have an x64 mcs tool available. # so point the non-x64 builds at the x64 artifacts. - name: SuperPmiMcsPath - value: $(_runtimeDownloadPath) + value: $(Build.SourcesDirectory)/artifacts/bin/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper) - ${{ if ne(parameters.archType, 'x64') }}: - - name: _runtimeX64DownloadPath - value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/${{ parameters.osGroup }}.x64.$(buildConfigUpper)/' - name: SuperPmiMcsPath - value: $(_runtimeX64DownloadPath) + value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/${{ parameters.osGroup }}.x64.$(buildConfigUpper)/' - ${{ if eq(parameters.osGroup, 'windows') }}: - name: MchFilesLocation value: '$(Build.SourcesDirectory)\artifacts\helixresults\' diff --git a/eng/pipelines/libraries/superpmi-postprocess-step.yml b/eng/pipelines/libraries/superpmi-postprocess-step.yml index e4c47fcc2092a..5184564266d42 100644 --- a/eng/pipelines/libraries/superpmi-postprocess-step.yml +++ b/eng/pipelines/libraries/superpmi-postprocess-step.yml @@ -36,10 +36,11 @@ steps: mkdir ${{ parameters.MergedMchFileLocation }} mkdir ${{ parameters.SpmiLogsLocation }} displayName: 'Create SuperPMI directories' - condition: always() + condition: always() - script: ${{ parameters.PythonSetupScript }} displayName: Enable python venv + condition: always() - script: ${{ parameters.PythonScript }} $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch -log_level DEBUG -pattern ${{ parameters.MchFilesLocation }}${{ parameters.SuperPmiCollectionName }}.${{ parameters.SuperPmiCollectionType }}*.mch -output_mch_path ${{ parameters.MergedMchFileLocation }}${{ parameters.SuperPmiCollectionName }}.${{ parameters.SuperPmiCollectionType }}.${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}.mch -core_root ${{ parameters.SuperPmiMcsPath }} displayName: 'Merge ${{ parameters.SuperPmiCollectionName }}-${{ parameters.SuperPmiCollectionType }} SuperPMI collections' @@ -88,4 +89,4 @@ steps: inputs: targetPath: ${{ parameters.SpmiLogsLocation }} artifactName: 'SuperPMI_Logs_${{ parameters.SuperPmiCollectionName }}_${{ parameters.SuperPmiCollectionType }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' - condition: always() \ No newline at end of file + condition: always() diff --git a/src/libraries/sendtohelix-superpmi-collect.targets b/src/libraries/sendtohelix-superpmi-collect.targets index 9a48cc9a34e11..6c02e16db7c5f 100644 --- a/src/libraries/sendtohelix-superpmi-collect.targets +++ b/src/libraries/sendtohelix-superpmi-collect.targets @@ -58,7 +58,7 @@ - + From 496adeb2a1f83d05f82f146d8e364f3930cfb245 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 12 Mar 2024 16:21:56 -0700 Subject: [PATCH 73/79] Now that we're consistently using a venv, install the package in the venv --- eng/pipelines/common/templates/runtimes/run-test-job.yml | 2 +- eng/pipelines/libraries/superpmi-postprocess-step.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 0f0cf346eeca8..7176df0decd8a 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -615,7 +615,7 @@ jobs: condition: always() # Ensure the Python azure-storage-blob package is installed before doing the upload. - - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall + - script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall displayName: Upgrade Pip to latest and install azure-storage-blob Python package condition: always() diff --git a/eng/pipelines/libraries/superpmi-postprocess-step.yml b/eng/pipelines/libraries/superpmi-postprocess-step.yml index 5184564266d42..e9ae5d13f32f2 100644 --- a/eng/pipelines/libraries/superpmi-postprocess-step.yml +++ b/eng/pipelines/libraries/superpmi-postprocess-step.yml @@ -66,7 +66,7 @@ steps: condition: always() # Ensure the Python azure-storage-blob package is installed before doing the upload. - - script: ${{ parameters.PipScript }} install --user --upgrade pip && ${{ parameters.PipScript }} install --user azure.storage.blob==12.5.0 --force-reinstall + - script: ${{ parameters.PipScript }} install --upgrade pip && ${{ parameters.PipScript }} install azure.storage.blob==12.5.0 --force-reinstall displayName: Upgrade Pip to latest and install azure-storage-blob Python package condition: always() From 879dcc86d61d7a36618da84e875b7f1aaf1afb28 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 13 Mar 2024 13:17:38 -0700 Subject: [PATCH 74/79] We're passing the correct runtime type now. Update the condition for the .dotnet-mono folder to explicitly include wasm (instead of relying on an unset runtimeFlavorName --- eng/pipelines/coreclr/templates/perf-job.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index bd7bfbefad86e..b94179197f47f 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -266,14 +266,14 @@ jobs: # Create Core_Root - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) $(_crossBuildPropertyArg) displayName: Create Core_Root - condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono'), ne('${{ parameters.runtimeType }}', 'wasm'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) + condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono'), not(in('${{ parameters.runtimeType }}', 'wasm', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) - task: CopyFiles@2 displayName: Create .dotnet-mono folder inputs: SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/testhost/${{ parameters.framework }}-$(osGroup)-$(buildConfigUpper)-$(archType) TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono - condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) + condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'wasm', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) - task: CopyFiles@2 displayName: Copy corerun to .dotnet-mono @@ -281,4 +281,4 @@ jobs: SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper) Contents: 'corerun*' TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono - condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) + condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'wasm', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) From 0f336d3746795caa9995ceac1b709a82610ead20 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 13 Mar 2024 14:10:44 -0700 Subject: [PATCH 75/79] Specify testGroup into our variable template when using xplat-pipeline-job.yml --- eng/pipelines/coreclr/templates/xplat-pipeline-job.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index 2bb667b8b512b..baad6dca1a953 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -54,6 +54,7 @@ jobs: - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml parameters: runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} - ${{ each variable in parameters.variables }}: From b38ed639ad4d33acfe829c92519f1fb273c3ea29 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 13 Mar 2024 14:34:18 -0700 Subject: [PATCH 76/79] Remove xplat-pipeline-job template and insert the "native test assets" template in the jobs that were using the variables provided. Remove unused variables and move variables that were only used by the perf jobs into the perf jobs themselves --- .../templates/runtimes/build-test-job.yml | 8 ++- .../runtimes/native-test-assets-variables.yml | 10 +-- .../templates/runtimes/run-test-job.yml | 10 ++- .../crossgen2-comparison-build-job.yml | 2 +- .../templates/crossgen2-comparison-job.yml | 2 +- .../coreclr/templates/format-job.yml | 4 +- eng/pipelines/coreclr/templates/perf-job.yml | 9 +++ .../coreclr/templates/run-performance-job.yml | 2 +- .../coreclr/templates/run-scenarios-job.yml | 5 +- ...-superpmi-asmdiffs-checked-release-job.yml | 8 ++- .../templates/run-superpmi-collect-job.yml | 8 ++- .../templates/run-superpmi-diffs-job.yml | 8 ++- .../templates/run-superpmi-replay-job.yml | 8 ++- .../coreclr/templates/xplat-pipeline-job.yml | 63 ------------------- .../mono/templates/workloads-build.yml | 4 +- 15 files changed, 64 insertions(+), 87 deletions(-) delete mode 100644 eng/pipelines/coreclr/templates/xplat-pipeline-job.yml diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index 1b8ad3138a78c..65a7004a5cf1a 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -19,7 +19,7 @@ parameters: ### of the product build job). jobs: -- template: /eng/pipelines/${{ parameters.runtimeFlavor }}/templates/xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -82,6 +82,12 @@ jobs: - name: testTreeFilterArg value: 'tree GC/Scenarios/GCSimulator' + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + steps: # Install test build dependencies diff --git a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml index bb828a64cae1a..6351952b16954 100644 --- a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml +++ b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -27,7 +27,7 @@ variables: - name: nativeTestArtifactConfig value: $(_BuildConfig) - + - ${{ if ne(parameters.configOverride, '') }}: - name: nativeTestArtifactConfig value: ${{ parameters.configOverride }} @@ -44,20 +44,12 @@ variables: - name: microsoftNetSdkIlArtifactName value: 'MicrosoftNetSdkIlPackage_AnyOS_AnyCPU_$(buildConfig)' - - name: librariesBuildArtifactName - value : '' - name: librariesOverrideArg value : '' - - name: librariesDownloadDir - value : '' - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - name: librariesBuildArtifactName - value : ${{ format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - name: librariesOverrideArg value : ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' - - name: librariesDownloadDir - value : $(Build.SourcesDirectory)/artifacts - name: priorityArg value: '' diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 7176df0decd8a..3fce853aa4a0f 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -34,7 +34,7 @@ parameters: ### buildConfig and archType. jobs: -- template: /eng/pipelines/${{ parameters.runtimeFlavor }}/templates/xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -47,7 +47,7 @@ jobs: helixType: 'build/tests/' runtimeVariant: ${{ parameters.runtimeVariant }} pool: ${{ parameters.pool }} - condition: ${{ parameters.condition }} + condition: and(succeeded(), ${{ parameters.condition }}) dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} # Test jobs should continue on error for internal builds @@ -128,6 +128,12 @@ jobs: - name: testTreeFilterArg value: 'tree GC/Scenarios/GCSimulator' + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + # Variables used for SuperPMI collection - ${{ if eq(parameters.SuperPmiCollect, true) }}: - MchFileTag: '${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}' diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml index ea328a9c543ec..4964581693db5 100644 --- a/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml @@ -24,7 +24,7 @@ parameters: ### crossgen matches that of native, e.g. arm-hosted-arm-targeting, crossgen. jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml index 85a303c6e9dd6..25753737006b6 100644 --- a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml @@ -20,7 +20,7 @@ parameters: ### crossgen matches that of native, e.g. arm-hosted-arm-targeting, crossgen. jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} diff --git a/eng/pipelines/coreclr/templates/format-job.yml b/eng/pipelines/coreclr/templates/format-job.yml index bc749d4f84cdf..e310625b3ee83 100644 --- a/eng/pipelines/coreclr/templates/format-job.yml +++ b/eng/pipelines/coreclr/templates/format-job.yml @@ -13,7 +13,7 @@ parameters: ### Format job jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -41,7 +41,7 @@ jobs: - name: LinuxCrossArg value: '' - condition: ${{ parameters.condition }} + condition: and(succeeded(), ${{ parameters.condition }}) steps: diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index b94179197f47f..4f81977885a4b 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -139,6 +139,15 @@ jobs: - ${{ if eq(parameters.crossBuild, true) }}: - _crossBuildPropertyArg: '-cross' + - name: librariesDownloadDir + value: '$(Build.SourcesDirectory)/artifacts' + + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + frameworks: - ${{ parameters.framework }} steps: diff --git a/eng/pipelines/coreclr/templates/run-performance-job.yml b/eng/pipelines/coreclr/templates/run-performance-job.yml index 51eb0ce0e9676..9c665bbb1a7e1 100644 --- a/eng/pipelines/coreclr/templates/run-performance-job.yml +++ b/eng/pipelines/coreclr/templates/run-performance-job.yml @@ -31,7 +31,7 @@ parameters: additionalSetupParameters: '' # optional -- additional setup parameters that are job-specific jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} diff --git a/eng/pipelines/coreclr/templates/run-scenarios-job.yml b/eng/pipelines/coreclr/templates/run-scenarios-job.yml index 44910148e6527..2c1b3efe8d989 100644 --- a/eng/pipelines/coreclr/templates/run-scenarios-job.yml +++ b/eng/pipelines/coreclr/templates/run-scenarios-job.yml @@ -23,7 +23,7 @@ parameters: additionalSetupParameters: '' # optional -- additional setup parameters that are job-specific jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} @@ -108,6 +108,9 @@ jobs: ${{ if ne(parameters.runtimeType, 'wasm') }}: value: --install-dir $(PayloadDirectory)/dotnet + - name: librariesDownloadDir + value: '$(Build.SourcesDirectory)/artifacts' + workspace: clean: all pool: diff --git a/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml index 791a71c734838..8b2993f00d189 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml @@ -18,7 +18,7 @@ parameters: dependOnEvaluatePaths: false jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} @@ -49,6 +49,12 @@ jobs: - name: HelixResultLocation value: '$(Build.SourcesDirectory)\artifacts\helixresults\' + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 5580169122022..8273ba4892d00 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -19,7 +19,7 @@ parameters: dependOnEvaluatePaths: false jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} @@ -46,6 +46,12 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} variables: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} - HelixApiAccessToken: '' diff --git a/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml index 2e00e66d7f9c9..f5a0d23f6cb00 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml @@ -21,7 +21,7 @@ parameters: diffJitOptions: '' jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} @@ -51,6 +51,12 @@ jobs: parameters: osGroup: ${{ parameters.osGroup }} + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + - ${{ if eq(parameters.osGroup, 'windows') }}: - name: SpmiCollectionLocation value: '$(Build.SourcesDirectory)\artifacts\spmi\' diff --git a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml index 25fabb2f7670e..9331d905cd386 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml @@ -18,7 +18,7 @@ parameters: dependOnEvaluatePaths: false jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} @@ -38,6 +38,12 @@ jobs: displayName: '${{ parameters.jobName }}' variables: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml deleted file mode 100644 index baad6dca1a953..0000000000000 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ /dev/null @@ -1,63 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osSubgroup: '' - name: '' - helixType: '(unspecified)' - container: '' - testGroup: '' - crossBuild: false - liveLibrariesBuildConfig: '' - strategy: '' - pool: '' - - # arcade-specific parameters - condition: true - continueOnError: false - dependsOn: '' - dependOnEvaluatePaths: false - displayName: '' - timeoutInMinutes: '' - enableMicrobuild: '' - gatherAssetManifests: false - disableComponentGovernance: false - - variables: {} ## any extra variables to add to the defaults defined below - -jobs: -- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - name: ${{ parameters.name }} - helixType: ${{ parameters.helixType }} - container: ${{ parameters.container }} - crossBuild: ${{ parameters.crossBuild }} - strategy: ${{ parameters.strategy }} - pool: ${{ parameters.pool }} - - # arcade-specific parameters - condition: and(succeeded(), ${{ parameters.condition }}) - continueOnError: ${{ parameters.continueOnError }} - dependsOn: ${{ parameters.dependsOn }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - displayName: ${{ parameters.displayName }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - enableMicrobuild: ${{ parameters.enableMicrobuild }} - gatherAssetManifests: ${{ parameters.gatherAssetManifests }} - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - - variables: - - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml - parameters: - runtimeFlavor: coreclr - testGroup: ${{ parameters.testGroup }} - liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} - - - ${{ each variable in parameters.variables }}: - - ${{insert}}: ${{ variable }} - - steps: ${{ parameters.steps }} diff --git a/eng/pipelines/mono/templates/workloads-build.yml b/eng/pipelines/mono/templates/workloads-build.yml index 20ff5c29d5d89..1f8087287ae78 100644 --- a/eng/pipelines/mono/templates/workloads-build.yml +++ b/eng/pipelines/mono/templates/workloads-build.yml @@ -15,12 +15,12 @@ parameters: variables: {} jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/xplat-job.yml parameters: archType: ${{ parameters.archType }} buildConfig: ${{ parameters.buildConfig }} container: ${{ parameters.container }} - condition: ${{ parameters.isOfficialBuild }} + condition: and(succeeded(), ${{ parameters.isOfficialBuild }}) helixType: 'build/product/' osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup }} From 4ce33d5b0efda8e5db6c0c44fda3b00cacc13346 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 13 Mar 2024 14:42:47 -0700 Subject: [PATCH 77/79] Fix condition indentation --- eng/pipelines/coreclr/templates/perf-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 4f81977885a4b..a54cb58dd8da3 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -282,7 +282,7 @@ jobs: inputs: SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/testhost/${{ parameters.framework }}-$(osGroup)-$(buildConfigUpper)-$(archType) TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono - condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'wasm', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) + condition: and(succeeded(), eq(variables.runtimeFlavorName, 'Mono'), ne(variables.osGroup, 'windows'), notIn('${{ parameters.runtimeType }}', 'wasm', 'AndroidMono', 'iOSMono', 'iOSNativeAOT')) - task: CopyFiles@2 displayName: Copy corerun to .dotnet-mono @@ -290,4 +290,4 @@ jobs: SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper) Contents: 'corerun*' TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono - condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'wasm', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) + condition: and(succeeded(), eq(variables.runtimeFlavorName, 'Mono'), ne(variables.osGroup, 'windows'), notIn('${{ parameters.runtimeType }}', 'wasm', 'AndroidMono', 'iOSMono', 'iOSNativeAOT')) From 126839feb42fbbc87cd61b39a3a13fe9fb89e240 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 13 Mar 2024 14:45:53 -0700 Subject: [PATCH 78/79] Fix xplat-job.yml path --- eng/pipelines/common/templates/runtimes/build-test-job.yml | 2 +- eng/pipelines/common/templates/runtimes/run-test-job.yml | 2 +- .../coreclr/templates/crossgen2-comparison-build-job.yml | 2 +- eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml | 2 +- eng/pipelines/coreclr/templates/format-job.yml | 2 +- eng/pipelines/coreclr/templates/run-performance-job.yml | 2 +- eng/pipelines/coreclr/templates/run-scenarios-job.yml | 2 +- .../templates/run-superpmi-asmdiffs-checked-release-job.yml | 2 +- eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml | 2 +- eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml | 2 +- eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml | 2 +- eng/pipelines/mono/templates/workloads-build.yml | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index 65a7004a5cf1a..4966d87b9ada2 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -19,7 +19,7 @@ parameters: ### of the product build job). jobs: -- template: /eng/pipelines/common/templates/xplat-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 3fce853aa4a0f..5dcb4e840296e 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -34,7 +34,7 @@ parameters: ### buildConfig and archType. jobs: -- template: /eng/pipelines/common/templates/xplat-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml index 4964581693db5..41d1541c61057 100644 --- a/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml @@ -24,7 +24,7 @@ parameters: ### crossgen matches that of native, e.g. arm-hosted-arm-targeting, crossgen. jobs: -- template: /eng/pipelines/common/templates/xplat-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml index 25753737006b6..fbf10b45ea8f3 100644 --- a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml @@ -20,7 +20,7 @@ parameters: ### crossgen matches that of native, e.g. arm-hosted-arm-targeting, crossgen. jobs: -- template: /eng/pipelines/common/templates/xplat-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} diff --git a/eng/pipelines/coreclr/templates/format-job.yml b/eng/pipelines/coreclr/templates/format-job.yml index e310625b3ee83..3b360aa355fdc 100644 --- a/eng/pipelines/coreclr/templates/format-job.yml +++ b/eng/pipelines/coreclr/templates/format-job.yml @@ -13,7 +13,7 @@ parameters: ### Format job jobs: -- template: /eng/pipelines/common/templates/xplat-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} diff --git a/eng/pipelines/coreclr/templates/run-performance-job.yml b/eng/pipelines/coreclr/templates/run-performance-job.yml index 9c665bbb1a7e1..9f33585fd7591 100644 --- a/eng/pipelines/coreclr/templates/run-performance-job.yml +++ b/eng/pipelines/coreclr/templates/run-performance-job.yml @@ -31,7 +31,7 @@ parameters: additionalSetupParameters: '' # optional -- additional setup parameters that are job-specific jobs: -- template: /eng/pipelines/common/templates/xplat-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} diff --git a/eng/pipelines/coreclr/templates/run-scenarios-job.yml b/eng/pipelines/coreclr/templates/run-scenarios-job.yml index 2c1b3efe8d989..ffe4f5c16482e 100644 --- a/eng/pipelines/coreclr/templates/run-scenarios-job.yml +++ b/eng/pipelines/coreclr/templates/run-scenarios-job.yml @@ -23,7 +23,7 @@ parameters: additionalSetupParameters: '' # optional -- additional setup parameters that are job-specific jobs: -- template: /eng/pipelines/common/templates/xplat-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} diff --git a/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml index 8b2993f00d189..1b315b328927d 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml @@ -18,7 +18,7 @@ parameters: dependOnEvaluatePaths: false jobs: -- template: /eng/pipelines/common/templates/xplat-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 8273ba4892d00..b6f48f78535db 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -19,7 +19,7 @@ parameters: dependOnEvaluatePaths: false jobs: -- template: /eng/pipelines/common/templates/xplat-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} diff --git a/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml index f5a0d23f6cb00..8f70fe36407aa 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml @@ -21,7 +21,7 @@ parameters: diffJitOptions: '' jobs: -- template: /eng/pipelines/common/templates/xplat-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} diff --git a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml index 9331d905cd386..6f7c4529a2137 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml @@ -18,7 +18,7 @@ parameters: dependOnEvaluatePaths: false jobs: -- template: /eng/pipelines/common/templates/xplat-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} diff --git a/eng/pipelines/mono/templates/workloads-build.yml b/eng/pipelines/mono/templates/workloads-build.yml index 1f8087287ae78..4824f2f1203f2 100644 --- a/eng/pipelines/mono/templates/workloads-build.yml +++ b/eng/pipelines/mono/templates/workloads-build.yml @@ -15,7 +15,7 @@ parameters: variables: {} jobs: -- template: /eng/pipelines/common/templates/xplat-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: archType: ${{ parameters.archType }} buildConfig: ${{ parameters.buildConfig }} From 994e7cea04f1898b2b493e9853e3c55d0149b878 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 14 Mar 2024 17:27:44 -0700 Subject: [PATCH 79/79] Undo jit change --- src/coreclr/jit/jit.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/coreclr/jit/jit.h b/src/coreclr/jit/jit.h index b94e73eb6c196..1df8c034d0c1d 100644 --- a/src/coreclr/jit/jit.h +++ b/src/coreclr/jit/jit.h @@ -6,7 +6,6 @@ #define _JIT_H_ /*****************************************************************************/ - // // clr.sln only defines _DEBUG // The jit uses DEBUG rather than _DEBUG