diff --git a/eng/pipelines/common/build-coreclr-and-libraries-job.yml b/eng/pipelines/common/build-coreclr-and-libraries-job.yml deleted file mode 100644 index 71cefc5dbbd07..0000000000000 --- a/eng/pipelines/common/build-coreclr-and-libraries-job.yml +++ /dev/null @@ -1,53 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osSubgroup: '' - targetRid: '' - container: '' - testGroup: '' - crossBuild: false - timeoutInMinutes: '' - signBinaries: false - variables: {} - pool: '' - platform: '' - testBuildPlatforms: [] - disableComponentGovernance: false - -jobs: -- template: /eng/pipelines/coreclr/templates/build-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - targetRid: ${{ parameters.targetRid }} - container: ${{ parameters.container }} - testGroup: ${{ parameters.testGroup }} - crossBuild: ${{ parameters.crossBuild }} - timeoutInminutes: ${{ parameters.timeoutInMinutes }} - signBinaries: ${{ parameters.signBinaries }} - variables: ${{ parameters.variables }} - pool: ${{ parameters.pool }} - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - -- template: /eng/pipelines/libraries/build-job.yml - parameters: - ${{ if eq(parameters.buildConfig, 'debug') }}: - buildConfig: Debug - ${{ if ne(parameters.buildConfig, 'debug') }}: - buildConfig: Release - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - container: ${{ parameters.container }} - testGroup: ${{ parameters.testGroup }} - crossBuild: ${{ parameters.crossBuild }} - timeoutInminutes: ${{ parameters.timeoutInMinutes }} - variables: ${{ parameters.variables }} - pool: ${{ parameters.pool }} - liveRuntimeBuildConfig: ${{ parameters.buildConfig }} - platform: ${{ parameters.platform }} - testBuildPlatforms: ${{ parameters.testBuildPlatforms }} - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 6f98fba1029f1..cdcadcbc04802 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -32,6 +32,7 @@ parameters: preBuildSteps: [] enableRichCodeNavigation: false richCodeNavigationLanguage: 'csharp' + disableComponentGovernance: false jobs: - template: /eng/common/templates/job/job.yml @@ -55,6 +56,8 @@ jobs: # Component governance does not work on musl machines ${{ if eq(parameters.osSubGroup, '_musl') }}: disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} workspace: clean: all diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index a846faca46e3c..7245e52168046 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -272,7 +272,6 @@ jobs: buildConfig: ${{ parameters.buildConfig }} helixQueueGroup: ${{ parameters.helixQueueGroup }} crossBuild: true - disableClrTest: true ${{ insert }}: ${{ parameters.jobParameters }} # Runtime-dev-innerloop build @@ -463,7 +462,6 @@ jobs: buildConfig: ${{ parameters.buildConfig }} helixQueueGroup: ${{ parameters.helixQueueGroup }} crossBuild: true - disableClrTest: true ${{ insert }}: ${{ parameters.jobParameters }} # WASI WebAssembly @@ -920,7 +918,6 @@ jobs: buildConfig: ${{ parameters.buildConfig }} helixQueueGroup: ${{ parameters.helixQueueGroup }} crossBuild: true - disableClrTest: true ${{ insert }}: ${{ parameters.jobParameters }} # Windows x64 diff --git a/eng/pipelines/common/templates/global-build-step.yml b/eng/pipelines/common/templates/global-build-step.yml index 7f38a9fd11840..6597ac4feaefc 100644 --- a/eng/pipelines/common/templates/global-build-step.yml +++ b/eng/pipelines/common/templates/global-build-step.yml @@ -4,9 +4,12 @@ parameters: shouldContinueOnError: false archParameter: $(_archParameter) displayName: Build product + container: '' steps: - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) $(crossArg) ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) displayName: ${{ parameters.displayName }} ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}: continueOnError: ${{ parameters.shouldContinueOnError }} + ${{ if ne(parameters.container, '') }}: + target: ${{ parameters.container }} diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index e249e8ac922a7..4966d87b9ada2 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -18,14 +18,8 @@ parameters: ### Build managed test components (native components are getting built as part ### of the product build job). -### TODO: As of today, build of managed test components requires the product build -### as a prerequisite due to dependency on System.Private.Corelib. After switching -### over to its reference assembly we should be able to remove this dependency and -### run managed test builds in parallel with the product build job. - - jobs: -- template: /eng/pipelines/${{ parameters.runtimeFlavor }}/templates/xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -37,7 +31,6 @@ jobs: testGroup: ${{ parameters.testGroup }} pool: ${{ parameters.pool }} dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - compilerArg: '' # Test jobs should continue on error for internal builds ${{ if eq(variables['System.TeamProject'], 'internal') }}: @@ -69,8 +62,6 @@ jobs: - ${{ variable }} - name: liveRuntimeBuildParams value: 'libs+clr.iltools -c Release -ci' - - name: compilerArg - value: '' - name: runtimeFlavorArgs value: '' @@ -91,6 +82,12 @@ jobs: - name: testTreeFilterArg value: 'tree GC/Scenarios/GCSimulator' + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + steps: # Install test build dependencies @@ -99,7 +96,7 @@ jobs: displayName: Install native dependencies # Build core/libraries dependencies of test build - - script: $(Build.SourcesDirectory)/build$(scriptExt) $(liveRuntimeBuildParams) $(crossArg) $(compilerArg) + - script: $(Build.SourcesDirectory)/build$(scriptExt) $(liveRuntimeBuildParams) $(crossArg) displayName: Build coreclr/libs components needed by test build - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: diff --git a/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml new file mode 100644 index 0000000000000..6351952b16954 --- /dev/null +++ b/eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml @@ -0,0 +1,65 @@ +# Variables to resolve the paths and artifact names for the test assets used and produced to run the tests +# in the src/tests tree. +parameters: + runtimeFlavor: 'coreclr' + runtimeVariant: '' + liveLibrariesBuildConfig: '' + testGroup: 'innerloop' + configOverride: '' + +variables: + - name: binTestsPath + value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr' + + # Build product defines what we are trying to build, either coreclr or mono + - name: buildProductRootFolderPath + value: '$(Build.SourcesDirectory)/artifacts/bin/${{ parameters.runtimeFlavor }}/$(osGroup).$(archType).$(buildConfigUpper)' + + - ${{ if ne(parameters.runtimeFlavor, 'mono') }}: + - name: managedGenericTestArtifactName + value: 'CoreCLRManagedTestArtifacts_AnyOS_AnyCPU_$(buildConfig)' + - ${{ else }}: + - name: managedGenericTestArtifactName + value: 'MonoManagedTestArtifacts_AnyOS_AnyCPU_$(buildConfig)' + + - name: managedTestArtifactRootFolderPath + value: '$(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper)' + + - name: nativeTestArtifactConfig + value: $(_BuildConfig) + + - ${{ if ne(parameters.configOverride, '') }}: + - name: nativeTestArtifactConfig + value: ${{ parameters.configOverride }} + + - name: nativeTestArtifactName + value: 'CoreCLRNativeTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(nativeTestArtifactConfig)' + + - name: nativeTestArtifactRootFolderPath + value: '$(binTestsPath)/obj/$(osGroup).$(archType).$(nativeTestArtifactConfig)' + + - name: microsoftNetSdkIlFolderPath + value: '$(Build.SourcesDirectory)/.packages/microsoft.net.sdk.il' + + - name: microsoftNetSdkIlArtifactName + value: 'MicrosoftNetSdkIlPackage_AnyOS_AnyCPU_$(buildConfig)' + + - name: librariesOverrideArg + value : '' + + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - name: librariesOverrideArg + value : ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' + + - name: priorityArg + value: '' + + # 'innerloop' and 'clrinterpreter' jobs run the Priority 0 tests; everything else runs the Priority 1 tests. + # 'gc-standalone' is forced to run pri0 as well to start with. + - ${{ if and(ne(parameters.testGroup, 'innerloop'), ne(parameters.testGroup, 'clrinterpreter'), ne(parameters.testGroup, 'gc-standalone'), ne(parameters.testGroup, 'gc-standalone-server') ) }}: + - ${{ if ne(parameters.osGroup, 'windows') }}: + - name: priorityArg + value: 'priority1' + - ${{ if eq(parameters.osGroup, 'windows') }}: + - name: priorityArg + value: '-priority 1' diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index a063c2127ecc7..5dcb4e840296e 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -24,6 +24,9 @@ parameters: dependsOn: [] dependOnEvaluatePaths: false SuperPmiCollect: false + unifiedArtifactsName: '' + unifiedBuildNameSuffix: '' + unifiedBuildConfigOverride: '' ### Test run job @@ -31,7 +34,7 @@ parameters: ### buildConfig and archType. jobs: -- template: /eng/pipelines/${{ parameters.runtimeFlavor }}/templates/xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -44,7 +47,7 @@ jobs: helixType: 'build/tests/' runtimeVariant: ${{ parameters.runtimeVariant }} pool: ${{ parameters.pool }} - condition: ${{ parameters.condition }} + condition: and(succeeded(), ${{ parameters.condition }}) dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} # Test jobs should continue on error for internal builds @@ -60,22 +63,8 @@ jobs: - '${{ parameters.runtimeFlavor }}_common_test_build_p0_AnyOS_AnyCPU_${{parameters.buildConfig }}' - ${{ if notIn(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: - '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}' - - ${{ if or( eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter'), eq(parameters.runtimeVariant, 'llvmaot'), eq(parameters.runtimeVariant, 'llvmfullaot')) }}: - # This is needed for creating a CORE_ROOT in the current design. - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', '', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if or( eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter')) }} : - # minijit and mono interpreter runtimevariants do not require any special build of the runtime - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, '', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if not(or(eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter'))) }}: - - ${{ if eq(parameters.runtimeVariant, 'llvmfullaot') }}: - - ${{ format('{0}_llvmaot_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.runtimeVariant, 'llvmfullaot') }}: - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - # SuperPMI collection needs to run mcs.exe on the AzDO machine. Assume that's an x64 machine, and download an x64 product build if needed. - - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}: - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', '', parameters.osGroup, parameters.osSubgroup, 'x64', parameters.buildConfig) }} + - ${{ if ne(parameters.unifiedArtifactsName, '')}}: + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ coalesce(parameters.unifiedBuildConfigOverride, parameters.buildConfig) }}_${{ parameters.unifiedBuildNameSuffix }}' # Compute job name from template parameters ${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}: @@ -87,6 +76,13 @@ jobs: variables: + - name: osGroup + value: ${{ parameters.osGroup }} + - name: osSubgroup + value: ${{ parameters.osSubgroup }} + - name: archType + value: ${{ parameters.archType }} + - name: monoAotBuildshCommand value: '' @@ -132,6 +128,12 @@ jobs: - name: testTreeFilterArg value: 'tree GC/Scenarios/GCSimulator' + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + # Variables used for SuperPMI collection - ${{ if eq(parameters.SuperPmiCollect, true) }}: - MchFileTag: '${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}' @@ -139,11 +141,10 @@ jobs: value: 'run' - name: CollectionName value: 'coreclr_tests' + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' - name: MchFilesLocation value: '$(Build.SourcesDirectory)\artifacts\helixresults\' - name: MergedMchFileLocation @@ -151,10 +152,6 @@ jobs: - name: SpmiLogsLocation value: '$(Build.SourcesDirectory)\artifacts\spmi_logs\' - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'python3' - - name: PipScript - value: 'pip3' - name: MchFilesLocation value: '$(Build.SourcesDirectory)/artifacts/helixresults/' - name: MergedMchFileLocation @@ -199,43 +196,12 @@ jobs: steps: - # Optionally download live-built libraries - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir) - cleanUnpackFolder: false - artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' - artifactName: '$(librariesBuildArtifactName)' - displayName: 'live-built libraries' - - - # Download and unzip managed test artifacts - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: '$(managedTestArtifactRootFolderPath)' - artifactFileName: '$(managedGenericTestArtifactName).tar.gz' - artifactName: '$(managedGenericTestArtifactName)' - displayName: 'generic managed test artifacts' - - - # Download product binaries directory - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' - displayName: 'product build' - - - - ${{ if eq(parameters.runtimeFlavor, 'mono') }}: - # We need to explicitly download CoreCLR for Mono - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(coreClrProductRootFolderPath) - artifactFileName: '$(coreClrProductArtifactName)$(archiveExtension)' - artifactName: '$(coreClrProductArtifactName)' - displayName: 'CoreCLR product download for Mono' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: '${{ parameters.unifiedArtifactsName }}$(archiveExtension)' + artifactName: '${{ parameters.unifiedArtifactsName }}' + displayName: 'unified artifacts' # Download and unzip the Microsoft.NET.Sdk.IL package needed for traversing # ilproj test projects during copynativeonly. @@ -246,6 +212,13 @@ jobs: artifactName: '$(microsoftNetSdkIlArtifactName)' displayName: 'Microsoft.NET.Sdk.IL package' + # Download and unzip managed test artifacts + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: '$(managedTestArtifactRootFolderPath)' + artifactFileName: '$(managedGenericTestArtifactName).tar.gz' + artifactName: '$(managedGenericTestArtifactName)' + displayName: 'generic managed test artifacts' # Download and unzip native test artifacts - template: /eng/pipelines/common/download-artifact-step.yml @@ -255,17 +228,6 @@ jobs: artifactName: '$(nativeTestArtifactName)' displayName: 'native test artifacts' - - # SuperPMI collection: Download x64 coreclr if running on non-x64 configuration (needed for mcs.exe) - - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper)' - artifactFileName: 'CoreCLRProduct__$(osGroup)$(osSubgroup)_x64_$(buildConfig)$(archiveExtension)' - artifactName: 'CoreCLRProduct__$(osGroup)$(osSubgroup)_x64_$(buildConfig)' - displayName: 'CoreCLR product build (x64)' - - # Publish native test components to test output folder. Sadly we cannot do this # during product build (so that we could zip up the files in their final test location # and directly unzip them there after download). Unfortunately the logic to copy @@ -632,6 +594,9 @@ jobs: displayName: Create SuperPMI directories condition: always() + - script: $(PythonSetupScript) + displayName: Enable python venv + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch displayName: 'Merge $(CollectionName)-$(CollectionType) SuperPMI collections' condition: always() @@ -656,7 +621,7 @@ jobs: condition: always() # Ensure the Python azure-storage-blob package is installed before doing the upload. - - script: $(PipScript) install --user --upgrade pip && $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall + - script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall displayName: Upgrade Pip to latest and install azure-storage-blob Python package condition: always() diff --git a/eng/pipelines/common/templates/runtimes/xplat-job.yml b/eng/pipelines/common/templates/runtimes/xplat-job.yml index 23e74c70e57ac..f97ca08e5162e 100644 --- a/eng/pipelines/common/templates/runtimes/xplat-job.yml +++ b/eng/pipelines/common/templates/runtimes/xplat-job.yml @@ -77,6 +77,9 @@ jobs: - name: buildConfig value: ${{ parameters.buildConfig }} + - name: _BuildConfig + value: ${{ parameters.buildConfig }} + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - name: _HelixSource value: official/dotnet/runtime/$(Build.SourceBranch) diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml index 1b7b180a85f1f..8ce9cf6a4deed 100644 --- a/eng/pipelines/coreclr/ci.yml +++ b/eng/pipelines/coreclr/ci.yml @@ -44,7 +44,7 @@ extends: # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + jobTemplate: /eng/pipelines/common/templates/global-build-job.yml buildConfig: debug platforms: - linux_arm @@ -56,29 +56,20 @@ extends: - osx_x64 - windows_arm64 jobParameters: - testGroup: outerloop - - # - # Checked builds - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: checked - platformGroup: all - platforms: - # It is too early to include osx_arm64 in platform group all - # Adding it here will enable it also - - osx_arm64 - jobParameters: - testGroup: outerloop + buildArgs: -s clr -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop # # Release builds # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + jobTemplate: /eng/pipelines/common/templates/global-build-job.yml buildConfig: release platforms: - linux_arm @@ -88,23 +79,43 @@ extends: - osx_x64 - windows_x86 jobParameters: - testGroup: outerloop + buildArgs: -s clr -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop # - # Release library builds + # Checked builds # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Release + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked platformGroup: all platforms: # It is too early to include osx_arm64 in platform group all # Adding it here will enable it also - osx_arm64 jobParameters: - isOfficialBuild: false - liveRuntimeBuildConfig: checked + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop # # Checked test builds @@ -134,6 +145,7 @@ extends: jobParameters: testGroup: outerloop liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) # # Checked R2R test runs @@ -158,6 +170,7 @@ extends: readyToRun: true displayNameArgs: R2R_CG2 liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) # # Formatting diff --git a/eng/pipelines/coreclr/clrinterpreter.yml b/eng/pipelines/coreclr/clrinterpreter.yml index eac505a9f7072..288e297c6f905 100644 --- a/eng/pipelines/coreclr/clrinterpreter.yml +++ b/eng/pipelines/coreclr/clrinterpreter.yml @@ -21,7 +21,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_arm @@ -32,7 +32,22 @@ extends: - windows_arm64 - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: clrinterpreter + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release /p:CMakeArgs="-DFEATURE_INTERPRETER=1" + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreCLRInterpreterBuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: clrinterpreter - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -59,3 +74,7 @@ extends: jobParameters: testGroup: clrinterpreter liveLibrariesBuildConfig: Release + dependsOn: + - build_$(osGroup)$(osSubgroup)_$(archType)_checked_ + - coreclr_common_test_build_pri0_AnyOS_AnyCPU_checked + unifiedArtifactsName: CoreCLRInterpreterBuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/crossgen2-composite.yml b/eng/pipelines/coreclr/crossgen2-composite.yml index 2830cdc47ded1..0dbe78f5907f3 100644 --- a/eng/pipelines/coreclr/crossgen2-composite.yml +++ b/eng/pipelines/coreclr/crossgen2-composite.yml @@ -20,7 +20,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_arm @@ -31,9 +31,23 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: innerloop + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: innerloop - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -64,3 +78,4 @@ extends: compositeBuildMode: true displayNameArgs: Composite liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/crossgen2-gcstress.yml b/eng/pipelines/coreclr/crossgen2-gcstress.yml index a004e7904a824..a509018edc0b1 100644 --- a/eng/pipelines/coreclr/crossgen2-gcstress.yml +++ b/eng/pipelines/coreclr/crossgen2-gcstress.yml @@ -20,7 +20,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_x64 @@ -30,9 +30,23 @@ extends: # - osx_x64 - windows_x64 - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: gcstress-extra + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: gcstress-extra - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -62,3 +76,4 @@ extends: compositeBuildMode: true displayNameArgs: Composite liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/crossgen2-outerloop.yml b/eng/pipelines/coreclr/crossgen2-outerloop.yml index 978143090b427..833513ea532e2 100644 --- a/eng/pipelines/coreclr/crossgen2-outerloop.yml +++ b/eng/pipelines/coreclr/crossgen2-outerloop.yml @@ -20,41 +20,10 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_arm - - linux_x64 - - linux_arm64 - - osx_arm64 - - osx_x64 - - windows_x86 - - windows_x64 - - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: outerloop - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: Release - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - osx_arm64 - - windows_x86 - - windows_x64 - jobParameters: - testGroup: outerloop - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Release - platforms: - - linux_arm - linux_arm64 - linux_x64 - osx_arm64 @@ -62,10 +31,23 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - isOfficialBuild: false - liveRuntimeBuildConfig: Release + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Checked_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -95,6 +77,7 @@ extends: compositeBuildMode: true displayNameArgs: R2R_Composite liveLibrariesBuildConfig: Release + unifiedArtifactsName: Checked_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) # Outerloop testing in non-composite mode - template: /eng/pipelines/common/platform-matrix.yml @@ -116,6 +99,34 @@ extends: readyToRun: true displayNameArgs: R2R liveLibrariesBuildConfig: Release + unifiedArtifactsName: Checked_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + + # Build release CoreCLR for Crossgen2 baseline generation + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: + - linux_arm + - linux_arm64 + - linux_x64 + - osx_arm64 + - osx_x64 + - windows_x86 + - windows_x64 + - windows_arm64 + jobParameters: + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Release_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets # Build Crossgen2 baselines # These are the various crossgen2 targets that are supported, and cover all major diff --git a/eng/pipelines/coreclr/crossgen2.yml b/eng/pipelines/coreclr/crossgen2.yml index 53ea5878d32ce..a52fbae3a091b 100644 --- a/eng/pipelines/coreclr/crossgen2.yml +++ b/eng/pipelines/coreclr/crossgen2.yml @@ -20,7 +20,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_x64 @@ -29,9 +29,23 @@ extends: - osx_x64 - windows_x64 - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: innerloop + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: innerloop - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -59,6 +73,7 @@ extends: readyToRun: true displayNameArgs: R2R_CG2 liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -100,3 +115,4 @@ extends: hotColdSplitting: true displayNameArgs: R2R_CG2_HotColdSplitting liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/exploratory.yml b/eng/pipelines/coreclr/exploratory.yml index af76dec9da249..1c0b6a7680bd4 100644 --- a/eng/pipelines/coreclr/exploratory.yml +++ b/eng/pipelines/coreclr/exploratory.yml @@ -21,27 +21,11 @@ extends: stages: - stage: Build jobs: - - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - # Linux tests are built on the OSX machines. - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - - osx_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: outerloop - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/jit-exploratory-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml buildConfig: checked platforms: # Linux tests are built on the OSX machines. @@ -52,9 +36,17 @@ extends: - windows_x86 - windows_arm64 - osx_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - toolName: ${{ variables.toolName }} + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + timeoutInMinutes: 360 + postBuildSteps: + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(_BuildConfig) $(archType) generatelayoutonly + displayName: Create Core_Root + condition: succeeded() + - template: /eng/pipelines/coreclr/templates/jit-exploratory-steps.yml + parameters: + toolName: ${{ variables.toolName }} + extraVariablesTemplates: + - template: /eng/pipelines/coreclr/templates/jit-exploratory-variables.yml + parameters: + toolName: ${{ variables.toolName }} diff --git a/eng/pipelines/coreclr/gc-longrunning.yml b/eng/pipelines/coreclr/gc-longrunning.yml index c58e6bd0ab5b1..0dbd66cd9ea61 100644 --- a/eng/pipelines/coreclr/gc-longrunning.yml +++ b/eng/pipelines/coreclr/gc-longrunning.yml @@ -20,7 +20,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - linux_x64 @@ -28,9 +28,23 @@ extends: - windows_x64 - windows_arm64 - osx_x64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: gc-longrunning + buildArgs: -s clr+libs -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: gc-longrunning - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -55,3 +69,4 @@ extends: jobParameters: testGroup: gc-longrunning liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/gc-simulator.yml b/eng/pipelines/coreclr/gc-simulator.yml index fd4b700a7053a..a0bda3d462105 100644 --- a/eng/pipelines/coreclr/gc-simulator.yml +++ b/eng/pipelines/coreclr/gc-simulator.yml @@ -20,7 +20,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: # disable Linux x64 for now until OOMs are resolved. @@ -29,9 +29,23 @@ extends: - windows_x64 - windows_arm64 - osx_x64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: gc-simulator + buildArgs: -s clr+libs -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: gc-simulator - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -57,3 +71,4 @@ extends: jobParameters: testGroup: gc-simulator liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/gc-standalone.yml b/eng/pipelines/coreclr/gc-standalone.yml index 7ebf86566f3b4..6089ac89178e8 100644 --- a/eng/pipelines/coreclr/gc-standalone.yml +++ b/eng/pipelines/coreclr/gc-standalone.yml @@ -20,15 +20,30 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_arm64 + - linux_x64 - windows_arm64 - windows_x64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: gc-standalone + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: gc-standalone - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -54,6 +69,7 @@ extends: testGroup: gc-standalone displayNameArgs: GCStandAlone liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -70,3 +86,4 @@ extends: testGroup: gc-standalone-server displayNameArgs: GCStandAloneServer liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/gcstress-extra.yml b/eng/pipelines/coreclr/gcstress-extra.yml index 61e5f3651f08b..d981efa133d1a 100644 --- a/eng/pipelines/coreclr/gcstress-extra.yml +++ b/eng/pipelines/coreclr/gcstress-extra.yml @@ -20,16 +20,30 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platformGroup: gcstress platforms: # It is too early to include osx_arm64 in platform group gcstress # Adding it here will enable it also - osx_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: gcstress-extra + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: gcstress-extra - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -54,3 +68,4 @@ extends: jobParameters: testGroup: gcstress-extra liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml b/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml index 2e501b2e18abf..5376898b7bf32 100644 --- a/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml +++ b/eng/pipelines/coreclr/gcstress0x3-gcstress0xc.yml @@ -20,16 +20,30 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platformGroup: gcstress platforms: # It is too early to include osx_arm64 in platform group gcstress # Adding it here will enable it also - osx_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: gcstress0x3-gcstress0xc + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: gcstress0x3-gcstress0xc - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -54,3 +68,4 @@ extends: jobParameters: testGroup: gcstress0x3-gcstress0xc liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/ilasm.yml b/eng/pipelines/coreclr/ilasm.yml index c57709a37d1b5..4747727284393 100644 --- a/eng/pipelines/coreclr/ilasm.yml +++ b/eng/pipelines/coreclr/ilasm.yml @@ -19,55 +19,16 @@ schedules: variables: - template: /eng/pipelines/common/variables.yml - extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: ilasm - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: ilasm - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: ilasm - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - osx_x64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + testGroup: ilasm diff --git a/eng/pipelines/coreclr/jit-cfg.yml b/eng/pipelines/coreclr/jit-cfg.yml index 87ae66fbe4329..80fe44da119d0 100644 --- a/eng/pipelines/coreclr/jit-cfg.yml +++ b/eng/pipelines/coreclr/jit-cfg.yml @@ -10,47 +10,12 @@ schedules: variables: - template: /eng/pipelines/common/variables.yml - extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jit-cfg - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jit-cfg - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jit-cfg - liveLibrariesBuildConfig: Release + platforms: + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_arm64 + testGroup: jit-cfg \ No newline at end of file diff --git a/eng/pipelines/coreclr/jit-experimental.yml b/eng/pipelines/coreclr/jit-experimental.yml index a599f343e310f..56453d894796d 100644 --- a/eng/pipelines/coreclr/jit-experimental.yml +++ b/eng/pipelines/coreclr/jit-experimental.yml @@ -12,49 +12,13 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jit-experimental - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jit-experimental - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jit-experimental - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - osx_x64 + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_arm64 + testGroup: jit-experimental \ No newline at end of file diff --git a/eng/pipelines/coreclr/jitrollingbuild.yml b/eng/pipelines/coreclr/jitrollingbuild.yml index b2d2b86f2a1c2..c99557f4c3cc9 100644 --- a/eng/pipelines/coreclr/jitrollingbuild.yml +++ b/eng/pipelines/coreclr/jitrollingbuild.yml @@ -22,10 +22,9 @@ extends: stages: - stage: Build jobs: - - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - osx_arm64 @@ -38,10 +37,15 @@ extends: - windows_arm64 jobParameters: disableComponentGovernance: true # Not a shipping artifact + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/upload-jits-steps.yml + extraVariablesTemplates: + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - osx_arm64 @@ -54,3 +58,8 @@ extends: - windows_arm64 jobParameters: disableComponentGovernance: true # Not a shipping artifact + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) /p:NoPgoOptimize=true + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/upload-jits-steps.yml + extraVariablesTemplates: + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml diff --git a/eng/pipelines/coreclr/jitstress-isas-arm.yml b/eng/pipelines/coreclr/jitstress-isas-arm.yml index 7e178b4f42595..0733c08ab8b49 100644 --- a/eng/pipelines/coreclr/jitstress-isas-arm.yml +++ b/eng/pipelines/coreclr/jitstress-isas-arm.yml @@ -12,43 +12,10 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - linux_arm64 - - osx_arm64 - - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-isas-arm - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-isas-arm - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_arm64 - - osx_arm64 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstress-isas-arm - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - linux_arm64 + - windows_arm64 + testGroup: jitstress-isas-arm diff --git a/eng/pipelines/coreclr/jitstress-isas-avx512.yml b/eng/pipelines/coreclr/jitstress-isas-avx512.yml index 4074c2b91358a..87b33c8479529 100644 --- a/eng/pipelines/coreclr/jitstress-isas-avx512.yml +++ b/eng/pipelines/coreclr/jitstress-isas-avx512.yml @@ -27,43 +27,10 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - linux_x64 - - windows_x64 - - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-isas-avx512 - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-isas-avx512 - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_x64 - - windows_x64 - - windows_x86 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstress-isas-avx512 - liveLibrariesBuildConfig: Release + platforms: + - linux_x64 + - windows_x64 + - windows_x86 + testGroup: jitstress-isas-avx512 diff --git a/eng/pipelines/coreclr/jitstress-isas-x86.yml b/eng/pipelines/coreclr/jitstress-isas-x86.yml index b5a1b9764fc51..0951fd572772e 100644 --- a/eng/pipelines/coreclr/jitstress-isas-x86.yml +++ b/eng/pipelines/coreclr/jitstress-isas-x86.yml @@ -12,45 +12,11 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - linux_x64 - - osx_x64 - - windows_x64 - - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-isas-x86 - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-isas-x86 - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_x64 - - osx_x64 - - windows_x64 - - windows_x86 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstress-isas-x86 - liveLibrariesBuildConfig: Release + platforms: + - osx_x64 + - linux_x64 + - windows_x64 + - windows_x86 + testGroup: jitstress-isas-x86 diff --git a/eng/pipelines/coreclr/jitstress-random.yml b/eng/pipelines/coreclr/jitstress-random.yml index f9437db5ea5cb..ffbda2acb4801 100644 --- a/eng/pipelines/coreclr/jitstress-random.yml +++ b/eng/pipelines/coreclr/jitstress-random.yml @@ -12,53 +12,15 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-random - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress-random - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstress-random - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - osx_x64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + testGroup: jitstress-random diff --git a/eng/pipelines/coreclr/jitstress.yml b/eng/pipelines/coreclr/jitstress.yml index 71263872c0929..0e9e0d052dfe8 100644 --- a/eng/pipelines/coreclr/jitstress.yml +++ b/eng/pipelines/coreclr/jitstress.yml @@ -12,53 +12,15 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstress - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - osx_x64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + testGroup: jitstress diff --git a/eng/pipelines/coreclr/jitstress2-jitstressregs.yml b/eng/pipelines/coreclr/jitstress2-jitstressregs.yml index ee672273e5647..0e2c961e4def2 100644 --- a/eng/pipelines/coreclr/jitstress2-jitstressregs.yml +++ b/eng/pipelines/coreclr/jitstress2-jitstressregs.yml @@ -12,53 +12,15 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstress2-jitstressregs - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: checked - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstress2-jitstressregs - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - osx_x64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + testGroup: jitstress2-jitstressregs diff --git a/eng/pipelines/coreclr/jitstressregs-x86.yml b/eng/pipelines/coreclr/jitstressregs-x86.yml index 8b8009e63cf55..63ba958217f47 100644 --- a/eng/pipelines/coreclr/jitstressregs-x86.yml +++ b/eng/pipelines/coreclr/jitstressregs-x86.yml @@ -12,43 +12,10 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - linux_x64 - - windows_x64 - - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstressregs-x86 - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstressregs-x86 - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_x64 - - windows_x64 - - windows_x86 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstressregs-x86 - liveLibrariesBuildConfig: Release + platforms: + - linux_x64 + - windows_x64 + - windows_x86 + testGroup: jitstressregs-x86 diff --git a/eng/pipelines/coreclr/jitstressregs.yml b/eng/pipelines/coreclr/jitstressregs.yml index d947e48efa2ee..25ab3ead6ba47 100644 --- a/eng/pipelines/coreclr/jitstressregs.yml +++ b/eng/pipelines/coreclr/jitstressregs.yml @@ -12,53 +12,15 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstressregs - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: jitstressregs - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - osx_x64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: jitstressregs - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - osx_x64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + testGroup: jitstressregs \ No newline at end of file diff --git a/eng/pipelines/coreclr/libraries-gcstress-extra.yml b/eng/pipelines/coreclr/libraries-gcstress-extra.yml index c87a99b56c1d2..c6bc819be20eb 100644 --- a/eng/pipelines/coreclr/libraries-gcstress-extra.yml +++ b/eng/pipelines/coreclr/libraries-gcstress-extra.yml @@ -19,35 +19,27 @@ extends: - stage: Build jobs: - # - # Build CoreCLR checked and libraries Release - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked - platformGroup: gcstress - jobParameters: - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 - - # - # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platformGroup: gcstress helixQueueGroup: libraries helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + platformGroup: gcstress jobParameters: # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. - timeoutInMinutes: 600 - testScope: innerloop - liveRuntimeBuildConfig: checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - coreclrTestGroup: gcstress-extra + timeoutInMinutes: 660 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + - heapverify1 + - gcstress0xc_disabler2r + - gcstress0xc_disabler2r_jitstress2 + - gcstress0xc_disabler2r_heapverify1 + - gcstress0xc_jitstress1 + - gcstress0xc_jitstress2 + - gcstress0xc_jitminopts_heapverify1 diff --git a/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml b/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml index 1acc98df23927..90f9803f74517 100644 --- a/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml +++ b/eng/pipelines/coreclr/libraries-gcstress0x3-gcstress0xc.yml @@ -12,6 +12,12 @@ trigger: none variables: - template: /eng/pipelines/common/variables.yml +extends: + template: /eng/pipelines/common/templates/pipeline-with-resources.yml + parameters: + stages: + - stage: Build + jobs: extends: template: /eng/pipelines/common/templates/pipeline-with-resources.yml parameters: @@ -19,35 +25,24 @@ extends: - stage: Build jobs: - # - # Build CoreCLR checked and libraries Release - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked - platformGroup: gcstress - jobParameters: - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 - - # - # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platformGroup: gcstress helixQueueGroup: libraries helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + platformGroup: gcstress jobParameters: # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. - timeoutInMinutes: 600 - testScope: innerloop - liveRuntimeBuildConfig: checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - coreclrTestGroup: gcstress0x3-gcstress0xc + timeoutInMinutes: 660 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + # Disable gcstress0x3 for now; it causes lots of test timeouts. Investigate this after + # gcstress0xc runs are clean. Tracking issue: https://github.com/dotnet/runtime/issues/38903. + # - gcstress0x3 + - gcstress0xc diff --git a/eng/pipelines/coreclr/libraries-jitstress-random.yml b/eng/pipelines/coreclr/libraries-jitstress-random.yml index eba6fb360f948..b9ab14fc30804 100644 --- a/eng/pipelines/coreclr/libraries-jitstress-random.yml +++ b/eng/pipelines/coreclr/libraries-jitstress-random.yml @@ -18,13 +18,12 @@ extends: - stage: Build jobs: - # - # Build CoreCLR checked and libraries Release - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml platforms: - linux_x64 - linux_arm @@ -32,33 +31,15 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - jobParameters: - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 - - # - # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - - windows_x86 - helixQueueGroup: libraries - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. - timeoutInMinutes: 300 - testScope: innerloop - liveRuntimeBuildConfig: checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - coreclrTestGroup: jitstress-random + timeoutInMinutes: 360 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + - jitstress_random_1 + - jitstress_random_2 diff --git a/eng/pipelines/coreclr/libraries-jitstress.yml b/eng/pipelines/coreclr/libraries-jitstress.yml index 4efcd59377237..501f233f47137 100644 --- a/eng/pipelines/coreclr/libraries-jitstress.yml +++ b/eng/pipelines/coreclr/libraries-jitstress.yml @@ -18,13 +18,12 @@ extends: - stage: Build jobs: - # - # Build CoreCLR checked and libraries Release - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml platforms: - linux_x64 - linux_arm @@ -32,33 +31,21 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - jobParameters: - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 - - # - # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - - windows_x86 - helixQueueGroup: libraries - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. - timeoutInMinutes: 300 - testScope: innerloop - liveRuntimeBuildConfig: checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - coreclrTestGroup: jitstress + timeoutInMinutes: 360 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + - no_tiered_compilation + - jitminopts + - jitstress1 + - jitstress1_tiered + - jitstress2 + - jitstress2_tiered + - disabler2r + - tailcallstress diff --git a/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml b/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml index 2a80b2757be90..9c50b8d5f4588 100644 --- a/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml +++ b/eng/pipelines/coreclr/libraries-jitstress2-jitstressregs.yml @@ -18,13 +18,12 @@ extends: - stage: Build jobs: - # - # Build CoreCLR checked and libraries Release - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml platforms: - linux_x64 - linux_arm @@ -32,33 +31,22 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - jobParameters: - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 - - # - # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - - windows_x86 - helixQueueGroup: libraries - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. - timeoutInMinutes: 300 - testScope: innerloop - liveRuntimeBuildConfig: checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - coreclrTestGroup: jitstress2-jitstressregs + timeoutInMinutes: 360 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + - jitstress2_jitstressregs1 + - jitstress2_jitstressregs2 + - jitstress2_jitstressregs3 + - jitstress2_jitstressregs4 + - jitstress2_jitstressregs8 + - jitstress2_jitstressregs0x10 + - jitstress2_jitstressregs0x80 + - jitstress2_jitstressregs0x1000 + - jitstress2_jitstressregs0x2000 \ No newline at end of file diff --git a/eng/pipelines/coreclr/libraries-jitstressregs.yml b/eng/pipelines/coreclr/libraries-jitstressregs.yml index 57ab5bae5326f..5ab04e9a27add 100644 --- a/eng/pipelines/coreclr/libraries-jitstressregs.yml +++ b/eng/pipelines/coreclr/libraries-jitstressregs.yml @@ -18,13 +18,12 @@ extends: - stage: Build jobs: - # - # Build CoreCLR checked and libraries Release - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml platforms: - linux_x64 - linux_arm @@ -32,33 +31,22 @@ extends: - windows_x86 - windows_x64 - windows_arm64 - jobParameters: - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 - - # - # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - - windows_x86 - helixQueueGroup: libraries - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. - timeoutInMinutes: 300 - testScope: innerloop - liveRuntimeBuildConfig: checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - coreclrTestGroup: jitstressregs + timeoutInMinutes: 360 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + - jitstressregs1 + - jitstressregs2 + - jitstressregs3 + - jitstressregs4 + - jitstressregs8 + - jitstressregs0x10 + - jitstressregs0x80 + - jitstressregs0x1000 + - jitstressregs0x2000 diff --git a/eng/pipelines/coreclr/libraries-pgo.yml b/eng/pipelines/coreclr/libraries-pgo.yml index 46643af71d2a2..2ac83670dd512 100644 --- a/eng/pipelines/coreclr/libraries-pgo.yml +++ b/eng/pipelines/coreclr/libraries-pgo.yml @@ -18,46 +18,56 @@ extends: - stage: Build jobs: - # - # Build CoreCLR checked and libraries Release - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml platforms: - - linux_x64 - - linux_arm - - linux_arm64 - - windows_x86 - - windows_x64 - windows_arm64 jobParameters: - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 + # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. + timeoutInMinutes: 660 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + - defaultpgo - # - # Libraries Test Run using Release libraries, Checked CoreCLR, and stress modes - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + helixQueueGroup: libraries + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml platforms: + - linux_x64 - linux_arm - linux_arm64 - - linux_x64 - - windows_arm64 - - windows_x64 - windows_x86 - helixQueueGroup: libraries - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + - windows_x64 jobParameters: - timeoutInMinutes: 600 - testScope: innerloop - liveRuntimeBuildConfig: checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - coreclrTestGroup: pgo + # Default timeout is 150 minutes (2.5 hours), which is not enough for stress. + timeoutInMinutes: 660 + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /maxcpucount:10 + scenarios: + - defaultpgo + - fullpgo + - fullpgo_methodprofiling + - fullpgo_random_gdv + - fullpgo_random_gdv_methodprofiling_only + - fullpgo_random_gdv_edge + - jitosr_stress + - jitosr_stress_random + - syntheticpgo + - syntheticpgo_blend + - jitrlcse diff --git a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml index c72da916d6f09..3a3d14b70a052 100644 --- a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml +++ b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml @@ -5,7 +5,7 @@ jobs: # build coreclr and libraries - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - linux_x64 @@ -13,7 +13,19 @@ jobs: - windows_x86 - linux_musl_x64 jobParameters: - testGroup: perf + nameSuffix: coreclr + buildArgs: -s clr+libs+host+packs -c $(_BuildConfig) + isOfficialBuild: false + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr + displayName: Build Assets # build mono for AOT - template: /eng/pipelines/common/platform-matrix.yml @@ -64,11 +76,25 @@ jobs: # build mono - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release + runtimeFlavor: mono platforms: - linux_x64 + jobParameters: + nameSuffix: mono + buildArgs: -s mono+clr.iltools+clr.hosts+libs+host+packs -c $(_BuildConfig) + isOfficialBuild: false + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_mono + displayName: Build Assets # Build and run iOS Mono and NativeAOT scenarios - template: /eng/pipelines/coreclr/templates/build-and-run-perf-ios-scenarios.yml diff --git a/eng/pipelines/coreclr/perf-wasm-jobs.yml b/eng/pipelines/coreclr/perf-wasm-jobs.yml index 9b6fa299752b1..3f33e90cbc8f1 100644 --- a/eng/pipelines/coreclr/perf-wasm-jobs.yml +++ b/eng/pipelines/coreclr/perf-wasm-jobs.yml @@ -67,7 +67,6 @@ jobs: jobparameters: testgroup: perf livelibrariesbuildconfig: Release - skipLiveLibrariesDownload: true runtimetype: wasm codegentype: 'aot' projectfile: microbenchmarks.proj @@ -109,7 +108,6 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - skipLiveLibrariesDownload: true runtimeType: wasm codeGenType: 'wasm' projectFile: microbenchmarks.proj @@ -138,7 +136,6 @@ jobs: jobparameters: testgroup: perf livelibrariesbuildconfig: Release - skipLiveLibrariesDownload: true runtimetype: wasm codegentype: 'aot' projectfile: microbenchmarks.proj @@ -165,7 +162,6 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - skipLiveLibrariesDownload: true runtimeType: wasm projectFile: blazor_perf.proj runKind: blazor_scenarios @@ -188,7 +184,6 @@ jobs: jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - skipLiveLibrariesDownload: true runtimeType: wasm projectFile: blazor_perf.proj runKind: blazor_scenarios diff --git a/eng/pipelines/coreclr/perf_slow.yml b/eng/pipelines/coreclr/perf_slow.yml index 49afdad10f577..92bd74ab5c513 100644 --- a/eng/pipelines/coreclr/perf_slow.yml +++ b/eng/pipelines/coreclr/perf_slow.yml @@ -36,24 +36,28 @@ extends: - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')) }}: - # build mono - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono - buildConfig: release - platforms: - - linux_arm64 - - # build coreclr and libraries - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release + runtimeFlavor: mono + runtimeVariant: monointerpreter platforms: - linux_arm64 jobParameters: - testGroup: perf + nameSuffix: mono + buildArgs: -s mono+clr.iltools+clr.hosts+libs+host+packs -c $(_BuildConfig) + isOfficialBuild: false + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_mono + displayName: Build Assets # run arm64 interpreter jobs for mono - template: /eng/pipelines/common/platform-matrix.yml @@ -79,7 +83,7 @@ extends: # build coreclr and libraries - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - linux_x64 @@ -87,7 +91,19 @@ extends: - linux_arm64 - windows_arm64 jobParameters: - testGroup: perf + nameSuffix: coreclr + buildArgs: -s clr+libs+host+packs -c $(_BuildConfig) + isOfficialBuild: false + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr + displayName: Build Assets - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/coreclr/pgo.yml b/eng/pipelines/coreclr/pgo.yml index f6d00e6fea502..26b2846d1a196 100644 --- a/eng/pipelines/coreclr/pgo.yml +++ b/eng/pipelines/coreclr/pgo.yml @@ -12,51 +12,14 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - osx_arm64 - - windows_arm64 - - windows_x64 - - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: pgo - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: pgo - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - osx_arm64 - - windows_arm64 - - windows_x64 - - windows_x86 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: pgo - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + testGroup: pgo diff --git a/eng/pipelines/coreclr/pgostress.yml b/eng/pipelines/coreclr/pgostress.yml index 228b3b582e5d6..fdd8e964082dd 100644 --- a/eng/pipelines/coreclr/pgostress.yml +++ b/eng/pipelines/coreclr/pgostress.yml @@ -12,51 +12,14 @@ variables: - template: /eng/pipelines/common/variables.yml extends: - template: /eng/pipelines/common/templates/pipeline-with-resources.yml + template: /eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml parameters: - stages: - - stage: Build - jobs: - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: checked - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - osx_arm64 - - windows_arm64 - - windows_x64 - - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: pgostress - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: pgostress - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_arm - - linux_arm64 - - linux_x64 - - osx_arm64 - - windows_arm64 - - windows_x64 - - windows_x86 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: pgostress - liveLibrariesBuildConfig: Release + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + testGroup: pgostress diff --git a/eng/pipelines/coreclr/r2r-extra.yml b/eng/pipelines/coreclr/r2r-extra.yml index acc1c6a962079..71324e3224f38 100644 --- a/eng/pipelines/coreclr/r2r-extra.yml +++ b/eng/pipelines/coreclr/r2r-extra.yml @@ -20,16 +20,30 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platformGroup: gcstress platforms: # It is too early to include osx_arm64 in platform group gcstress # Adding it here will enable it also - osx_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: r2r-extra + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: r2r-extra - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -56,3 +70,4 @@ extends: readyToRun: true displayNameArgs: R2R liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/r2r.yml b/eng/pipelines/coreclr/r2r.yml index 4f94eb6a1570b..977580703f71d 100644 --- a/eng/pipelines/coreclr/r2r.yml +++ b/eng/pipelines/coreclr/r2r.yml @@ -20,7 +20,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_arm @@ -30,9 +30,23 @@ extends: - windows_arm64 - windows_x64 - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: outerloop + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -62,3 +76,4 @@ extends: readyToRun: true displayNameArgs: R2R liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/release-tests.yml b/eng/pipelines/coreclr/release-tests.yml index 9ccf725463912..d25c3069b7cf6 100644 --- a/eng/pipelines/coreclr/release-tests.yml +++ b/eng/pipelines/coreclr/release-tests.yml @@ -23,7 +23,7 @@ extends: # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platformGroup: all platforms: @@ -31,8 +31,22 @@ extends: # Adding it here will enable it also - osx_arm64 jobParameters: - testGroup: outerloop - isOfficialBuild: false + buildArgs: -s clr+libs -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop # # Release test builds @@ -63,6 +77,7 @@ extends: jobParameters: testGroup: outerloop liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) # # Release R2R test runs @@ -79,4 +94,5 @@ extends: liveLibrariesBuildConfig: Release readyToRun: true displayNameArgs: R2R + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/runincontext.yml b/eng/pipelines/coreclr/runincontext.yml index 9135532fd5f93..6ade58de8fe5a 100644 --- a/eng/pipelines/coreclr/runincontext.yml +++ b/eng/pipelines/coreclr/runincontext.yml @@ -20,15 +20,29 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_x64 - windows_x64 - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: outerloop + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -54,3 +68,4 @@ extends: runInUnloadableContext: true displayNameArgs: RunInContext liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml b/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml index a356acb4fd9cf..4d25196bdc8b1 100644 --- a/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml +++ b/eng/pipelines/coreclr/superpmi-asmdiffs-checked-release.yml @@ -17,26 +17,46 @@ extends: stages: - stage: Build jobs: - - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - windows_x64 - windows_x86 jobParameters: - uploadAs: 'pipelineArtifacts' + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: CheckedJIT_$(osGroup)$(osSubgroup)_$(archType) + displayName: JIT and SuperPMI Assets - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - windows_x64 - windows_x86 + - linux_x64 jobParameters: - uploadAs: 'pipelineArtifacts' + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) /p:NoPgoOptimize=true + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType) + displayName: JIT and SuperPMI Assets - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index 336125d43bc13..e784fe45be716 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -35,36 +35,116 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - windows_x64 + - linux_x64 + jobParameters: + testGroup: outerloop + buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop + disableComponentGovernance: true # No shipping artifacts produced by this pipeline + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + platforms: - windows_x86 - windows_arm64 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 + - osx_arm64 jobParameters: testGroup: outerloop - # libraries test build platforms - testBuildPlatforms: - - linux_x64 - - windows_x64 + buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true + postBuildSteps: + # Build CLR assets for x64 as well as the target as we need an x64 mcs + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.spmi -c $(_BuildConfig) + archParameter: -arch x64 + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop disableComponentGovernance: true # No shipping artifacts produced by this pipeline - # superpmi-collect-job that targets macOS/arm64 depends on coreclr binaries produced by the macOS/x64 job - # We don't collect osx-x64 (it's essentially the same as linux-x64). If we did, we'd add osx_x64 in the - # build-coreclr-and-libraries-job.yml above, and remove this. - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - - osx_x64 + - linux_arm + - linux_arm64 jobParameters: testGroup: outerloop + buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true + postBuildSteps: + # Build CLR assets for x64 as well as the target as we need an x64 mcs + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.spmi -c $(_BuildConfig) + archParameter: -arch x64 + container: linux_x64 + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop disableComponentGovernance: true # No shipping artifacts produced by this pipeline - template: /eng/pipelines/common/platform-matrix.yml @@ -217,6 +297,7 @@ extends: testGroup: outerloop liveLibrariesBuildConfig: Release SuperPmiCollect: true + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -255,12 +336,16 @@ extends: helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testScope: innerloop - liveRuntimeBuildConfig: checked + liveRuntimeBuildConfig: Checked dependsOnTestBuildConfiguration: Release dependsOnTestArchitecture: x64 - coreclrTestGroup: superpmi_collection + scenarios: + - normal SuperPmiCollect: true SuperPmiCollectionName: libraries_tests + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + unifiedBuildConfigOverride: checked # # Collection of libraries test run: no_tiered_compilation @@ -282,9 +367,13 @@ extends: helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testScope: innerloop - liveRuntimeBuildConfig: checked + liveRuntimeBuildConfig: Checked dependsOnTestBuildConfiguration: Release dependsOnTestArchitecture: x64 - coreclrTestGroup: superpmi_collection_no_tiered_compilation + scenarios: + - no_tiered_compilation SuperPmiCollect: true SuperPmiCollectionName: libraries_tests_no_tiered_compilation + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + unifiedBuildConfigOverride: checked diff --git a/eng/pipelines/coreclr/superpmi-diffs.yml b/eng/pipelines/coreclr/superpmi-diffs.yml index 423f1e8b1dd1a..1fd327f838c70 100644 --- a/eng/pipelines/coreclr/superpmi-diffs.yml +++ b/eng/pipelines/coreclr/superpmi-diffs.yml @@ -11,7 +11,7 @@ parameters: # This pipeline only runs on GitHub PRs, not on merges. trigger: none -# Only run on changes to the JIT directory. +# Only run on changes to the JIT directory. pr: branches: include: @@ -45,25 +45,45 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - windows_x64 - windows_x86 jobParameters: - uploadAs: 'pipelineArtifacts' + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: CheckedJIT_$(osGroup)$(osSubgroup)_$(archType) + displayName: JIT and SuperPMI Assets condition: not(eq(dependencies.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true)) - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - windows_x64 - windows_x86 - linux_x64 jobParameters: - uploadAs: 'pipelineArtifacts' + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) /p:NoPgoOptimize=true + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType) + displayName: JIT and SuperPMI Assets condition: not(eq(dependencies.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true)) - template: /eng/pipelines/common/platform-matrix.yml diff --git a/eng/pipelines/coreclr/superpmi-replay.yml b/eng/pipelines/coreclr/superpmi-replay.yml index 88e575b37d72a..638f9840b5391 100644 --- a/eng/pipelines/coreclr/superpmi-replay.yml +++ b/eng/pipelines/coreclr/superpmi-replay.yml @@ -35,13 +35,23 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - windows_x64 - windows_x86 jobParameters: - uploadAs: 'pipelineArtifacts' + buildArgs: -s clr.alljits+clr.spmi -c $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: CheckedJIT_$(osGroup)$(osSubgroup)_$(archType) + displayName: JIT and SuperPMI Assets condition: not(eq(dependencies.evaluate_paths.outputs['SetPathVars_jiteeversionguid.containsChange'], true)) - template: /eng/pipelines/common/platform-matrix.yml diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml deleted file mode 100644 index 1c131f3fc8f14..0000000000000 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ /dev/null @@ -1,144 +0,0 @@ -parameters: - archType: '' - buildConfig: '' - container: '' - crossBuild: false - osGroup: '' - osSubgroup: '' - condition: true - pool: '' - timeoutInMinutes: '' - variables: {} - dependOnEvaluatePaths: false - disableComponentGovernance: false - uploadAs: 'azureBlob' - -### Product build -jobs: -- template: xplat-pipeline-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - condition: ${{ parameters.condition }} - helixType: 'build/product/' - enableMicrobuild: true - pool: ${{ parameters.pool }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - - # Compute job name from template parameters - name: ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('CoreCLR JIT Build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - # Run all steps in the container. - # Note that the containers are defined in platform-matrix.yml - container: ${{ parameters.container }} - - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - crossBuild: ${{ parameters.crossBuild }} - - gatherAssetManifests: true - - variables: - - name: osGroup - value: ${{ parameters.osGroup }} - - name: osSubgroup - value: ${{ parameters.osSubgroup }} - - - name: publishLogsArtifactPrefix - value: 'BuildLogs_CoreCLR_JIT' - - name: uploadAs - value: ${{ parameters.uploadAs }} - - - name: compilerArg - value: '' - - - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonSetupScript - value: 'py -3 -m venv $(Build.SourcesDirectory)\venv' - - name: PythonScript - value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe' - - name: PipScript - value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe -m pip' - - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonSetupScript - value: 'python3 -m venv $(Build.SourcesDirectory)/venv' - - name: PythonScript - value: '$(Build.SourcesDirectory)/venv/bin/python3' - - name: PipScript - value: '$(Build.SourcesDirectory)/venv/bin/pip3' - - - ${{ parameters.variables }} - - steps: - - # Install native dependencies - # Linux builds use docker images with dependencies preinstalled, - # and FreeBSD builds use a build agent with dependencies - # preinstalled, so we only need this step for OSX and Windows. - - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup) - displayName: Install native dependencies (OSX) - - # Install internal tools on official builds - # Since our internal tools are behind an authenticated feed, - # we need to use the DotNetCli AzDO task to restore from the feed using a service connection. - # We can't do this from within the build, so we need to do this as a separate step. - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/pipelines/common/restore-internal-tools.yml - - # Build/Generate native prerequisites - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.nativeprereqs $(crossArg) -arch $(archType) -c $(buildConfig) -ci /bl:$(Build.SourcesDirectory)artifacts/log/$(buildConfig)/CoreCLRNativePrereqs.binlog - displayName: Build and generate native prerequisites - - # Build CoreCLR JIT - - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) -ci $(compilerArg) -component alljits -component spmi - displayName: Build CoreCLR JIT - - ${{ if eq(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -component alljits -component spmi - displayName: Build CoreCLR JIT - - - ${{ if eq(parameters.uploadAs, 'azureBlob') }}: - # Add authenticated pip feed - - task: PipAuthenticate@1 - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: public/dotnet-public-pypi - onlyAddExtraIndex: false - - - script: $(PythonSetupScript) - displayName: Enable python venv - - # Ensure the Python azure-storage-blob package is installed before doing the upload. - - script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall - displayName: Upgrade Pip to latest and install azure-storage-blob Python package - - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(buildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion) --use_latest_jit_change - displayName: Upload JIT to Azure Storage - env: - CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline - - - ${{ if eq(parameters.uploadAs, 'pipelineArtifacts') }}: - # Publish product output directory for consumption by tests. - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(buildProductRootFolderPath) - includeRootFolder: false - archiveType: $(archiveType) - tarCompression: $(tarCompression) - archiveExtension: $(archiveExtension) - artifactName: $(buildProductArtifactName) - displayName: 'product build' - - # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: '$(publishLogsArtifactPrefix)_Attempt$(System.JobAttempt)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - continueOnError: true - condition: always() diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml deleted file mode 100644 index 9ed21ff433cd9..0000000000000 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ /dev/null @@ -1,262 +0,0 @@ -parameters: - archType: '' - buildConfig: '' - condition: true - container: '' - crossBuild: false - dependOnEvaluatePaths: false - disableComponentGovernance: false - disableClrTest: false - isOfficialBuild: false - osGroup: '' - osSubgroup: '' - platform: '' - pool: '' - runtimeVariant: '' - signBinaries: false - testGroup: '' - timeoutInMinutes: '' - variables: {} - -### Product build -jobs: -- template: xplat-pipeline-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - runtimeVariant: ${{ parameters.runtimeVariant }} - testGroup: ${{ parameters.testGroup }} - helixType: 'build/product/' - enableMicrobuild: true - pool: ${{ parameters.pool }} - condition: ${{ parameters.condition }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - disableClrTest: ${{ parameters.disableClrTest }} - - # Compute job name from template parameters - name: ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', - parameters.runtimeVariant, - parameters.osGroup, - parameters.osSubgroup, - parameters.archType, - parameters.buildConfig) }} - displayName: ${{ format('CoreCLR {0} Product Build {1}{2} {3} {4}', - parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, - parameters.archType, - parameters.buildConfig) }} - - # Run all steps in the container. - # Note that the containers are defined in platform-matrix.yml - container: ${{ parameters.container }} - - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - crossBuild: ${{ parameters.crossBuild }} - - gatherAssetManifests: true - variables: - - name: osGroup - value: ${{ parameters.osGroup }} - - name: osSubgroup - value: ${{ parameters.osSubgroup }} - - name: compilerArg - value: '' - - name: publishLogsArtifactPrefix - value: 'BuildLogs_CoreCLR' - - name: officialBuildIdArg - value: '' - - ${{ if eq(parameters.isOfficialBuild, true) }}: - - name: officialBuildIdArg - value: '/p:OfficialBuildId=$(Build.BuildNumber)' - - name: enforcePgoArg - value: '' - # The EnforcePGO script is only supported on Windows and is not supported on arm64. - - ${{ if and(eq(parameters.buildConfig, 'Release'), and(eq(parameters.osGroup, 'windows'), ne(parameters.archType, 'arm64'))) }}: - - name: enforcePgoArg - value: '-enforcepgo' - - - name: clrInterpreterBuildArg - value: '' - - ${{ if eq(parameters.testGroup, 'clrinterpreter') }}: - - name: clrInterpreterBuildArg - value: '-cmakeargs "-DFEATURE_INTERPRETER=1"' - - - name: clrRuntimeComponentsBuildArg - value: '' - - ${{ if ne(parameters.testGroup, 'innerloop') }}: - - name: clrRuntimeComponentsBuildArg - value: '-component runtime -component alljits -component nativeaot -component spmi ' - - - name: SignType - value: $[ coalesce(variables.OfficialSignType, 'real') ] - - # Set a default empty argument for the pgo path. - # This will be set during the 'native prerequisites' step if PGO optimization is enabled. - - name: CoreClrPgoDataArg - value: '' - - - name: nativeSymbols - value: '' - - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - name: nativeSymbols - value: '--keepnativesymbols' - - - ${{ parameters.variables }} - - steps: - # Install native dependencies - # Linux builds use docker images with dependencies preinstalled, - # and FreeBSD builds use a build agent with dependencies - # preinstalled, so we only need this step for OSX and Windows. - - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup) - displayName: Install native dependencies - - # Install internal tools on official builds - # Since our internal tools are behind an authenticated feed, - # we need to use the DotNetCli AzDO task to restore from the feed using a service connection. - # We can't do this from within the build, so we need to do this as a separate step. - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/pipelines/common/restore-internal-tools.yml - - # Install MicroBuild for signing the DAC and DBI - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.signBinaries, true), eq(parameters.osGroup, 'windows')) }}: - - task: MicroBuildSigningPlugin@2 - displayName: Install MicroBuild plugin for Signing - inputs: - signType: $(SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - continueOnError: false - condition: and(succeeded(), in(variables['SignType'], 'real', 'test')) - - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - ${{ if ne(parameters.osGroup, 'windows') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if eq(parameters.osGroup, 'windows') }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage before Build - - # Build/Generate native prerequisites - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.nativeprereqs $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) -ci /bl:$(Build.SourcesDirectory)artifacts/log/$(buildConfig)/CoreCLRNativePrereqs.binlog - displayName: Build and generate native prerequisites - - # Build CoreCLR Runtime - - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) $(nativeSymbols) - displayName: Build CoreCLR Runtime - - ${{ if eq(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) - displayName: Build CoreCLR Runtime - - - ${{ if or(eq(parameters.crossBuild, 'true'), ne(parameters.archType, 'x64')) }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -hostarch x64 $(osArg) -ci $(compilerArg) -component crosscomponents -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" $(officialBuildIdArg) - displayName: Build CoreCLR Cross-Arch Tools (Tools that run on x64 targeting x86) - - - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage after Build - - # Build CoreCLR Managed Components - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages $(crossArg) $(compilerArg) -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) -ci - displayName: Build managed product components and packages - - # Build native test components - - ${{ if and(ne(parameters.isOfficialBuild, true), ne(parameters.disableClrTest, true)) }}: - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(osArg) $(priorityArg) $(compilerArg) - displayName: Build native test components - - # Sign and add entitlements to these MacOS binaries - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(parameters.osGroup, 'osx') }}: - - - template: /eng/pipelines/common/macos-sign-with-entitlements.yml - parameters: - filesToSign: - - name: createdump - path: $(buildProductRootFolderPath) - - name: corerun - path: $(buildProductRootFolderPath) - - - task: CopyFiles@2 - displayName: 'Copy signed createdump to sharedFramework' - inputs: - contents: createdump - sourceFolder: $(buildProductRootFolderPath) - targetFolder: $(buildProductRootFolderPath)/sharedFramework - overWrite: true - - - ${{ if and(eq(parameters.osGroup, 'windows'), eq(parameters.signBinaries, true)) }}: - - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml - parameters: - basePath: $(buildProductRootFolderPath) - isOfficialBuild: ${{ parameters.signBinaries }} - timeoutInMinutes: 30 - - - ${{ if ne(parameters.disableClrTest, true) }}: - # Publish product output directory for consumption by tests. - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(buildProductRootFolderPath) - includeRootFolder: false - archiveType: $(archiveType) - tarCompression: $(tarCompression) - archiveExtension: $(archiveExtension) - artifactName: $(buildProductArtifactName) - displayName: 'product build' - - - ${{ if and(ne(parameters.testGroup, ''), ne(parameters.disableClrTest, true)) }}: - # Publish test native components for consumption by test execution. - - ${{ if ne(parameters.isOfficialBuild, true) }}: - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(nativeTestArtifactRootFolderPath) - includeRootFolder: false - archiveType: $(archiveType) - tarCompression: $(tarCompression) - archiveExtension: $(archiveExtension) - artifactName: $(nativeTestArtifactName) - displayName: 'native test components' - - # Save packages using the prepare-signed-artifacts format. - - ${{ if eq(parameters.isOfficialBuild, true) }}: - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: ${{ parameters.platform }} - - - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: - - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log - displayName: Collect vslogs on exit - condition: always() - - - # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: '$(publishLogsArtifactPrefix)_Attempt$(System.JobAttempt)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - continueOnError: true - condition: always() diff --git a/eng/pipelines/coreclr/templates/build-native-test-assets-step.yml b/eng/pipelines/coreclr/templates/build-native-test-assets-step.yml new file mode 100644 index 0000000000000..e8d1fcfe69cc1 --- /dev/null +++ b/eng/pipelines/coreclr/templates/build-native-test-assets-step.yml @@ -0,0 +1,16 @@ +# Build the native assets for the tests in the src/tests +parameters: + compiler: '' + +steps: + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(nativeTestArtifactConfig) $(archType) $(crossArg) $(priorityArg) ${{ parameters.compiler }} + displayName: Build native test components + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(nativeTestArtifactRootFolderPath) + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: $(nativeTestArtifactName) + displayName: 'native test components' diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml index 3ca668998b133..41d1541c61057 100644 --- a/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml @@ -24,7 +24,7 @@ parameters: ### crossgen matches that of native, e.g. arm-hosted-arm-targeting, crossgen. jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -82,9 +82,7 @@ jobs: # Test job depends on the corresponding build job dependsOn: - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_ # Run all steps in the container. # Note that the containers are defined in platform-matrix.yml @@ -96,21 +94,11 @@ jobs: # Download product build - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: 'Release_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactName: 'Release_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)' displayName: 'product build' - # Optionally download live-built libraries - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir) - cleanUnpackFolder: false - artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' - artifactName: '$(librariesBuildArtifactName)' - displayName: 'live-built libraries' - # Populate Core_Root - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) $(crossArg) generatelayoutonly displayName: Populate Core_Root diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml index 4faa6c501cfe1..fbf10b45ea8f3 100644 --- a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml @@ -12,12 +12,6 @@ parameters: pool: '' targetarch: '' targetos: '' - - # When set to a non-empty value (Debug / Release), it determines libraries - # build configuration to use for the tests. Setting this property implies - # a dependency of this job on the appropriate libraries build and is used - # to construct the name of the Azure artifact representing libraries build - # to use for building the tests. liveLibrariesBuildConfig: '' ### Crossgen-comparison job @@ -26,7 +20,7 @@ parameters: ### crossgen matches that of native, e.g. arm-hosted-arm-targeting, crossgen. jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -84,9 +78,7 @@ jobs: # Test job depends on the corresponding build job dependsOn: - ${{ format('test_crossgen2_comparison_build_{0}_{1}_Release', parameters.targetos, parameters.targetarch)}} - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_ # Run all steps in the container. # Note that the containers are defined in platform-matrix.yml @@ -98,21 +90,11 @@ jobs: # Download product build - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: 'Release_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactName: 'Release_CoreCLR_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)' displayName: 'product build' - # Optionally download live-built libraries - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir) - cleanUnpackFolder: false - artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' - artifactName: '$(librariesBuildArtifactName)' - displayName: 'live-built libraries' - # Populate Core_Root - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) $(crossArg) generatelayoutonly displayName: Populate Core_Root diff --git a/eng/pipelines/coreclr/templates/format-job.yml b/eng/pipelines/coreclr/templates/format-job.yml index 88e5184db032c..3b360aa355fdc 100644 --- a/eng/pipelines/coreclr/templates/format-job.yml +++ b/eng/pipelines/coreclr/templates/format-job.yml @@ -13,7 +13,7 @@ parameters: ### Format job jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -28,17 +28,12 @@ jobs: helixType: 'format' pool: ${{ parameters.pool }} variables: - + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} - - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'py -3' - - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'python3' - - ${{ if eq(parameters.osGroup, 'linux') }}: - name: LinuxCrossArg value: '--cross' @@ -46,7 +41,7 @@ jobs: - name: LinuxCrossArg value: '' - condition: ${{ parameters.condition }} + condition: and(succeeded(), ${{ parameters.condition }}) steps: @@ -59,6 +54,9 @@ jobs: includePreviewVersions: true installationPath: $(Agent.ToolsDirectory)/dotnet + - script: $(PythonSetupScript) + displayName: Enable python venv + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitformat.py -r $(Build.SourcesDirectory) -o $(osGroup) -a $(archType) $(LinuxCrossArg) displayName: Run jitformat.py diff --git a/eng/pipelines/coreclr/templates/jit-exploratory-job.yml b/eng/pipelines/coreclr/templates/jit-exploratory-job.yml deleted file mode 100644 index 94771613c6668..0000000000000 --- a/eng/pipelines/coreclr/templates/jit-exploratory-job.yml +++ /dev/null @@ -1,75 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osSubgroup: '' - runtimeVariant: '' - testGroup: '' - framework: net6.0 # Specify the appropriate framework when running release branches (ie netcoreapp3.0 for release/3.0) - liveLibrariesBuildConfig: '' - variables: {} - helixQueues: '' - runtimeType: 'coreclr' - pool: '' - codeGenType: 'JIT' - runJobTemplate: '/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml' - additionalSetupParameters: '' - toolName: '' # Antigen or Fuzzlyn - -### Exploratory job - -### Each exploratory job depends on a corresponding build job with the same -### buildConfig and archType. - -jobs: -- template: ${{ parameters.runJobTemplate }} - parameters: - # Compute job name from template parameters - jobName: ${{ format('exploratory_{0}{1}_{2}_{3}_{4}_{5}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType) }} - displayName: ${{ format('Exploratory {0}{1} {2} {3} {4} {5}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType) }} - pool: ${{ parameters.pool }} - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - runtimeVariant: ${{ parameters.runtimeVariant }} - liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} - runtimeType: ${{ parameters.runtimeType }} - codeGenType: ${{ parameters.codeGenType }} - testGroup: ${{ parameters.testGroup }} - helixQueues: ${{ parameters.helixQueues }} - additionalSetupParameters: ${{ parameters.additionalSetupParameters }} - toolName: ${{ parameters.toolName }} - # Test job depends on the corresponding build job - dependsOn: - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - - variables: ${{ parameters.variables }} - - frameworks: - - ${{ parameters.framework }} - steps: - # Extra steps that will be passed to the exploratory template and run before sending the job to helix (all of which is done in the template) - - # Optionally download live-built libraries - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir) - cleanUnpackFolder: false - artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' - artifactName: '$(librariesBuildArtifactName)' - displayName: 'live-built libraries' - - # Download coreclr - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' - displayName: 'Coreclr product build' - - # Create Core_Root - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) - displayName: Create Core_Root - condition: succeeded() diff --git a/eng/pipelines/coreclr/templates/jit-exploratory-steps.yml b/eng/pipelines/coreclr/templates/jit-exploratory-steps.yml new file mode 100644 index 0000000000000..b26906ded6ad0 --- /dev/null +++ b/eng/pipelines/coreclr/templates/jit-exploratory-steps.yml @@ -0,0 +1,90 @@ +parameters: + osGroup: '' # required -- operating system for the job + osSubgroup: '' # optional -- operating system subgroup + archType: '' # required -- targeting CPU architecture + buildConfig: '' # required -- build configuration + helixQueues: '' # required -- Helix queues + toolName: '' # required -- which tool to use: Antigen or Fuzzlyn? + +steps: +- script: $(PythonSetupScript) + displayName: Enable python venv + +- script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/fuzzer_setup.py -tool_name $(toolName) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) + displayName: ${{ format('{0} setup ({1}-{2})', parameters.toolName, parameters.osGroup, parameters.archType) }} + + # Run exploratory tool in helix +- template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml + parameters: + displayName: 'Send job to Helix' + helixBuild: $(Build.BuildNumber) + helixSource: $(_HelixSource) + helixType: 'build/tests/' + helixQueues: ${{ join(',', parameters.helixQueues) }} + creator: dotnet-bot + WorkItemTimeout: 2:30 # 2.5 hours + WorkItemDirectory: '$(WorkItemDirectory)' + CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)' + helixProjectArguments: '$(Build.SourcesDirectory)/src/coreclr/scripts/exploratory.proj' + BuildConfig: ${{ parameters.buildConfig }} + osGroup: ${{ parameters.osGroup }} + RunConfiguration: '$(RunConfiguration)' + ToolName: ${{ parameters.toolName }} + RunReason: '$(RunReason)' + +# Always upload the available issues-summary.txt files +- task: CopyFiles@2 + displayName: Copying issues-summary.txt of all partitions + inputs: + sourceFolder: '$(HelixResults)' + contents: '**/issues-summary-*.txt' + targetFolder: '$(IssuesLocation)' + continueOnError: true + condition: always() + +# Always upload the available AllIssues-*.zip files to be used for summarization +- task: CopyFiles@2 + displayName: Copying AllIssues.zip of all partitions + inputs: + sourceFolder: '$(HelixResults)' + contents: '**/AllIssues-*.zip' + targetFolder: '$(IssuesLocation)' + continueOnError: true + condition: always() + +- task: PublishPipelineArtifact@1 + displayName: Publish issue files + inputs: + targetPath: $(IssuesLocation) + artifactName: 'Issues_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)' + continueOnError: true + condition: always() + +# Always upload the available log files +- task: CopyFiles@2 + displayName: Copying fuzzer logs of all partitions + inputs: + sourceFolder: '$(HelixResults)' + contents: '**/*.log' + targetFolder: '$(FuzzerLogsLocation)' + continueOnError: true + condition: always() + +- task: PublishPipelineArtifact@1 + displayName: Publish fuzzer log files + inputs: + targetPath: '$(FuzzerLogsLocation)' + artifactName: '$(toolName)_Logs_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)' + continueOnError: true + condition: always() + +- script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/$(SummarizeScript) -issues_directory $(IssuesLocation) -arch $(archType) -platform $(osGroup)$(osSubgroup) -build_config $(_BuildConfig) + displayName: ${{ format('Summarize ({0}{1} {2})', parameters.osGroup, parameters.osSubgroup, parameters.archType) }} + condition: always() + +- task: PublishPipelineArtifact@1 + displayName: ${{ format('Publish {0} build logs', parameters.toolName) }} + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + artifactName: '$(toolName)_BuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)' + condition: always() diff --git a/eng/pipelines/coreclr/templates/jit-exploratory-variables.yml b/eng/pipelines/coreclr/templates/jit-exploratory-variables.yml new file mode 100644 index 0000000000000..9af139026c5d9 --- /dev/null +++ b/eng/pipelines/coreclr/templates/jit-exploratory-variables.yml @@ -0,0 +1,53 @@ +parameters: + osGroup: '' # required -- operating system for the job + osSubgroup: '' # optional -- operating system subgroup + archType: '' # required -- targeting CPU architecture + buildConfig: '' # required -- build configuration + toolName: '' # required -- which tool to use: Antigen or Fuzzlyn? + +variables: +- template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} + +- ${{ if in(variables['Build.Reason'], 'Schedule') }}: + - name: RunReason + value: 'Scheduled' +- ${{ if notin(variables['Build.Reason'], 'Schedule') }}: + - name: RunReason + value: 'PR' +- ${{ if eq(parameters.osGroup, 'windows') }}: + - name: Core_Root_Dir + value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' + - name: HelixResults + value: '$(Build.SourcesDirectory)\artifacts\helixresults\' + - name: IssuesLocation + value: '$(Build.SourcesDirectory)\artifacts\issues\' + - name: AntigenLogsLocation + value: '$(Build.SourcesDirectory)\artifacts\antigen_logs\' + - name: FuzzlynLogsLocation + value: '$(Build.SourcesDirectory)\artifacts\fuzzlyn_logs\' + +- ${{ if ne(parameters.osGroup, 'windows') }}: + - name: Core_Root_Dir + value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Tests/Core_Root' + - name: HelixResults + value: '$(Build.SourcesDirectory)/artifacts/helixresults/' + - name: IssuesLocation + value: '$(Build.SourcesDirectory)/artifacts/issues/' + - name: AntigenLogsLocation + value: '$(Build.SourcesDirectory)/artifacts/antigen_logs/' + - name: FuzzlynLogsLocation + value: '$(Build.SourcesDirectory)/artifacts/fuzzlyn_logs/' + +- ${{ if eq(parameters.toolName, 'Antigen') }}: + - name: SummarizeScript + value: 'antigen_summarize.py' + - name: FuzzerLogsLocation + value: '$(AntigenLogsLocation)' + +- ${{ if eq(parameters.toolName, 'Fuzzlyn') }}: + - name: SummarizeScript + value: 'fuzzlyn_summarize.py' + - name: FuzzerLogsLocation + value: '$(FuzzlynLogsLocation)' diff --git a/eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml b/eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml new file mode 100644 index 0000000000000..1e9c70114c561 --- /dev/null +++ b/eng/pipelines/coreclr/templates/jit-outerloop-pipeline.yml @@ -0,0 +1,57 @@ +parameters: + - name: platforms + type: object + - name: testGroup + type: string + default: outerloop + +extends: + template: /eng/pipelines/common/templates/pipeline-with-resources.yml + parameters: + stages: + - stage: Build + jobs: + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + platforms: ${{ parameters.platforms }} + jobParameters: + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: ${{ parameters.testGroup }} + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml + buildConfig: checked + platforms: + - CoreClrTestBuildHost # Either osx_x64 or linux_x64 + jobParameters: + testGroup: ${{ parameters.testGroup }} + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + buildConfig: checked + platforms: ${{ parameters.platforms }} + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/coreclr/templates/jit-python-variables.yml b/eng/pipelines/coreclr/templates/jit-python-variables.yml new file mode 100644 index 0000000000000..3ca90600cfb09 --- /dev/null +++ b/eng/pipelines/coreclr/templates/jit-python-variables.yml @@ -0,0 +1,18 @@ +parameters: + osGroup: '' + +variables: +- ${{ if eq(parameters.osGroup, 'windows') }}: + - name: PythonSetupScript + value: 'py -3 -m venv $(Build.SourcesDirectory)\venv' + - name: PythonScript + value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe' + - name: PipScript + value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe -m pip' +- ${{ if ne(parameters.osGroup, 'windows') }}: + - name: PythonSetupScript + value: 'python3 -m venv $(Build.SourcesDirectory)/venv' + - name: PythonScript + value: '$(Build.SourcesDirectory)/venv/bin/python3' + - name: PipScript + value: '$(Build.SourcesDirectory)/venv/bin/pip3' diff --git a/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml b/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml deleted file mode 100644 index 91472d75b57de..0000000000000 --- a/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml +++ /dev/null @@ -1,185 +0,0 @@ -parameters: - steps: [] # optional -- any additional steps that need to happen before pulling down the jitutils repo and sending the jitutils to helix (ie building your repo) - variables: [] # optional -- list of additional variables to send to the template - jobName: '' # required -- job name - displayName: '' # optional -- display name for the job. Will use jobName if not passed - pool: '' # required -- name of the Build pool - container: '' # required -- name of the container - buildConfig: '' # required -- build configuration - archType: '' # required -- targeting CPU architecture - osGroup: '' # required -- operating system for the job - osSubgroup: '' # optional -- operating system subgroup - continueOnError: 'false' # optional -- determines whether to continue the build if the step errors - dependsOn: '' # optional -- dependencies of the job - timeoutInMinutes: 320 # optional -- timeout for the job - enableTelemetry: false # optional -- enable for telemetry - liveLibrariesBuildConfig: '' # optional -- live-live libraries configuration to use for the run - runtimeType: 'coreclr' # optional -- Sets the runtime as coreclr or mono - codeGenType: 'JIT' # optional -- Decides on the codegen technology if running on mono - helixQueues: '' # required -- Helix queues - dependOnEvaluatePaths: false - toolName: '' # required -- which tool to use: Antigen or Fuzzlyn? - -jobs: -- template: xplat-pipeline-job.yml - parameters: - dependsOn: ${{ parameters.dependsOn }} - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} - enableTelemetry: ${{ parameters.enableTelemetry }} - enablePublishBuildArtifacts: true - continueOnError: ${{ parameters.continueOnError }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: '${{ parameters.displayName }}' - ${{ if eq(parameters.displayName, '') }}: - displayName: '${{ parameters.jobName }}' - - variables: - - ${{ each variable in parameters.variables }}: - - ${{insert}}: ${{ variable }} - - - HelixApiAccessToken: '' - - HelixPreCommand: '' - - - ${{ if in(variables['Build.Reason'], 'Schedule') }}: - - name: RunReason - value: 'Scheduled' - - ${{ if notin(variables['Build.Reason'], 'Schedule') }}: - - name: RunReason - value: 'PR' - - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' - - name: Core_Root_Dir - value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' - - name: HelixResults - value: '$(Build.SourcesDirectory)\artifacts\helixresults\' - - name: IssuesLocation - value: '$(Build.SourcesDirectory)\artifacts\issues\' - - name: AntigenLogsLocation - value: '$(Build.SourcesDirectory)\artifacts\antigen_logs\' - - name: FuzzlynLogsLocation - value: '$(Build.SourcesDirectory)\artifacts\fuzzlyn_logs\' - - - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'python3' - - name: PipScript - value: 'pip3' - - name: Core_Root_Dir - value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Tests/Core_Root' - - name: HelixResults - value: '$(Build.SourcesDirectory)/artifacts/helixresults/' - - name: IssuesLocation - value: '$(Build.SourcesDirectory)/artifacts/issues/' - - name: AntigenLogsLocation - value: '$(Build.SourcesDirectory)/artifacts/antigen_logs/' - - name: FuzzlynLogsLocation - value: '$(Build.SourcesDirectory)/artifacts/fuzzlyn_logs/' - - - ${{ if eq(parameters.toolName, 'Antigen') }}: - - name: SummarizeScript - value: 'antigen_summarize.py' - - name: FuzzerLogsLocation - value: '$(AntigenLogsLocation)' - - - ${{ if eq(parameters.toolName, 'Fuzzlyn') }}: - - name: SummarizeScript - value: 'fuzzlyn_summarize.py' - - name: FuzzerLogsLocation - value: '$(FuzzlynLogsLocation)' - - workspace: - clean: all - pool: - ${{ parameters.pool }} - container: ${{ parameters.container }} - steps: - - ${{ parameters.steps }} - - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/fuzzer_setup.py -tool_name $(toolName) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) - displayName: ${{ format('{0} setup ({1}-{2})', parameters.toolName, parameters.osGroup, parameters.archType) }} - - # Run exploratory tool in helix - - template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml - parameters: - displayName: 'Send job to Helix' - helixBuild: $(Build.BuildNumber) - helixSource: $(_HelixSource) - helixType: 'build/tests/' - helixQueues: ${{ join(',', parameters.helixQueues) }} - creator: dotnet-bot - WorkItemTimeout: 2:30 # 2.5 hours - WorkItemDirectory: '$(WorkItemDirectory)' - CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)' - helixProjectArguments: '$(Build.SourcesDirectory)/src/coreclr/scripts/exploratory.proj' - BuildConfig: ${{ parameters.buildConfig }} - osGroup: ${{ parameters.osGroup }} - RunConfiguration: '$(RunConfiguration)' - ToolName: ${{ parameters.toolName }} - RunReason: '$(RunReason)' - - # Always upload the available issues-summary.txt files - - task: CopyFiles@2 - displayName: Copying issues-summary.txt of all partitions - inputs: - sourceFolder: '$(HelixResults)' - contents: '**/issues-summary-*.txt' - targetFolder: '$(IssuesLocation)' - continueOnError: true - condition: always() - - # Always upload the available AllIssues-*.zip files to be used for summarization - - task: CopyFiles@2 - displayName: Copying AllIssues.zip of all partitions - inputs: - sourceFolder: '$(HelixResults)' - contents: '**/AllIssues-*.zip' - targetFolder: '$(IssuesLocation)' - continueOnError: true - condition: always() - - - task: PublishPipelineArtifact@1 - displayName: Publish issue files - inputs: - targetPath: $(IssuesLocation) - artifactName: 'Issues_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - continueOnError: true - condition: always() - - # Always upload the available log files - - task: CopyFiles@2 - displayName: Copying fuzzer logs of all partitions - inputs: - sourceFolder: '$(HelixResults)' - contents: '**/*.log' - targetFolder: '$(FuzzerLogsLocation)' - continueOnError: true - condition: always() - - - task: PublishPipelineArtifact@1 - displayName: Publish fuzzer log files - inputs: - targetPath: '$(FuzzerLogsLocation)' - artifactName: '$(toolName)_Logs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - continueOnError: true - condition: always() - - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/$(SummarizeScript) -issues_directory $(IssuesLocation) -arch $(archType) -platform $(osGroup)$(osSubgroup) -build_config $(buildConfig) - displayName: ${{ format('Summarize ({0}{1} {2})', parameters.osGroup, parameters.osSubgroup, parameters.archType) }} - condition: always() - - - task: PublishPipelineArtifact@1 - displayName: ${{ format('Publish {0} build logs', parameters.toolName) }} - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: '$(toolName)_BuildLogs_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - condition: always() diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 450cd2799fa0c..a54cb58dd8da3 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -24,7 +24,6 @@ parameters: iOSLlvmBuild: 'False' iOSStripSymbols: 'False' hybridGlobalization: 'False' - skipLiveLibrariesDownload: false collectHelixLogsScript: '' timeoutInMinutes: 320 compare: false @@ -79,11 +78,9 @@ jobs: ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: dependsOn: - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono', 'iOSNativeAOT', 'wasm')) }}: - - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if and(ne(parameters.liveLibrariesBuildConfig, ''), eq(parameters.skipLiveLibrariesDownload, 'false')) }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, 'coreclr') }} - ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}: - - ${{ format('mono_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, 'mono') }} - ${{ if eq(parameters.runtimeType, 'wasm')}}: - ${{ format('build_{0}{1}_{2}_{3}_{4}_{5}', 'browser', '', 'wasm', 'linux', parameters.buildConfig, parameters.runtimeType) }} - ${{ if and(eq(parameters.codeGenType, 'AOT'), ne(parameters.runtimeType, 'wasm'))}}: @@ -142,6 +139,15 @@ jobs: - ${{ if eq(parameters.crossBuild, true) }}: - _crossBuildPropertyArg: '-cross' + - name: librariesDownloadDir + value: '$(Build.SourcesDirectory)/artifacts' + + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + frameworks: - ${{ parameters.framework }} steps: @@ -150,35 +156,6 @@ jobs: $(Build.SourcesDirectory)/eng/common/msbuild.sh $(Build.SourcesDirectory)/eng/testing/performance/add_properties_to_pipeline.proj /t:SetVariables displayName: Add Properties To Pipeline Env - # Optionally download live-built libraries - - ${{ if and(ne(parameters.liveLibrariesBuildConfig, ''), eq(parameters.skipLiveLibrariesDownload, 'false')) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir) - cleanUnpackFolder: false - artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' - artifactName: '$(librariesBuildArtifactName)' - displayName: 'live-built libraries' - - # Download coreclr - - ${{ if not(in(parameters.runtimeType, 'AndroidMono', 'iOSMono', 'iOSNativeAOT', 'wasm')) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' - displayName: 'Coreclr product build' - - # Download mono - - ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) - cleanUnpackFolder: false - artifactFileName: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)_$(archType)_$(buildConfig)$(archiveExtension)' - artifactName: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)_$(archType)_$(buildConfig)' - displayName: 'Mono runtime' - # Download wasm - ${{ if eq(parameters.runtimeType, 'wasm') }}: - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: @@ -206,9 +183,15 @@ jobs: find $(librariesDownloadDir)/bin/wasm -type d && find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \; displayName: "Create wasm directory (Linux)" + - ${{ elseif eq(parameters.codeGenType, 'AOT') }}: + # Download mono AOT + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: 'BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr$(archiveExtension)' + artifactName: 'BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_coreclr' + displayName: 'Runtime artifacts' - # Download mono AOT - - ${{ if and(eq(parameters.codeGenType, 'AOT'), ne(parameters.runtimeType, 'wasm')) }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/LinuxMonoAOT @@ -218,9 +201,16 @@ jobs: - script: "mkdir -p $(librariesDownloadDir)/bin/aot;mkdir -p $(librariesDownloadDir)/bin/aot/pack;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/mono/linux.${{ parameters.archType }}.Release/cross/linux-${{ parameters.archType }}/* $(librariesDownloadDir)/bin/aot;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/microsoft.netcore.app.runtime.linux-${{ parameters.archType }}/Release/* $(librariesDownloadDir)/bin/aot/pack" displayName: "Create aot directory (Linux)" + - ${{ elseif in(parameters.runtimeType, 'coreclr', 'mono') }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: 'BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_${{ parameters.runtimeType }}$(archiveExtension)' + artifactName: 'BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)_${{ parameters.runtimeType }}' + displayName: 'Runtime artifacts' - # Download artifacts for Android Testing - - ${{ if eq(parameters.runtimeType, 'AndroidMono')}}: + - ${{ elseif eq(parameters.runtimeType, 'AndroidMono')}}: + # Download artifacts for Android Testing - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/androidHelloWorld @@ -237,7 +227,7 @@ jobs: displayName: 'Mono Android BDN Apk' # Download iOSMono and Native AOT tests - - ${{ if or(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.runtimeType, 'iOSNativeAOT')) }}: + - ${{ elseif or(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.runtimeType, 'iOSNativeAOT')) }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(Build.SourcesDirectory)/iosHelloWorld @@ -285,13 +275,19 @@ jobs: # Create Core_Root - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) $(_crossBuildPropertyArg) displayName: Create Core_Root - condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono'), ne('${{ parameters.runtimeType }}', 'wasm'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) + condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono'), not(in('${{ parameters.runtimeType }}', 'wasm', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) - # Copy the runtime directory into the testhost folder to include OOBs. - - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\runtime\\${{parameters.framework}}-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\${{parameters.framework}}-$(osGroup)-$(buildConfigUpper)-$(archType)\\shared\\Microsoft.NETCore.App\\$(productVersion) /E /I /Y;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\${{parameters.framework}}-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y;copy $(Build.SourcesDirectory)\\artifacts\\bin\\coreclr\\$(osGroup).$(archType).$(buildConfigUpper)\\corerun.exe $(Build.SourcesDirectory)\\.dotnet-mono\\shared\\Microsoft.NETCore.App\\$(productVersion)\\corerun.exe" - displayName: "Create mono dotnet (Windows)" - condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), eq(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) + - task: CopyFiles@2 + displayName: Create .dotnet-mono folder + inputs: + SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/testhost/${{ parameters.framework }}-$(osGroup)-$(buildConfigUpper)-$(archType) + TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono + condition: and(succeeded(), eq(variables.runtimeFlavorName, 'Mono'), ne(variables.osGroup, 'windows'), notIn('${{ parameters.runtimeType }}', 'wasm', 'AndroidMono', 'iOSMono', 'iOSNativeAOT')) - - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)/artifacts/bin/runtime/${{parameters.framework}}-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/artifacts/bin/testhost/${{parameters.framework}}-$(osGroup)-$(buildConfigUpper)-$(archType)/shared/Microsoft.NETCore.App/$(productVersion) -rf;cp $(Build.SourcesDirectory)/artifacts/bin/testhost/${{parameters.framework}}-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono -r;cp $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)/corerun $(Build.SourcesDirectory)/.dotnet-mono/shared/Microsoft.NETCore.App/$(productVersion)/corerun" - displayName: "Create mono dotnet (Linux)" - condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'windows'), not(in('${{ parameters.runtimeType }}', 'AndroidMono', 'iOSMono', 'iOSNativeAOT'))) + - task: CopyFiles@2 + displayName: Copy corerun to .dotnet-mono + inputs: + SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper) + Contents: 'corerun*' + TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono + condition: and(succeeded(), eq(variables.runtimeFlavorName, 'Mono'), ne(variables.osGroup, 'windows'), notIn('${{ parameters.runtimeType }}', 'wasm', 'AndroidMono', 'iOSMono', 'iOSNativeAOT')) diff --git a/eng/pipelines/coreclr/templates/run-performance-job.yml b/eng/pipelines/coreclr/templates/run-performance-job.yml index 51eb0ce0e9676..9f33585fd7591 100644 --- a/eng/pipelines/coreclr/templates/run-performance-job.yml +++ b/eng/pipelines/coreclr/templates/run-performance-job.yml @@ -31,7 +31,7 @@ parameters: additionalSetupParameters: '' # optional -- additional setup parameters that are job-specific jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} diff --git a/eng/pipelines/coreclr/templates/run-scenarios-job.yml b/eng/pipelines/coreclr/templates/run-scenarios-job.yml index 44910148e6527..ffe4f5c16482e 100644 --- a/eng/pipelines/coreclr/templates/run-scenarios-job.yml +++ b/eng/pipelines/coreclr/templates/run-scenarios-job.yml @@ -23,7 +23,7 @@ parameters: additionalSetupParameters: '' # optional -- additional setup parameters that are job-specific jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} @@ -108,6 +108,9 @@ jobs: ${{ if ne(parameters.runtimeType, 'wasm') }}: value: --install-dir $(PayloadDirectory)/dotnet + - name: librariesDownloadDir + value: '$(Build.SourcesDirectory)/artifacts' + workspace: clean: all pool: diff --git a/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml index 155d82205938c..1b315b328927d 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml @@ -18,7 +18,7 @@ parameters: dependOnEvaluatePaths: false jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} @@ -38,11 +38,10 @@ jobs: displayName: '${{ parameters.jobName }}' variables: + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' - name: SpmiCollectionLocation value: '$(Build.SourcesDirectory)\artifacts\spmi\' - name: SpmiLogsLocation @@ -50,6 +49,12 @@ jobs: - name: HelixResultLocation value: '$(Build.SourcesDirectory)\artifacts\helixresults\' + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} @@ -65,6 +70,9 @@ jobs: mkdir -p $(SpmiCollectionLocation) displayName: Create directory for SPMI collection + - script: $(PythonSetupScript) + displayName: Enable python venv + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_asmdiffs_checked_release_setup.py -source_directory $(Build.SourcesDirectory) -checked_directory $(buildProductRootFolderPath) -release_directory $(releaseProductRootFolderPath) -arch $(archType) displayName: ${{ format('SuperPMI asmdiffs checked release setup ({0} {1})', parameters.osGroup, parameters.archType) }} diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index d83f6e7e089da..b6f48f78535db 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -19,7 +19,7 @@ parameters: dependOnEvaluatePaths: false jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} @@ -46,6 +46,12 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} variables: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} - HelixApiAccessToken: '' @@ -54,13 +60,11 @@ jobs: - CollectionType: ${{ parameters.collectionType }} - CollectionName: ${{ parameters.collectionName }} + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} + - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonSetupScript - value: 'py -3 -m venv $(Build.SourcesDirectory)\venv' - - name: PythonScript - value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe' - - name: PipScript - value: '$(Build.SourcesDirectory)\venv\Scripts\python.exe -m pip' - name: Core_Root_Dir value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' - name: MchFilesLocation @@ -72,12 +76,6 @@ jobs: - name: PayloadLocation value: '$(Build.SourcesDirectory)\payload' - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonSetupScript - value: 'python3 -m venv $(Build.SourcesDirectory)/venv' - - name: PythonScript - value: '$(Build.SourcesDirectory)/venv/bin/python3' - - name: PipScript - value: '$(Build.SourcesDirectory)/venv/bin/pip3' - name: Core_Root_Dir value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Tests/Core_Root' - name: MchFilesLocation @@ -97,7 +95,7 @@ jobs: value: '$(Core_Root_Dir)' - ${{ if eq(parameters.collectionName, 'realworld') }}: - name: InputDirectory - value: '$(Core_Root_Dir)' + value: '$(Core_Root_Dir)' - ${{ if eq(parameters.collectionName, 'coreclr_tests') }}: - name: InputDirectory value: '$(managedTestArtifactRootFolderPath)' @@ -118,6 +116,7 @@ jobs: - script: $(PythonSetupScript) displayName: Enable python venv + condition: always() - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} diff --git a/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml index da61351842a58..8f70fe36407aa 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml @@ -21,7 +21,7 @@ parameters: diffJitOptions: '' jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} @@ -47,11 +47,17 @@ jobs: - name: diffType value: ${{ parameters.diffType }} + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} + + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' - name: SpmiCollectionLocation value: '$(Build.SourcesDirectory)\artifacts\spmi\' - name: SpmiLogsLocation @@ -61,10 +67,6 @@ jobs: - name: HelixResultLocation value: '$(Build.SourcesDirectory)\artifacts\helixresults\' - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'python3' - - name: PipScript - value: 'pip3' - name: SpmiCollectionLocation value: '$(Build.SourcesDirectory)/artifacts/spmi/' - name: SpmiLogsLocation @@ -94,6 +96,9 @@ jobs: steps: - ${{ parameters.steps }} + - script: $(PythonSetupScript) + displayName: Enable python venv + - ${{ if ne(parameters.osGroup, 'windows') }}: - script: | mkdir -p $(SpmiCollectionLocation) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml index 3083a080c230b..6f7c4529a2137 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml @@ -18,7 +18,7 @@ parameters: dependOnEvaluatePaths: false jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: dependsOn: ${{ parameters.dependsOn }} buildConfig: ${{ parameters.buildConfig }} @@ -38,13 +38,19 @@ jobs: displayName: '${{ parameters.jobName }}' variables: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + runtimeFlavor: coreclr + testGroup: ${{ parameters.testGroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + - ${{ each variable in parameters.variables }}: - ${{insert}}: ${{ variable }} - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} + - name: SpmiCollectionLocation value: '$(Build.SourcesDirectory)\artifacts\spmi\' - name: SpmiLogsLocation @@ -60,6 +66,9 @@ jobs: steps: - ${{ parameters.steps }} + - script: $(PythonSetupScript) + displayName: Enable python venv + - script: | mkdir $(SpmiCollectionLocation) mkdir $(SpmiLogsLocation) diff --git a/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml b/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml index 659483c9bc302..3ef86c53ca91e 100644 --- a/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml @@ -24,8 +24,8 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} helixQueues: ${{ parameters.helixQueues }} dependsOn: - - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, 'checked') }} - - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, 'release') }} + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_checked_' + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_release_' variables: - ${{ each variable in parameters.variables }}: @@ -33,23 +33,22 @@ jobs: - name: releaseProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).Release' - - name: releaseProductArtifactName - value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_release' steps: - # Download jit checked builds - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + artifactFileName: 'CheckedJIT_$(osGroup)$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'CheckedJIT_$(osGroup)$(osSubgroup)_$(archType)' displayName: 'JIT checked build' + cleanupUnpackFolder: false - #Download jit release builds + # Download jit release builds - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(releaseProductRootFolderPath) - artifactFileName: '$(releaseProductArtifactName)$(archiveExtension)' - artifactName: '$(releaseProductArtifactName)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + artifactFileName: 'ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType)' displayName: 'JIT release build' + cleanupUnpackFolder: false diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml index b77ec27e6e5ce..6af16c3db8764 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml @@ -29,12 +29,7 @@ jobs: collectionName: ${{ parameters.collectionName }} # Test job depends on the corresponding build job dependsOn: - - ${{ format('coreclr__product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - # Depend on coreclr x64 so we can download it and use mcs.exe from it while publishing non-x64 arch SPMI collection - - ${{ if ne(parameters.archType, 'x64') }}: - - ${{ format('coreclr__product_build_{0}{1}_x64_{2}', parameters.osGroup, parameters.osSubgroup, parameters.buildConfig) }} - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_' - ${{ if eq(parameters.collectionName, 'coreclr_tests') }}: - 'coreclr_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}' @@ -43,32 +38,29 @@ jobs: steps: # Extra steps that will be passed to the superpmi template and run before sending the job to helix (all of which is done in the template) - # Optionally download live-built libraries - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(librariesDownloadDir) - cleanUnpackFolder: false - artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' - artifactName: '$(librariesBuildArtifactName)' - displayName: 'live-built libraries' - - # Download coreclr + # Download runtime artifacts - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' - displayName: 'Coreclr product build' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: 'BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: 'Runtime build artifacts' - # Download x64 coreclr if running on non-x64 configuration - - ${{ if ne(parameters.archType, 'x64') }}: + # Unzip individual test projects + - ${{ if eq(parameters.collectionName, 'libraries_tests') }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper)' - artifactFileName: 'CoreCLRProduct__$(osGroup)$(osSubgroup)_x64_$(buildConfig)$(archiveExtension)' - artifactName: 'CoreCLRProduct__$(osGroup)$(osSubgroup)_x64_$(buildConfig)' - displayName: 'Coreclr product build (x64)' + unpackFolder: '$(Build.SourcesDirectory)/artifacts/helix/' + artifactFileName: 'LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)$(archiveExtension)' + artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: 'Libraries test artifacts' + - task: ExtractFiles@1 + displayName: 'Unzip libraries tests' + inputs: + archiveFilePatterns: '$(Build.SourcesDirectory)/artifacts/helix/**/*.zip' + destinationFolder: '$(Build.SourcesDirectory)/artifacts/tests/libraries/$(osGroup).$(archType).$(buildConfigUpper)' + cleanDestinationFolder: true + overwriteExistingFiles: true # Download and unzip managed test artifacts - ${{ if eq(parameters.collectionName, 'coreclr_tests') }}: @@ -79,25 +71,6 @@ jobs: artifactName: '$(managedGenericTestArtifactName)' displayName: 'generic managed test artifacts' - # Download and unzip libraries test artifacts - - ${{ if eq(parameters.collectionName, 'libraries_tests') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: '$(Build.SourcesDirectory)/artifacts/tests/libraries_zipped/$(osGroup).$(archType).$(buildConfigUpper)' - artifactFileName: 'libraries_test_assets_${{ parameters.osGroup }}_$(archType)_Release$(archiveExtension)' - artifactName: ${{ format('libraries_test_assets_{0}_$(archType)_Release', parameters.osGroup) }} - displayName: 'generic libraries test artifacts' - - # Unzip individual test projects - - ${{ if eq(parameters.collectionName, 'libraries_tests') }}: - - task: ExtractFiles@1 - displayName: 'Unzip Tests.zip' - inputs: - archiveFilePatterns: '$(Build.SourcesDirectory)/artifacts/tests/libraries_zipped/$(osGroup).$(archType).$(buildConfigUpper)/**/*.zip' - destinationFolder: '$(Build.SourcesDirectory)/artifacts/tests/libraries/$(osGroup).$(archType).$(buildConfigUpper)' - cleanDestinationFolder: true - overwriteExistingFiles: true - # Create Core_Root - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) displayName: Create Core_Root diff --git a/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml b/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml index 2375089e250b9..6bab325a0b21e 100644 --- a/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml @@ -33,9 +33,9 @@ jobs: diffJitOptions: ${{ parameters.diffJitOptions }} dependsOn: - ${{ if in(parameters.diffType, 'asmdiffs', 'all') }}: - - ${{ format('coreclr_jit_build_{0}{1}_{2}_checked', parameters.osGroup, parameters.osSubgroup, parameters.archType) }} + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_checked_' - ${{ if in(parameters.diffType, 'tpdiff', 'all') }}: - - ${{ format('coreclr_jit_build_{0}{1}_{2}_release', parameters.osGroup, parameters.osSubgroup, parameters.archType) }} + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_release_' variables: - ${{ each variable in parameters.variables }}: @@ -48,8 +48,6 @@ jobs: - ${{ if ne(parameters.osGroup, 'windows') }}: - name: releaseProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).Release' - - name: releaseProductArtifactName - value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_release' steps: @@ -57,16 +55,18 @@ jobs: # Download jit checked builds - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + artifactFileName: 'CheckedJIT_$(osGroup)$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'CheckedJIT_$(osGroup)$(osSubgroup)_$(archType)' displayName: 'JIT checked build' + cleanupUnpackFolder: false - ${{ if in(parameters.diffType, 'tpdiff', 'all') }}: # Download jit release builds - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(releaseProductRootFolderPath) - artifactFileName: '$(releaseProductArtifactName)$(archiveExtension)' - artifactName: '$(releaseProductArtifactName)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + artifactFileName: 'ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'ReleaseJIT_$(osGroup)$(osSubgroup)_$(archType)' displayName: 'JIT release build' + cleanupUnpackFolder: false diff --git a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml index 979f95ce685d8..59b7ef9ded0c6 100644 --- a/eng/pipelines/coreclr/templates/superpmi-replay-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-replay-job.yml @@ -26,7 +26,7 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} helixQueues: ${{ parameters.helixQueues }} dependsOn: - - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_' variables: ${{ parameters.variables }} @@ -35,7 +35,8 @@ jobs: # Download jit builds - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(buildProductRootFolderPath) - artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' - artifactName: '$(buildProductArtifactName)' - displayName: 'JIT product build' \ No newline at end of file + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + artifactFileName: 'CheckedJIT_$(osGroup)$(osSubgroup)_$(archType)$(archiveExtension)' + artifactName: 'CheckedJIT_$(osGroup)$(osSubgroup)_$(archType)' + displayName: 'JIT checked build' + cleanupUnpackFolder: false diff --git a/eng/pipelines/coreclr/templates/upload-jits-steps.yml b/eng/pipelines/coreclr/templates/upload-jits-steps.yml new file mode 100644 index 0000000000000..912732b90b847 --- /dev/null +++ b/eng/pipelines/coreclr/templates/upload-jits-steps.yml @@ -0,0 +1,19 @@ +steps: +# Add authenticated pip feed +- task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: public/dotnet-public-pypi + onlyAddExtraIndex: false + +- script: $(PythonSetupScript) + displayName: Enable python venv + +# Ensure the Python azure-storage-blob package is installed before doing the upload. +- script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall + displayName: Upgrade Pip to latest and install azure-storage-blob Python package + +- script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(_BuildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion) --use_latest_jit_change + displayName: Upload JIT to Azure Storage + env: + CLRJIT_AZ_KEY: $(clrjit_key1) # secret key stored as variable in pipeline diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml deleted file mode 100644 index b6c57be0c7fba..0000000000000 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ /dev/null @@ -1,132 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osSubgroup: '' - name: '' - helixType: '(unspecified)' - container: '' - testGroup: '' - crossBuild: false - liveLibrariesBuildConfig: '' - strategy: '' - pool: '' - - # arcade-specific parameters - condition: true - continueOnError: false - dependsOn: '' - dependOnEvaluatePaths: false - displayName: '' - timeoutInMinutes: '' - enableMicrobuild: '' - gatherAssetManifests: false - disableComponentGovernance: false - - variables: {} ## any extra variables to add to the defaults defined below - -jobs: -- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - name: ${{ parameters.name }} - helixType: ${{ parameters.helixType }} - container: ${{ parameters.container }} - crossBuild: ${{ parameters.crossBuild }} - strategy: ${{ parameters.strategy }} - pool: ${{ parameters.pool }} - - # arcade-specific parameters - condition: and(succeeded(), ${{ parameters.condition }}) - continueOnError: ${{ parameters.continueOnError }} - dependsOn: ${{ parameters.dependsOn }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - displayName: ${{ parameters.displayName }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - enableMicrobuild: ${{ parameters.enableMicrobuild }} - gatherAssetManifests: ${{ parameters.gatherAssetManifests }} - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - - variables: - - ${{ if ne(parameters.testGroup, '') }}: - - name: testArtifactRootName - value: ${{ parameters.Group }}${{ parameters.Subgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.testGroup }} - - - ${{ if eq(parameters.testGroup, '') }}: - - name: testArtifactRootName - value: ${{ parameters.Group }}${{ parameters.Subgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }} - - - name: binTestsPath - value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr' - - # Build product defines what we are trying to build, either coreclr or mono - - name: buildProductArtifactName - value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: buildProductRootFolderPath - value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' - - # We need this because both mono and coreclr build currently depends on CoreClr - - name: coreClrProductArtifactName - value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: coreClrProductRootFolderPath - value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' - - - name: corelibProductArtifactName - value: 'CoreLib_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: managedGenericTestArtifactName - value: 'CoreCLRManagedTestArtifacts_AnyOS_AnyCPU_$(buildConfig)' - - - name: managedTestArtifactRootFolderPath - value: '$(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper)' - - - name: nativeTestArtifactName - value: 'CoreCLRNativeTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: nativeTestArtifactRootFolderPath - value: '$(binTestsPath)/obj/$(osGroup).$(archType).$(buildConfigUpper)' - - - name: microsoftNetSdkIlFolderPath - value: '$(Build.SourcesDirectory)/.packages/microsoft.net.sdk.il' - - - name: microsoftNetSdkIlArtifactName - value: 'MicrosoftNetSdkIlPackage_AnyOS_AnyCPU_$(buildConfig)' - - - name: priorityArg - value: '' - - # 'innerloop' and 'clrinterpreter' jobs run the Priority 0 tests; everything else runs the Priority 1 tests. - # 'gc-standalone' is forced to run pri0 as well to start with. - - ${{ if and(ne(parameters.testGroup, 'innerloop'), ne(parameters.testGroup, 'clrinterpreter'), ne(parameters.testGroup, 'gc-standalone'), ne(parameters.testGroup, 'gc-standalone-server') ) }}: - - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: priorityArg - value: 'priority1' - - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: priorityArg - value: '-priority 1' - - - librariesBuildArtifactName: '' - - librariesOverrideArg: '' - - librariesDownloadDir: '' - - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - librariesBuildArtifactName: ${{ format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - - librariesDownloadDir: $(Build.SourcesDirectory)/artifacts - - librariesOverrideArg: ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' - - - ${{ each variable in parameters.variables }}: - - ${{insert}}: ${{ variable }} - - - name: osArg - value: '' - - - ${{ if eq(parameters.osGroup, 'freebsd') }}: - - name: osArg - value: -os freebsd - - steps: ${{ parameters.steps }} diff --git a/eng/pipelines/coreclr/tieringtest.yml b/eng/pipelines/coreclr/tieringtest.yml index 117fac1b5ded7..2ce790dcc26f9 100644 --- a/eng/pipelines/coreclr/tieringtest.yml +++ b/eng/pipelines/coreclr/tieringtest.yml @@ -20,15 +20,29 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - linux_x64 - windows_x64 - windows_x86 - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - testGroup: outerloop + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -54,3 +68,4 @@ extends: tieringTest: true displayNameArgs: TieringTest liveLibrariesBuildConfig: Release + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml b/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml index 10b3e9baca01f..4b16485873a83 100644 --- a/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml +++ b/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml @@ -7,50 +7,13 @@ parameters: jobs: -# -# Build CoreCLR release -# Always as they are needed by Installer and we always build and test the Installer. -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: release - platforms: - - linux_x64 - - linux_arm - - linux_arm64 - - linux_musl_x64 - - osx_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - jobParameters: - testGroup: innerloop - -# -# Build libraries using live CoreLib -# -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Release - platforms: - - linux_x64 - - linux_arm - - linux_arm64 - - linux_musl_x64 - - osx_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - # # Libraries Release Test Execution against a release coreclr runtime # Only when the PR contains a libraries change # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: Release platforms: - linux_x64 @@ -63,10 +26,14 @@ jobs: - windows_arm64 helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml jobParameters: - isOfficialBuild: false - isExtraPlatforms: ${{ parameters.isExtraPlatformsBuild }} - testScope: innerloop - liveRuntimeBuildConfig: release + buildArgs: -s tools+clr+libs+libs.tests -c $(_BuildConfig) -testscope innerloop /p:ArchiveTests=true + nameSuffix: Libraries_Release_CoreCLR + timeoutInMinutes: 150 + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Libraries_Release_CoreCLR condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), @@ -138,67 +105,6 @@ jobs: eq(variables['monoContainsChange'], true), eq(variables['isRollingBuild'], true)) -# -# Build the whole product using Mono and run runtime tests -# Build Mono release -# Only when libraries, mono, or the runtime tests changed -# Currently only these architectures are needed for the runtime tests. -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono - buildConfig: release - platforms: - - linux_arm64 - jobParameters: - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(variables['isRollingBuild'], true)) - -# -# Mono Test builds with CoreCLR runtime tests using live libraries debug build -# Only when Mono is changed -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: innerloop - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) - -# -# Mono CoreCLR runtime Test executions using live libraries in jit mode -# Only when Mono is changed -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - linux_arm64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: innerloop - liveLibrariesBuildConfig: Release - liveRuntimeBuildConfig: release - runtimeVariant: minijit - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) - # # Mono CoreCLR runtime Test executions using live libraries and LLVM Full AOT # Only when Mono is changed @@ -241,28 +147,42 @@ jobs: # # Mono CoreCLR runtime Test executions using live libraries in interpreter mode # Only when Mono is changed + - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: release + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + buildConfig: Release runtimeFlavor: mono platforms: - - linux_arm64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + - linux_arm64 + - osx_arm64 + variables: + - name: timeoutPerTestInMinutes + value: 60 + - name: timeoutPerTestCollectionInMinutes + value: 180 jobParameters: testGroup: innerloop - liveLibrariesBuildConfig: Release - liveRuntimeBuildConfig: release + nameSuffix: AllSubsets_Mono_Interpreter_RuntimeTests runtimeVariant: monointerpreter + buildArgs: -s mono+libs+clr.hosts+clr.iltools -c Release + timeoutInMinutes: 180 condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), eq(variables['isRollingBuild'], true)) + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_Release + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/test-variables.yml # -# Mono CoreCLR runtime Test executions using live libraries in interpreter mode +# Mono CoreCLR runtime Test executions using live libraries in JIT mode # Only when Mono is changed - template: /eng/pipelines/common/platform-matrix.yml @@ -272,7 +192,7 @@ jobs: buildConfig: Release runtimeFlavor: mono platforms: - - osx_arm64 + - linux_arm64 variables: - name: timeoutPerTestInMinutes value: 60 @@ -280,8 +200,8 @@ jobs: value: 180 jobParameters: testGroup: innerloop - nameSuffix: AllSubsets_Mono_Interpreter_RuntimeTests - runtimeVariant: monointerpreter + nameSuffix: AllSubsets_Mono_Minijit_RuntimeTests + runtimeVariant: minijit buildArgs: -s mono+libs+clr.hosts+clr.iltools -c Release timeoutInMinutes: 180 condition: >- diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml index 3bc803b2ba87a..a59953c64310e 100644 --- a/eng/pipelines/installer/jobs/build-job.yml +++ b/eng/pipelines/installer/jobs/build-job.yml @@ -18,7 +18,6 @@ parameters: displayName: '' runtimeVariant: '' pool: '' - runOnlyIfDependenciesSucceeded: false # The target names here should match container names in the resources section in our pipelines, like runtime.yml packageDistroList: @@ -35,6 +34,9 @@ parameters: liveRuntimeBuildConfig: '' liveLibrariesBuildConfig: '' runtimeFlavor: 'coreclr' + unifiedArtifactsName: '' + unifiedBuildNameSuffix: '' + unifiedBuildConfigOverride: '' ### Product build jobs: @@ -50,17 +52,11 @@ jobs: enableMicrobuild: true pool: ${{ parameters.pool }} - ${{ if eq(parameters.runOnlyIfDependenciesSucceeded, true) }}: - condition: and(succeeded(), ${{ parameters.condition }}) - ${{ if ne(parameters.runOnlyIfDependenciesSucceeded, true) }}: - condition: ${{ parameters.condition }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - disableClrTest: ${{ parameters.disableClrTest }} # Compute job name from template parameters - name: ${{ format('installer_{0}_{1}_{2}_{3}_', parameters.runtimeFlavor, parameters.runtimeVariant, coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} - displayName: ${{ format('Installer Build and Test {0} {1} {2} {3}', parameters.runtimeFlavor, parameters.runtimeVariant, coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} + name: ${{ format('installer_{0}_{1}', coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} + displayName: ${{ format('Installer Build and Test {0} {1}', coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} # Run all steps in the container. # Note that the containers are defined in platform-matrix.yml @@ -80,44 +76,18 @@ jobs: - ${{ each variable in parameters.variables }}: - ${{ variable }} - - name: OfficialBuildArg - value: '' - - # Explicitly enable tests for linux even though it is a cross build using mariner - # They still work in this configuration and until they run on Helix, it is our only - # linux test coverage in the installer pipeline - - name: SkipTests - value: ${{ or( - not(in(parameters.archType, 'x64', 'x86')), - eq(parameters.runtimeFlavor, 'mono'), - eq(parameters.isOfficialBuild, true), - and( - eq(parameters.crossBuild, true), - not(and( - eq(parameters.osGroup, 'linux'), - eq(parameters.osSubgroup, '')) - ))) }} - - name: BuildAction value: -test - - ${{ if eq(or(not(in(parameters.archType, 'x64', 'x86')), eq(parameters.runtimeFlavor, 'mono'), eq(parameters.isOfficialBuild, true), and(eq(parameters.crossBuild, true), not(and(eq(parameters.osGroup, 'linux'), eq(parameters.osSubgroup, ''))))), true) }}: - - name: BuildAction - value: '' - - - name: SignType - value: test - - # Set up non-PR build from internal project - - ${{ if eq(parameters.isOfficialBuild, true) }}: - - name: SignType - value: $[ coalesce(variables.OfficialSignType, 'real') ] - - name: OfficialBuildArg - value: /p:OfficialBuildId=$(Build.BuildNumber) - - name: buildCommandSourcesDirectory value: '$(Build.SourcesDirectory)/' + - name: librariesBuildConfigArg + value: -lc ${{ coalesce(parameters.liveLibrariesBuildConfig, parameters.buildConfig) }} + + - name: runtimeBuildConfigArg + value: -rc ${{ coalesce(parameters.liveRuntimeBuildConfig, parameters.buildConfig) }} + ### ### Platform-specific variable setup ### @@ -128,13 +98,7 @@ jobs: value: >- /p:TargetArchitecture=${{ parameters.archType }} /p:PortableBuild=true - /p:SkipTests=$(SkipTests) /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} - $(OfficialBuildArg) - - name: MsbuildSigningArguments - value: >- - /p:CertificateId=400 - /p:DotNetSignType=$(SignType) - name: TargetArchitecture value: ${{ parameters.archType }} @@ -143,57 +107,36 @@ jobs: build.cmd -subset host+packs -ci $(BuildAction) -configuration $(_BuildConfig) - $(LiveOverridePathArgs) + $(librariesBuildConfigArg) + $(runtimeBuildConfigArg) $(CommonMSBuildArgs) - $(MsbuildSigningArguments) - - ${{ if eq(parameters.osGroup, 'osx') }}: + - ${{ elseif eq(parameters.osGroup, 'osx') }}: - name: CommonMSBuildArgs value: >- /p:PortableBuild=true - /p:SkipTests=$(SkipTests) /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /p:TargetArchitecture=${{ parameters.archType }} /p:CrossBuild=${{ parameters.crossBuild }} - name: BaseJobBuildCommand value: >- - $(Build.SourcesDirectory)/build.sh -ci - $(BuildAction) - -configuration $(_BuildConfig) - -arch ${{ parameters.archType }} - $(LiveOverridePathArgs) - $(CommonMSBuildArgs) - $(OfficialBuildArg) - - - ${{ if in(parameters.osGroup, 'ios', 'tvos', 'android', 'browser') }}: - - - name: CommonMSBuildArgs - value: >- - /p:PortableBuild=true - /p:SkipTests=$(SkipTests) - - - name: BaseJobBuildCommand - value: >- - $(Build.SourcesDirectory)/build.sh -subset packs -ci + $(Build.SourcesDirectory)/build.sh -subset host+packs -ci $(BuildAction) -configuration $(_BuildConfig) - -os ${{ parameters.osGroup }} -arch ${{ parameters.archType }} - /p:StripSymbols=true - $(LiveOverridePathArgs) + $(librariesBuildConfigArg) + $(runtimeBuildConfigArg) $(CommonMSBuildArgs) - $(OfficialBuildArg) - - ${{ if in(parameters.osGroup, 'linux', 'freebsd') }}: + - ${{ elseif in(parameters.osGroup, 'linux', 'freebsd') }}: - name: CommonMSBuildArgs value: >- /p:Configuration=$(_BuildConfig) /p:TargetOS=${{ parameters.osGroup }} /p:TargetArchitecture=${{ parameters.archType }} /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} - $(OfficialBuildArg) - name: _PortableBuild value: ${{ eq(parameters.osSubgroup, '') }} @@ -210,83 +153,21 @@ jobs: value: >- $(Build.SourcesDirectory)/build.sh -subset host+packs -ci $(BuildAction) + $(librariesBuildConfigArg) + $(runtimeBuildConfigArg) $(ArchArg) /p:CrossBuild=${{ parameters.crossBuild }} /p:PortableBuild=$(_PortableBuild) - /p:SkipTests=$(SkipTests) - $(LiveOverridePathArgs) $(CommonMSBuildArgs) - name: installersSubsetArg value: --subset packs.installers - ### - ### Common Live build override variable setup - ### - - - name: LiveOverridePathArgs - value: >- - $(RuntimeArtifactsArgs) - $(LibrariesConfigurationArg) - - - name: RuntimeArtifactsArgs - value: '' - - name: LibrariesConfigurationArg - value: '' - - - name: RuntimeDownloadPath - value: '' - - name: LibrariesDownloadPath - value: '' - - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - name: liveRuntimeLegName - value: ${{ format('{0}{1}_{2}_{3}', - parameters.osGroup, - parameters.osSubgroup, - parameters.archType, - parameters.liveRuntimeBuildConfig) }} - - name: RuntimeDownloadPath - value: 'artifacts/transport/${{ parameters.runtimeFlavor }}' - - name: RuntimeArtifactsArgs - value: >- - /p:RuntimeArtifactsPath=$(buildCommandSourcesDirectory)$(RuntimeDownloadPath) - /p:RuntimeConfiguration=${{ parameters.liveRuntimeBuildConfig }} - - name: RuntimeArtifactName - value: $(runtimeFlavorName)Product_${{ parameters.runtimeVariant }}_$(liveRuntimeLegName) - - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - name: liveLibrariesLegName - value: ${{ format('{0}{1}_{2}_{3}', - parameters.osGroup, - parameters.osSubgroup, - parameters.archType, - parameters.liveLibrariesBuildConfig) }} - - name: LibrariesDownloadPath - value: 'artifacts' - - name: LibrariesArtifactName - value: libraries_bin_$(liveLibrariesLegName) - - name: LibrariesConfigurationArg - value: ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' - dependsOn: - ${{ if eq(parameters.dependOnEvaluatePaths, true) }}: - evaluate_paths + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ coalesce(parameters.unifiedBuildConfigOverride, parameters.buildConfig) }}_${{ parameters.unifiedBuildNameSuffix }}' - ${{ parameters.dependsOn }} - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', - parameters.runtimeFlavor, - parameters.runtimeVariant, - parameters.osGroup, - parameters.osSubgroup, - parameters.archType, - parameters.liveRuntimeBuildConfig) }} - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - libraries_build_${{ format('{0}{1}_{2}_{3}', - parameters.osGroup, - parameters.osSubgroup, - parameters.archType, - parameters.liveLibrariesBuildConfig) }} steps: - ${{ if eq(parameters.osGroup, 'windows') }}: - template: /eng/pipelines/common/templates/disable-vsupdate-or-failfast.yml @@ -312,22 +193,12 @@ jobs: env: Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(Build.SourcesDirectory)/$(RuntimeDownloadPath) - artifactFileName: '$(RuntimeArtifactName)$(archiveExtension)' - artifactName: '$(RuntimeArtifactName)' - displayName: '$(runtimeFlavorName) artifacts' - - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(Build.SourcesDirectory)/$(LibrariesDownloadPath) - artifactFileName: '$(LibrariesArtifactName)$(archiveExtension)' - artifactName: '$(LibrariesArtifactName)' - displayName: 'Libraries artifacts' - cleanUnpackFolder: false + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: '${{ parameters.unifiedArtifactsName }}$(archiveExtension)' + artifactName: '${{ parameters.unifiedArtifactsName }}' + displayName: 'unified artifacts' - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} @@ -340,7 +211,7 @@ jobs: - script: $(BaseJobBuildCommand) displayName: Build - continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} + continueOnError: ${{ eq(parameters.shouldContinueOnError, true) }} - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log @@ -374,8 +245,6 @@ jobs: name: ${{ coalesce(parameters.name, parameters.platform) }} runtimeFlavor: ${{ parameters.runtimeFlavor }} runtimeVariant: ${{ parameters.runtimeVariant }} - isOfficialBuild: ${{ eq(parameters.isOfficialBuild, true) }} - skipTests: ${{ eq(variables.SkipTests, true) }} - ${{ if ne(parameters.osGroup, 'windows') }}: - script: set -x && df -h diff --git a/eng/pipelines/installer/jobs/steps/build-linux-package.yml b/eng/pipelines/installer/jobs/steps/build-linux-package.yml index 102eab770c278..7f8320b55dfe3 100644 --- a/eng/pipelines/installer/jobs/steps/build-linux-package.yml +++ b/eng/pipelines/installer/jobs/steps/build-linux-package.yml @@ -17,7 +17,8 @@ steps: ${{ parameters.subsetArg }} \ ${{ parameters.packagingArgs }} \ $(CommonMSBuildArgs) \ - $(LiveOverridePathArgs) \ + $(librariesBuildConfigArg) \ + $(runtimeBuildConfigArg) \ /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog displayName: Package ${{ parameters.packageStepDescription }} - ${{ parameters.packageType }} target: ${{ parameters.target }} diff --git a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml b/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml index 4012b9a4fa34c..8cdf6e8fb7778 100644 --- a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml +++ b/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml @@ -1,16 +1,7 @@ parameters: name: '' - runtimeFlavor: 'coreclr' - runtimeVariant: '' - isOfficialBuild: false steps: -# Upload build artifacts (packages) to pipeline only if official, to save storage space. -- ${{ if eq(parameters.isOfficialBuild, true) }}: - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: ${{ parameters.name }} - - task: PublishTestResults@2 displayName: Publish Test Results inputs: @@ -18,41 +9,40 @@ steps: testResultsFiles: '*.trx' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' mergeTestResults: true - testRunTitle: Installer-${{ parameters.runtimeFlavor }}-${{ parameters.name }}-$(_BuildConfig) + testRunTitle: Installer-${{ parameters.name }}-$(_BuildConfig) continueOnError: true - condition: eq(variables.SkipTests, false) + condition: always() # Upload binaries and symbols on failure to allow debugging issues -- ${{ if eq(parameters.skipTests, false) }}: - - task: CopyFiles@2 - displayName: Prepare binaries to publish - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/bin' - Contents: | - */corehost/** - */corehost_test/** - TargetFolder: '$(Build.StagingDirectory)/Binaries' - continueOnError: true - condition: failed() +- task: CopyFiles@2 + displayName: Prepare binaries to publish + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/bin' + Contents: | + */corehost/** + */corehost_test/** + TargetFolder: '$(Build.StagingDirectory)/Binaries' + continueOnError: true + condition: failed() - - task: ArchiveFiles@2 - displayName: Zip binaries - inputs: - rootFolderOrFile: '$(Build.StagingDirectory)/Binaries' - archiveFile: '$(Build.StagingDirectory)/corehost-bin-${{ parameters.name }}-$(_BuildConfig)$(archiveExtension)' - archiveType: $(archiveType) - tarCompression: $(tarCompression) - includeRootFolder: false - continueOnError: true - condition: failed() +- task: ArchiveFiles@2 + displayName: Zip binaries + inputs: + rootFolderOrFile: '$(Build.StagingDirectory)/Binaries' + archiveFile: '$(Build.StagingDirectory)/corehost-bin-${{ parameters.name }}-$(_BuildConfig)$(archiveExtension)' + archiveType: $(archiveType) + tarCompression: $(tarCompression) + includeRootFolder: false + continueOnError: true + condition: failed() - - task: PublishBuildArtifacts@1 - displayName: Publish binaries - inputs: - pathtoPublish: '$(Build.StagingDirectory)/corehost-bin-${{ parameters.name }}-$(_BuildConfig)$(archiveExtension)' - artifactName: Installer-Binaries-${{ parameters.runtimeFlavor }}-${{ parameters.runtimeVariant }}-${{ parameters.name }}-$(_BuildConfig) - continueOnError: true - condition: failed() +- task: PublishBuildArtifacts@1 + displayName: Publish binaries + inputs: + pathtoPublish: '$(Build.StagingDirectory)/corehost-bin-${{ parameters.name }}-$(_BuildConfig)$(archiveExtension)' + artifactName: Installer-Binaries-${{ parameters.name }}-$(_BuildConfig) + continueOnError: true + condition: failed() - task: CopyFiles@2 displayName: Prepare BuildLogs staging directory diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml deleted file mode 100644 index f41dd67d02a42..0000000000000 --- a/eng/pipelines/libraries/base-job.yml +++ /dev/null @@ -1,146 +0,0 @@ -parameters: - buildConfig: '' - osGroup: '' - archType: '' - osSubgroup: '' - crossBuild: false - framework: 'net9.0' - isSourceBuild: false - liveRuntimeBuildConfig: '' - runtimeFlavor: 'coreclr' - timeoutInMinutes: 150 - condition: true - container: '' - steps: [] - dependsOn: [] - dependOnEvaluatePaths: false - disableComponentGovernance: false - variables: {} - name: '' - displayName: '' - testDisplayName: '' - testScope: '' - pool: '' - runTests: false - SuperPmiCollect: false - -jobs: - - template: /eng/common/templates/job/job.yml - parameters: - displayName: ${{ format('Libraries {0} {1}{2} {3} {4}', parameters.displayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - name: ${{ format('libraries_{0}_{1}{2}_{3}_{4}', parameters.name, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - enableTelemetry: ${{ parameters.isOfficialBuild }} # TODO: figure out if it's needed - container: ${{ parameters.container }} - condition: and(succeeded(), ${{ parameters.condition }}) - helixRepo: dotnet/runtime - pool: ${{ parameters.pool }} - - # Disable component governance if requested or on musl machines where it does not work well - ${{ if or(eq(parameters.disableComponentGovernance, true), eq(parameters.osSubGroup, '_musl')) }}: - disableComponentGovernance: true - - variables: - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNet-HelixApi-Access - - group: AzureDevOps-Artifact-Feeds-Pats - - - _buildScriptFileName: build - - - _msbuildCommonParameters: '' - # rename this variable, due to collision with build-native.proj - - _osArg: '' - - _finalFrameworkArg: '' - - _testModeArg: '' - - _buildScript: $(_buildScriptFileName)$(scriptExt) - - _testScopeArg: '' - - _extraHelixArguments: '' - - _crossBuildPropertyArg: '' - - _testRunNamePrefixSuffix: '' - - - librariesBuildArtifactName: ${{ format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - - ${{ if ne(parameters.testScope, '') }}: - - _testScopeArg: -testscope ${{ parameters.testScope }} - - - ${{ if eq(parameters.crossBuild, true) }}: - - _crossBuildPropertyArg: /p:CrossBuild=true - - # force a value for OS when cross-building - - ${{ if in(parameters.osGroup, 'browser', 'ios', 'tvos', 'android', 'freebsd') }}: - - _osArg: -os ${{ parameters.osGroup }} - - - ${{ if ne(parameters.framework, '') }}: - - _finalFrameworkArg: -framework ${{ parameters.framework }} - - _extraHelixArguments: /p:BuildTargetFramework=${{ parameters.framework }} - - - ${{ if eq(parameters.isOfficialBuild, true) }}: - - _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber) - - - _runtimeArtifactName: '' - - _runtimeDownloadPath: '' - - _runtimeArtifactsPathArg: '' - - _runtimeConfigurationArg: '' - - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}' - - _runtimeConfigurationArg: -rc ${{ parameters.liveRuntimeBuildConfig }} - - ${{ if eq(parameters.runTests, true) }}: - - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.runtimeVariant}}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' - - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)' - - ${{ if eq(parameters.testDisplayName, '') }}: - - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }} - - ${{ if ne(parameters.testDisplayName, '') }}: - - _testRunNamePrefixSuffix: ${{ parameters.testDisplayName }} - - - ${{ if ne(parameters.osGroup, 'windows') }}: - - _buildScript: ./$(_buildScriptFileName)$(scriptExt) - - - _buildArguments: $(_runtimeConfigurationArg) -configuration ${{ parameters.buildConfig }} -ci -arch ${{ parameters.archType }} $(_finalFrameworkArg) $(_testModeArg) $(_testScopeArg) $(_osArg) $(_msbuildCommonParameters) $(_runtimeArtifactsPathArg) $(_crossBuildPropertyArg) - - ${{ parameters.variables }} - - # we need to override this value to support build-coreclr-and-libraries-job.yml - - _BuildConfig: ${{ parameters.buildConfig }} - - dependsOn: - - ${{ if eq(parameters.dependOnEvaluatePaths, true) }}: - - evaluate_paths - - ${{ parameters.dependsOn }} - workspace: - clean: all - - enablePublishBuildArtifacts: true - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - steps: - - checkout: self - clean: true - fetchDepth: $(checkoutFetchDepth) - - - ${{ if and(ne(parameters.liveRuntimeBuildConfig, ''), eq(parameters.runTests, true)) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(_runtimeDownloadPath) - artifactFileName: '$(_runtimeArtifactName)$(archiveExtension)' - artifactName: '$(_runtimeArtifactName)' - displayName: '$(runtimeFlavorName) build drop' - - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - ${{ if ne(parameters.osGroup, 'windows') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - ${{ if eq(parameters.osGroup, 'windows') }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - - ${{ parameters.steps }} diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml deleted file mode 100644 index b73c8d09f0a6e..0000000000000 --- a/eng/pipelines/libraries/build-job.yml +++ /dev/null @@ -1,98 +0,0 @@ -parameters: - buildConfig: '' - osGroup: '' - osSubgroup: '' - archType: '' - targetRid: '' - crossBuild: false - framework: 'net9.0' - isOfficialBuild: false - runtimeVariant: '' - platform: '' - testScope: '' - - timeoutInMinutes: 150 - container: '' - condition: true - dependOnEvaluatePaths: false - disableComponentGovernance: false - shouldContinueOnError: false - variables: {} - pool: '' - -jobs: - - template: /eng/pipelines/libraries/base-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - archType: ${{ parameters.archType }} - crossBuild: ${{ parameters.crossBuild }} - framework: ${{ parameters.framework }} - isOfficialBuild: ${{ parameters.isOfficialBuild }} - runtimeFlavor: ${{ parameters.runtimeFlavor }} - runTests: false - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - container: ${{ parameters.container }} - condition: ${{ parameters.condition }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - pool: ${{ parameters.pool }} - runtimeVariant: ${{ parameters.runtimeVariant }} - testScope: ${{ parameters.testScope }} - name: build - displayName: 'Build' - - variables: - - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - _subset: tools+libs+libs.tests - - _buildAction: '' - - _additionalBuildArguments: '/p:ArchiveTests=true' - - ${{ parameters.variables }} - - steps: - - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} - displayName: Install Build Dependencies - - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage before Build - - - script: $(_buildScript) - -subset $(_subset) - $(_buildAction) - $(_buildArguments) - $(_additionalBuildArguments) - displayName: Restore and Build Product - - - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage after Build - - - template: /eng/pipelines/libraries/prepare-for-bin-publish.yml - - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.ArtifactStagingDirectory)/artifacts - includeRootFolder: false - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: $(librariesBuildArtifactName) - displayName: Build Assets - - # Upload test assets - # We'll pull them down in another job to send to Helix - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/helix - includeRootFolder: true - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: $(librariesTestsArtifactName) - displayName: Test Assets \ No newline at end of file diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index f6f452199d627..78dd68907c7d6 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -4,7 +4,6 @@ parameters: osSubgroup: '' archType: '' targetRid: '' - framework: 'net9.0' isOfficialBuild: false liveRuntimeBuildConfig: '' runtimeFlavor: 'coreclr' @@ -19,68 +18,64 @@ parameters: condition: true shouldContinueOnError: false variables: {} - # coreclrTestGroup: if empty, then a normal, default test run is created. If set, it indicates a set of - # stress modes that each test will be run with. This is the same usage as 'testGroup' in - # eng/pipelines/common/templates/runtimes/run-test-job.yml. - coreclrTestGroup: '' + scenarios: [] SuperPmiCollect: false SuperPmiCollectionType: 'run' SuperPmiCollectionName: 'libraries_tests' dependsOn: [] + unifiedArtifactsName: '' + helixArtifactsName: '' + unifiedBuildNameSuffix: '' + unifiedBuildConfigOverride: '' jobs: - - template: /eng/pipelines/libraries/base-job.yml + - template: /eng/common/templates/job/job.yml parameters: - buildConfig: ${{ parameters.buildConfig }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - archType: ${{ parameters.archType }} - crossBuild: ${{ parameters.crossBuild }} - framework: ${{ parameters.framework }} - isOfficialBuild: ${{ parameters.isOfficialBuild }} - liveRuntimeBuildConfig: ${{ parameters.liveRuntimeBuildConfig }} - runtimeFlavor: ${{ parameters.runtimeFlavor }} - runtimeVariant: ${{ parameters.runtimeVariant }} + enablePublishBuildArtifacts: true timeoutInMinutes: ${{ parameters.timeoutInMinutes }} container: ${{ parameters.container }} - condition: ${{ parameters.condition }} - testScope: ${{ parameters.testScope }} - SuperPmiCollect: ${{ parameters.SuperPmiCollect }} - runTests: true + condition: and(succeeded(), ${{ parameters.condition }}) + helixRepo: dotnet/runtime ${{ if eq(parameters.SuperPmiCollect, true) }}: - displayName: ${{ format('SuperPMI collection {0} {1} {2}', parameters.SuperPmiCollectionName, parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }} - name: ${{ format('spmi_{0}_{1}_{2}', parameters.SuperPmiCollectionName, parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }} + displayName: ${{ format('Libraries SuperPMI collection {0} {1} {2} {3}{4} {5} {6}', parameters.SuperPmiCollectionName, parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + name: ${{ format('spmi_{0}_{1}_{2}_{3}{4}_{5}_{6}', parameters.SuperPmiCollectionName, parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} ${{ else }}: ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - displayName: ${{ format('Test Run {0} {1}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }} - name: ${{ format('test_run_{0}_{1}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName) }} + displayName: ${{ format('Libraries Test Run {0} {1} {2}{3} {4} {5}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + name: ${{ format('libraries_test_run_{0}_{1}_{2}{3}_{4}_{5}', parameters.liveRuntimeBuildConfig, parameters.runtimeDisplayName, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} ${{ if eq(parameters.liveRuntimeBuildConfig, '') }}: displayName: 'Test Run' name: test_run - ${{ if eq(parameters.interpreter, 'true') }}: - testDisplayName: ${{ parameters.runtimeFlavor }}_interpreter_${{ parameters.liveRuntimeBuildConfig }} # To run the tests we just send to helix and wait, use ubuntu hosted pools for faster providing and to not back up our build pools pool: ${{ parameters.pool }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} + # Disable component governance if requested or on musl machines where it does not work well + ${{ if or(eq(parameters.disableComponentGovernance, true), eq(parameters.osSubGroup, '_musl')) }}: + disableComponentGovernance: true + dependsOn: + - ${{ if eq(parameters.dependOnEvaluatePaths, true) }}: + - evaluate_paths - ${{ if ne(parameters.dependsOn[0], '') }}: - ${{ parameters.dependsOn }} - - ${{ if eq(parameters.dependsOn[0], '') }}: - - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveRuntimeBuildConfig) }} - # SuperPMI collection needs to run mcs.exe on the AzDO machine. Assume that's an x64 machine, and download an x64 product build if needed. - - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}: - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', 'coreclr', '', parameters.osGroup, parameters.osSubgroup, 'x64', parameters.liveRuntimeBuildConfig) }} + - ${{ else }}: + - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ coalesce(parameters.unifiedBuildConfigOverride, parameters.buildConfig) }}_${{ parameters.unifiedBuildNameSuffix }}' + + workspace: + clean: all variables: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - group: DotNet-HelixApi-Access + - group: AzureDevOps-Artifact-Feeds-Pats - - librariesTestsArtifactName: ${{ format('libraries_test_assets_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - _archiveTestsParameter: /p:ArchiveTests=true - ${{ if eq(parameters.SuperPmiCollect, true) }}: + - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml + parameters: + osGroup: ${{ parameters.osGroup }} - template: /eng/pipelines/libraries/superpmi-collect-variables.yml parameters: buildConfig: ${{ parameters.buildConfig }} @@ -91,172 +86,85 @@ jobs: - ${{ parameters.variables }} steps: - - # SuperPMI collection: Download x64 coreclr if running on non-x64 configuration (needed for mcs.exe on AzDO machine; see `SuperPmiMcsPath`. - - ${{ if and(eq(parameters.SuperPmiCollect, true), ne(parameters.archType, 'x64')) }}: - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(_runtimeX64DownloadPath) - artifactFileName: 'CoreCLRProduct__${{ parameters.osGroup }}${{ parameters.osSubgroup }}_x64_${{ parameters.liveRuntimeBuildConfig }}$(archiveExtension)' - artifactName: 'CoreCLRProduct__${{ parameters.osGroup }}${{ parameters.osSubgroup }}_x64_${{ parameters.liveRuntimeBuildConfig }}' - displayName: 'CoreCLR product build (x64)' - - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - displayName: Build Assets - cleanUnpackFolder: false - artifactName: $(librariesBuildArtifactName) - artifactFileName: $(librariesBuildArtifactName)$(archiveExtension) - unpackFolder: $(Build.SourcesDirectory)/artifacts - - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - displayName: Test Assets - cleanUnpackFolder: false - artifactName: $(librariesTestsArtifactName) - artifactFileName: $(librariesTestsArtifactName)$(archiveExtension) - unpackFolder: $(Build.SourcesDirectory)/artifacts - - - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} - displayName: Install Build Dependencies - - - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - script: $(_buildScript) - -subset host.native+libs.pretest - $(_buildArguments) - /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} - /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/overrideRuntimeFromLiveDrop.binlog - displayName: Prepare TestHost with runtime $(runtimeFlavorName) - - - template: /eng/pipelines/libraries/helix.yml + - checkout: self + clean: true + fetchDepth: $(checkoutFetchDepth) + + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - ${{ if ne(parameters.osGroup, 'windows') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - ${{ if eq(parameters.osGroup, 'windows') }}: + - task: PowerShell@2 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + + - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: + - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} + displayName: Install Build Dependencies + + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(Build.SourcesDirectory)/artifacts/bin + artifactFileName: '${{ parameters.unifiedArtifactsName }}$(archiveExtension)' + artifactName: '${{ parameters.unifiedArtifactsName }}' + displayName: 'unified artifacts' + + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + displayName: Test Assets + artifactName: ${{ parameters.helixArtifactsName }} + artifactFileName: '${{ parameters.helixArtifactsName }}$(archiveExtension)' + unpackFolder: $(Build.SourcesDirectory)/artifacts/helix + + - template: /eng/pipelines/libraries/helix.yml + parameters: + runtimeFlavor: ${{ parameters.runtimeFlavor }} + osGroup: ${{ parameters.osGroup }} + targetRid: ${{ parameters.targetRid }} + archType: ${{ parameters.archType }} + buildConfig: ${{ parameters.buildConfig }} + helixQueues: ${{ parameters.helixQueues }} + testScope: ${{ parameters.testScope }} + interpreter: ${{ parameters.interpreter }} + shouldContinueOnError: ${{ parameters.shouldContinueOnError }} + creator: dotnet-bot + ${{ if eq(parameters.interpreter, 'true') }}: + testRunNamePrefixSuffix: ${{ parameters.runtimeFlavor }}_interpreter_${{ parameters.liveRuntimeBuildConfig }} + ${{ else }}: + testRunNamePrefixSuffix: ${{ parameters.runtimeDisplayName }}_${{ parameters.liveRuntimeBuildConfig }} + SuperPmiCollect: ${{ parameters.SuperPmiCollect }} + SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} + SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} + ${{ if eq(parameters.SuperPmiCollect, true) }}: + extraHelixArguments: /p:RuntimeConfiguration=${{ parameters.liveRuntimeBuildConfig }} + + ${{ if ne(parameters.scenarios[0], '') }}: + scenarios: ${{ parameters.scenarios }} + + - ${{ if eq(parameters.SuperPmiCollect, true) }}: + - template: /eng/pipelines/libraries/superpmi-postprocess-step.yml parameters: - runtimeFlavor: ${{ parameters.runtimeFlavor }} + buildConfig: ${{ parameters.buildConfig }} + buildConfigUpper: $(buildConfigUpper) osGroup: ${{ parameters.osGroup }} - targetRid: ${{ parameters.targetRid }} + osSubgroup: ${{ parameters.osSubgroup }} archType: ${{ parameters.archType }} - buildConfig: ${{ parameters.buildConfig }} - helixQueues: ${{ parameters.helixQueues }} - testScope: ${{ parameters.testScope }} - interpreter: ${{ parameters.interpreter }} - shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - creator: dotnet-bot - testRunNamePrefixSuffix: $(_testRunNamePrefixSuffix) - SuperPmiCollect: ${{ parameters.SuperPmiCollect }} SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} - - # coreclrTestGroup: The following mappings of 'coreclrTestGroup' to 'scenarios' is copied from - # eng/pipelines/common/templates/runtimes/run-test-job.yml (with 'testGroup' replaced by 'coreclrTestGroup' - # for clarity), and should remain in sync. This is only a subset; only the testGroups that are - # used to test the libraries have been added here. More could be added if we decided to test the - # libraries with more stress modes. The scenario tags are interpreted by - # src\tests\Common\testenvironment.proj. - # - # The one difference here compared to eng/pipelines/common/templates/runtimes/run-test-job.yml is - # that 'jitstress' contains 'no_tiered_compilation'. The 'normal' (default) test mode - # is run in a regular CI job, so there is no need to duplicate it here. So, add 'no_tiered_compilation' - # to the 'jitstress' job instead of adding a new job just for 'no_tiered_compilation'. - - # src/libraries/sendtohelix.proj processes one scenario per parallel MSBuild invocation. Each invocation only - # creates Helix work items and them waits for their completion on the remote Helix machines, so is not - # computationally intensive. We want Helix to be provided with all the possible work items in up front, - # so can do as much work in parallel as possible. Thus, increase the amount of allowed MSBuild parallelism - # to at least the maximum number of scenarios to be processed in a coreclrTestGroup. If there is no - # coreclrTestGroup then there is only one scenario (the default scenario), so don't change the MSBuild argument. - ${{ if ne(parameters.coreclrTestGroup, '') }}: - extraHelixArguments: $(_extraHelixArguments) /maxcpucount:10 - ${{ else }}: - extraHelixArguments: $(_extraHelixArguments) - - ${{ if in(parameters.coreclrTestGroup, 'superpmi_collection') }}: - scenarios: - - normal - ${{ if in(parameters.coreclrTestGroup, 'superpmi_collection_no_tiered_compilation') }}: - scenarios: - - no_tiered_compilation - ${{ if in(parameters.coreclrTestGroup, 'jitstress') }}: - scenarios: - - no_tiered_compilation - - jitminopts - - jitstress1 - - jitstress1_tiered - - jitstress2 - - jitstress2_tiered - - disabler2r - - tailcallstress - ${{ if in(parameters.coreclrTestGroup, 'jitstress-random') }}: - scenarios: - - jitstress_random_1 - - jitstress_random_2 - ${{ if in(parameters.coreclrTestGroup, 'jitstressregs' ) }}: - scenarios: - - jitstressregs1 - - jitstressregs2 - - jitstressregs3 - - jitstressregs4 - - jitstressregs8 - - jitstressregs0x10 - - jitstressregs0x80 - - jitstressregs0x1000 - - jitstressregs0x2000 - ${{ if in(parameters.coreclrTestGroup, 'jitstress2-jitstressregs') }}: - scenarios: - - jitstress2_jitstressregs1 - - jitstress2_jitstressregs2 - - jitstress2_jitstressregs3 - - jitstress2_jitstressregs4 - - jitstress2_jitstressregs8 - - jitstress2_jitstressregs0x10 - - jitstress2_jitstressregs0x80 - - jitstress2_jitstressregs0x1000 - - jitstress2_jitstressregs0x2000 - ${{ if in(parameters.coreclrTestGroup, 'gcstress0x3-gcstress0xc') }}: - scenarios: - # Disable gcstress0x3 for now; it causes lots of test timeouts. Investigate this after - # gcstress0xc runs are clean. Tracking issue: https://github.com/dotnet/runtime/issues/38903. - # - gcstress0x3 - - gcstress0xc - ${{ if in(parameters.coreclrTestGroup, 'gcstress-extra') }}: - scenarios: - - heapverify1 - - gcstress0xc_disabler2r - - gcstress0xc_disabler2r_jitstress2 - - gcstress0xc_disabler2r_heapverify1 - - gcstress0xc_jitstress1 - - gcstress0xc_jitstress2 - - gcstress0xc_jitminopts_heapverify1 - ${{ if in(parameters.coreclrTestGroup, 'pgo') }}: - ${{ if and(eq(parameters.osGroup, 'windows'), eq(parameters.archType, 'arm64')) }}: - scenarios: - - defaultpgo - ${{ else }}: - scenarios: - - defaultpgo - - fullpgo - - fullpgo_methodprofiling - - fullpgo_random_gdv - - fullpgo_random_gdv_methodprofiling_only - - fullpgo_random_gdv_edge - - jitosr_stress - - jitosr_stress_random - - syntheticpgo - - syntheticpgo_blend - - jitrlcse - - - ${{ if eq(parameters.SuperPmiCollect, true) }}: - - template: /eng/pipelines/libraries/superpmi-postprocess-step.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - buildConfigUpper: $(buildConfigUpper) - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - archType: ${{ parameters.archType }} - SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} - SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} - MergedMchFileLocation: $(MergedMchFileLocation) - MchFilesLocation: $(MchFilesLocation) - SpmiLogsLocation: $(SpmiLogsLocation) - SuperPmiMcsPath: $(SuperPmiMcsPath) - PythonScript: $(PythonScript) - PipScript: $(PipScript) + MergedMchFileLocation: $(MergedMchFileLocation) + MchFilesLocation: $(MchFilesLocation) + SpmiLogsLocation: $(SpmiLogsLocation) + SuperPmiMcsPath: $(SuperPmiMcsPath) + PythonSetupScript: $(PythonSetupScript) + PythonScript: $(PythonScript) + PipScript: $(PipScript) diff --git a/eng/pipelines/libraries/superpmi-collect-variables.yml b/eng/pipelines/libraries/superpmi-collect-variables.yml index 509bd60942450..82ead1c26ca79 100644 --- a/eng/pipelines/libraries/superpmi-collect-variables.yml +++ b/eng/pipelines/libraries/superpmi-collect-variables.yml @@ -13,20 +13,14 @@ variables: value: 'Release' - name: _runtimeX64DownloadPath value: '' - # superpmi.py 'merge-mch' needs to be able to find the mcs tool. Point SuperPmiMcsPath at the downloaded CoreCLR binaries. For non-x64 targets, download an x64 - # build and point at that. Pass this to superpmi.py as the '-core_root' argument. It's not actually a "Core_Root" directory, but all it needs is to find mcs. + # superpmi.py 'merge-mch' needs to be able to find the mcs tool. Point SuperPmiMcsPath at the downloaded CoreCLR binaries. For non-x64 targets, we'll also have an x64 mcs tool available. + # so point the non-x64 builds at the x64 artifacts. - name: SuperPmiMcsPath - value: $(_runtimeDownloadPath) + value: $(Build.SourcesDirectory)/artifacts/bin/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper) - ${{ if ne(parameters.archType, 'x64') }}: - - name: _runtimeX64DownloadPath - value: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}.x64' - name: SuperPmiMcsPath - value: $(_runtimeX64DownloadPath) + value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/${{ parameters.osGroup }}.x64.$(buildConfigUpper)/' - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'py -3' - - name: PipScript - value: 'py -3 -m pip' - name: MchFilesLocation value: '$(Build.SourcesDirectory)\artifacts\helixresults\' - name: MergedMchFileLocation @@ -34,13 +28,9 @@ variables: - name: SpmiLogsLocation value: '$(Build.SourcesDirectory)\artifacts\spmi_logs\' - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: PythonScript - value: 'python3' - - name: PipScript - value: 'pip3' - name: MchFilesLocation value: '$(Build.SourcesDirectory)/artifacts/helixresults/' - name: MergedMchFileLocation value: '$(Build.SourcesDirectory)/artifacts/spmi_collection/' - name: SpmiLogsLocation - value: '$(Build.SourcesDirectory)/artifacts/spmi_logs/' \ No newline at end of file + value: '$(Build.SourcesDirectory)/artifacts/spmi_logs/' diff --git a/eng/pipelines/libraries/superpmi-postprocess-step.yml b/eng/pipelines/libraries/superpmi-postprocess-step.yml index ef29b332c7517..e9ae5d13f32f2 100644 --- a/eng/pipelines/libraries/superpmi-postprocess-step.yml +++ b/eng/pipelines/libraries/superpmi-postprocess-step.yml @@ -18,6 +18,7 @@ parameters: MchFilesLocation: '' SpmiLogsLocation: '' SuperPmiMcsPath: '' + PythonSetupScript: '' PythonScript: '' PipScript: '' @@ -37,6 +38,10 @@ steps: displayName: 'Create SuperPMI directories' condition: always() + - script: ${{ parameters.PythonSetupScript }} + displayName: Enable python venv + condition: always() + - script: ${{ parameters.PythonScript }} $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch -log_level DEBUG -pattern ${{ parameters.MchFilesLocation }}${{ parameters.SuperPmiCollectionName }}.${{ parameters.SuperPmiCollectionType }}*.mch -output_mch_path ${{ parameters.MergedMchFileLocation }}${{ parameters.SuperPmiCollectionName }}.${{ parameters.SuperPmiCollectionType }}.${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}.mch -core_root ${{ parameters.SuperPmiMcsPath }} displayName: 'Merge ${{ parameters.SuperPmiCollectionName }}-${{ parameters.SuperPmiCollectionType }} SuperPMI collections' condition: always() @@ -61,7 +66,7 @@ steps: condition: always() # Ensure the Python azure-storage-blob package is installed before doing the upload. - - script: ${{ parameters.PipScript }} install --user --upgrade pip && ${{ parameters.PipScript }} install --user azure.storage.blob==12.5.0 --force-reinstall + - script: ${{ parameters.PipScript }} install --upgrade pip && ${{ parameters.PipScript }} install azure.storage.blob==12.5.0 --force-reinstall displayName: Upgrade Pip to latest and install azure-storage-blob Python package condition: always() @@ -84,4 +89,4 @@ steps: inputs: targetPath: ${{ parameters.SpmiLogsLocation }} artifactName: 'SuperPMI_Logs_${{ parameters.SuperPmiCollectionName }}_${{ parameters.SuperPmiCollectionType }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}' - condition: always() \ No newline at end of file + condition: always() diff --git a/eng/pipelines/mono/templates/build-job.yml b/eng/pipelines/mono/templates/build-job.yml deleted file mode 100644 index 86e0813c7c7e5..0000000000000 --- a/eng/pipelines/mono/templates/build-job.yml +++ /dev/null @@ -1,184 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osSubgroup: '' - platform: '' - container: '' - timeoutInMinutes: '' - variables: {} - pool: '' - condition: true - runtimeVariant: '' - isOfficialBuild: false - crossBuild: false - dependsOn: [] - monoCrossAOTTargetOS: [] - dependOnEvaluatePaths: false - -### Product build -jobs: -- template: xplat-pipeline-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - helixType: 'build/product/' - enableMicrobuild: true - pool: ${{ parameters.pool }} - runtimeVariant: ${{ parameters.runtimeVariant }} - crossBuild: ${{ parameters.crossBuild }} - monoCrossAOTTargetOS: ${{ parameters.monoCrossAOTTargetOS }} - condition: ${{ parameters.condition }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - - # Compute job name from template parameters - name: ${{ format('mono_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Mono {0} Product Build {1}{2} {3} {4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - - # Run all steps in the container. - # Note that the containers are defined in platform-matrix.yml - container: ${{ parameters.container }} - - dependsOn: ${{ parameters.dependsOn }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - gatherAssetManifests: true - variables: - - name: osGroup - value: ${{ parameters.osGroup }} - - name: osSubgroup - value: ${{ parameters.osSubgroup }} - - name: officialBuildIdArg - value: '' - - name: osOverride - value: '' - - name: aotCrossParameter - value: '' - - name: llvmParameter - value: '' - - name: darwinFrameworks - value: '' - - ${{ if eq(parameters.isOfficialBuild, true) }}: - - name: officialBuildIdArg - value: '/p:OfficialBuildId=$(Build.BuildNumber)' - - ${{ if and(eq(parameters.osSubgroup, '_musl'), eq(parameters.osGroup, 'linux')) }}: - # Set output RID manually: musl isn't properly detected. Make sure to also convert linux to - # lowercase for RID format. (Detection normally converts, but we're preventing it.) - - name: OutputRidArg - value: /p:OutputRID=linux-musl-${{ parameters.archType }} - - name: _PortableBuild - value: true - - ${{ if eq(parameters.osGroup, 'tvos') }}: - - name: osOverride - value: -os tvos - - name: darwinFrameworks - value: /p:BuildDarwinFrameworks=true - - ${{ if eq(parameters.osGroup, 'tvossimulator') }}: - - name: osOverride - value: -os tvossimulator - - name: darwinFrameworks - value: /p:BuildDarwinFrameworks=true - - ${{ if eq(parameters.osGroup, 'ios') }}: - - name: osOverride - value: -os ios - - name: darwinFrameworks - value: /p:BuildDarwinFrameworks=true - - ${{ if eq(parameters.osGroup, 'iossimulator') }}: - - name: osOverride - value: -os iossimulator - - name: darwinFrameworks - value: /p:BuildDarwinFrameworks=true - - ${{ if eq(parameters.osGroup, 'android') }}: - - name: osOverride - value: -os android - - ${{ if eq(parameters.osGroup, 'browser') }}: - - name: archType - value: wasm - - name: osOverride - value: '-os browser' - - ${{ if eq(parameters.osGroup, 'wasi') }}: - - name: archType - value: wasm - - name: osOverride - value: '-os wasi' - - ${{ if eq(parameters.runtimeVariant, 'llvmjit') }}: - - name: llvmParameter - value: /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=false - - ${{ if eq(parameters.runtimeVariant, 'llvmaot') }}: - - name: llvmParameter - value: /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=true - - ${{ if gt(length(parameters.monoCrossAOTTargetOS),0) }}: - - name: aotCrossParameter - value: /p:MonoCrossAOTTargetOS=${{join('+',parameters.monoCrossAOTTargetOS)}} /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true - - ${{ parameters.variables }} - - steps: - - # Install native dependencies - # Linux builds use docker images with dependencies preinstalled, - # and FreeBSD builds use a build agent with dependencies - # preinstalled, so we only need this step for OSX and Windows. - - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: - - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup) - displayName: Install native dependencies - - - ${{ each monoCrossAOTTargetOS in parameters.monoCrossAOTTargetOS }}: - - task: DownloadPipelineArtifact@2 - displayName: Download ${{monoCrossAOTTargetOS}} AOT offset files - inputs: - artifact: Mono_Offsets_${{monoCrossAOTTargetOS}} - path: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles' - - - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage before Build - - # Build - - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: ./build$(scriptExt) -subset mono+clr.hosts $(crossArg) -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter) $(darwinFrameworks) - displayName: Build product - - ${{ if eq(parameters.osGroup, 'windows') }}: - - script: build$(scriptExt) -subset mono+clr.hosts -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter) - displayName: Build product - - - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: - - script: | - du -sh $(Build.SourcesDirectory)/* - df -h - displayName: Disk Usage after Build - - # Publish product output directory for consumption by tests. - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(buildProductRootFolderPath) - includeRootFolder: false - archiveType: $(archiveType) - tarCompression: $(tarCompression) - archiveExtension: $(archiveExtension) - artifactName: $(buildProductArtifactName) - displayName: 'product build' - - # Build packages - - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: ./build$(scriptExt) -subset mono+clr.hosts $(crossArg) -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter) -pack $(OutputRidArg) - displayName: Build nupkg - - ${{ if eq(parameters.osGroup, 'windows') }}: - - script: build$(scriptExt) -subset mono+clr.hosts -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter) -pack $(OutputRidArg) - displayName: Build nupkg - - - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: - - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log - displayName: Collect vslogs on exit - condition: always() - # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'BuildLogs_Attempt$(System.JobAttempt)_Mono_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - continueOnError: true - condition: always() diff --git a/eng/pipelines/mono/templates/generate-offsets.yml b/eng/pipelines/mono/templates/generate-offsets.yml index c68adfe67a9e6..2573b65e1d27f 100644 --- a/eng/pipelines/mono/templates/generate-offsets.yml +++ b/eng/pipelines/mono/templates/generate-offsets.yml @@ -12,7 +12,7 @@ parameters: ### Product build jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: buildConfig: ${{ parameters.buildConfig }} osGroup: ${{ parameters.osGroup }} diff --git a/eng/pipelines/mono/templates/workloads-build.yml b/eng/pipelines/mono/templates/workloads-build.yml index 20ff5c29d5d89..4824f2f1203f2 100644 --- a/eng/pipelines/mono/templates/workloads-build.yml +++ b/eng/pipelines/mono/templates/workloads-build.yml @@ -15,12 +15,12 @@ parameters: variables: {} jobs: -- template: xplat-pipeline-job.yml +- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: archType: ${{ parameters.archType }} buildConfig: ${{ parameters.buildConfig }} container: ${{ parameters.container }} - condition: ${{ parameters.isOfficialBuild }} + condition: and(succeeded(), ${{ parameters.isOfficialBuild }}) helixType: 'build/product/' osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup }} diff --git a/eng/pipelines/mono/templates/xplat-pipeline-job.yml b/eng/pipelines/mono/templates/xplat-pipeline-job.yml deleted file mode 100644 index 1ca84d9caac1b..0000000000000 --- a/eng/pipelines/mono/templates/xplat-pipeline-job.yml +++ /dev/null @@ -1,128 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osSubgroup: '' - name: '' - helixType: '(unspecified)' - container: '' - crossBuild: false - liveLibrariesBuildConfig: '' - strategy: '' - pool: '' - runtimeVariant: '' - liveRuntimeBuildConfig: 'release' - - # arcade-specific parameters - condition: true - continueOnError: false - dependsOn: '' - dependOnEvaluatePaths: false - displayName: '' - timeoutInMinutes: '' - enableMicrobuild: '' - gatherAssetManifests: false - - variables: {} ## any extra variables to add to the defaults defined below - -jobs: -- template: /eng/pipelines/common/templates/runtimes/xplat-job.yml - parameters: - buildConfig: ${{ parameters.buildConfig }} - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - name: ${{ parameters.name }} - helixType: ${{ parameters.helixType }} - container: ${{ parameters.container }} - crossBuild: ${{ parameters.crossBuild }} - strategy: ${{ parameters.strategy }} - pool: ${{ parameters.pool }} - runtimeVariant: ${{ parameters.runtimeVariant }} - - # arcade-specific parameters - condition: and(succeeded(), ${{ parameters.condition }}) - continueOnError: ${{ parameters.continueOnError }} - dependsOn: ${{ parameters.dependsOn }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - displayName: ${{ parameters.displayName }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - enableMicrobuild: ${{ parameters.enableMicrobuild }} - gatherAssetManifests: ${{ parameters.gatherAssetManifests }} - - variables: - - name: coreClrProductArtifactName - value: 'CoreCLRProduct__$(osGroup)$(osSubgroup)_$(archType)_${{ parameters.liveRuntimeBuildConfig }}' - - - name: coreClrProductRootFolderPath - value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(liveRuntimeBuildConfigUpper)' - - - name: buildProductArtifactName - value: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - # minijit and monointerpreter do not use separate product builds. - - ${{ if or(eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter')) }}: - - name : buildProductArtifactName - value : 'MonoProduct__$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - ${{ if eq(parameters.runtimeVariant, 'llvmfullaot') }}: - - name : buildProductArtifactName - value : 'MonoProduct_llvmaot_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: binTestsPath - value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr' - - - name: buildProductRootFolderPath - value: '$(Build.SourcesDirectory)/artifacts/bin/mono/$(osGroup).$(archType).$(buildConfigUpper)' - - - name: managedTestArtifactRootFolderPath - value: '$(binTestsPath)/$(osGroup).$(archType).$(buildConfigUpper)' - - - name: managedGenericTestArtifactName - value: 'MonoManagedTestArtifacts_AnyOS_AnyCPU_$(buildConfig)' - - - name: microsoftNetSdkIlFolderPath - value: '$(Build.SourcesDirectory)/.packages/microsoft.net.sdk.il' - - - name: microsoftNetSdkIlArtifactName - value: 'MicrosoftNetSdkIlPackage_AnyOS_AnyCPU_$(buildConfig)' - - - name: monoRepoRoot - value: '$(Build.SourcesDirectory)/src/mono' - - - name: nativeTestArtifactName - value: 'CoreCLRNativeTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - - - name: nativeTestArtifactRootFolderPath - value: '$(binTestsPath)/obj/$(osGroup).$(archType).$(buildConfigUpper)' - - - name: workloadPackagesPath - value: $(Build.SourcesDirectory)/artifacts/workloadPackages - - - name: workloadArtifactsPath - value: $(Build.SourcesDirectory)/artifacts/workloads - - - name: liveRuntimeBuildConfigUpper - ${{ if eq(parameters.liveRuntimeBuildConfig, 'release') }}: - value: 'Release' - ${{ if eq(parameters.liveRuntimeBuildConfig, 'checked') }}: - value: 'Checked' - ${{ if eq(parameters.liveRuntimeBuildConfig, 'debug') }}: - value: 'Debug' - - - name: priorityArg - value: '' - - - librariesBuildArtifactName: '' - - librariesOverrideArg: '' - - librariesDownloadDir: '' - - - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - librariesBuildArtifactName: ${{ format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - - librariesDownloadDir: $(Build.SourcesDirectory)/artifacts - - librariesOverrideArg: ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' - - - ${{ each variable in parameters.variables }}: - - ${{insert}}: ${{ variable }} - - steps: ${{ parameters.steps }} diff --git a/eng/pipelines/runtime-cet.yml b/eng/pipelines/runtime-cet.yml index c756a3c83d05c..459905bd89ee6 100644 --- a/eng/pipelines/runtime-cet.yml +++ b/eng/pipelines/runtime-cet.yml @@ -38,35 +38,34 @@ extends: jobs: # - # Build CoreCLR checked - # Only when CoreCLR is changed + # Build CoreCLR and Libraries # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: checked platforms: - windows_x64 jobParameters: - testGroup: innerloop + buildArgs: -s clr+libs -c $(_BuildConfig) -lc Release + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: innerloop # - # Build CoreCLR release - # Always as they are needed by Installer and we always build and test the Installer. - # - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: release - platforms: - - windows_x64 - jobParameters: - testScope: innerloop - - # - # CoreCLR Test builds using live libraries release build - # Only when CoreCLR is changed + # CoreCLR Test build # - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -77,10 +76,8 @@ extends: jobParameters: testGroup: innerloop - # # CoreCLR Test executions using live libraries - # Only when CoreCLR is changed # - template: /eng/pipelines/common/platform-matrix.yml @@ -95,3 +92,4 @@ extends: testGroup: innerloop liveLibrariesBuildConfig: release useCodeFlowEnforcement: true + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) diff --git a/eng/pipelines/runtime-llvm.yml b/eng/pipelines/runtime-llvm.yml index 9d358e5f79308..15be109bc9acd 100644 --- a/eng/pipelines/runtime-llvm.yml +++ b/eng/pipelines/runtime-llvm.yml @@ -146,73 +146,59 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isRollingBuild'], true)) - # - # Build Mono release with LLVM AOT - # Only when mono, or the runtime tests changed - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono - buildConfig: release - platforms: - - linux_x64 - - linux_arm64 - jobParameters: - runtimeVariant: llvmaot - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # Mono CoreCLR runtime Test executions using live libraries and LLVM AOT - # Only when Mono is changed - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml buildConfig: release runtimeFlavor: mono platforms: - - linux_x64_llvmaot + - linux_x64 # Disabled pending outcome of https://github.com/dotnet/runtime/issues/60234 investigation #- linux_arm64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testGroup: innerloop - liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - liveRuntimeBuildConfig: release - runtimeVariant: llvmaot + nameSuffix: AllSubsets_Mono_LLVMAOT_RuntimeTests + buildArgs: -s mono+clr.iltools+clr.hosts+libs+host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} + /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_Release + llvmAotStepContainer: linux_x64_llvmaot + runtimeVariant: llvmaot condition: >- or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isRollingBuild'], true)) - # - # Mono CoreCLR runtime Test executions using live libraries and LLVM Full AOT - # Only when Mono is changed - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml buildConfig: release runtimeFlavor: mono platforms: - - linux_x64_llvmaot + - linux_x64 - linux_arm64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testGroup: innerloop - liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - liveRuntimeBuildConfig: release - runtimeVariant: llvmfullaot + nameSuffix: AllSubsets_Mono_LLVMFullAOT_RuntimeTests + buildArgs: -s mono+clr.iltools+clr.hosts+libs+host+packs -c $(_BuildConfig) -lc ${{ variables.debugOnPrReleaseOnRolling }} + /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_Release + llvmAotStepContainer: linux_x64_llvmaot + runtimeVariant: llvmfullaot condition: >- or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isRollingBuild'], true)) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index d7a1f1847eb11..9611a7e3a773c 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -67,72 +67,157 @@ extends: - template: /eng/pipelines/common/evaluate-default-paths.yml # - # Build CoreCLR checked - # Only when CoreCLR is changed + # Build CoreCLR verticals where we don't run host tests # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: checked + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - - linux_x64 - - linux_arm - - linux_arm64 - - linux_riscv64 - linux_musl_arm - linux_musl_arm64 - - linux_musl_x64 - - osx_arm64 - - windows_x86 - - windows_x64 - windows_arm64 + - linux_arm jobParameters: - testGroup: innerloop + nameSuffix: AllSubsets_CoreCLR_ReleaseRuntimeLibs + buildArgs: -s clr+libs+host+packs -rc Release -lc Release -c $(_BuildConfig) + timeoutInMinutes: 120 condition: >- or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + platforms: + - osx_arm64 + - linux_arm64 + - linux_musl_x64 + jobParameters: + nameSuffix: AllSubsets_CoreCLR + buildArgs: -s clr+libs+host+packs -rc Release -c Release -lc $(_BuildConfig) + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) # - # Build the whole product using GNU compiler toolchain - # When CoreCLR, Mono, Libraries, Installer and src/tests are changed + # Build CoreCLR and Libraries with Libraries tests + # For running libraries tests and installer tests # - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: checked + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - - gcc_linux_x64 + - linux_x64 + - linux_musl_x64 + - osx_x64 + - windows_x64 jobParameters: - testGroup: innerloop - nameSuffix: Native_GCC - buildArgs: -s clr.native+libs.native+mono+host.native -c $(_BuildConfig) -gcc + nameSuffix: CoreCLR_Libraries + buildArgs: -s clr+libs+libs.tests -rc Release -c $(_BuildConfig) /p:ArchiveTests=true + timeoutInMinutes: 120 postBuildSteps: - - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests.yml + - template: /eng/pipelines/common/upload-artifact-step.yml parameters: - testBuildArgs: skipmanaged skipgeneratelayout skiprestorepackages -gcc + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreCLR_Libraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) condition: >- or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: + - windows_x86 + jobParameters: + nameSuffix: CoreCLR_Libraries + buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreCLR_Libraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), eq(variables['isRollingBuild'], true)) # - # Build CoreCLR osx_x64 checked - # Only when CoreCLR or Libraries is changed + # Build CoreCLR and Libraries with the respective tests + # for the test configurations we run. # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: checked + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: + - linux_arm64 - osx_x64 jobParameters: - testGroup: innerloop + nameSuffix: Libraries_CheckedCoreCLR + buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:ArchiveTests=true + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: innerloop + configOverride: Checked condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), @@ -140,34 +225,197 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), eq(variables['isRollingBuild'], true)) - # - # Build CoreCLR release - # Always as they are needed by Installer and we always build and test the Installer. - # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: release + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - - osx_arm64 - - osx_x64 - linux_x64 - - linux_arm - - linux_arm64 + - windows_x64 + jobParameters: + nameSuffix: Libraries_CheckedCoreCLR + buildArgs: -s clr+libs -c $(_BuildConfig) -rc Checked + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: innerloop + configOverride: Checked + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + platforms: - linux_musl_x64 + - windows_x86 + jobParameters: + nameSuffix: Libraries_CheckedCoreCLR + buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:ArchiveTests=true + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + platforms: + - osx_arm64 + jobParameters: + nameSuffix: Libraries_CheckedCoreCLR + buildArgs: -s clr+libs -c $(_BuildConfig) -rc Checked + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: innerloop + configOverride: Checked + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr_AppleSilicon.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + platforms: + - linux_arm + - windows_arm64 + - windows_x86 + jobParameters: + nameSuffix: CoreCLR_ReleaseLibraries + buildArgs: -s clr+libs -rc $(_BuildConfig) -c Release + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreCLR_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: innerloop + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + platforms: + - linux_x64 - linux_musl_arm - linux_musl_arm64 - windows_x64 - - windows_x86 - - windows_arm64 - - freebsd_x64 + jobParameters: + nameSuffix: CoreCLR_ReleaseLibraries + buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreCLR_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + displayName: Build Assets + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: CoreCLR_ReleaseLibraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + # + # Build the whole product using GNU compiler toolchain + # When CoreCLR, Mono, Libraries, Installer and src/tests are changed + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + platforms: + - gcc_linux_x64 jobParameters: testGroup: innerloop - # Mono/runtimetests also need this, but skip for wasm - condition: + nameSuffix: Native_GCC + buildArgs: -s clr.native+libs.native+mono+host.native -c $(_BuildConfig) -gcc + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests.yml + parameters: + testBuildArgs: skipmanaged skipgeneratelayout skiprestorepackages -gcc + condition: >- or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), eq(variables['isRollingBuild'], true)) @@ -234,6 +482,48 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), eq(variables['isRollingBuild'], true)) + # + # Build CoreCLR without building test assets + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: checked + runtimeFlavor: coreclr + platforms: + - linux_riscv64 + jobParameters: + testScope: innerloop + nameSuffix: CoreCLR + buildArgs: -s clr.native+clr.tools+clr.corelib+clr.nativecorelib+clr.aot+clr.packages -c $(_BuildConfig) + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + # + # Build CoreCLR + Libs + Host + Packs + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + runtimeFlavor: coreclr + platforms: + - freebsd_x64 + jobParameters: + testScope: innerloop + nameSuffix: CoreCLR + buildArgs: -s clr+libs+host+packs -c $(_BuildConfig) -rc Checked + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + # # CoreCLR NativeAOT debug build and smoke tests # Only when CoreCLR is changed @@ -892,7 +1182,7 @@ extends: # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml runtimeFlavor: mono buildConfig: debug platforms: @@ -905,6 +1195,8 @@ extends: - windows_x86 # - windows_arm64 jobParameters: + nameSuffix: Mono_Runtime + buildArgs: -s mono -c $(_BuildConfig) condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), @@ -912,11 +1204,10 @@ extends: # # Build Mono release AOT cross-compilers - # Only when mono changed # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml runtimeFlavor: mono buildConfig: release platforms: @@ -924,9 +1215,10 @@ extends: - linux_musl_x64 - linux_arm64 - linux_musl_arm64 - - windows_arm64 - - windows_x64 jobParameters: + buildArgs: -s mono+packs -c $(_BuildConfig) + /p:MonoCrossAOTTargetOS=android+browser+wasi /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true + nameSuffix: CrossAOT_Mono runtimeVariant: crossaot dependsOn: - mono_android_offsets @@ -944,177 +1236,68 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml runtimeFlavor: mono buildConfig: release platforms: - - osx_x64 - - osx_arm64 + - windows_arm64 + - windows_x64 jobParameters: + buildArgs: -s mono+packs -c $(_BuildConfig) + /p:MonoCrossAOTTargetOS=android+browser+wasi /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true + nameSuffix: CrossAOT_Mono runtimeVariant: crossaot dependsOn: - mono_android_offsets - mono_browser_offsets - mono_wasi_offsets - - mono_tvos_offsets - - mono_ios_offsets - - mono_maccatalyst_offsets monoCrossAOTTargetOS: - android - browser - wasi - - tvos - - ios - - maccatalyst condition: >- or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isRollingBuild'], true)) - # - # Build Mono release - # Only when libraries or mono changed + # Build Mono release AOT cross-compilers + # Only when mono changed # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono - buildConfig: release - platforms: - - linux_x64 - # - linux_musl_arm64 - - windows_x64 - - windows_x86 - # - windows_arm64 - jobParameters: - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(variables['isRollingBuild'], true)) - # - # Build Mono release - # Only when libraries, mono, or the runtime tests changed - # Currently only these architectures are needed for the runtime tests. - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml runtimeFlavor: mono buildConfig: release platforms: - osx_x64 - - linux_arm64 - jobParameters: - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # Build Mono release with LLVM AOT - # Only when mono, or the runtime tests changed - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/mono/templates/build-job.yml - runtimeFlavor: mono - buildConfig: release - platforms: - - linux_x64 - - linux_arm64 - jobParameters: - runtimeVariant: llvmaot - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # Build libraries using live CoreLib - # These set of libraries are built always no matter what changed - # The reason for that is because Corelib and Installer needs it and - # These are part of the test matrix for Libraries changes. - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Release - platforms: - - linux_arm - - linux_musl_arm - - linux_musl_arm64 - - windows_arm64 - - windows_x86 - jobParameters: - condition: - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - platforms: - - linux_arm64 - - linux_musl_x64 - - linux_x64 - osx_arm64 - - osx_x64 - - windows_x64 - - freebsd_x64 jobParameters: - testScope: innerloop - condition: + buildArgs: -s mono+packs -c $(_BuildConfig) + /p:MonoCrossAOTTargetOS=android+browser+wasi+tvos+ios+maccatalyst /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true + nameSuffix: CrossAOT_Mono + runtimeVariant: crossaot + dependsOn: + - mono_android_offsets + - mono_browser_offsets + - mono_wasi_offsets + - mono_tvos_offsets + - mono_ios_offsets + - mono_maccatalyst_offsets + monoCrossAOTTargetOS: + - android + - browser + - wasi + - tvos + - ios + - maccatalyst + condition: >- or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) - # - # Libraries debug build that only runs when coreclr is changed - # Only do this on PR builds since we use the Release builds for these test runs in CI - # and those are already built above - # - - ${{ if eq(variables['isRollingBuild'], false) }}: - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Debug - platforms: - - windows_x86 - jobParameters: - condition: >- - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true) - - # - # Libraries release build that only runs when coreclr is changed in PRs - # We need these for checked coreclr + release libraries tests runs. - # - - ${{ if eq(variables['isRollingBuild'], false) }}: - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Release - platforms: - - linux_x64 - - windows_x64 - jobParameters: - condition: >- - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true) - # # Build and test libraries for .NET Framework # @@ -1170,15 +1353,13 @@ extends: jobTemplate: /eng/pipelines/installer/jobs/build-job.yml buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - - linux_musl_arm - - linux_musl_arm64 - windows_x86 - - windows_arm64 - - linux_arm jobParameters: liveRuntimeBuildConfig: release liveLibrariesBuildConfig: Release - runOnlyIfDependenciesSucceeded: true + unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Release + unifiedBuildConfigOverride: release + unifiedBuildNameSuffix: CoreCLR_Libraries condition: or( eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), @@ -1189,136 +1370,56 @@ extends: jobTemplate: /eng/pipelines/installer/jobs/build-job.yml buildConfig: Release platforms: - - osx_arm64 - osx_x64 - linux_x64 - - linux_arm64 - - linux_musl_x64 - windows_x64 - - freebsd_x64 jobParameters: liveRuntimeBuildConfig: release liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - runOnlyIfDependenciesSucceeded: true + unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} + unifiedBuildNameSuffix: CoreCLR_Libraries condition: or( eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) # - # CoreCLR Test builds using live libraries release build - # Only when CoreCLR is changed - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: innerloop - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # CoreCLR Test executions using live libraries - # Only when CoreCLR is changed + # Build the whole product using Mono and run runtime tests # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - linux_arm - - windows_x86 - - windows_arm64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: innerloop - liveLibrariesBuildConfig: Release - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_x64 - - linux_x64 - - linux_arm64 - - windows_x64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: innerloop - liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - helixQueueGroup: pr + jobTemplate: /eng/pipelines/common/global-build-job.yml helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: innerloop - liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr_AppleSilicon.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # Mono Test builds with CoreCLR runtime tests using live libraries debug build - # Only when Mono is changed - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: innerloop - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: release + buildConfig: Release runtimeFlavor: mono platforms: - - windows_x64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + - osx_x64 + - linux_arm64 + variables: + - name: timeoutPerTestInMinutes + value: 60 + - name: timeoutPerTestCollectionInMinutes + value: 180 jobParameters: testGroup: innerloop - liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} - liveRuntimeBuildConfig: release + nameSuffix: AllSubsets_Mono_Minijit_RuntimeTests runtimeVariant: minijit + buildArgs: -s mono+libs+clr.hosts+clr.iltools -c Release + timeoutInMinutes: 180 condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_Release + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/test-variables.yml # # Build the whole product using Mono and run runtime tests @@ -1330,8 +1431,7 @@ extends: buildConfig: Release runtimeFlavor: mono platforms: - - osx_x64 - - linux_arm64 + - windows_x64 variables: - name: timeoutPerTestInMinutes value: 60 @@ -1341,7 +1441,7 @@ extends: testGroup: innerloop nameSuffix: AllSubsets_Mono_Minijit_RuntimeTests runtimeVariant: minijit - buildArgs: -s mono+libs+clr.hosts+clr.iltools -c Release + buildArgs: -s mono+libs+clr.hosts+clr.iltools -c Release -lc ${{ variables.debugOnPrReleaseOnRolling }} timeoutInMinutes: 180 condition: >- or( @@ -1356,6 +1456,8 @@ extends: testRunNamePrefixSuffix: Mono_Release extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml + parameters: + liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} # # Mono CoreCLR runtime Test executions using live libraries in interpreter mode @@ -1433,55 +1535,88 @@ extends: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml # - # Libraries Release Test Execution against a release mono runtime. - # Only when libraries or mono changed + # CoreCLR Test builds using live libraries release build + # Only when CoreCLR is changed # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - runtimeFlavor: mono - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml + buildConfig: checked platforms: - # - windows_x64 - - osx_x64 - - linux_arm64 - - linux_x64 - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + - CoreClrTestBuildHost # Either osx_x64 or linux_x64 jobParameters: - isOfficialBuild: false - runtimeDisplayName: mono - testScope: innerloop - liveRuntimeBuildConfig: release + testGroup: innerloop condition: >- or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), eq(variables['isRollingBuild'], true)) # - # Libraries Release Test Execution against a release mono interpreter runtime. - # Only when libraries or mono changed + # CoreCLR Test executions using live libraries + # Only when CoreCLR is changed # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - runtimeFlavor: mono - buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + buildConfig: checked platforms: - # - windows_x64 - #- osx_x64 + - linux_arm + - windows_x86 + - windows_arm64 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: innerloop + liveLibrariesBuildConfig: Release + unifiedArtifactsName: CoreCLR_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedBuildNameSuffix: CoreCLR_ReleaseLibraries + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + buildConfig: checked + platforms: + - osx_x64 - linux_x64 - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + - linux_arm64 + - windows_x64 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: - isOfficialBuild: false - interpreter: true - runtimeDisplayName: mono_interpreter - testScope: innerloop - liveRuntimeBuildConfig: release + testGroup: innerloop + liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedBuildNameSuffix: Libraries_CheckedCoreCLR + unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} condition: >- or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + buildConfig: checked + platforms: + - osx_arm64 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: innerloop + liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedBuildNameSuffix: Libraries_CheckedCoreCLR + unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr_AppleSilicon.containsChange'], true), eq(variables['isRollingBuild'], true)) # @@ -1499,6 +1634,9 @@ extends: isOfficialBuild: false testScope: innerloop liveRuntimeBuildConfig: release + unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + helixArtifactsName: CoreCLR_Libraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedBuildNameSuffix: CoreCLR_Libraries condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), @@ -1522,11 +1660,14 @@ extends: isOfficialBuild: false testScope: innerloop liveRuntimeBuildConfig: release + unifiedArtifactsName: CoreCLR_Libraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + helixArtifactsName: CoreCLR_Libraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(debugOnPrReleaseOnRolling) + unifiedBuildNameSuffix: CoreCLR_Libraries + unifiedBuildConfigOverride: ${{ variables.debugOnPrReleaseOnRolling }} condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(variables['isRollingBuild'], true)) - # The next three jobs run checked coreclr + libraries tests. # The matrix looks like the following, where the right columns specify which configurations # the libraries tests are built in. @@ -1559,6 +1700,9 @@ extends: jobParameters: testScope: innerloop liveRuntimeBuildConfig: checked + unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + helixArtifactsName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedBuildNameSuffix: Libraries_CheckedCoreCLR condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), @@ -1582,6 +1726,10 @@ extends: jobParameters: testScope: innerloop liveRuntimeBuildConfig: checked + unifiedArtifactsName: CoreCLR_ReleaseLibraries_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + helixArtifactsName: CoreCLR_ReleaseLibraries_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + unifiedBuildNameSuffix: CoreCLR_ReleaseLibraries + unifiedBuildConfigOverride: checked condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), @@ -1598,12 +1746,78 @@ extends: jobParameters: testScope: innerloop liveRuntimeBuildConfig: checked + unifiedArtifactsName: Libraries_CheckedCoreCLR_BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + helixArtifactsName: Libraries_CheckedCoreCLR_TestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + unifiedBuildNameSuffix: Libraries_CheckedCoreCLR condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(variables['isRollingBuild'], true)) + # + # Build and test Mono Interpreter with the libraries testss + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + runtimeFlavor: mono + runtimeVariant: monointerpreter + platforms: + - linux_x64 + #- osx_x64 + #- windows_x64 + jobParameters: + testGroup: innerloop + nameSuffix: Mono_Interpreter_LibrariesTests + buildArgs: -s mono+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true + timeoutInMinutes: 480 + # extra steps, run tests + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_Interpreter_$(_BuildConfig) + interpreter: true + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + # + # Build and test Mono Minijit with the libraries testss + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + runtimeFlavor: mono + platforms: + - linux_arm64 + - linux_x64 + - osx_x64 + #- windows_x64 + jobParameters: + testGroup: innerloop + nameSuffix: Mono_MiniJIT_LibrariesTests + buildArgs: -s mono+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true + timeoutInMinutes: 480 + # extra steps, run tests + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_Minijit_$(_BuildConfig) + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) + # # Sourcebuild legs # We have 3 important legs for source-build: diff --git a/eng/pipelines/runtimelab.yml b/eng/pipelines/runtimelab.yml index 7c34126757d73..b1b65f5343de9 100644 --- a/eng/pipelines/runtimelab.yml +++ b/eng/pipelines/runtimelab.yml @@ -49,175 +49,103 @@ extends: stages: - stage: Build jobs: + - ${{ if ne(variables.isOfficialBuild, true) }}: + # + # Build the whole product with Checked CoreCLR and run runtime tests + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + buildConfig: checked + platforms: + - linux_x64 + - windows_x64 + jobParameters: + timeoutInMinutes: 200 + buildArgs: -s clr+libs+hosts+packs -c debug -rc $(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) - # - # Build with Debug config and Checked runtimeConfiguration - # - - ${{ if ne(variables.isOfficialBuild, true) }}: - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: Checked - platforms: - - linux_x64 - - windows_x64 - jobParameters: - timeoutInMinutes: 100 - testGroup: innerloop - buildArgs: -s clr+libs+host+packs -c debug -runtimeConfiguration Checked - postBuildSteps: - - template: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml - parameters: - uploadRuntimeTests: true + # + # Build the whole product with Release CoreCLR and run libraries tests + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: release + platforms: + - linux_x64 + - windows_x64 + jobParameters: + timeoutInMinutes: 180 + buildArgs: -s clr+libs+libs.tests+hosts+packs -c $(_BuildConfig) /p:ArchiveTests=true + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Libraries_$(_BuildConfig) - # - # Build with Release config and Release runtimeConfiguration - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: Release - platforms: - - linux_x64 - - windows_x64 - jobParameters: - timeoutInMinutes: 100 - isOfficialBuild: ${{ variables.isOfficialBuild }} - testGroup: innerloop - postBuildSteps: - - template: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml - parameters: - uploadLibrariesTests: ${{ eq(variables.isOfficialBuild, false) }} - uploadIntermediateArtifacts: false - ${{ if eq(variables.isOfficialBuild, false) }}: - buildArgs: -s clr+libs+libs.tests+host+packs -c $(_BuildConfig) /p:ArchiveTests=true - ${{ if eq(variables.isOfficialBuild, true) }}: - buildArgs: -s clr+libs -c $(_BuildConfig) + # + # Build and test libraries AllConfigurations + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + platforms: + - windows_x64 + jobParameters: + buildArgs: -test -s tools+libs+libs.tests -allConfigurations -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + nameSuffix: Libraries_AllConfigurations + timeoutInMinutes: 150 - # - # Build with Release allConfigurations to produce packages - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: Release - platforms: - - windows_x64 - jobParameters: - isOfficialBuild: ${{ variables.isOfficialBuild }} - testGroup: innerloop - nameSuffix: AllConfigurations - buildArgs: -s libs -c $(_BuildConfig) -allConfigurations - ${{ if eq(variables.isOfficialBuild, true) }}: - postBuildSteps: - - template: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml + - ${{ else }}: + # + # Build the whole product with Release CoreCLR + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: release + platforms: + - linux_x64 + - windows_x64 + jobParameters: + isOfficialBuild: true + timeoutInMinutes: 180 + buildArgs: -s clr+libs+hosts+packs -c $(_BuildConfig) + postBuildSteps: + # Upload the results. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml parameters: - uploadIntermediateArtifacts: true - isOfficialBuild: true - librariesBinArtifactName: libraries_bin_official_allconfigurations - - # Installer official builds need to build installers and need the libraries all configurations build - - ${{ if eq(variables.isOfficialBuild, true) }}: - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/installer/jobs/build-job.yml - jobParameters: - liveRuntimeBuildConfig: Release - liveLibrariesBuildConfig: Release - isOfficialBuild: ${{ variables.isOfficialBuild }} - useOfficialAllConfigurations: true - dependsOnGlobalBuild: true - platforms: - - linux_x64 - - windows_x64 - - - ${{ if ne(variables.isOfficialBuild, true) }}: - # - # CoreCLR Test builds using live libraries release build - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml - buildConfig: Checked - platforms: - - CoreClrTestBuildHost # Either osx_x64 or linux_x64 - jobParameters: - testGroup: innerloop - dependsOn: - - build_linux_x64_Checked_ - - build_linux_x64_Release_ - - # - # CoreCLR Test executions using live libraries - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: Checked - platforms: - - linux_x64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: innerloop - liveLibrariesBuildConfig: Release - dependsOn: - - coreclr_common_test_build_p0_AnyOS_AnyCPU_Checked - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: Checked - platforms: - - windows_x64 - helixQueueGroup: pr - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: innerloop - liveLibrariesBuildConfig: Release - dependsOn: - - coreclr_common_test_build_p0_AnyOS_AnyCPU_Checked - - build_windows_x64_Checked_ - - build_windows_x64_Release_ - - # - # Libraries Release Test Execution against a release coreclr runtime - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - linux_x64 - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - jobParameters: - isRollingBuild: false - isOfficialBuild: false - testScope: innerloop - liveRuntimeBuildConfig: Release - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - dependsOn: - - build_linux_x64_Release_ + name: $(osGroup)$(osSubgroup)_$(archType) - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - windows_x64 - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - jobParameters: - isRollingBuild: false - isOfficialBuild: false - testScope: innerloop - liveRuntimeBuildConfig: Release - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - dependsOn: - - build_windows_x64_Release_ + # + # Build libraries AllConfigurations for packages + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: Release + platforms: + - windows_x64 + jobParameters: + buildArgs: -s tools+libs -allConfigurations -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + nameSuffix: Libraries_AllConfigurations + isOfficialBuild: true + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: Libraries_AllConfigurations + timeoutInMinutes: 95 - ${{ if eq(variables.isOfficialBuild, true) }}: - template: /eng/pipelines/official/stages/publish.yml parameters: - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: true \ No newline at end of file diff --git a/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml b/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml deleted file mode 100644 index 97c007ba4b307..0000000000000 --- a/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml +++ /dev/null @@ -1,76 +0,0 @@ -parameters: - buildConfig: '' - archType: '' - osGroup: '' - osSubgroup: '' - nameSuffix: '' - platform: '' - runtimeVariant: '' - librariesBinArtifactName: '' - isOfficialBuild: false - uploadLibrariesTests: false - uploadRuntimeTests: false - uploadIntermediateArtifacts: false - -steps: - # Build coreclr native test output - - ${{ if eq(parameters.uploadRuntimeTests, true) }}: - - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) skipmanaged skipgeneratelayout $(buildConfigUpper) ${{ parameters.archType }} - displayName: Build native test components - - # Copy all build output into artifacts staging directory - - template: /eng/pipelines/libraries/prepare-for-bin-publish.yml - parameters: - isOfficialBuild: ${{ parameters.isOfficialBuild }} - - # Zip CoreCLR Build Output - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper) - archiveType: $(archiveType) - tarCompression: $(tarCompression) - includeRootFolder: false - archiveExtension: $(archiveExtension) - artifactName: CoreCLRProduct_${{ parameters.runtimeVariant }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }} - displayName: 'CoreCLR product build' - - # Zip Test Build - - ${{ if eq(parameters.uploadLibrariesTests, true) }}: - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/helix - includeRootFolder: true - archiveType: $(archiveType) - archiveExtension: $(archiveExtension) - tarCompression: $(tarCompression) - artifactName: libraries_test_assets_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }} - displayName: Test Assets - - # Zip runtime native assets for use by Tests - - ${{ if eq(parameters.uploadRuntimeTests, true) }}: - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/tests/coreclr/obj/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper) - includeRootFolder: false - archiveType: $(archiveType) - tarCompression: $(tarCompression) - archiveExtension: $(archiveExtension) - artifactName: CoreCLRNativeTestArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }} - displayName: 'native test components' - - # Zip Libraries Build Output - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.ArtifactStagingDirectory)/artifacts - archiveType: $(archiveType) - tarCompression: $(tarCompression) - includeRootFolder: false - archiveExtension: $(archiveExtension) - artifactName: ${{ coalesce(parameters.librariesBinArtifactName, format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig)) }} - displayName: Build Assets - - # Upload unsigned artifacts - - ${{ if eq(parameters.uploadIntermediateArtifacts, true) }}: - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: ${{ parameters.platform }}${{ parameters.nameSuffix }} diff --git a/src/libraries/sendtohelix-superpmi-collect.targets b/src/libraries/sendtohelix-superpmi-collect.targets index 5affc58dd36ef..6c02e16db7c5f 100644 --- a/src/libraries/sendtohelix-superpmi-collect.targets +++ b/src/libraries/sendtohelix-superpmi-collect.targets @@ -56,4 +56,9 @@ + + + + + diff --git a/src/libraries/sendtohelix.proj b/src/libraries/sendtohelix.proj index 44a02cd5b828e..17aa91b1597bb 100644 --- a/src/libraries/sendtohelix.proj +++ b/src/libraries/sendtohelix.proj @@ -23,7 +23,7 @@ SetStressModes_$(Scenario).sh - + @@ -201,7 +201,8 @@ + Condition="'$(SuperPmiCollect)' == 'true'" + DependsOnTargets="ResolveRuntimeFilesFromLocalBuild"> @@ -220,12 +221,16 @@ (superpmi, mcs, superpmi-shim-collector, clrjit), but it's easier to just copy everything. --> - + - + - + + +