From d9bfc742de6da7330ebe395012155a259390e53f Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 14:24:16 +0200 Subject: [PATCH 01/19] Add support for Native AOT runs --- azure-pipelines.yml | 15 +++++++++++++++ eng/performance/maui_scenarios_ios.proj | 7 ++++--- eng/performance/scenarios.yml | 7 ++++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8878d1a3499..3cd380c970f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -333,6 +333,21 @@ jobs: projectFile: maui_scenarios_ios.proj dotnetVersionsLinks: main: https://aka.ms/dotnet/sdk/maui/net8.0.json + runtimeFlavor: mono + + # Maui iOS scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + kind: maui_scenarios_ios + projectFile: maui_scenarios_ios.proj + dotnetVersionsLinks: + main: https://aka.ms/dotnet/sdk/maui/net8.0.json + runtimeFlavor: coreclr ## Maui scenario benchmarks #- template: /eng/performance/build_machine_matrix.yml diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index 393529f1020..0a4d36c8c00 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -24,7 +24,7 @@ - + mauiios $(ScenariosDir)%(ScenarioDirectoryName) MauiiOSDefault @@ -37,7 +37,7 @@ MauiiOSPodcast net.dot.net.dot.netconf2021.maui --> - + mauiblazorios $(ScenariosDir)%(ScenarioDirectoryName) MauiBlazoriOSDefault @@ -47,7 +47,8 @@ - $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) + /p:PublishAot=true /p:PublishAotUsingRuntimePack=true + $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 $(OptionalCommandProps) -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) %(PreparePayloadWorkItem.PayloadDirectory) diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index 4c32a1a172f..558c64cc5b8 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -13,6 +13,7 @@ parameters: affinity: '0' # optional -- Affinity bitmask to a specific machine in the pool as a decimal number (e.g. 1, 2, etc) runEnvVars: [] # optional -- Environment variables to set for the benchmark run in the form of a multi-line object like: "runEnvVars:\n - DOTNET_GCgen0size=1E00000 \n - DOTNET_GCHeapCount=4 \n - DOTNET_GCTotalPhysicalMemory=200000000" additionalJobIdentifier: '' # optional -- Additional identifier to append to the job name (no spaces) + runtimeFlavor: '' # optional -- Runtime flavor used for the scenarios jobs: - template: ../common/templates/jobs/jobs.yml @@ -158,7 +159,7 @@ jobs: $(Python) -m pip install urllib3==1.26.15 $(Python) -m pip install requests .\src\scenarios\init.ps1 -DotnetDirectory $(CorrelationStaging)dotnet - dotnet msbuild .\eng\performance\${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:.\artifacts\log\$(_BuildConfig)\PrepareWorkItemPayloads.binlog + dotnet msbuild .\eng\performance\${{ parameters.projectFile }} /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /restore /t:PreparePayloadWorkItems /bl:.\artifacts\log\$(_BuildConfig)\PrepareWorkItemPayloads.binlog displayName: Prepare scenarios env: CorrelationPayloadDirectory: $(CorrelationStaging) @@ -193,7 +194,7 @@ jobs: $(Python) -m pip install --user urllib3==1.26.15 $(Python) -m pip install --user requests . ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet - dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog + dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog displayName: Prepare scenarios env: CorrelationPayloadDirectory: $(CorrelationStaging) @@ -221,7 +222,7 @@ jobs: $(Python) -m pip install --user urllib3==1.26.15 $(Python) -m pip install --user requests . ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet - dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog + dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog displayName: Prepare scenarios env: CorrelationPayloadDirectory: $(CorrelationStaging) From 07f3bb312412fb2211f0540b75febf0e3741ad4e Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 14:26:44 +0200 Subject: [PATCH 02/19] Disable perf jobs --- azure-pipelines.yml | 1026 +++++++++++++++++++++---------------------- 1 file changed, 513 insertions(+), 513 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3cd380c970f..f26450670e1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -57,269 +57,269 @@ jobs: # Public correctness jobs ###################################################### -- ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: - - # Scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: scenarios - projectFile: scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - ## MAUI scenario benchmarks - #- template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/scenarios.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # isPublic: true - # jobParameters: - # kind: maui_scenarios - # projectFile: maui_scenarios.proj - # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - # - main - - # Blazor scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: blazor_scenarios - projectFile: blazor_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - # SDK scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - win-x86 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: sdk_scenarios - projectFile: sdk_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main +# - ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: + +# # Scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: scenarios +# projectFile: scenarios.proj +# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks +# - main + +# ## MAUI scenario benchmarks +# #- template: /eng/performance/build_machine_matrix.yml +# # parameters: +# # jobTemplate: /eng/performance/scenarios.yml +# # buildMachines: +# # - win-x64 +# # - ubuntu-x64 +# # isPublic: true +# # jobParameters: +# # kind: maui_scenarios +# # projectFile: maui_scenarios.proj +# # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks +# # - main + +# # Blazor scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: blazor_scenarios +# projectFile: blazor_scenarios.proj +# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks +# - main + +# # SDK scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - win-x86 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: sdk_scenarios +# projectFile: sdk_scenarios.proj +# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks +# - main - # micro benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - win-x86 - isPublic: true - jobParameters: - kind: micro - csproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - main - - # Ubuntux64 Default and NativeAOT micro benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - ubuntu-x64 - isPublic: true - jobParameters: - kind: micro - csproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - main - - nativeaot8.0 - - # net462 micro benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-rs5-x64 - isPublic: true - jobParameters: - kind: micro_net462 - csproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - net462 - - # ML.NET benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: mlnet - csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj - runCategories: 'mldotnet' - channels: - - main - - # F# benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: fsharp - csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj - runCategories: 'fsharp' - channels: - - main - - # bepuphysics benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: bepuphysics - csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj - runCategories: 'BepuPhysics' - channels: - - main - - # ImageSharp benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: imagesharp - csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj - runCategories: 'ImageSharp' - channels: - - main - - # Roslyn benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: roslyn - csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj - runCategories: 'roslyn' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - # ILLink benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: illink - csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj - runCategories: 'illink' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - # NativeAOT scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: nativeaot_scenarios - projectFile: nativeaot_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - -########################################### -# Private Jobs -########################################### - -- ${{ if or(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest', 'Manual')), parameters.runPrivateJobs) }}: - - # Scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: scenarios - projectFile: scenarios.proj - channels: - - main - - # Affinitized Scenario benchmarks (Initially just PDN) - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - win-arm64 - - win-arm64-ampere - isPublic: false - jobParameters: - kind: scenarios - projectFile: scenarios_affinitized.proj - channels: - - main - additionalJobIdentifier: 'Affinity_85' - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - - # Maui Android scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - kind: maui_scenarios_android - projectFile: maui_scenarios_android.proj - dotnetVersionsLinks: - main: https://aka.ms/dotnet/sdk/maui/net8.0.json +# # micro benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - win-x86 +# isPublic: true +# jobParameters: +# kind: micro +# csproj: src\benchmarks\micro\MicroBenchmarks.csproj +# runCategories: 'runtime libraries' +# channels: +# - main + +# # Ubuntux64 Default and NativeAOT micro benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: micro +# csproj: src\benchmarks\micro\MicroBenchmarks.csproj +# runCategories: 'runtime libraries' +# channels: +# - main +# - nativeaot8.0 + +# # net462 micro benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-rs5-x64 +# isPublic: true +# jobParameters: +# kind: micro_net462 +# csproj: src\benchmarks\micro\MicroBenchmarks.csproj +# runCategories: 'runtime libraries' +# channels: +# - net462 + +# # ML.NET benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: mlnet +# csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj +# runCategories: 'mldotnet' +# channels: +# - main + +# # F# benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: fsharp +# csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj +# runCategories: 'fsharp' +# channels: +# - main + +# # bepuphysics benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: bepuphysics +# csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj +# runCategories: 'BepuPhysics' +# channels: +# - main + +# # ImageSharp benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: imagesharp +# csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj +# runCategories: 'ImageSharp' +# channels: +# - main + +# # Roslyn benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: roslyn +# csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj +# runCategories: 'roslyn' +# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only +# - main + +# # ILLink benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: illink +# csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj +# runCategories: 'illink' +# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only +# - main + +# # NativeAOT scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: nativeaot_scenarios +# projectFile: nativeaot_scenarios.proj +# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks +# - main + +# ########################################### +# # Private Jobs +# ########################################### + +# - ${{ if or(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest', 'Manual')), parameters.runPrivateJobs) }}: + +# # Scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# - win-arm64 +# - ubuntu-arm64-ampere +# isPublic: false +# jobParameters: +# kind: scenarios +# projectFile: scenarios.proj +# channels: +# - main + +# # Affinitized Scenario benchmarks (Initially just PDN) +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - win-arm64 +# - win-arm64-ampere +# isPublic: false +# jobParameters: +# kind: scenarios +# projectFile: scenarios_affinitized.proj +# channels: +# - main +# additionalJobIdentifier: 'Affinity_85' +# affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account +# runEnvVars: +# - DOTNET_GCgen0size=410000 # ~4MB +# - DOTNET_GCHeapCount=4 +# - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + +# # Maui Android scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64-android-arm64-pixel +# - win-x64-android-arm64-galaxy +# isPublic: false +# jobParameters: +# kind: maui_scenarios_android +# projectFile: maui_scenarios_android.proj +# dotnetVersionsLinks: +# main: https://aka.ms/dotnet/sdk/maui/net8.0.json # Maui iOS scenario benchmarks - template: /eng/performance/build_machine_matrix.yml @@ -349,254 +349,254 @@ jobs: main: https://aka.ms/dotnet/sdk/maui/net8.0.json runtimeFlavor: coreclr - ## Maui scenario benchmarks - #- template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/scenarios.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # - win-arm64 - # - ubuntu-arm64-ampere - # isPublic: false - # jobParameters: - # kind: maui_scenarios - # projectFile: maui_scenarios.proj - # channels: - # - main - - # NativeAOT scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - isPublic: false - jobParameters: - kind: nativeaot_scenarios - projectFile: nativeaot_scenarios.proj - channels: - - main - -################################################ -# Scheduled Private jobs -################################################ - -# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -- ${{ if or(and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), not(contains(variables['Build.QueuedBy'], 'Weekly'))), parameters.runScheduledPrivateJobs) }}: - - # SDK scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - win-x86 - - ubuntu-x64 - isPublic: false - jobParameters: - kind: sdk_scenarios - projectFile: sdk_scenarios.proj - channels: - - main - - # Blazor 3.2 scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - isPublic: false - jobParameters: - kind: blazor_scenarios - projectFile: blazor_scenarios.proj - channels: - - main - - # F# benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: fsharp - csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj - runCategories: 'fsharp' - channels: - - main - - # bepuphysics benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: bepuphysics - csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj - runCategories: 'BepuPhysics' - channels: - - main - - # ImageSharp benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: imagesharp - csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj - runCategories: 'ImageSharp' - channels: - - main - - # ML.NET benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: mlnet - csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj - runCategories: 'mldotnet' - channels: - - main - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - - # Roslyn benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: roslyn - csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj - runCategories: 'roslyn' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - - # ILLink benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) - isPublic: false - jobParameters: - kind: illink - csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj - runCategories: 'illink' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - # Secret Sync - - job: Synchronize - pool: - name: NetCore1ESPool-Internal-NoMSI - demands: ImageOverride -equals 1es-windows-2019 - steps: - - task: UseDotNet@2 - displayName: Install .NET 6.0 runtime - inputs: - version: 6.x - - - script: dotnet tool restore - - - task: AzureCLI@2 - inputs: - azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} - -################################################ -# Manually Triggered Job -################################################ - - -- ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: - - job: Synchronize - pool: - name: NetCore1ESPool-Internal-NoMSI - demands: ImageOverride -equals 1es-windows-2019 - steps: - - task: UseDotNet@2 - displayName: Install .NET 6.0 runtime - inputs: - version: 6.x - - - script: dotnet tool restore - - - task: AzureCLI@2 - inputs: - azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} - -################################################ -# Scheduled Private jobs (Weekly) -################################################ - -# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -- ${{ if and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), contains(variables['Build.QueuedBy'], 'Weekly')) }}: - - job: Synchronize - pool: - name: NetCore1ESPool-Internal-NoMSI - demands: ImageOverride -equals 1es-windows-2019 - steps: - - task: UseDotNet@2 - displayName: Install .NET 6.0 runtime - inputs: - version: 6.x - - - script: dotnet tool restore - - - task: AzureCLI@2 - inputs: - azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} +# ## Maui scenario benchmarks +# #- template: /eng/performance/build_machine_matrix.yml +# # parameters: +# # jobTemplate: /eng/performance/scenarios.yml +# # buildMachines: +# # - win-x64 +# # - ubuntu-x64 +# # - win-arm64 +# # - ubuntu-arm64-ampere +# # isPublic: false +# # jobParameters: +# # kind: maui_scenarios +# # projectFile: maui_scenarios.proj +# # channels: +# # - main + +# # NativeAOT scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# - win-arm64 +# isPublic: false +# jobParameters: +# kind: nativeaot_scenarios +# projectFile: nativeaot_scenarios.proj +# channels: +# - main + +# ################################################ +# # Scheduled Private jobs +# ################################################ + +# # Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job +# - ${{ if or(and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), not(contains(variables['Build.QueuedBy'], 'Weekly'))), parameters.runScheduledPrivateJobs) }}: + +# # SDK scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - win-x86 +# - ubuntu-x64 +# isPublic: false +# jobParameters: +# kind: sdk_scenarios +# projectFile: sdk_scenarios.proj +# channels: +# - main + +# # Blazor 3.2 scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# isPublic: false +# jobParameters: +# kind: blazor_scenarios +# projectFile: blazor_scenarios.proj +# channels: +# - main + +# # F# benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# - win-arm64 +# - ubuntu-arm64-ampere +# isPublic: false +# jobParameters: +# kind: fsharp +# csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj +# runCategories: 'fsharp' +# channels: +# - main + +# # bepuphysics benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# - win-arm64 +# - ubuntu-arm64-ampere +# isPublic: false +# jobParameters: +# kind: bepuphysics +# csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj +# runCategories: 'BepuPhysics' +# channels: +# - main + +# # ImageSharp benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# - win-arm64 +# - ubuntu-arm64-ampere +# isPublic: false +# jobParameters: +# kind: imagesharp +# csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj +# runCategories: 'ImageSharp' +# channels: +# - main + +# # ML.NET benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# - win-arm64 +# - win-arm64-ampere +# - ubuntu-arm64-ampere +# isPublic: false +# jobParameters: +# kind: mlnet +# csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj +# runCategories: 'mldotnet' +# channels: +# - main +# affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account +# runEnvVars: +# - DOTNET_GCgen0size=410000 # ~4MB +# - DOTNET_GCHeapCount=4 +# - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + +# # Roslyn benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# - win-arm64 +# - win-arm64-ampere +# - ubuntu-arm64-ampere +# isPublic: false +# jobParameters: +# kind: roslyn +# csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj +# runCategories: 'roslyn' +# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only +# - main +# affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account +# runEnvVars: +# - DOTNET_GCgen0size=410000 # ~4MB +# - DOTNET_GCHeapCount=4 +# - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + +# # ILLink benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) +# isPublic: false +# jobParameters: +# kind: illink +# csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj +# runCategories: 'illink' +# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only +# - main + +# # Secret Sync +# - job: Synchronize +# pool: +# name: NetCore1ESPool-Internal-NoMSI +# demands: ImageOverride -equals 1es-windows-2019 +# steps: +# - task: UseDotNet@2 +# displayName: Install .NET 6.0 runtime +# inputs: +# version: 6.x + +# - script: dotnet tool restore + +# - task: AzureCLI@2 +# inputs: +# azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) +# scriptType: ps +# scriptLocation: inlineScript +# inlineScript: | +# Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + +# ################################################ +# # Manually Triggered Job +# ################################################ + + +# - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: +# - job: Synchronize +# pool: +# name: NetCore1ESPool-Internal-NoMSI +# demands: ImageOverride -equals 1es-windows-2019 +# steps: +# - task: UseDotNet@2 +# displayName: Install .NET 6.0 runtime +# inputs: +# version: 6.x + +# - script: dotnet tool restore + +# - task: AzureCLI@2 +# inputs: +# azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) +# scriptType: ps +# scriptLocation: inlineScript +# inlineScript: | +# Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + +# ################################################ +# # Scheduled Private jobs (Weekly) +# ################################################ + +# # Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job +# - ${{ if and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), contains(variables['Build.QueuedBy'], 'Weekly')) }}: +# - job: Synchronize +# pool: +# name: NetCore1ESPool-Internal-NoMSI +# demands: ImageOverride -equals 1es-windows-2019 +# steps: +# - task: UseDotNet@2 +# displayName: Install .NET 6.0 runtime +# inputs: +# version: 6.x + +# - script: dotnet tool restore + +# - task: AzureCLI@2 +# inputs: +# azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) +# scriptType: ps +# scriptLocation: inlineScript +# inlineScript: | +# Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} From 911501b1eda3e69c37ec7915f4afd7630e9345f7 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 14:29:17 +0200 Subject: [PATCH 03/19] Add runtimeFlavor to the job name --- eng/performance/scenarios.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index 558c64cc5b8..3069d494075 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -22,8 +22,8 @@ jobs: enablePublishBuildArtifacts: true helixRepo: dotnet/performance jobs: - - job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}_${{ parameters.machinePool }}${{ parameters.additionalJobIdentifier }}' - displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }} ${{ parameters.machinePool }} ${{ parameters.additionalJobIdentifier }}' + - job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}_${{ parameters.runtimeFlavor }}_${{ parameters.machinePool }}${{ parameters.additionalJobIdentifier }}' + displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }} ${{ parameters.runtimeFlavor }} ${{ parameters.machinePool }} ${{ parameters.additionalJobIdentifier }}' timeoutInMinutes: 320 variables: - name: AffinityValue # Make the affinity value available to the csproj generation From 290524acc6f097783c1c7b995866c807a7851b1c Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 15:02:28 +0200 Subject: [PATCH 04/19] Add RuntimeFlavor parameter to the restore scenario command --- eng/performance/scenarios.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index 3069d494075..543fbb3fe23 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -159,7 +159,7 @@ jobs: $(Python) -m pip install urllib3==1.26.15 $(Python) -m pip install requests .\src\scenarios\init.ps1 -DotnetDirectory $(CorrelationStaging)dotnet - dotnet msbuild .\eng\performance\${{ parameters.projectFile }} /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /restore /t:PreparePayloadWorkItems /bl:.\artifacts\log\$(_BuildConfig)\PrepareWorkItemPayloads.binlog + dotnet msbuild .\eng\performance\${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /bl:.\artifacts\log\$(_BuildConfig)\PrepareWorkItemPayloads.binlog displayName: Prepare scenarios env: CorrelationPayloadDirectory: $(CorrelationStaging) @@ -194,7 +194,7 @@ jobs: $(Python) -m pip install --user urllib3==1.26.15 $(Python) -m pip install --user requests . ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet - dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog + dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog displayName: Prepare scenarios env: CorrelationPayloadDirectory: $(CorrelationStaging) @@ -222,7 +222,7 @@ jobs: $(Python) -m pip install --user urllib3==1.26.15 $(Python) -m pip install --user requests . ./src/scenarios/init.sh -dotnetdir $(CorrelationStaging)dotnet - dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /restore /t:PreparePayloadWorkItems /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog + dotnet msbuild ./eng/performance/${{ parameters.projectFile }} /restore /t:PreparePayloadWorkItems /p:RuntimeFlavor=${{ parameters.runtimeFlavor }} /bl:./artifacts/log/$(_BuildConfig)/PrepareWorkItemPayloads.binlog displayName: Prepare scenarios env: CorrelationPayloadDirectory: $(CorrelationStaging) From 76e42537702455f11e5a6149cde9feb9c8dbfcdf Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 15:57:29 +0200 Subject: [PATCH 05/19] Define optional command props as a property --- eng/performance/maui_scenarios_ios.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index 0a4d36c8c00..9e68fcff7fa 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -10,6 +10,7 @@ scenarios_out $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\ $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/ + /p:PublishAot=true /p:PublishAotUsingRuntimePack=true @@ -47,7 +48,6 @@ - /p:PublishAot=true /p:PublishAotUsingRuntimePack=true $(Python) pre.py publish -f $(PERFLAB_Framework)-ios --self-contained -c Release -r ios-arm64 $(OptionalCommandProps) -o $(PreparePayloadWorkItemBaseDirectory)%(PreparePayloadWorkItem.ScenarioDirectoryName); cd ../; zip -r %(PreparePayloadWorkItem.ScenarioDirectoryName).zip %(PreparePayloadWorkItem.ScenarioDirectoryName) %(PreparePayloadWorkItem.PayloadDirectory) From f5630def4a9d9ae940060b4204698e11b3515ae7 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 17:45:45 +0200 Subject: [PATCH 06/19] Add Xamarin iOS scenario --- eng/performance/maui_scenarios_ios.proj | 50 +++++++++++++++++++++---- src/scenarios/xamarinios/post.py | 7 ++++ src/scenarios/xamarinios/pre.py | 34 +++++++++++++++++ src/scenarios/xamarinios/test.py | 16 ++++++++ 4 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 src/scenarios/xamarinios/post.py create mode 100644 src/scenarios/xamarinios/pre.py create mode 100644 src/scenarios/xamarinios/test.py diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index 9e68fcff7fa..c884e3620ea 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -25,12 +25,24 @@ + + xamarinios + $(ScenariosDir)%(ScenarioDirectoryName) + XamariniOSDefault + net.dot.xamarintesting + mauiios $(ScenariosDir)%(ScenarioDirectoryName) MauiiOSDefault net.dot.mauitesting + + mauiblazorios + $(ScenariosDir)%(ScenarioDirectoryName) + MauiBlazoriOSDefault + net.dot.mauiblazortesting + - - mauiblazorios - $(ScenariosDir)%(ScenarioDirectoryName) - MauiBlazoriOSDefault - net.dot.mauiblazortesting - @@ -62,7 +68,35 @@ cp -r $HELIX_CORRELATION_PAYLOAD/$(PreparePayloadOutDirectoryName)/%(HelixWorkItem.ScenarioDirectoryName) $HELIX_WORKITEM_ROOT/pub; mv $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).ipa $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; unzip -d $HELIX_WORKITEM_ROOT/pub $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip; rm $HELIX_WORKITEM_ROOT/pub/%(HelixWorkItem.IPAName).zip $(Python) test.py sod --scenario-name "%(Identity)" - + + $(ScenariosDir)xamarinios.zip + 00:15:00 + ios-device + + + + + $(ScenariosDir)mauiios.zip 00:15:00 ios-device @@ -90,7 +124,7 @@ ]]> - + $(ScenariosDir)mauiblazorios.zip 00:15:00 ios-device diff --git a/src/scenarios/xamarinios/post.py b/src/scenarios/xamarinios/post.py new file mode 100644 index 00000000000..c87a49b4b7d --- /dev/null +++ b/src/scenarios/xamarinios/post.py @@ -0,0 +1,7 @@ +''' +post cleanup script +''' + +from shared.postcommands import clean_directories + +clean_directories() diff --git a/src/scenarios/xamarinios/pre.py b/src/scenarios/xamarinios/pre.py new file mode 100644 index 00000000000..69c21a353fa --- /dev/null +++ b/src/scenarios/xamarinios/pre.py @@ -0,0 +1,34 @@ +''' +pre-command +''' +import shutil +import sys +import subprocess +from performance.logger import setup_loggers, getLogger +from shared import const +from shared.mauisharedpython import remove_aab_files +from shared.precommands import PreCommands +from shared.versionmanager import versions_write_json, get_version_from_dll_powershell_ios +from test import EXENAME + +setup_loggers(True) + +precommands = PreCommands() +precommands.install_workload('ios') + +# Setup the Maui folder +precommands.new(template='ios', + output_dir=const.APPDIR, + bin_dir=const.BINDIR, + exename=EXENAME, + working_directory=sys.path[0], + no_restore=False) + +# Build the APK +precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.mauitesting']) + +# Remove the aab files as we don't need them, this saves space +output_dir = const.PUBDIR +if precommands.output: + output_dir = precommands.output +remove_aab_files(output_dir) \ No newline at end of file diff --git a/src/scenarios/xamarinios/test.py b/src/scenarios/xamarinios/test.py new file mode 100644 index 00000000000..3be3df35083 --- /dev/null +++ b/src/scenarios/xamarinios/test.py @@ -0,0 +1,16 @@ +''' +Mobile Xamarin App +''' +from shared.const import PUBDIR +from shared.runner import TestTraits, Runner +from shared.versionmanager import versions_read_json_file_save_env + +EXENAME = 'XamariniOSDefault' + +if __name__ == "__main__": + versions_read_json_file_save_env(rf"./{PUBDIR}/versions.json") + + traits = TestTraits(exename=EXENAME, + guiapp='false', + ) + Runner(traits).run() From ad0f60ad950cb9dbd56ed98ca668f4324ec81840 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 17:58:36 +0200 Subject: [PATCH 07/19] Add support for Native AOT publish --- eng/performance/maui_scenarios_ios.proj | 2 +- src/scenarios/shared/precommands.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index c884e3620ea..2a333085fba 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -10,7 +10,7 @@ scenarios_out $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)\ $(CorrelationPayloadDirectory)$(PreparePayloadOutDirectoryName)/ - /p:PublishAot=true /p:PublishAotUsingRuntimePack=true + --nativeaot true diff --git a/src/scenarios/shared/precommands.py b/src/scenarios/shared/precommands.py index 7fe274a490b..d0a4a27667b 100644 --- a/src/scenarios/shared/precommands.py +++ b/src/scenarios/shared/precommands.py @@ -91,6 +91,7 @@ def __init__(self): self.operation = args.operation self.framework = args.framework self.runtime_identifier = args.runtime + self.nativeaot = args.nativeaot self.msbuild = args.msbuild print(self.msbuild) self.msbuildstatic = args.msbuildstatic @@ -149,6 +150,10 @@ def add_common_arguments(self, parser: ArgumentParser): dest='runtime', metavar='runtime', help='runtime for build or publish - ex: win-x64') + parser.add_argument('-n', '--nativeaot', + dest='nativeaot', + metavar='nativeaot', + help='use Native AOT runtime for build or publish') parser.add_argument('--msbuild', dest='msbuild', metavar='msbuild', @@ -202,6 +207,8 @@ def execute(self, build_args: list = []): build_args.append('--self-contained') elif self.no_self_contained: build_args.append('--no-self-contained') + if self.nativeaot: + build_args.append(['/p:PublishAot=true', '/p:PublishAotUsingRuntimePack=true']) self._publish(configuration=self.configuration, runtime_identifier=self.runtime_identifier, framework=self.framework, output=self.output, build_args=build_args) if self.operation == CROSSGEN: startup_args = [ From 37fd153ae62a1eda08844fa723f548e06f1780cf Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 18:07:50 +0200 Subject: [PATCH 08/19] Test the perf CI --- src/scenarios/shared/precommands.py | 2 -- src/scenarios/xamarinios/pre.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/scenarios/shared/precommands.py b/src/scenarios/shared/precommands.py index d0a4a27667b..275f11e6532 100644 --- a/src/scenarios/shared/precommands.py +++ b/src/scenarios/shared/precommands.py @@ -207,8 +207,6 @@ def execute(self, build_args: list = []): build_args.append('--self-contained') elif self.no_self_contained: build_args.append('--no-self-contained') - if self.nativeaot: - build_args.append(['/p:PublishAot=true', '/p:PublishAotUsingRuntimePack=true']) self._publish(configuration=self.configuration, runtime_identifier=self.runtime_identifier, framework=self.framework, output=self.output, build_args=build_args) if self.operation == CROSSGEN: startup_args = [ diff --git a/src/scenarios/xamarinios/pre.py b/src/scenarios/xamarinios/pre.py index 69c21a353fa..a9137b87ad3 100644 --- a/src/scenarios/xamarinios/pre.py +++ b/src/scenarios/xamarinios/pre.py @@ -25,7 +25,7 @@ no_restore=False) # Build the APK -precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.mauitesting']) +precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.xamarintesting']) # Remove the aab files as we don't need them, this saves space output_dir = const.PUBDIR From 2d1a9d2a6523543efbf940ed8f1f603642bd249d Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 18:15:18 +0200 Subject: [PATCH 09/19] Test the perf CI --- src/scenarios/shared/precommands.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scenarios/shared/precommands.py b/src/scenarios/shared/precommands.py index 275f11e6532..7dfb7b28dc6 100644 --- a/src/scenarios/shared/precommands.py +++ b/src/scenarios/shared/precommands.py @@ -207,6 +207,8 @@ def execute(self, build_args: list = []): build_args.append('--self-contained') elif self.no_self_contained: build_args.append('--no-self-contained') + if self.nativeaot: + build_args.append('/p:PublishAot=true /p:PublishAotUsingRuntimePack=true') self._publish(configuration=self.configuration, runtime_identifier=self.runtime_identifier, framework=self.framework, output=self.output, build_args=build_args) if self.operation == CROSSGEN: startup_args = [ From 90f00c3bb0e3ac0640eaa3f114e6bbba2d09f41e Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 18:22:52 +0200 Subject: [PATCH 10/19] Test the perf CI --- src/scenarios/shared/precommands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scenarios/shared/precommands.py b/src/scenarios/shared/precommands.py index 7dfb7b28dc6..9bb21eb647f 100644 --- a/src/scenarios/shared/precommands.py +++ b/src/scenarios/shared/precommands.py @@ -208,7 +208,8 @@ def execute(self, build_args: list = []): elif self.no_self_contained: build_args.append('--no-self-contained') if self.nativeaot: - build_args.append('/p:PublishAot=true /p:PublishAotUsingRuntimePack=true') + build_args.append('/p:PublishAot=true') + build_args.append('/p:PublishAotUsingRuntimePack=true') self._publish(configuration=self.configuration, runtime_identifier=self.runtime_identifier, framework=self.framework, output=self.output, build_args=build_args) if self.operation == CROSSGEN: startup_args = [ From 28ad3456a499e7ace1117068c69eaafc46ac9267 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 18:45:16 +0200 Subject: [PATCH 11/19] Implement install_versioned_xamarin function --- src/scenarios/shared/xamarinsharedpython.py | 24 +++++++++++++++++++++ src/scenarios/xamarinios/pre.py | 15 +++++++++---- 2 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 src/scenarios/shared/xamarinsharedpython.py diff --git a/src/scenarios/shared/xamarinsharedpython.py b/src/scenarios/shared/xamarinsharedpython.py new file mode 100644 index 00000000000..53a76c74e29 --- /dev/null +++ b/src/scenarios/shared/xamarinsharedpython.py @@ -0,0 +1,24 @@ +import subprocess +import os +import requests +from shared.precommands import PreCommands + +# Remove the aab files as we don't need them, this saves space in the correlation payload +def remove_aab_files(output_dir="."): + file_list = os.listdir(output_dir) + for file in file_list: + if file.endswith(".aab"): + os.remove(os.path.join(output_dir, file)) + +def install_versioned_xamarin(precommands): + target_framework_wo_platform = precommands.framework.split('-')[0] + + # Download what we need + with open ("XamarinNuGet.config", "wb") as f: + f.write(requests.get(f'https://raw.githubusercontent.com/xamarin/xamarin-macios/{target_framework_wo_platform}/NuGet.config', allow_redirects=True).content) + + workload_install_args = ['--configfile', 'XamarinNuGet.config'] + # if int(target_framework_wo_platform.split('.')[0][3:]) > 7: # Use the rollback file for versions greater than 7 + # workload_install_args += ['--from-rollback-file', f'https://aka.ms/dotnet/maui/{target_framework_wo_platform}.json'] + + precommands.install_workload('ios', workload_install_args) diff --git a/src/scenarios/xamarinios/pre.py b/src/scenarios/xamarinios/pre.py index a9137b87ad3..94ef2df3b66 100644 --- a/src/scenarios/xamarinios/pre.py +++ b/src/scenarios/xamarinios/pre.py @@ -6,7 +6,7 @@ import subprocess from performance.logger import setup_loggers, getLogger from shared import const -from shared.mauisharedpython import remove_aab_files +from shared.xamarinsharedpython import remove_aab_files, install_versioned_xamarin from shared.precommands import PreCommands from shared.versionmanager import versions_write_json, get_version_from_dll_powershell_ios from test import EXENAME @@ -14,9 +14,9 @@ setup_loggers(True) precommands = PreCommands() -precommands.install_workload('ios') +install_versioned_xamarin(precommands) -# Setup the Maui folder +# Setup the Xamarin folder precommands.new(template='ios', output_dir=const.APPDIR, bin_dir=const.BINDIR, @@ -25,10 +25,17 @@ no_restore=False) # Build the APK +shutil.copy('./XamarinNuGet.config', './app/Nuget.config') precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.xamarintesting']) # Remove the aab files as we don't need them, this saves space output_dir = const.PUBDIR if precommands.output: output_dir = precommands.output -remove_aab_files(output_dir) \ No newline at end of file +remove_aab_files(output_dir) + +# Copy the XamarinVersion to a file so we have it on the machine +xamarin_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.iOS.dll") +version_dict = { "xamarinVersion": xamarin_version } +versions_write_json(version_dict, rf"{output_dir}/versions.json") +print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.iOS.dll") From 683fe2d28be8a25633a25f9f907def274df64a72 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 18:55:50 +0200 Subject: [PATCH 12/19] Test the perf CI --- src/scenarios/shared/xamarinsharedpython.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scenarios/shared/xamarinsharedpython.py b/src/scenarios/shared/xamarinsharedpython.py index 53a76c74e29..ccdb6267e53 100644 --- a/src/scenarios/shared/xamarinsharedpython.py +++ b/src/scenarios/shared/xamarinsharedpython.py @@ -18,7 +18,7 @@ def install_versioned_xamarin(precommands): f.write(requests.get(f'https://raw.githubusercontent.com/xamarin/xamarin-macios/{target_framework_wo_platform}/NuGet.config', allow_redirects=True).content) workload_install_args = ['--configfile', 'XamarinNuGet.config'] - # if int(target_framework_wo_platform.split('.')[0][3:]) > 7: # Use the rollback file for versions greater than 7 - # workload_install_args += ['--from-rollback-file', f'https://aka.ms/dotnet/maui/{target_framework_wo_platform}.json'] + if int(target_framework_wo_platform.split('.')[0][3:]) > 7: # Use the rollback file for versions greater than 7 + workload_install_args += ['--from-rollback-file', f'https://aka.ms/dotnet/maui/{target_framework_wo_platform}.json'] precommands.install_workload('ios', workload_install_args) From d1264683a5540b6a2fa11a8143a697b0740b4887 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 19:36:47 +0200 Subject: [PATCH 13/19] Test the perf CI --- src/scenarios/shared/xamarinsharedpython.py | 2 +- src/scenarios/xamarinios/pre.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/scenarios/shared/xamarinsharedpython.py b/src/scenarios/shared/xamarinsharedpython.py index ccdb6267e53..33ea3a55c1c 100644 --- a/src/scenarios/shared/xamarinsharedpython.py +++ b/src/scenarios/shared/xamarinsharedpython.py @@ -21,4 +21,4 @@ def install_versioned_xamarin(precommands): if int(target_framework_wo_platform.split('.')[0][3:]) > 7: # Use the rollback file for versions greater than 7 workload_install_args += ['--from-rollback-file', f'https://aka.ms/dotnet/maui/{target_framework_wo_platform}.json'] - precommands.install_workload('ios', workload_install_args) + precommands.install_workload('ios', workload_install_args) \ No newline at end of file diff --git a/src/scenarios/xamarinios/pre.py b/src/scenarios/xamarinios/pre.py index 94ef2df3b66..147a967913f 100644 --- a/src/scenarios/xamarinios/pre.py +++ b/src/scenarios/xamarinios/pre.py @@ -6,7 +6,7 @@ import subprocess from performance.logger import setup_loggers, getLogger from shared import const -from shared.xamarinsharedpython import remove_aab_files, install_versioned_xamarin +from shared.mauisharedpython import remove_aab_files, install_versioned_maui from shared.precommands import PreCommands from shared.versionmanager import versions_write_json, get_version_from_dll_powershell_ios from test import EXENAME @@ -14,7 +14,7 @@ setup_loggers(True) precommands = PreCommands() -install_versioned_xamarin(precommands) +install_versioned_maui(precommands) # Setup the Xamarin folder precommands.new(template='ios', @@ -25,7 +25,7 @@ no_restore=False) # Build the APK -shutil.copy('./XamarinNuGet.config', './app/Nuget.config') +shutil.copy('./MauiNuGet.config', './app/Nuget.config') precommands.execute(['/p:_RequireCodeSigning=false', '/p:ApplicationId=net.dot.xamarintesting']) # Remove the aab files as we don't need them, this saves space From 943cd86bbb9f851beb54f8a493cb3d37002dbc5a Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 23:06:39 +0200 Subject: [PATCH 14/19] Use runtime flavor on Helix --- eng/performance/maui_scenarios_ios.proj | 2 +- eng/performance/scenarios.yml | 5 +++-- eng/performance/send-to-helix.yml | 5 +++-- src/scenarios/shared/xamarinsharedpython.py | 24 --------------------- 4 files changed, 7 insertions(+), 29 deletions(-) delete mode 100644 src/scenarios/shared/xamarinsharedpython.py diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index 2a333085fba..299a547e6cc 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -87,7 +87,7 @@ sign XamariniOSDefault.app # Testing commands - $(Python) test.py devicestartup --device-type ios --package-path XamariniOSDefault.app --package-name net.dot.mauitesting --scenario-name "%(Identity)" + $(Python) test.py devicestartup --device-type ios --package-path XamariniOSDefault.app --package-name net.dot.xamarintesting --scenario-name "%(Identity)" ((result=$?)) # Post commands diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml index 543fbb3fe23..f56a1460150 100644 --- a/eng/performance/scenarios.yml +++ b/eng/performance/scenarios.yml @@ -13,7 +13,7 @@ parameters: affinity: '0' # optional -- Affinity bitmask to a specific machine in the pool as a decimal number (e.g. 1, 2, etc) runEnvVars: [] # optional -- Environment variables to set for the benchmark run in the form of a multi-line object like: "runEnvVars:\n - DOTNET_GCgen0size=1E00000 \n - DOTNET_GCHeapCount=4 \n - DOTNET_GCTotalPhysicalMemory=200000000" additionalJobIdentifier: '' # optional -- Additional identifier to append to the job name (no spaces) - runtimeFlavor: '' # optional -- Runtime flavor used for the scenarios + runtimeFlavor: '' # optional -- Runtime flavor used for scenarios jobs: - template: ../common/templates/jobs/jobs.yml @@ -244,4 +244,5 @@ jobs: Creator: $(Creator) Architecture: ${{ parameters.architecture }} WorkItemTimeout: 4:00 # 4 hours - Python: $(Python) \ No newline at end of file + Python: $(Python) + RuntimeFlavor: ${{ parameters.runtimeFlavor }} \ No newline at end of file diff --git a/eng/performance/send-to-helix.yml b/eng/performance/send-to-helix.yml index 69e1b88b074..a37804673c3 100644 --- a/eng/performance/send-to-helix.yml +++ b/eng/performance/send-to-helix.yml @@ -23,9 +23,10 @@ parameters: condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false ProjectFile: helix.proj # optional -- override default project used to send benchmarks + RuntimeFlavor: '' # optional -- runtime flavor used for scenarios steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 -warnaserror 0 $env:BUILD_SOURCESDIRECTORY\eng\performance\${{ parameters.ProjectFile }} /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 -warnaserror 0 $env:BUILD_SOURCESDIRECTORY\eng\performance\${{ parameters.ProjectFile }} /restore /t:Test /p:RuntimeFlavor=${{ parameters.RuntimeFlavor }} /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' displayName: ${{ parameters.DisplayNamePrefix }} (Windows) env: Python: ${{ parameters.Python }} @@ -51,7 +52,7 @@ steps: SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh --warnaserror false $BUILD_SOURCESDIRECTORY/eng/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh --warnaserror false $BUILD_SOURCESDIRECTORY/eng/performance/${{ parameters.ProjectFile }} /restore /t:Test /p:RuntimeFlavor=${{ parameters.RuntimeFlavor }} /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} (Unix) env: Python: ${{ parameters.Python }} diff --git a/src/scenarios/shared/xamarinsharedpython.py b/src/scenarios/shared/xamarinsharedpython.py deleted file mode 100644 index 33ea3a55c1c..00000000000 --- a/src/scenarios/shared/xamarinsharedpython.py +++ /dev/null @@ -1,24 +0,0 @@ -import subprocess -import os -import requests -from shared.precommands import PreCommands - -# Remove the aab files as we don't need them, this saves space in the correlation payload -def remove_aab_files(output_dir="."): - file_list = os.listdir(output_dir) - for file in file_list: - if file.endswith(".aab"): - os.remove(os.path.join(output_dir, file)) - -def install_versioned_xamarin(precommands): - target_framework_wo_platform = precommands.framework.split('-')[0] - - # Download what we need - with open ("XamarinNuGet.config", "wb") as f: - f.write(requests.get(f'https://raw.githubusercontent.com/xamarin/xamarin-macios/{target_framework_wo_platform}/NuGet.config', allow_redirects=True).content) - - workload_install_args = ['--configfile', 'XamarinNuGet.config'] - if int(target_framework_wo_platform.split('.')[0][3:]) > 7: # Use the rollback file for versions greater than 7 - workload_install_args += ['--from-rollback-file', f'https://aka.ms/dotnet/maui/{target_framework_wo_platform}.json'] - - precommands.install_workload('ios', workload_install_args) \ No newline at end of file From a4adf76376fd9e102fc0c0b6c52873ccbbb8687d Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 18 Jul 2023 23:22:42 +0200 Subject: [PATCH 15/19] Enable perf jobs --- azure-pipelines.yml | 1026 +++++++++++++++++++++---------------------- 1 file changed, 513 insertions(+), 513 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f26450670e1..3cd380c970f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -57,269 +57,269 @@ jobs: # Public correctness jobs ###################################################### -# - ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: - -# # Scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: scenarios -# projectFile: scenarios.proj -# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks -# - main - -# ## MAUI scenario benchmarks -# #- template: /eng/performance/build_machine_matrix.yml -# # parameters: -# # jobTemplate: /eng/performance/scenarios.yml -# # buildMachines: -# # - win-x64 -# # - ubuntu-x64 -# # isPublic: true -# # jobParameters: -# # kind: maui_scenarios -# # projectFile: maui_scenarios.proj -# # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks -# # - main - -# # Blazor scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: blazor_scenarios -# projectFile: blazor_scenarios.proj -# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks -# - main - -# # SDK scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - win-x86 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: sdk_scenarios -# projectFile: sdk_scenarios.proj -# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks -# - main +- ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: + + # Scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: scenarios + projectFile: scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + + ## MAUI scenario benchmarks + #- template: /eng/performance/build_machine_matrix.yml + # parameters: + # jobTemplate: /eng/performance/scenarios.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # kind: maui_scenarios + # projectFile: maui_scenarios.proj + # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + # - main + + # Blazor scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: blazor_scenarios + projectFile: blazor_scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + + # SDK scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - win-x86 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: sdk_scenarios + projectFile: sdk_scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main -# # micro benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - win-x86 -# isPublic: true -# jobParameters: -# kind: micro -# csproj: src\benchmarks\micro\MicroBenchmarks.csproj -# runCategories: 'runtime libraries' -# channels: -# - main - -# # Ubuntux64 Default and NativeAOT micro benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: micro -# csproj: src\benchmarks\micro\MicroBenchmarks.csproj -# runCategories: 'runtime libraries' -# channels: -# - main -# - nativeaot8.0 - -# # net462 micro benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-rs5-x64 -# isPublic: true -# jobParameters: -# kind: micro_net462 -# csproj: src\benchmarks\micro\MicroBenchmarks.csproj -# runCategories: 'runtime libraries' -# channels: -# - net462 - -# # ML.NET benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: mlnet -# csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj -# runCategories: 'mldotnet' -# channels: -# - main - -# # F# benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: fsharp -# csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj -# runCategories: 'fsharp' -# channels: -# - main - -# # bepuphysics benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: bepuphysics -# csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj -# runCategories: 'BepuPhysics' -# channels: -# - main - -# # ImageSharp benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: imagesharp -# csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj -# runCategories: 'ImageSharp' -# channels: -# - main - -# # Roslyn benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: roslyn -# csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj -# runCategories: 'roslyn' -# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only -# - main - -# # ILLink benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: illink -# csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj -# runCategories: 'illink' -# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only -# - main - -# # NativeAOT scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: nativeaot_scenarios -# projectFile: nativeaot_scenarios.proj -# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks -# - main - -# ########################################### -# # Private Jobs -# ########################################### - -# - ${{ if or(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest', 'Manual')), parameters.runPrivateJobs) }}: - -# # Scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# - win-arm64 -# - ubuntu-arm64-ampere -# isPublic: false -# jobParameters: -# kind: scenarios -# projectFile: scenarios.proj -# channels: -# - main - -# # Affinitized Scenario benchmarks (Initially just PDN) -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - win-arm64 -# - win-arm64-ampere -# isPublic: false -# jobParameters: -# kind: scenarios -# projectFile: scenarios_affinitized.proj -# channels: -# - main -# additionalJobIdentifier: 'Affinity_85' -# affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account -# runEnvVars: -# - DOTNET_GCgen0size=410000 # ~4MB -# - DOTNET_GCHeapCount=4 -# - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - -# # Maui Android scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64-android-arm64-pixel -# - win-x64-android-arm64-galaxy -# isPublic: false -# jobParameters: -# kind: maui_scenarios_android -# projectFile: maui_scenarios_android.proj -# dotnetVersionsLinks: -# main: https://aka.ms/dotnet/sdk/maui/net8.0.json + # micro benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - win-x86 + isPublic: true + jobParameters: + kind: micro + csproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: + - main + + # Ubuntux64 Default and NativeAOT micro benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - ubuntu-x64 + isPublic: true + jobParameters: + kind: micro + csproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: + - main + - nativeaot8.0 + + # net462 micro benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-rs5-x64 + isPublic: true + jobParameters: + kind: micro_net462 + csproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: + - net462 + + # ML.NET benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: mlnet + csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj + runCategories: 'mldotnet' + channels: + - main + + # F# benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: fsharp + csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj + runCategories: 'fsharp' + channels: + - main + + # bepuphysics benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: bepuphysics + csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj + runCategories: 'BepuPhysics' + channels: + - main + + # ImageSharp benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: imagesharp + csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj + runCategories: 'ImageSharp' + channels: + - main + + # Roslyn benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: roslyn + csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + runCategories: 'roslyn' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + + # ILLink benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: illink + csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj + runCategories: 'illink' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + + # NativeAOT scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: nativeaot_scenarios + projectFile: nativeaot_scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + +########################################### +# Private Jobs +########################################### + +- ${{ if or(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest', 'Manual')), parameters.runPrivateJobs) }}: + + # Scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: scenarios + projectFile: scenarios.proj + channels: + - main + + # Affinitized Scenario benchmarks (Initially just PDN) + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - win-arm64 + - win-arm64-ampere + isPublic: false + jobParameters: + kind: scenarios + projectFile: scenarios_affinitized.proj + channels: + - main + additionalJobIdentifier: 'Affinity_85' + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + + # Maui Android scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + kind: maui_scenarios_android + projectFile: maui_scenarios_android.proj + dotnetVersionsLinks: + main: https://aka.ms/dotnet/sdk/maui/net8.0.json # Maui iOS scenario benchmarks - template: /eng/performance/build_machine_matrix.yml @@ -349,254 +349,254 @@ jobs: main: https://aka.ms/dotnet/sdk/maui/net8.0.json runtimeFlavor: coreclr -# ## Maui scenario benchmarks -# #- template: /eng/performance/build_machine_matrix.yml -# # parameters: -# # jobTemplate: /eng/performance/scenarios.yml -# # buildMachines: -# # - win-x64 -# # - ubuntu-x64 -# # - win-arm64 -# # - ubuntu-arm64-ampere -# # isPublic: false -# # jobParameters: -# # kind: maui_scenarios -# # projectFile: maui_scenarios.proj -# # channels: -# # - main - -# # NativeAOT scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# - win-arm64 -# isPublic: false -# jobParameters: -# kind: nativeaot_scenarios -# projectFile: nativeaot_scenarios.proj -# channels: -# - main - -# ################################################ -# # Scheduled Private jobs -# ################################################ - -# # Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -# - ${{ if or(and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), not(contains(variables['Build.QueuedBy'], 'Weekly'))), parameters.runScheduledPrivateJobs) }}: - -# # SDK scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - win-x86 -# - ubuntu-x64 -# isPublic: false -# jobParameters: -# kind: sdk_scenarios -# projectFile: sdk_scenarios.proj -# channels: -# - main - -# # Blazor 3.2 scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# isPublic: false -# jobParameters: -# kind: blazor_scenarios -# projectFile: blazor_scenarios.proj -# channels: -# - main - -# # F# benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# - win-arm64 -# - ubuntu-arm64-ampere -# isPublic: false -# jobParameters: -# kind: fsharp -# csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj -# runCategories: 'fsharp' -# channels: -# - main - -# # bepuphysics benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# - win-arm64 -# - ubuntu-arm64-ampere -# isPublic: false -# jobParameters: -# kind: bepuphysics -# csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj -# runCategories: 'BepuPhysics' -# channels: -# - main - -# # ImageSharp benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# - win-arm64 -# - ubuntu-arm64-ampere -# isPublic: false -# jobParameters: -# kind: imagesharp -# csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj -# runCategories: 'ImageSharp' -# channels: -# - main - -# # ML.NET benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# - win-arm64 -# - win-arm64-ampere -# - ubuntu-arm64-ampere -# isPublic: false -# jobParameters: -# kind: mlnet -# csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj -# runCategories: 'mldotnet' -# channels: -# - main -# affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account -# runEnvVars: -# - DOTNET_GCgen0size=410000 # ~4MB -# - DOTNET_GCHeapCount=4 -# - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - -# # Roslyn benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# - win-arm64 -# - win-arm64-ampere -# - ubuntu-arm64-ampere -# isPublic: false -# jobParameters: -# kind: roslyn -# csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj -# runCategories: 'roslyn' -# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only -# - main -# affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account -# runEnvVars: -# - DOTNET_GCgen0size=410000 # ~4MB -# - DOTNET_GCHeapCount=4 -# - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - -# # ILLink benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) -# isPublic: false -# jobParameters: -# kind: illink -# csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj -# runCategories: 'illink' -# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only -# - main - -# # Secret Sync -# - job: Synchronize -# pool: -# name: NetCore1ESPool-Internal-NoMSI -# demands: ImageOverride -equals 1es-windows-2019 -# steps: -# - task: UseDotNet@2 -# displayName: Install .NET 6.0 runtime -# inputs: -# version: 6.x - -# - script: dotnet tool restore - -# - task: AzureCLI@2 -# inputs: -# azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) -# scriptType: ps -# scriptLocation: inlineScript -# inlineScript: | -# Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} - -# ################################################ -# # Manually Triggered Job -# ################################################ - - -# - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: -# - job: Synchronize -# pool: -# name: NetCore1ESPool-Internal-NoMSI -# demands: ImageOverride -equals 1es-windows-2019 -# steps: -# - task: UseDotNet@2 -# displayName: Install .NET 6.0 runtime -# inputs: -# version: 6.x - -# - script: dotnet tool restore - -# - task: AzureCLI@2 -# inputs: -# azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) -# scriptType: ps -# scriptLocation: inlineScript -# inlineScript: | -# Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} - -# ################################################ -# # Scheduled Private jobs (Weekly) -# ################################################ - -# # Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -# - ${{ if and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), contains(variables['Build.QueuedBy'], 'Weekly')) }}: -# - job: Synchronize -# pool: -# name: NetCore1ESPool-Internal-NoMSI -# demands: ImageOverride -equals 1es-windows-2019 -# steps: -# - task: UseDotNet@2 -# displayName: Install .NET 6.0 runtime -# inputs: -# version: 6.x - -# - script: dotnet tool restore - -# - task: AzureCLI@2 -# inputs: -# azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) -# scriptType: ps -# scriptLocation: inlineScript -# inlineScript: | -# Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + ## Maui scenario benchmarks + #- template: /eng/performance/build_machine_matrix.yml + # parameters: + # jobTemplate: /eng/performance/scenarios.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # - win-arm64 + # - ubuntu-arm64-ampere + # isPublic: false + # jobParameters: + # kind: maui_scenarios + # projectFile: maui_scenarios.proj + # channels: + # - main + + # NativeAOT scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + isPublic: false + jobParameters: + kind: nativeaot_scenarios + projectFile: nativeaot_scenarios.proj + channels: + - main + +################################################ +# Scheduled Private jobs +################################################ + +# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job +- ${{ if or(and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), not(contains(variables['Build.QueuedBy'], 'Weekly'))), parameters.runScheduledPrivateJobs) }}: + + # SDK scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - win-x86 + - ubuntu-x64 + isPublic: false + jobParameters: + kind: sdk_scenarios + projectFile: sdk_scenarios.proj + channels: + - main + + # Blazor 3.2 scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + isPublic: false + jobParameters: + kind: blazor_scenarios + projectFile: blazor_scenarios.proj + channels: + - main + + # F# benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: fsharp + csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj + runCategories: 'fsharp' + channels: + - main + + # bepuphysics benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: bepuphysics + csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj + runCategories: 'BepuPhysics' + channels: + - main + + # ImageSharp benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: imagesharp + csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj + runCategories: 'ImageSharp' + channels: + - main + + # ML.NET benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: mlnet + csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj + runCategories: 'mldotnet' + channels: + - main + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + + # Roslyn benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: roslyn + csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + runCategories: 'roslyn' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + + # ILLink benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) + isPublic: false + jobParameters: + kind: illink + csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj + runCategories: 'illink' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + + # Secret Sync + - job: Synchronize + pool: + name: NetCore1ESPool-Internal-NoMSI + demands: ImageOverride -equals 1es-windows-2019 + steps: + - task: UseDotNet@2 + displayName: Install .NET 6.0 runtime + inputs: + version: 6.x + + - script: dotnet tool restore + + - task: AzureCLI@2 + inputs: + azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + +################################################ +# Manually Triggered Job +################################################ + + +- ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: + - job: Synchronize + pool: + name: NetCore1ESPool-Internal-NoMSI + demands: ImageOverride -equals 1es-windows-2019 + steps: + - task: UseDotNet@2 + displayName: Install .NET 6.0 runtime + inputs: + version: 6.x + + - script: dotnet tool restore + + - task: AzureCLI@2 + inputs: + azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + +################################################ +# Scheduled Private jobs (Weekly) +################################################ + +# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job +- ${{ if and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), contains(variables['Build.QueuedBy'], 'Weekly')) }}: + - job: Synchronize + pool: + name: NetCore1ESPool-Internal-NoMSI + demands: ImageOverride -equals 1es-windows-2019 + steps: + - task: UseDotNet@2 + displayName: Install .NET 6.0 runtime + inputs: + version: 6.x + + - script: dotnet tool restore + + - task: AzureCLI@2 + inputs: + azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} From 93a22cdce183ff5281ca407771af5c37eaad53b4 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Wed, 19 Jul 2023 12:16:14 +0200 Subject: [PATCH 16/19] Use correct package name for the Xamarin startup job --- eng/performance/maui_scenarios_ios.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index 299a547e6cc..f0f1a1de07f 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -29,7 +29,7 @@ xamarinios $(ScenariosDir)%(ScenarioDirectoryName) XamariniOSDefault - net.dot.xamarintesting + com.companyname.XamariniOSDefault mauiios From 2f52855ccd25c46328e164dec7ea083d7f3e159b Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Wed, 19 Jul 2023 12:17:08 +0200 Subject: [PATCH 17/19] Test perf jobs --- azure-pipelines.yml | 1034 +++++++++++++++++++++---------------------- 1 file changed, 517 insertions(+), 517 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3cd380c970f..864543af3c5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -53,273 +53,273 @@ schedules: jobs: -###################################################### -# Public correctness jobs -###################################################### - -- ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: - - # Scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: scenarios - projectFile: scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - ## MAUI scenario benchmarks - #- template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/scenarios.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # isPublic: true - # jobParameters: - # kind: maui_scenarios - # projectFile: maui_scenarios.proj - # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - # - main - - # Blazor scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: blazor_scenarios - projectFile: blazor_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - # SDK scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - win-x86 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: sdk_scenarios - projectFile: sdk_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main +# ###################################################### +# # Public correctness jobs +# ###################################################### + +# - ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: + +# # Scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: scenarios +# projectFile: scenarios.proj +# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks +# - main + +# ## MAUI scenario benchmarks +# #- template: /eng/performance/build_machine_matrix.yml +# # parameters: +# # jobTemplate: /eng/performance/scenarios.yml +# # buildMachines: +# # - win-x64 +# # - ubuntu-x64 +# # isPublic: true +# # jobParameters: +# # kind: maui_scenarios +# # projectFile: maui_scenarios.proj +# # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks +# # - main + +# # Blazor scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: blazor_scenarios +# projectFile: blazor_scenarios.proj +# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks +# - main + +# # SDK scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - win-x86 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: sdk_scenarios +# projectFile: sdk_scenarios.proj +# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks +# - main - # micro benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - win-x86 - isPublic: true - jobParameters: - kind: micro - csproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - main - - # Ubuntux64 Default and NativeAOT micro benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - ubuntu-x64 - isPublic: true - jobParameters: - kind: micro - csproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - main - - nativeaot8.0 - - # net462 micro benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-rs5-x64 - isPublic: true - jobParameters: - kind: micro_net462 - csproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - net462 - - # ML.NET benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: mlnet - csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj - runCategories: 'mldotnet' - channels: - - main - - # F# benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: fsharp - csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj - runCategories: 'fsharp' - channels: - - main - - # bepuphysics benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: bepuphysics - csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj - runCategories: 'BepuPhysics' - channels: - - main - - # ImageSharp benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: imagesharp - csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj - runCategories: 'ImageSharp' - channels: - - main - - # Roslyn benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: roslyn - csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj - runCategories: 'roslyn' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - # ILLink benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: illink - csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj - runCategories: 'illink' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - # NativeAOT scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: nativeaot_scenarios - projectFile: nativeaot_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - -########################################### -# Private Jobs -########################################### - -- ${{ if or(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest', 'Manual')), parameters.runPrivateJobs) }}: - - # Scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: scenarios - projectFile: scenarios.proj - channels: - - main - - # Affinitized Scenario benchmarks (Initially just PDN) - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - win-arm64 - - win-arm64-ampere - isPublic: false - jobParameters: - kind: scenarios - projectFile: scenarios_affinitized.proj - channels: - - main - additionalJobIdentifier: 'Affinity_85' - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - - # Maui Android scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - kind: maui_scenarios_android - projectFile: maui_scenarios_android.proj - dotnetVersionsLinks: - main: https://aka.ms/dotnet/sdk/maui/net8.0.json +# # micro benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - win-x86 +# isPublic: true +# jobParameters: +# kind: micro +# csproj: src\benchmarks\micro\MicroBenchmarks.csproj +# runCategories: 'runtime libraries' +# channels: +# - main + +# # Ubuntux64 Default and NativeAOT micro benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: micro +# csproj: src\benchmarks\micro\MicroBenchmarks.csproj +# runCategories: 'runtime libraries' +# channels: +# - main +# - nativeaot8.0 + +# # net462 micro benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-rs5-x64 +# isPublic: true +# jobParameters: +# kind: micro_net462 +# csproj: src\benchmarks\micro\MicroBenchmarks.csproj +# runCategories: 'runtime libraries' +# channels: +# - net462 + +# # ML.NET benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: mlnet +# csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj +# runCategories: 'mldotnet' +# channels: +# - main + +# # F# benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: fsharp +# csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj +# runCategories: 'fsharp' +# channels: +# - main + +# # bepuphysics benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: bepuphysics +# csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj +# runCategories: 'BepuPhysics' +# channels: +# - main + +# # ImageSharp benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: imagesharp +# csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj +# runCategories: 'ImageSharp' +# channels: +# - main + +# # Roslyn benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: roslyn +# csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj +# runCategories: 'roslyn' +# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only +# - main + +# # ILLink benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: illink +# csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj +# runCategories: 'illink' +# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only +# - main + +# # NativeAOT scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# isPublic: true +# jobParameters: +# kind: nativeaot_scenarios +# projectFile: nativeaot_scenarios.proj +# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks +# - main + +# ########################################### +# # Private Jobs +# ########################################### + +# - ${{ if or(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest', 'Manual')), parameters.runPrivateJobs) }}: + +# # Scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# - win-arm64 +# - ubuntu-arm64-ampere +# isPublic: false +# jobParameters: +# kind: scenarios +# projectFile: scenarios.proj +# channels: +# - main + +# # Affinitized Scenario benchmarks (Initially just PDN) +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - win-arm64 +# - win-arm64-ampere +# isPublic: false +# jobParameters: +# kind: scenarios +# projectFile: scenarios_affinitized.proj +# channels: +# - main +# additionalJobIdentifier: 'Affinity_85' +# affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account +# runEnvVars: +# - DOTNET_GCgen0size=410000 # ~4MB +# - DOTNET_GCHeapCount=4 +# - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + +# # Maui Android scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64-android-arm64-pixel +# - win-x64-android-arm64-galaxy +# isPublic: false +# jobParameters: +# kind: maui_scenarios_android +# projectFile: maui_scenarios_android.proj +# dotnetVersionsLinks: +# main: https://aka.ms/dotnet/sdk/maui/net8.0.json # Maui iOS scenario benchmarks - template: /eng/performance/build_machine_matrix.yml @@ -349,254 +349,254 @@ jobs: main: https://aka.ms/dotnet/sdk/maui/net8.0.json runtimeFlavor: coreclr - ## Maui scenario benchmarks - #- template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/scenarios.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # - win-arm64 - # - ubuntu-arm64-ampere - # isPublic: false - # jobParameters: - # kind: maui_scenarios - # projectFile: maui_scenarios.proj - # channels: - # - main - - # NativeAOT scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - isPublic: false - jobParameters: - kind: nativeaot_scenarios - projectFile: nativeaot_scenarios.proj - channels: - - main - -################################################ -# Scheduled Private jobs -################################################ - -# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -- ${{ if or(and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), not(contains(variables['Build.QueuedBy'], 'Weekly'))), parameters.runScheduledPrivateJobs) }}: - - # SDK scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - win-x86 - - ubuntu-x64 - isPublic: false - jobParameters: - kind: sdk_scenarios - projectFile: sdk_scenarios.proj - channels: - - main - - # Blazor 3.2 scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - isPublic: false - jobParameters: - kind: blazor_scenarios - projectFile: blazor_scenarios.proj - channels: - - main - - # F# benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: fsharp - csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj - runCategories: 'fsharp' - channels: - - main - - # bepuphysics benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: bepuphysics - csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj - runCategories: 'BepuPhysics' - channels: - - main - - # ImageSharp benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: imagesharp - csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj - runCategories: 'ImageSharp' - channels: - - main - - # ML.NET benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: mlnet - csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj - runCategories: 'mldotnet' - channels: - - main - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - - # Roslyn benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: roslyn - csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj - runCategories: 'roslyn' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - - # ILLink benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) - isPublic: false - jobParameters: - kind: illink - csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj - runCategories: 'illink' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - # Secret Sync - - job: Synchronize - pool: - name: NetCore1ESPool-Internal-NoMSI - demands: ImageOverride -equals 1es-windows-2019 - steps: - - task: UseDotNet@2 - displayName: Install .NET 6.0 runtime - inputs: - version: 6.x - - - script: dotnet tool restore - - - task: AzureCLI@2 - inputs: - azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} - -################################################ -# Manually Triggered Job -################################################ - - -- ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: - - job: Synchronize - pool: - name: NetCore1ESPool-Internal-NoMSI - demands: ImageOverride -equals 1es-windows-2019 - steps: - - task: UseDotNet@2 - displayName: Install .NET 6.0 runtime - inputs: - version: 6.x - - - script: dotnet tool restore - - - task: AzureCLI@2 - inputs: - azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} - -################################################ -# Scheduled Private jobs (Weekly) -################################################ - -# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -- ${{ if and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), contains(variables['Build.QueuedBy'], 'Weekly')) }}: - - job: Synchronize - pool: - name: NetCore1ESPool-Internal-NoMSI - demands: ImageOverride -equals 1es-windows-2019 - steps: - - task: UseDotNet@2 - displayName: Install .NET 6.0 runtime - inputs: - version: 6.x - - - script: dotnet tool restore - - - task: AzureCLI@2 - inputs: - azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} +# ## Maui scenario benchmarks +# #- template: /eng/performance/build_machine_matrix.yml +# # parameters: +# # jobTemplate: /eng/performance/scenarios.yml +# # buildMachines: +# # - win-x64 +# # - ubuntu-x64 +# # - win-arm64 +# # - ubuntu-arm64-ampere +# # isPublic: false +# # jobParameters: +# # kind: maui_scenarios +# # projectFile: maui_scenarios.proj +# # channels: +# # - main + +# # NativeAOT scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# - win-arm64 +# isPublic: false +# jobParameters: +# kind: nativeaot_scenarios +# projectFile: nativeaot_scenarios.proj +# channels: +# - main + +# ################################################ +# # Scheduled Private jobs +# ################################################ + +# # Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job +# - ${{ if or(and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), not(contains(variables['Build.QueuedBy'], 'Weekly'))), parameters.runScheduledPrivateJobs) }}: + +# # SDK scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# - win-x86 +# - ubuntu-x64 +# isPublic: false +# jobParameters: +# kind: sdk_scenarios +# projectFile: sdk_scenarios.proj +# channels: +# - main + +# # Blazor 3.2 scenario benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/scenarios.yml +# buildMachines: +# - win-x64 +# isPublic: false +# jobParameters: +# kind: blazor_scenarios +# projectFile: blazor_scenarios.proj +# channels: +# - main + +# # F# benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# - win-arm64 +# - ubuntu-arm64-ampere +# isPublic: false +# jobParameters: +# kind: fsharp +# csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj +# runCategories: 'fsharp' +# channels: +# - main + +# # bepuphysics benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# - win-arm64 +# - ubuntu-arm64-ampere +# isPublic: false +# jobParameters: +# kind: bepuphysics +# csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj +# runCategories: 'BepuPhysics' +# channels: +# - main + +# # ImageSharp benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# - win-arm64 +# - ubuntu-arm64-ampere +# isPublic: false +# jobParameters: +# kind: imagesharp +# csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj +# runCategories: 'ImageSharp' +# channels: +# - main + +# # ML.NET benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# - win-arm64 +# - win-arm64-ampere +# - ubuntu-arm64-ampere +# isPublic: false +# jobParameters: +# kind: mlnet +# csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj +# runCategories: 'mldotnet' +# channels: +# - main +# affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account +# runEnvVars: +# - DOTNET_GCgen0size=410000 # ~4MB +# - DOTNET_GCHeapCount=4 +# - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + +# # Roslyn benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# - win-arm64 +# - win-arm64-ampere +# - ubuntu-arm64-ampere +# isPublic: false +# jobParameters: +# kind: roslyn +# csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj +# runCategories: 'roslyn' +# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only +# - main +# affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account +# runEnvVars: +# - DOTNET_GCgen0size=410000 # ~4MB +# - DOTNET_GCHeapCount=4 +# - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + +# # ILLink benchmarks +# - template: /eng/performance/build_machine_matrix.yml +# parameters: +# jobTemplate: /eng/performance/benchmark_jobs.yml +# buildMachines: +# - win-x64 +# - ubuntu-x64 +# # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) +# isPublic: false +# jobParameters: +# kind: illink +# csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj +# runCategories: 'illink' +# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only +# - main + +# # Secret Sync +# - job: Synchronize +# pool: +# name: NetCore1ESPool-Internal-NoMSI +# demands: ImageOverride -equals 1es-windows-2019 +# steps: +# - task: UseDotNet@2 +# displayName: Install .NET 6.0 runtime +# inputs: +# version: 6.x + +# - script: dotnet tool restore + +# - task: AzureCLI@2 +# inputs: +# azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) +# scriptType: ps +# scriptLocation: inlineScript +# inlineScript: | +# Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + +# ################################################ +# # Manually Triggered Job +# ################################################ + + +# - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: +# - job: Synchronize +# pool: +# name: NetCore1ESPool-Internal-NoMSI +# demands: ImageOverride -equals 1es-windows-2019 +# steps: +# - task: UseDotNet@2 +# displayName: Install .NET 6.0 runtime +# inputs: +# version: 6.x + +# - script: dotnet tool restore + +# - task: AzureCLI@2 +# inputs: +# azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) +# scriptType: ps +# scriptLocation: inlineScript +# inlineScript: | +# Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + +# ################################################ +# # Scheduled Private jobs (Weekly) +# ################################################ + +# # Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job +# - ${{ if and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), contains(variables['Build.QueuedBy'], 'Weekly')) }}: +# - job: Synchronize +# pool: +# name: NetCore1ESPool-Internal-NoMSI +# demands: ImageOverride -equals 1es-windows-2019 +# steps: +# - task: UseDotNet@2 +# displayName: Install .NET 6.0 runtime +# inputs: +# version: 6.x + +# - script: dotnet tool restore + +# - task: AzureCLI@2 +# inputs: +# azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) +# scriptType: ps +# scriptLocation: inlineScript +# inlineScript: | +# Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} From 8f057881d8f642d987159738877de88a0320908e Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Thu, 20 Jul 2023 09:52:11 +0200 Subject: [PATCH 18/19] Use correct package name for Xamarin startup measurement --- azure-pipelines.yml | 1034 +++++++++++------------ eng/performance/maui_scenarios_ios.proj | 2 +- 2 files changed, 518 insertions(+), 518 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 864543af3c5..3cd380c970f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -53,273 +53,273 @@ schedules: jobs: -# ###################################################### -# # Public correctness jobs -# ###################################################### - -# - ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: - -# # Scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: scenarios -# projectFile: scenarios.proj -# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks -# - main - -# ## MAUI scenario benchmarks -# #- template: /eng/performance/build_machine_matrix.yml -# # parameters: -# # jobTemplate: /eng/performance/scenarios.yml -# # buildMachines: -# # - win-x64 -# # - ubuntu-x64 -# # isPublic: true -# # jobParameters: -# # kind: maui_scenarios -# # projectFile: maui_scenarios.proj -# # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks -# # - main - -# # Blazor scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: blazor_scenarios -# projectFile: blazor_scenarios.proj -# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks -# - main - -# # SDK scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - win-x86 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: sdk_scenarios -# projectFile: sdk_scenarios.proj -# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks -# - main +###################################################### +# Public correctness jobs +###################################################### + +- ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: + + # Scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: scenarios + projectFile: scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + + ## MAUI scenario benchmarks + #- template: /eng/performance/build_machine_matrix.yml + # parameters: + # jobTemplate: /eng/performance/scenarios.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # isPublic: true + # jobParameters: + # kind: maui_scenarios + # projectFile: maui_scenarios.proj + # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + # - main + + # Blazor scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: blazor_scenarios + projectFile: blazor_scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + + # SDK scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - win-x86 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: sdk_scenarios + projectFile: sdk_scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main -# # micro benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - win-x86 -# isPublic: true -# jobParameters: -# kind: micro -# csproj: src\benchmarks\micro\MicroBenchmarks.csproj -# runCategories: 'runtime libraries' -# channels: -# - main - -# # Ubuntux64 Default and NativeAOT micro benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: micro -# csproj: src\benchmarks\micro\MicroBenchmarks.csproj -# runCategories: 'runtime libraries' -# channels: -# - main -# - nativeaot8.0 - -# # net462 micro benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-rs5-x64 -# isPublic: true -# jobParameters: -# kind: micro_net462 -# csproj: src\benchmarks\micro\MicroBenchmarks.csproj -# runCategories: 'runtime libraries' -# channels: -# - net462 - -# # ML.NET benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: mlnet -# csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj -# runCategories: 'mldotnet' -# channels: -# - main - -# # F# benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: fsharp -# csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj -# runCategories: 'fsharp' -# channels: -# - main - -# # bepuphysics benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: bepuphysics -# csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj -# runCategories: 'BepuPhysics' -# channels: -# - main - -# # ImageSharp benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: imagesharp -# csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj -# runCategories: 'ImageSharp' -# channels: -# - main - -# # Roslyn benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: roslyn -# csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj -# runCategories: 'roslyn' -# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only -# - main - -# # ILLink benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: illink -# csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj -# runCategories: 'illink' -# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only -# - main - -# # NativeAOT scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# isPublic: true -# jobParameters: -# kind: nativeaot_scenarios -# projectFile: nativeaot_scenarios.proj -# channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks -# - main - -# ########################################### -# # Private Jobs -# ########################################### - -# - ${{ if or(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest', 'Manual')), parameters.runPrivateJobs) }}: - -# # Scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# - win-arm64 -# - ubuntu-arm64-ampere -# isPublic: false -# jobParameters: -# kind: scenarios -# projectFile: scenarios.proj -# channels: -# - main - -# # Affinitized Scenario benchmarks (Initially just PDN) -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - win-arm64 -# - win-arm64-ampere -# isPublic: false -# jobParameters: -# kind: scenarios -# projectFile: scenarios_affinitized.proj -# channels: -# - main -# additionalJobIdentifier: 'Affinity_85' -# affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account -# runEnvVars: -# - DOTNET_GCgen0size=410000 # ~4MB -# - DOTNET_GCHeapCount=4 -# - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - -# # Maui Android scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64-android-arm64-pixel -# - win-x64-android-arm64-galaxy -# isPublic: false -# jobParameters: -# kind: maui_scenarios_android -# projectFile: maui_scenarios_android.proj -# dotnetVersionsLinks: -# main: https://aka.ms/dotnet/sdk/maui/net8.0.json + # micro benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - win-x86 + isPublic: true + jobParameters: + kind: micro + csproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: + - main + + # Ubuntux64 Default and NativeAOT micro benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - ubuntu-x64 + isPublic: true + jobParameters: + kind: micro + csproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: + - main + - nativeaot8.0 + + # net462 micro benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-rs5-x64 + isPublic: true + jobParameters: + kind: micro_net462 + csproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: + - net462 + + # ML.NET benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: mlnet + csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj + runCategories: 'mldotnet' + channels: + - main + + # F# benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: fsharp + csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj + runCategories: 'fsharp' + channels: + - main + + # bepuphysics benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: bepuphysics + csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj + runCategories: 'BepuPhysics' + channels: + - main + + # ImageSharp benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: imagesharp + csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj + runCategories: 'ImageSharp' + channels: + - main + + # Roslyn benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: roslyn + csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + runCategories: 'roslyn' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + + # ILLink benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: illink + csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj + runCategories: 'illink' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + + # NativeAOT scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + kind: nativeaot_scenarios + projectFile: nativeaot_scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + +########################################### +# Private Jobs +########################################### + +- ${{ if or(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest', 'Manual')), parameters.runPrivateJobs) }}: + + # Scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: scenarios + projectFile: scenarios.proj + channels: + - main + + # Affinitized Scenario benchmarks (Initially just PDN) + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - win-arm64 + - win-arm64-ampere + isPublic: false + jobParameters: + kind: scenarios + projectFile: scenarios_affinitized.proj + channels: + - main + additionalJobIdentifier: 'Affinity_85' + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + + # Maui Android scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + kind: maui_scenarios_android + projectFile: maui_scenarios_android.proj + dotnetVersionsLinks: + main: https://aka.ms/dotnet/sdk/maui/net8.0.json # Maui iOS scenario benchmarks - template: /eng/performance/build_machine_matrix.yml @@ -349,254 +349,254 @@ jobs: main: https://aka.ms/dotnet/sdk/maui/net8.0.json runtimeFlavor: coreclr -# ## Maui scenario benchmarks -# #- template: /eng/performance/build_machine_matrix.yml -# # parameters: -# # jobTemplate: /eng/performance/scenarios.yml -# # buildMachines: -# # - win-x64 -# # - ubuntu-x64 -# # - win-arm64 -# # - ubuntu-arm64-ampere -# # isPublic: false -# # jobParameters: -# # kind: maui_scenarios -# # projectFile: maui_scenarios.proj -# # channels: -# # - main - -# # NativeAOT scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# - win-arm64 -# isPublic: false -# jobParameters: -# kind: nativeaot_scenarios -# projectFile: nativeaot_scenarios.proj -# channels: -# - main - -# ################################################ -# # Scheduled Private jobs -# ################################################ - -# # Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -# - ${{ if or(and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), not(contains(variables['Build.QueuedBy'], 'Weekly'))), parameters.runScheduledPrivateJobs) }}: - -# # SDK scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# - win-x86 -# - ubuntu-x64 -# isPublic: false -# jobParameters: -# kind: sdk_scenarios -# projectFile: sdk_scenarios.proj -# channels: -# - main - -# # Blazor 3.2 scenario benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/scenarios.yml -# buildMachines: -# - win-x64 -# isPublic: false -# jobParameters: -# kind: blazor_scenarios -# projectFile: blazor_scenarios.proj -# channels: -# - main - -# # F# benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# - win-arm64 -# - ubuntu-arm64-ampere -# isPublic: false -# jobParameters: -# kind: fsharp -# csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj -# runCategories: 'fsharp' -# channels: -# - main - -# # bepuphysics benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# - win-arm64 -# - ubuntu-arm64-ampere -# isPublic: false -# jobParameters: -# kind: bepuphysics -# csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj -# runCategories: 'BepuPhysics' -# channels: -# - main - -# # ImageSharp benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# - win-arm64 -# - ubuntu-arm64-ampere -# isPublic: false -# jobParameters: -# kind: imagesharp -# csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj -# runCategories: 'ImageSharp' -# channels: -# - main - -# # ML.NET benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# - win-arm64 -# - win-arm64-ampere -# - ubuntu-arm64-ampere -# isPublic: false -# jobParameters: -# kind: mlnet -# csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj -# runCategories: 'mldotnet' -# channels: -# - main -# affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account -# runEnvVars: -# - DOTNET_GCgen0size=410000 # ~4MB -# - DOTNET_GCHeapCount=4 -# - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - -# # Roslyn benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# - win-arm64 -# - win-arm64-ampere -# - ubuntu-arm64-ampere -# isPublic: false -# jobParameters: -# kind: roslyn -# csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj -# runCategories: 'roslyn' -# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only -# - main -# affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account -# runEnvVars: -# - DOTNET_GCgen0size=410000 # ~4MB -# - DOTNET_GCHeapCount=4 -# - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - -# # ILLink benchmarks -# - template: /eng/performance/build_machine_matrix.yml -# parameters: -# jobTemplate: /eng/performance/benchmark_jobs.yml -# buildMachines: -# - win-x64 -# - ubuntu-x64 -# # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) -# isPublic: false -# jobParameters: -# kind: illink -# csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj -# runCategories: 'illink' -# channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only -# - main - -# # Secret Sync -# - job: Synchronize -# pool: -# name: NetCore1ESPool-Internal-NoMSI -# demands: ImageOverride -equals 1es-windows-2019 -# steps: -# - task: UseDotNet@2 -# displayName: Install .NET 6.0 runtime -# inputs: -# version: 6.x - -# - script: dotnet tool restore - -# - task: AzureCLI@2 -# inputs: -# azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) -# scriptType: ps -# scriptLocation: inlineScript -# inlineScript: | -# Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} - -# ################################################ -# # Manually Triggered Job -# ################################################ - - -# - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: -# - job: Synchronize -# pool: -# name: NetCore1ESPool-Internal-NoMSI -# demands: ImageOverride -equals 1es-windows-2019 -# steps: -# - task: UseDotNet@2 -# displayName: Install .NET 6.0 runtime -# inputs: -# version: 6.x - -# - script: dotnet tool restore - -# - task: AzureCLI@2 -# inputs: -# azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) -# scriptType: ps -# scriptLocation: inlineScript -# inlineScript: | -# Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} - -# ################################################ -# # Scheduled Private jobs (Weekly) -# ################################################ - -# # Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -# - ${{ if and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), contains(variables['Build.QueuedBy'], 'Weekly')) }}: -# - job: Synchronize -# pool: -# name: NetCore1ESPool-Internal-NoMSI -# demands: ImageOverride -equals 1es-windows-2019 -# steps: -# - task: UseDotNet@2 -# displayName: Install .NET 6.0 runtime -# inputs: -# version: 6.x - -# - script: dotnet tool restore - -# - task: AzureCLI@2 -# inputs: -# azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) -# scriptType: ps -# scriptLocation: inlineScript -# inlineScript: | -# Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + ## Maui scenario benchmarks + #- template: /eng/performance/build_machine_matrix.yml + # parameters: + # jobTemplate: /eng/performance/scenarios.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # - win-arm64 + # - ubuntu-arm64-ampere + # isPublic: false + # jobParameters: + # kind: maui_scenarios + # projectFile: maui_scenarios.proj + # channels: + # - main + + # NativeAOT scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + isPublic: false + jobParameters: + kind: nativeaot_scenarios + projectFile: nativeaot_scenarios.proj + channels: + - main + +################################################ +# Scheduled Private jobs +################################################ + +# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job +- ${{ if or(and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), not(contains(variables['Build.QueuedBy'], 'Weekly'))), parameters.runScheduledPrivateJobs) }}: + + # SDK scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - win-x86 + - ubuntu-x64 + isPublic: false + jobParameters: + kind: sdk_scenarios + projectFile: sdk_scenarios.proj + channels: + - main + + # Blazor 3.2 scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + isPublic: false + jobParameters: + kind: blazor_scenarios + projectFile: blazor_scenarios.proj + channels: + - main + + # F# benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: fsharp + csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj + runCategories: 'fsharp' + channels: + - main + + # bepuphysics benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: bepuphysics + csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj + runCategories: 'BepuPhysics' + channels: + - main + + # ImageSharp benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: imagesharp + csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj + runCategories: 'ImageSharp' + channels: + - main + + # ML.NET benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: mlnet + csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj + runCategories: 'mldotnet' + channels: + - main + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + + # Roslyn benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: roslyn + csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + runCategories: 'roslyn' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + + # ILLink benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) + isPublic: false + jobParameters: + kind: illink + csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj + runCategories: 'illink' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + + # Secret Sync + - job: Synchronize + pool: + name: NetCore1ESPool-Internal-NoMSI + demands: ImageOverride -equals 1es-windows-2019 + steps: + - task: UseDotNet@2 + displayName: Install .NET 6.0 runtime + inputs: + version: 6.x + + - script: dotnet tool restore + + - task: AzureCLI@2 + inputs: + azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + +################################################ +# Manually Triggered Job +################################################ + + +- ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: + - job: Synchronize + pool: + name: NetCore1ESPool-Internal-NoMSI + demands: ImageOverride -equals 1es-windows-2019 + steps: + - task: UseDotNet@2 + displayName: Install .NET 6.0 runtime + inputs: + version: 6.x + + - script: dotnet tool restore + + - task: AzureCLI@2 + inputs: + azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + +################################################ +# Scheduled Private jobs (Weekly) +################################################ + +# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job +- ${{ if and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), contains(variables['Build.QueuedBy'], 'Weekly')) }}: + - job: Synchronize + pool: + name: NetCore1ESPool-Internal-NoMSI + demands: ImageOverride -equals 1es-windows-2019 + steps: + - task: UseDotNet@2 + displayName: Install .NET 6.0 runtime + inputs: + version: 6.x + + - script: dotnet tool restore + + - task: AzureCLI@2 + inputs: + azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} diff --git a/eng/performance/maui_scenarios_ios.proj b/eng/performance/maui_scenarios_ios.proj index f0f1a1de07f..b914dca8b93 100644 --- a/eng/performance/maui_scenarios_ios.proj +++ b/eng/performance/maui_scenarios_ios.proj @@ -87,7 +87,7 @@ sign XamariniOSDefault.app # Testing commands - $(Python) test.py devicestartup --device-type ios --package-path XamariniOSDefault.app --package-name net.dot.xamarintesting --scenario-name "%(Identity)" + $(Python) test.py devicestartup --device-type ios --package-path XamariniOSDefault.app --package-name com.companyname.XamariniOSDefault --scenario-name "%(Identity)" ((result=$?)) # Post commands From 3aa42ff6af521342a4ea91517e5299e9935966b2 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Thu, 20 Jul 2023 23:47:27 +0200 Subject: [PATCH 19/19] Retrieve version from an assembly in intermediate directory --- src/scenarios/mauiblazorios/pre.py | 4 ++-- src/scenarios/mauiios/pre.py | 4 ++-- src/scenarios/xamarinios/pre.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/scenarios/mauiblazorios/pre.py b/src/scenarios/mauiblazorios/pre.py index 38ba650897a..ab781a42ade 100644 --- a/src/scenarios/mauiblazorios/pre.py +++ b/src/scenarios/mauiblazorios/pre.py @@ -72,8 +72,8 @@ remove_aab_files(output_dir) # Copy the MauiVersion to a file so we have it on the machine -maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") +maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll") version_dict = { "mauiVersion": maui_version } versions_write_json(version_dict, rf"{output_dir}/versions.json") -print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") +print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll") diff --git a/src/scenarios/mauiios/pre.py b/src/scenarios/mauiios/pre.py index fce66fe9bb1..7e85e2aa116 100644 --- a/src/scenarios/mauiios/pre.py +++ b/src/scenarios/mauiios/pre.py @@ -35,7 +35,7 @@ remove_aab_files(output_dir) # Copy the MauiVersion to a file so we have it on the machine -maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") +maui_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll") version_dict = { "mauiVersion": maui_version } versions_write_json(version_dict, rf"{output_dir}/versions.json") -print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.Maui.dll") +print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.Maui.dll") diff --git a/src/scenarios/xamarinios/pre.py b/src/scenarios/xamarinios/pre.py index 147a967913f..4efc24aee4d 100644 --- a/src/scenarios/xamarinios/pre.py +++ b/src/scenarios/xamarinios/pre.py @@ -35,7 +35,7 @@ remove_aab_files(output_dir) # Copy the XamarinVersion to a file so we have it on the machine -xamarin_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.iOS.dll") +xamarin_version = get_version_from_dll_powershell_ios(rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.iOS.dll") version_dict = { "xamarinVersion": xamarin_version } versions_write_json(version_dict, rf"{output_dir}/versions.json") -print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/{precommands.framework}/ios-arm64/ipa/Payload/{EXENAME}.app/Microsoft.iOS.dll") +print(f"Versions: {version_dict} from location " + rf"./{const.APPDIR}/obj/Release/net8.0-ios/ios-arm64/linked/Microsoft.iOS.dll")