Implementing BDN Perf pipeline

.github/workflows/ci-bdnbenchmark.yml

@@ -1,18 +1,23 @@
 name: Garnet CI BDN.benchmark
 on:
   workflow_dispatch:
-
+  push:
+    branches:
+      - main
+
 env:
   DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
   DOTNET_NOLOGO: true
 
+permissions:
+  actions: write
+  deployments: write  #permission to deploy GitHub pages website
+  contents: write  # permission to update benchmark contents in gh-pages branch
+
 jobs:
   changes:
     name: Check for changes
-    runs-on: ubuntu-latest # don't need matrix to test where the changes were made in code
-    permissions:
-      pull-requests: read
-      contents: read
+    runs-on: ubuntu-latest
     steps:
     - name: Check out code
       uses: actions/checkout@v4
@@ -25,3 +30,76 @@ jobs:
             - 'website/**'
           bdnbenchmark:
             - '!((*.md)|(website/**))'
+
+  # Job to build and run BDN.benchmark
+  build-run-bdnbenchmark:
+    name: BDNBenchmark
+    needs: [changes]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        framework: [ 'net8.0' ]
+        configuration: [ 'Release' ]
+        test: [ 'Operations.BasicOperations', 'Operations.ObjectOperations', 'Cluster.ClusterMigrate', 'Cluster.ClusterOperations', 'Lua.LuaScripts' ]   
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: 8.0.x
+      - name: Install dependencies
+        run: dotnet restore
+
+      - name: Run BDN.benchmark Perf Test
+        run: ./test/BDNPerfTests/run_bdnperftest.ps1 ${{ matrix.test }}
+        shell: pwsh
+        continue-on-error: false
+
+      - name: Upload test results to artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: Results-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }}
+          path: |
+             ./test/BDNPerfTests/results
+        if: ${{ always() }}
+
+      - name: Upload Error Log to artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ErrorLog-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }}
+          path: |
+            ./test/BDNPerfTests/errorlog
+        if: ${{ always() }}
+
+      # Run `github-action-benchmark` action to get Commit Comment
+      - name: Store benchmark result for commit comment
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'benchmarkdotnet'
+          output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          alert-threshold: '150%' 
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: '@darrenge'
+          summary-always: false
+          gh-pages-branch: 'continuousbenchmark'
+          benchmark-data-dir-path: 'website/static/charts'
+          auto-push: true
+
+      # Run `github-action-benchmark` action for the Continuous Benchmarking Charts
+      - name: Store benchmark result for charts
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: ${{matrix.test}} (${{matrix.os}}  ${{matrix.framework}} ${{matrix.configuration}})
+          tool: 'benchmarkdotnet'
+          output-file-path: ./test/BDNPerfTests/BenchmarkDotNet.Artifacts/results/BDN.benchmark.${{ matrix.test }}-report-full-compressed.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          summary-always: true
+          gh-pages-branch: 'continuousbenchmark'
+          benchmark-data-dir-path: 'website/static/charts'
+          auto-push: true
+          max-items-in-chart: 50

benchmark/BDN.benchmark/Lua/LuaScripts.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license.
 
 using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Columns;
 using Garnet.server;
 
 namespace BDN.benchmark.Lua
@@ -10,6 +11,7 @@ namespace BDN.benchmark.Lua
     /// Benchmark for Lua
     /// </summary>
     [MemoryDiagnoser]
+    [HideColumns(Column.Gen0)]
     public unsafe class LuaScripts
     {
         /// <summary>

test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterMigrate.json

@@ -0,0 +1,19 @@
+{
+    "configuration": "Release",
+    "framework": "net8.0",
+    "filter": "BDN.benchmark.Cluster.ClusterMigrate.*",
+
+    "expectedGet_None_AllocatedValue_linux": 0,
+    "expectedSet_None_AllocatedValue_linux": 0,
+    "expectedMGet_None_AllocatedValue_linux": 0,
+    "expectedMSet_None_AllocatedValue_linux": 0,
+
+    "expectedGet_None_AllocatedValue_win": 0,
+    "expectedSet_None_AllocatedValue_win": 0,
+    "expectedMGet_None_AllocatedValue_win": 0,
+    "expectedMSet_None_AllocatedValue_win": 0,
+
+    "acceptableAllocatedRange": 10, 
+    "acceptableError": ".1",
+    "acceptableStdDev": ".1"
+}

test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Cluster.ClusterOperations.json

@@ -0,0 +1,31 @@
+{
+    "configuration": "Release",
+    "framework": "net8.0",
+    "filter": "BDN.benchmark.Cluster.ClusterOperations.*",
+
+    "expectedGet_DSV_AllocatedValue_linux": 0,
+    "expectedSet_DSV_AllocatedValue_linux": 0,
+    "expectedMGet_DSV_AllocatedValue_linux": 0,
+    "expectedMSet_DSV_AllocatedValue_linux": 0,
+    "expectedCPBSet_DSV_AllocatedValue_linux": 0,
+    "expectedGet_None_AllocatedValue_linux": 0,
+    "expectedSet_None_AllocatedValue_linux": 0,
+    "expectedMGet_None_AllocatedValue_linux": 0,
+    "expectedMSet_None_AllocatedValue_linux": 0,
+    "expectedCPBSet_None_AllocatedValue_linux": 0,
+
+    "expectedGet_DSV_AllocatedValue_win": 0,
+    "expectedSet_DSV_AllocatedValue_win": 0,
+    "expectedMGet_DSV_AllocatedValue_win": 0,
+    "expectedMSet_DSV_AllocatedValue_win": 0,
+    "expectedCPBSet_DSV_AllocatedValue_win": 0,
+    "expectedGet_None_AllocatedValue_win": 0,
+    "expectedSet_None_AllocatedValue_win": 0,
+    "expectedMGet_None_AllocatedValue_win": 0,
+    "expectedMSet_None_AllocatedValue_win": 0,
+    "expectedCPBSet_None_AllocatedValue_win": 0,
+
+    "acceptableAllocatedRange": 10, 
+    "acceptableError": ".1",
+    "acceptableStdDev": ".1"
+}

test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Lua.LuaScripts.json

@@ -0,0 +1,19 @@
+{
+    "configuration": "Release",
+    "framework": "net8.0",
+    "filter": "BDN.benchmark.Lua.*",
+
+    "expectedScript1_None_AllocatedValue_linux": 24,
+    "expectedScript2_None_AllocatedValue_linux": 144,
+    "expectedScript3_None_AllocatedValue_linux": 240,
+    "expectedScript4_None_AllocatedValue_linux": 776,
+
+    "expectedScript1_None_AllocatedValue_win": 24,
+    "expectedScript2_None_AllocatedValue_win": 144,
+    "expectedScript3_None_AllocatedValue_win": 240,
+    "expectedScript4_None_AllocatedValue_win": 776,
+
+    "acceptableAllocatedRange": 10,    
+    "acceptableError": ".1",
+    "acceptableStdDev": ".1"
+}

test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.BasicOperations.json

@@ -0,0 +1,17 @@
+{
+    "configuration": "Release",
+    "framework": "net8.0",
+    "filter": "BDN.benchmark.Operations.BasicOperations.*",
+
+    "expectedInlinePing_ACL_AllocatedValue_linux": 0,
+    "expectedInlinePing_AOF_AllocatedValue_linux": 0,
+    "expectedInlinePing_None_AllocatedValue_linux": 0,
+
+    "expectedInlinePing_ACL_AllocatedValue_win": 0,
+    "expectedInlinePing_AOF_AllocatedValue_win": 0,
+    "expectedInlinePing_None_AllocatedValue_win": 0,
+
+    "acceptableAllocatedRange": 10,    
+    "acceptableError": ".1",
+    "acceptableStdDev": ".1"
+}

test/BDNPerfTests/ConfigFiles/CI_CONFIG_BDN_Benchmark_Operations.ObjectOperations.json

@@ -0,0 +1,53 @@
+{
+    "configuration": "Release",
+    "framework": "net8.0",
+    "filter": "BDN.benchmark.Operations.ObjectOperations.*",
+
+    "expectedZAddRem_ACL_AllocatedValue_linux": 18400,
+    "expectedLPushPop_ACL_AllocatedValue_linux": 14400,
+    "expectedSAddRem_ACL_AllocatedValue_linux": 12800,
+    "expectedHSetDel_ACL_AllocatedValue_linux": 43201,
+    "expectedMyDictSetGet_ACL_AllocatedValue_linux": 24000,
+    "expectedCustomProcSet_ACL_AllocatedValue_linux": 0,
+
+    "expectedZAddRem_AOF_AllocatedValue_linux": 18400,
+    "expectedLPushPop_AOF_AllocatedValue_linux": 14400,
+    "expectedSAddRem_AOF_AllocatedValue_linux": 12800,
+    "expectedHSetDel_AOF_AllocatedValue_linux": 43201,
+    "expectedMyDictSetGet_AOF_AllocatedValue_linux": 24000,
+    "expectedCustomProcSet_AOF_AllocatedValue_linux": 0,
+
+    "expectedZAddRem_None_AllocatedValue_linux": 18400,
+    "expectedLPushPop_None_AllocatedValue_linux": 14400,
+    "expectedSAddRem_None_AllocatedValue_linux": 12800,
+    "expectedHSetDel_None_AllocatedValue_linux": 43201,
+    "expectedMyDictSetGet_None_AllocatedValue_linux": 24000,
+    "expectedCustomProcSet_None_AllocatedValue_linux": 0,
+
+
+    "expectedZAddRem_ACL_AllocatedValue_win": 18400,
+    "expectedLPushPop_ACL_AllocatedValue_win": 14400,
+    "expectedSAddRem_ACL_AllocatedValue_win": 12800,
+    "expectedHSetDel_ACL_AllocatedValue_win": 43201,
+    "expectedMyDictSetGet_ACL_AllocatedValue_win": 24000,
+    "expectedCustomProcSet_ACL_AllocatedValue_win": 0,
+
+    "expectedZAddRem_AOF_AllocatedValue_win": 18400,
+    "expectedLPushPop_AOF_AllocatedValue_win": 14400,
+    "expectedSAddRem_AOF_AllocatedValue_win": 12800,
+    "expectedHSetDel_AOF_AllocatedValue_win": 43201,
+    "expectedMyDictSetGet_AOF_AllocatedValue_win": 24000,
+    "expectedCustomProcSet_AOF_AllocatedValue_win": 0,
+
+    "expectedZAddRem_None_AllocatedValue_win": 18400,
+    "expectedLPushPop_None_AllocatedValue_win": 14400,
+    "expectedSAddRem_None_AllocatedValue_win": 12800,
+    "expectedHSetDel_None_AllocatedValue_win": 43201,
+    "expectedMyDictSetGet_None_AllocatedValue_win": 24000,
+    "expectedCustomProcSet_None_AllocatedValue_win": 0,
+
+    "acceptableAllocatedRange": 10,    
+    "acceptableError": ".1",
+    "acceptableStdDev": ".1"
+}
+