Report benchmark statistics and comparison

.github/workflows/dotnet-build.yml

@@ -35,4 +35,52 @@ jobs:
       run: |
         reportgenerator -reports:"../../coverage-outputs/**/*.xml" -targetdir:"../../coverage-outputs" -reporttypes:Cobertura
         rm -rfv ../../coverage-outputs/*/
-        ls -la ../../coverage-outputs
+        ls -la ../../coverage-outputs
+
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Determine folders
+      id: determine-folders
+      run: |
+        HEAD_FOLDER=$(pwd)
+        HEAD_BENCH_FOLDER=$HEAD_FOLDER/tmp/benchmarks/head
+        echo "head_folder=$HEAD_FOLDER" >> $GITHUB_OUTPUT
+        echo "head_benchmarks_folder=$HEAD_BENCH_FOLDER" >> $GITHUB_OUTPUT
+    - name: Git Checkout
+      uses: actions/checkout@v2
+      with:
+        ref: ${{github.event.pull_request.head.ref}}
+        repository: ${{github.event.pull_request.head.repo.full_name}}
+    - name: Create benchmarks results folder (HEAD)
+      run: mkdir -p ${{ steps.determine-folders.outputs.head_benchmarks_folder }}
+    - name: Restore dependencies (HEAD)
+      run: dotnet restore rules-framework.sln
+    - name: Build benchmarks (HEAD)
+      run: dotnet build -c Release tests/Rules.Framework.BenchmarkTests/Rules.Framework.BenchmarkTests.csproj --framework net6.0
+    - name: Run benchmarks (HEAD)
+      run: sudo dotnet run --project tests/Rules.Framework.BenchmarkTests/Rules.Framework.BenchmarkTests.csproj -c Release --framework net6.0 -- -a "${{ steps.determine-folders.outputs.head_benchmarks_folder }}/artifacts"
+    - name: Determine results file (HEAD)
+      id: determine-results-file
+      run: |
+        cd ${{ steps.determine-folders.outputs.head_benchmarks_folder }}
+        ls -lRA
+        sudo chmod -R 777 artifacts
+        MD_FILE=$(find artifacts/results -name '*.md')
+        echo "file=${{ steps.determine-folders.outputs.head_benchmarks_folder }}/$MD_FILE" >> $GITHUB_OUTPUT
+      # Tries to find a comment ID.
+    - name: Find comment - report results
+      uses: peter-evans/find-comment@v2
+      id: find-comment
+      with:
+        issue-number: ${{ github.event.number }}
+        comment-author: 'github-actions[bot]'
+        body-includes: Benchmark Results Report
+      # If a prior comment ID is not found, creates a new comment, otherwise updates the existent one.
+    - name: Create or update comment - report results
+      uses: peter-evans/create-or-update-comment@v2
+      with:
+        comment-id: ${{ steps.find-comment.outputs.comment-id }}
+        issue-number: ${{ github.event.number }}
+        body-file: ${{ steps.determine-results-file.outputs.file }}
+        edit-mode: replace

.gitignore

@@ -558,6 +558,7 @@ typings/
 # Specific ones added
 coverage-outputs/**
 .env
+tmp/**
 
 ### Rules Framework Web UI ###
 !src/Rules.Framework.WebUI/node_modules/

README.md

@@ -4,7 +4,7 @@ Rules.Framework is a generic rules framework that allows defining and evaluating
 
 Why use rules? Most of us at some point, while developing software to support a business, have come across fast paced business logic changes. Sometimes, business needs change overnight, which requires a fast response to changes by engineering teams. By using rules, changing a calculus formula, a value mapping or simply a toggle configuration no longer requires code changes/endless CI/CD pipelines, QA validation, and so on... Business logic changes can be offloaded to configuration scenarios, instead of development scenarios.
 
-[![Codacy Badge](https://api.codacy.com/project/badge/Grade/8b48f4541fba4d4b8bad2e9a8563ede3)](https://app.codacy.com/gh/Farfetch/rules-framework?utm_source=github.com&utm_medium=referral&utm_content=Farfetch/rules-framework&utm_campaign=Badge_Grade_Settings)
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/8b48f4541fba4d4b8bad2e9a8563ede3)](https://app.codacy.com/gh/Farfetch/rules-framework?utm_source=github.com\&utm_medium=referral\&utm_content=Farfetch/rules-framework\&utm_campaign=Badge_Grade_Settings)
 [![.NET build](https://github.com/luispfgarces/rules-framework/actions/workflows/dotnet-build.yml/badge.svg)](https://github.com/luispfgarces/rules-framework/actions/workflows/dotnet-build.yml)
 
 ## Packages
@@ -13,21 +13,22 @@ Why use rules? Most of us at some point, while developing software to support a
 |---------------------------------|----|---------|
 |Rules.Framework|[![Nuget Package](https://img.shields.io/nuget/v/Rules.Framework.svg?logo=nuget)](https://www.nuget.org/packages/Rules.Framework/)|[![Rules.Framework on fuget.org](https://www.fuget.org/packages/Rules.Framework/badge.svg)](https://www.fuget.org/packages/Rules.Framework)|
 |Rules.Framework.Providers.MongoDb|[![Nuget Package](https://img.shields.io/nuget/v/Rules.Framework.Providers.MongoDb?logo=nuget)](https://www.nuget.org/packages/Rules.Framework.Providers.MongoDb/)|[![Rules.Framework.Providers.MongoDb on fuget.org](https://www.fuget.org/packages/Rules.Framework.Providers.MongoDb/badge.svg)](https://www.fuget.org/packages/Rules.Framework.Providers.MongoDb)|
-|Rules.Framework.Providers.InMemory|[![Nuget Package](https://img.shields.io/nuget/v/Rules.Framework.Providers.InMemory?logo=nuget)](https://www.nuget.org/packages/Rules.Framework.Providers.InMemory/)|[![Rules.Framework.Providers.InMemory on fuget.org](https://www.fuget.org/packages/Rules.Framework.Providers.InMemory/badge.svg)](https://www.fuget.org/packages/Rules.Framework.Providers.InMemory)|
+|Rules.Framework.WebUI|[![Nuget Package](https://img.shields.io/nuget/v/Rules.Framework.WebUI?logo=nuget)](https://www.nuget.org/packages/Rules.Framework.WebUI/)|[![Rules.Framework.WebUI on fuget.org](https://www.fuget.org/packages/Rules.Framework.WebUI/badge.svg)](https://www.fuget.org/packages/Rules.Framework.WebUI)|
 
 ## Features
 
 The following listing presents features implemented and features to be implemented:
 
-- [x] Rules evaluation (match one)
-- [x] Rules evaluation (match many)
-- [x] Rules search
-- [x] Rules content serializarion
-- [ ] Rules data source caching
-- [x] Rules management (Create, Read, Update)
-- [X] In-memory data source support
-- [x] MongoDB data source support
-- [ ] SQL Server data source support
+*   [x] Rules evaluation (match one)
+*   [x] Rules evaluation (match many)
+*   [x] Rules search
+*   [x] Rules content serializarion
+*   [ ] Rules data source caching
+*   [x] Rules management (Create, Read, Update)
+*   [x] In-memory data source support
+*   [x] MongoDB data source support
+*   [ ] SQL Server data source support
+*   [x] Rules evaluation modes (interpreted, compiled)
 
 ## How it works
 
@@ -37,36 +38,36 @@ Starting with the basics, what are we considering a rule?
 
 For Rules.Framework, a valid rule accounts for the following conditions:
 
-- Categorized by a **content type**, which groups rules by those that will be evaluated together. Rules from different content types won't be evaluated together. Content type is a user defined type, which can be a value type or a object, depending on the requirements of usage.
-- Has a **name**, which must be unique by content type.
-- Is constrained in time by a **date begin** and a **date end**. Date begin must be always set, and date end can be null (meaning that rule is applied from date begin to _ad eternum_). Please note that date begin threshold is inclusive and date end threshold is exclusive, so if you define a rule with date begin as "2020-01-01" and date end as "2021-01-01", if evaluation date is set to "2020-01-01", rule will match, but if evaluation date is set to "2021-01-01", rule will not match.
-- Has a **priority** numeric value, which works as tiebreaker when many rules match on rules interval and given input conditions. Rules.Framework has the ability to configure if tiebreaker criteria is set to highest priority value or lowest priority value. This value must always be positive.
-- Also has a set of **conditions** disposed in tree. Conditions can be set combined by AND/OR operators and by using comparison operators to compare values set on rule (integer, boolean, string or decimal) to input conditions. Conditions are categorized by a condition type, which must be one of the user-defined types (either value types or objects).
-- And a **content** defined by user and totally up to the user to validate it (can virtually be anything the user wants, as long as the persistence mechanism used as data source supports it).
+*   Categorized by a **content type**, which groups rules by those that will be evaluated together. Rules from different content types won't be evaluated together. Content type is a user defined type, which can be a value type or a object, depending on the requirements of usage.
+*   Has a **name**, which must be unique by content type.
+*   Is constrained in time by a **date begin** and a **date end**. Date begin must be always set, and date end can be null (meaning that rule is applied from date begin to *ad eternum*). Please note that date begin threshold is inclusive and date end threshold is exclusive, so if you define a rule with date begin as "2020-01-01" and date end as "2021-01-01", if evaluation date is set to "2020-01-01", rule will match, but if evaluation date is set to "2021-01-01", rule will not match.
+*   Has a **priority** numeric value, which works as tiebreaker when many rules match on rules interval and given input conditions. Rules.Framework has the ability to configure if tiebreaker criteria is set to highest priority value or lowest priority value. This value must always be positive.
+*   Also has a set of **conditions** disposed in tree. Conditions can be set combined by AND/OR operators and by using comparison operators to compare values set on rule (integer, boolean, string or decimal) to input conditions. Conditions are categorized by a condition type, which must be one of the user-defined types (either value types or objects).
+*   And a **content** defined by user and totally up to the user to validate it (can virtually be anything the user wants, as long as the persistence mechanism used as data source supports it).
 
 Bellow you can see a simple sample for demonstration purposes:
 
 ![Rule Sample 1](docs/rule-sample-1.png)
 
 The sample rule presented:
 
-- Is described by it's name as "Body Mass default formula" - a simple human-readable description.
-- Has a content type "Body Mass formula" that categorizes it.
-- Begins at 1st January 2018 and never ends - which means that requesting on a date before 1st January 2018, rule is not matched, but after midnight at the same date, the rule will match.
-- Priority is set to 1. This would be used as tiebreaker criteria if there were more rules defined, but since there's only one rule, there's no difference on evaluation.
-- Rule has no conditions defined - which means, requesting on a date on rule dates range, it will always match.
+*   Is described by it's name as "Body Mass default formula" - a simple human-readable description.
+*   Has a content type "Body Mass formula" that categorizes it.
+*   Begins at 1st January 2018 and never ends - which means that requesting on a date before 1st January 2018, rule is not matched, but after midnight at the same date, the rule will match.
+*   Priority is set to 1. This would be used as tiebreaker criteria if there were more rules defined, but since there's only one rule, there's no difference on evaluation.
+*   Rule has no conditions defined - which means, requesting on a date on rule dates range, it will always match.
 
 Simple right? You got the basics covered, let's complicate this a bit by adding a new rule. The formula you saw on the first rule is used to calculate body mass when using kilograms and meters unit of measures, but what if we wanted to calculate using pounds and inches? Let's define a new rule for this:
 
 ![Rule Sample 2](docs/rule-sample-2.png)
 
 Newly defined rule (Rule #2):
 
-- Becomes the rule with priority 1.
-- Defines a new formula.
-- Defines a composed condition node specifying that a AND logical operator must be applied between child nodes conditions results.
-- Defines a condition node with data type string, having a condition type of "Mass unit of measure", operator equal and operand "pounds".
-- Defines a second condition node with data type string, having a condition type of "Height unit of measure", operator equal and operand "inches".
+*   Becomes the rule with priority 1.
+*   Defines a new formula.
+*   Defines a composed condition node specifying that a AND logical operator must be applied between child nodes conditions results.
+*   Defines a condition node with data type string, having a condition type of "Mass unit of measure", operator equal and operand "pounds".
+*   Defines a second condition node with data type string, having a condition type of "Height unit of measure", operator equal and operand "inches".
 
 If you request a rule for the content type "Body Mass formula" by specifying date 2019-01-01, "Mass unit of measure" as "pounds" and "Height unit of measure" as "inches", both rules will match (remember that Rule #1 has no conditions, so it matches anything). At this point is where priority is used to select the right one (by default, lowest priority values win to highest values, but this is configurable), so Rule #2 is chosen.
 
@@ -91,4 +92,4 @@ Head over to [CONTRIBUTING](CONTRIBUTING.md) for further details.
 
 ## License
 
-[MIT License](LICENSE.md)
+[MIT License](LICENSE.md)

run-benchmarks.ps1

@@ -0,0 +1,55 @@
+param ([switch] $KeepBenchmarksFiles)
+
+$originalDir = (Get-Location).Path
+$timestamp = [DateTime]::UtcNow.ToString("yyyyMMdd_hhmmss")
+
+$directoryFound = Get-ChildItem -Path $originalDir -Directory | Select-String -Pattern "tmp"
+if (!$directoryFound) {
+    New-Item -Name "tmp" -ItemType Directory > $null
+}
+
+$directoryFound = Get-ChildItem -Path "$originalDir\\tmp" -Directory | Select-String -Pattern "benchmarks"
+if (!$directoryFound) {
+    New-Item -Name "benchmarks" -ItemType Directory -Path "$originalDir\\tmp" > $null
+}
+
+$directoryFound = Get-ChildItem -Path "$originalDir\\tmp\\benchmarks" -Directory | Select-String -Pattern $timestamp
+if (!$directoryFound) {
+    New-Item -Name $timestamp -ItemType Directory -Path "$originalDir\\tmp\\benchmarks" > $null
+}
+
+$reportDir = "$originalDir\\tmp\\benchmarks\\$timestamp"
+
+# Ensure all packages restored before running benchmarks
+dotnet restore rules-framework.sln
+
+# Build benchmarks binaries
+dotnet build -c Release .\tests\Rules.Framework.BenchmarkTests\Rules.Framework.BenchmarkTests.csproj -o "$reportDir\\bin" --framework net6.0
+
+Set-Location -Path $reportDir
+
+# Run benchmarks
+bin\Rules.Framework.BenchmarkTests.exe -a artifacts
+
+# Determine results file
+$filteredResultsFiles = Get-ChildItem -Path artifacts/results -File -Filter *.md
+if ($filteredResultsFiles) {
+    $resultsFile = $filteredResultsFiles.Name
+
+    # Copy results file
+    Copy-Item -Path artifacts/results/$resultsFile -Destination .
+
+    # Rename file
+    Rename-Item -Path $resultsFile -NewName report.md
+}
+
+if (!$KeepBenchmarksFiles) {
+    if ($directoryFound = Get-ChildItem -Path $reportDir -Directory | Select-String -Pattern "artifacts") {
+        Remove-Item -Path artifacts -Recurse > $null
+    }
+    if ($directoryFound = Get-ChildItem -Path $reportDir -Directory | Select-String -Pattern "bin") {
+        Remove-Item -Path bin -Recurse > $null
+    }
+}
+
+Set-Location -Path $originalDir

tests/Rules.Framework.BenchmarkTests/CustomBaselineClassifierColumn.cs

@@ -0,0 +1,36 @@
+namespace Rules.Framework.BenchmarkTests
+{
+    using System;
+    using BenchmarkDotNet.Columns;
+    using BenchmarkDotNet.Reports;
+    using BenchmarkDotNet.Running;
+
+    internal class CustomBaselineClassifierColumn : IColumn
+    {
+        private readonly Func<BenchmarkCase, bool> classifyLogicFunc;
+
+        public CustomBaselineClassifierColumn(Func<BenchmarkCase, bool> classifyLogicFunc)
+        {
+            this.classifyLogicFunc = classifyLogicFunc;
+        }
+
+        public bool AlwaysShow => true;
+        public ColumnCategory Category => ColumnCategory.Baseline;
+        public string ColumnName => "Baseline";
+        public string Id => nameof(CustomBaselineClassifierColumn);
+        public bool IsNumeric => false;
+        public string Legend => "Sets wether a test case is a baseline.";
+        public int PriorityInCategory => 0;
+        public UnitType UnitType => UnitType.Dimensionless;
+
+        public string GetValue(Summary summary, BenchmarkCase benchmarkCase)
+            => this.classifyLogicFunc.Invoke(benchmarkCase) ? "Yes" : "No";
+
+        public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
+            => this.GetValue(summary, benchmarkCase);
+
+        public bool IsAvailable(Summary summary) => true;
+
+        public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false;
+    }
+}

tests/Rules.Framework.BenchmarkTests/Exporters/Common/BenchmarkReport.cs

@@ -0,0 +1,17 @@
+namespace Rules.Framework.BenchmarkTests.Exporters.Common
+{
+    using System;
+
+    internal class BenchmarkReport
+    {
+        public DateTime Date { get; set; }
+
+        public Environment? Environment { get; set; }
+
+        public IEnumerable<BenchmarkStatisticsItem>? Statistics { get; set; }
+
+        public IEnumerable<BenchmarkStatisticsComparisonItem>? StatisticsComparison { get; set; }
+
+        public string Title { get; set; } = string.Empty;
+    }
+}

tests/Rules.Framework.BenchmarkTests/Exporters/Common/BenchmarkStatisticsComparisonItem.cs

@@ -0,0 +1,23 @@
+namespace Rules.Framework.BenchmarkTests.Exporters.Common
+{
+    internal class BenchmarkStatisticsComparisonItem
+    {
+        public BenchmarkStatisticsValue? AllocatedMemoryRate { get; set; }
+
+        public BenchmarkStatisticsValue? BaselineAllocatedMemory { get; set; }
+
+        public BenchmarkStatisticsValue? BaselineMeanTimeTaken { get; set; }
+
+        public string? BaselineParameters { get; set; }
+
+        public BenchmarkStatisticsValue? CompareAllocatedMemory { get; set; }
+
+        public BenchmarkStatisticsValue? CompareMeanTimeTaken { get; set; }
+
+        public string? CompareParameters { get; set; }
+
+        public string? Key { get; set; }
+
+        public BenchmarkStatisticsValue? MeanTimeTakenCompareRate { get; set; }
+    }
+}