Fleshing out Generation System Tests

Reqnroll.TestProjectGenerator/Reqnroll.TestProjectGenerator/Driver/ProjectsDriver.cs

@@ -157,7 +157,7 @@ public void AddNuGetPackage(string nugetPackage, string nugetVersion)
             _solutionDriver.DefaultProject.AddNuGetPackage(nugetPackage, nugetVersion);
         }
 
-        public void AddFailingStepBinding(string scenarioBlock, string stepRegex)
+        public void AddFailingStepBinding(string scenarioBlock = "StepDefinition", string stepRegex = ".*")
         {
             AddStepBinding(scenarioBlock, stepRegex, @"throw new System.Exception(""simulated failure"");", @"Throw New System.Exception(""simulated failure"")");
         }

Tests/Reqnroll.SystemTests/Generation/GenerationTestBase.cs

@@ -1,10 +1,27 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using FluentAssertions;
+using Reqnroll.TestProjectGenerator.Driver;
+using Reqnroll.TestProjectGenerator;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
 
 namespace Reqnroll.SystemTests.Generation;
 
 [TestCategory("Generation")]
 public class GenerationTestBase : SystemTestBase
 {
+    private HooksDriver _hookDriver = null!;
+    private TestProjectGenerator.Driver.ConfigurationDriver _configDriver = null!;
+
+    protected override void TestInitialize()
+    {
+        base.TestInitialize();
+        _hookDriver = _testContainer.Resolve<HooksDriver>();
+        _configDriver = _testContainer.Resolve<TestProjectGenerator.Driver.ConfigurationDriver>();
+    }
+
     [TestMethod]
     public void GeneratorAllIn_sample_can_be_handled()
     {
@@ -30,18 +47,239 @@ When something happens with
             	| who          | when     |
             	| me           | today    |
             	| someone else | tomorrow |
+
+            Scenario Outline: Scenario outline with examples
+            When something happens to <person>
+            Examples:
+            	| person |
+            	| me     |
+            	| you    |
+            """);
+
+        AddPassingStepBinding();
+
+        ExecuteTests();
+
+        ShouldAllScenariosPass();
+    }
+
+    //test different outcomes: success, failure, pending, undefined, ignored (scenario & scenario outline)
+    [TestMethod]
+    public void FailingScenariosAreCountedAsFailures()
+    {
+        AddScenario(
+            """
+            Scenario: Sample Scenario
+                When something happens
+
+            Scenario Outline: Scenario outline with examples
+            When something happens to <person>
+            Examples:
+            	| person |
+            	| me     |
+            	| you    |
+            """);
+
+        AddFailingStepBinding();
+
+        ExecuteTests();
+
+        ShouldAllScenariosFail();
+    }
+
+    [TestMethod]
+    public void PendingScenariosAreCountedAsPending()
+    {
+        AddScenario(
+            """
+            Scenario: Sample Scenario
+                When something happens
+
+            Scenario Outline: Scenario outline with examples
+            When something happens to <person>
+            Examples:
+            	| person |
+            	| me     |
+            	| you    |
+            """);
+
+        AddPendingStepBinding();
+
+        ExecuteTests();
+
+        ShouldAllScenariosPend();
+    }
+
+    [TestMethod]
+    public void IgnoredScenariosAreCountedAsIgnored()
+    {
+        AddScenario(
+            """
+            @ignore
+            Scenario: Sample Scenario
+                When something happens
+
+            @ignore
+            Scenario Outline: Scenario outline with examples
+            When something happens to <person>
+            Examples:
+            	| person |
+            	| me     |
+            	| you    |
+            """);
+
+        AddPassingStepBinding();
+        _configDriver.SetIsRowTestsAllowed( false); //This is necessary as MSTest and Xunit count the number of physical Test methods.
+        ExecuteTests();
+
+        ShouldAllScenariosBeIgnored(3); 
+    }
+
+    [TestMethod]
+    public void UndefinedScenariosAreNotExecuted()
+    {
+        AddScenario(
+            """
+            Scenario: Sample Scenario
+                When something happens
+
+            Scenario Outline: Scenario outline with examples
+            When something happens to <person>
+            Examples:
+            	| person |
+            	| me     |
+            	| you    |
+            """);
+
+        //AddPassingStepBinding();
+
+        ExecuteTests();
+
+        ShouldAllUndefinedScenariosNotBeExecuted();
+    }
+
+
+    //test async steps (async steps are executed in order)
+    [TestMethod]
+    public void AsyncStepsAreExecutedInOrder()
+    {
+        AddScenario(
+            """
+            Scenario: Async Scenario Steps
+                Given a list to hold step numbers
+                When Async Step '1' is called
+                When Async Step '2' is called
+                When Async Step '3' is called
+                Then async step order should be '1,2,3'
+            """);
+
+        var _asyncBindingClassContent =
+            @"namespace AsyncSequence.StepDefinitions
+{
+    [Binding]
+    public class AsyncsequenceStepDefinitions
+    {
+        private ScenarioContext _scenarioContext;
+
+        public AsyncsequenceStepDefinitions(ScenarioContext scenarioContext) {
+            _scenarioContext = scenarioContext;
+        }
+        [Given(""a list to hold step numbers"")]
+        public async Task GivenAPlaceholder()
+        {
+            await Task.Run(() => global::Log.LogStep() );
+        }
+
+        [When(""Async Step {string} is called"")]
+        public async Task WhenStepIsTaken(string p0)
+        {
+            await Task.Run(() => global::Log.LogStep() );
+        }
+
+        [Then(""async step order should be {string}"")]
+        public async Task ThenStepSequenceIs( string p0)
+        {
+            await Task.Run( () => 
+            {   
+                global::Log.LogStep();
+            }
+            );
+        }
+    }
+}
+";
+        AddBindingClass(_asyncBindingClassContent);
+
+        ExecuteTests();
+        CheckAreStepsExecutedInOrder(new string[] { "GivenAPlaceholder", "WhenStepIsTaken", "ThenStepSequenceIs" });
+
+        ShouldAllScenariosPass();
+    }
+    //test hooks: before/after test run (require special handling by test frameworks)
+    //TODO: Consider adding a AddHookBinding method to SystemTestBase 
+    [TestMethod]
+    public void BeforeAndAfterTestHooksRun()
+    {
+        AddScenario(
+    """
+            Scenario: Sample Scenario
+                When something happens
+
+            Scenario Outline: Scenario outline with examples
+            When something happens to <person>
+            Examples:
+            	| person |
+            	| me     |
+            	| you    |
             """);
-        _projectsDriver.AddPassingStepBinding();
+
+
+        AddPassingStepBinding();
+
+        _projectsDriver.AddHookBinding("BeforeTestRun", "BeforeTestRun", null, null, null, "global::Log.LogHook();");
+        _projectsDriver.AddHookBinding("AfterTestRun", "AfterTestRun", null, null, null, "global::Log.LogHook();");
 
         ExecuteTests();
 
+        _hookDriver?.CheckIsHookExecutedInOrder(new string[] { "BeforeTestRun", "AfterTestRun" });
         ShouldAllScenariosPass();
+
+    }
+
+    //test hooks: before/after test feature & scenario (require special handling by test frameworks)
+    [TestMethod]
+    public void BeforeAndAfterFeatureAndScenarioHooksRun()
+    {
+        AddScenario(
+    """
+            Scenario: Sample Scenario
+                When something happens
+
+            Scenario Outline: Scenario outline with examples
+            When something happens to <person>
+            Examples:
+            	| person |
+            	| me     |
+            	| you    |
+            """);
+
+
+        AddPassingStepBinding();
+
+        _projectsDriver.AddHookBinding("BeforeFeature", "BeforeFeatureRun", null, null, null, "global::Log.LogHook();");
+        _projectsDriver.AddHookBinding("AfterFeature", "AfterFeatureRun", null, null, null, "global::Log.LogHook();");
+        _projectsDriver.AddHookBinding("BeforeScenario", "BeforeSenarioRun", null, null, null, "global::Log.LogHook();");
+        _projectsDriver.AddHookBinding("AfterScenario", "AfterScenarioRun", null, null, null, "global::Log.LogHook();");
+
+        ExecuteTests();
+
+        _hookDriver.CheckIsHookExecutedInOrder(new string[] { "BeforeFeatureRun", "BeforeSenarioRun", "AfterScenarioRun", "BeforeSenarioRun", "AfterScenarioRun", "BeforeSenarioRun", "AfterScenarioRun", "AfterFeatureRun" });
+        ShouldAllScenariosPass();
+
     }
 
-    //TODO: test different outcomes: success, failure, pending, undefined, ignored (scenario & scenario outline)
-    //TODO: test async steps (async steps are executed in order)
-    //TODO: test hooks: before/after test run (require special handling by test frameworks)
-    //TODO: test hooks: before/after test feature & scenario (require special handling by test frameworks)
     //TODO: test scenario outlines (nr of examples, params are available in ScenarioContext, allowRowTests=false, examples tags)
     //TODO: test parallel execution (details TBD) - maybe this should be in a separate test class
+
+
 }

Tests/Reqnroll.SystemTests/SystemTestBase.cs

@@ -1,4 +1,7 @@
 using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
 using System.Text.RegularExpressions;
 using FluentAssertions;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
@@ -23,6 +26,7 @@ public abstract class SystemTestBase
     protected CompilationDriver _compilationDriver = null!;
 
     protected int _preparedTests = 0;
+    private TestProjectFolders _testProjectFolders = null!;
 
     public TestContext TestContext { get; set; } = null!;
 
@@ -55,6 +59,8 @@ protected virtual void TestInitialize()
         _executionDriver = _testContainer.Resolve<ExecutionDriver>();
         _vsTestExecutionDriver = _testContainer.Resolve<VSTestExecutionDriver>();
         _compilationDriver = _testContainer.Resolve<CompilationDriver>();
+        _testProjectFolders = _testContainer.Resolve<TestProjectFolders>();
+
     }
 
     protected void AddFeatureFileFromResource(string fileName, int? preparedTests = null)
@@ -137,13 +143,89 @@ protected void ExecuteTests()
 
     protected void ShouldAllScenariosPass(int? expectedNrOfTestsSpec = null)
     {
-        if (expectedNrOfTestsSpec == null && _preparedTests == 0) 
+        int expectedNrOfTests = ConfirmAllTestsRan(expectedNrOfTestsSpec);
+        _vsTestExecutionDriver.LastTestExecutionResult.Succeeded.Should().Be(expectedNrOfTests, "all tests should pass");
+        _folderCleaner.CleanSolutionFolder();
+    }
+
+    protected void ShouldAllScenariosFail(int? expectedNrOfTestsSpec = null)
+    {
+        int expectedNrOfTests = ConfirmAllTestsRan(expectedNrOfTestsSpec);
+        _vsTestExecutionDriver.LastTestExecutionResult.Failed.Should().Be(expectedNrOfTests, "all tests should fail");
+        _folderCleaner.CleanSolutionFolder();
+
+    }
+
+    protected void ShouldAllScenariosPend(int? expectedNrOfTestsSpec = null)
+    {
+        int expectedNrOfTests = ConfirmAllTestsRan(expectedNrOfTestsSpec);
+        _vsTestExecutionDriver.LastTestExecutionResult.Pending.Should().Be(expectedNrOfTests, "all tests should Pend");
+        _folderCleaner.CleanSolutionFolder();
+    }
+
+    protected void ShouldAllScenariosBeIgnored(int? expectedNrOfTestsSpec = null)
+    {
+        int expectedNrOfTests = ConfirmAllTestsRan(expectedNrOfTestsSpec);
+        _vsTestExecutionDriver.LastTestExecutionResult.Ignored.Should().Be(expectedNrOfTests, "all tests should be Ignored");
+        _folderCleaner.CleanSolutionFolder();
+    }
+
+    protected void ShouldAllUndefinedScenariosNotBeExecuted(int? expectedNrOfTestsSpec = null)
+    {
+        int expectedNrOfTests = ConfirmAllTestsRan(expectedNrOfTestsSpec);
+        _vsTestExecutionDriver.LastTestExecutionResult.Succeeded.Should().Be(0, "none of the tests should pass");
+        _vsTestExecutionDriver.LastTestExecutionResult.Failed.Should().Be(0, "none of the tests should fail");
+
+        // MSTest treats tests with undefined steps as Ignored, the other two frameworks treat them as Pending
+        int pendingExpected = _testRunConfiguration.UnitTestProvider == UnitTestProvider.MSTest ? 0 : expectedNrOfTests;
+        _vsTestExecutionDriver.LastTestExecutionResult.Pending.Should().Be(pendingExpected, "All of the tests should Pend");
+
+        int IgnoredExpected = _testRunConfiguration.UnitTestProvider == UnitTestProvider.MSTest ? expectedNrOfTests : 0;
+        _vsTestExecutionDriver.LastTestExecutionResult.Ignored.Should().Be(IgnoredExpected, "None of the tests should be Ignored");
+
+
+        _folderCleaner.CleanSolutionFolder();
+    }
+
+    protected int ConfirmAllTestsRan(int? expectedNrOfTestsSpec)
+    {
+        if (expectedNrOfTestsSpec == null && _preparedTests == 0)
             throw new ArgumentException($"If {nameof(_preparedTests)} is not set, the {nameof(expectedNrOfTestsSpec)} is mandatory.", nameof(expectedNrOfTestsSpec));
         var expectedNrOfTests = expectedNrOfTestsSpec ?? _preparedTests;
 
         _vsTestExecutionDriver.LastTestExecutionResult.Should().NotBeNull();
         _vsTestExecutionDriver.LastTestExecutionResult.Total.Should().Be(expectedNrOfTests, $"the run should contain {expectedNrOfTests} tests");
-        _vsTestExecutionDriver.LastTestExecutionResult.Succeeded.Should().Be(expectedNrOfTests, "all tests should pass");
-        _folderCleaner.CleanSolutionFolder();
+        return expectedNrOfTests;
+    }
+
+    protected void AddPassingStepBinding(string scenarioBlock = "StepDefinition", string stepRegex = ".*")
+    {
+        _projectsDriver.AddPassingStepBinding(scenarioBlock, stepRegex);
+    }
+
+    protected void AddFailingStepBinding(string scenarioBlock = "StepDefinition", string stepRegex = ".*")
+    {
+        _projectsDriver.AddFailingStepBinding(scenarioBlock, stepRegex);
+    }
+
+    protected void AddPendingStepBinding(string scenarioBlock = "StepDefinition", string stepRegex = ".*")
+    {
+        _projectsDriver.AddStepBinding(scenarioBlock, stepRegex, "throw new PendingStepException();", "ScenarioContext.Current.Pending()");
+    }
+
+    protected void AddBindingClass(string content)
+    {
+        _projectsDriver.AddBindingClass(content);
+    }
+
+    //TODO: Consider moving this to the TestProjectGenerator as a driver method
+    public void CheckAreStepsExecutedInOrder(IEnumerable<string> methodNames)
+    {
+        _testProjectFolders.PathToSolutionDirectory.Should().NotBeNullOrWhiteSpace();
+
+        var pathToLogFile = Path.Combine(_testProjectFolders.PathToSolutionDirectory, "steps.log");
+        var lines = File.ReadAllLines(pathToLogFile);
+        var methodNameLines = methodNames.Select(m => $"-> step: {m}");
+        lines.Should().ContainInOrder(methodNameLines);
     }
 }