diff --git a/fmt_output_test.go b/fmt_output_test.go index c9f53fd1..4d7e8352 100644 --- a/fmt_output_test.go +++ b/fmt_output_test.go @@ -67,6 +67,13 @@ func fmtOutputTest(fmtName, testName, featureFilePath string) func(*testing.T) { s.Step(`^odd (\d+) and even (\d+) number$`, oddEvenStepDef) } + fmtOutputScenarioInitializer := func(ctx *godog.ScenarioContext) { + ctx.Step(`^(?:a )?failing step`, failingStepDef) + ctx.Step(`^(?:a )?pending step$`, pendingStepDef) + ctx.Step(`^(?:a )?passing step$`, passingStepDef) + ctx.Step(`^odd (\d+) and even (\d+) number$`, oddEvenStepDef) + } + return func(t *testing.T) { expectOutputPath := strings.Replace(featureFilePath, "features", fmtName, 1) expectOutputPath = strings.TrimSuffix(expectOutputPath, path.Ext(expectOutputPath)) @@ -91,6 +98,18 @@ func fmtOutputTest(fmtName, testName, featureFilePath string) func(*testing.T) { expected := string(expectedOutput) actual := buf.String() assert.Equalf(t, expected, actual, "path: %s", expectOutputPath) + + buf.Reset() + + godog.TestSuite{ + Name: fmtName, + ScenarioInitializer: fmtOutputScenarioInitializer, + Options: &opts, + }.Run() + + expected = string(expectedOutput) + actual = buf.String() + assert.Equalf(t, expected, actual, "path: %s", expectOutputPath) } } diff --git a/formatter-tests/cucumber/scenario_outline b/formatter-tests/cucumber/scenario_outline index 54233b44..295188c4 100644 --- a/formatter-tests/cucumber/scenario_outline +++ b/formatter-tests/cucumber/scenario_outline @@ -48,7 +48,7 @@ "name": "passing step", "line": 13, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -60,7 +60,7 @@ "name": "passing step", "line": 13, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -72,7 +72,7 @@ "name": "odd 1 and even 2 number", "line": 13, "match": { - "location": "fmt_output_test.go:99" + "location": "fmt_output_test.go:118" }, "result": { "status": "passed", @@ -112,7 +112,7 @@ "name": "passing step", "line": 14, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -124,7 +124,7 @@ "name": "passing step", "line": 14, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -136,7 +136,7 @@ "name": "odd 2 and even 0 number", "line": 14, "match": { - "location": "fmt_output_test.go:99" + "location": "fmt_output_test.go:118" }, "result": { "status": "failed", @@ -177,7 +177,7 @@ "name": "passing step", "line": 15, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -189,7 +189,7 @@ "name": "passing step", "line": 15, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -201,7 +201,7 @@ "name": "odd 3 and even 11 number", "line": 15, "match": { - "location": "fmt_output_test.go:99" + "location": "fmt_output_test.go:118" }, "result": { "status": "failed", @@ -242,7 +242,7 @@ "name": "passing step", "line": 20, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -254,7 +254,7 @@ "name": "passing step", "line": 20, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -266,7 +266,7 @@ "name": "odd 1 and even 14 number", "line": 20, "match": { - "location": "fmt_output_test.go:99" + "location": "fmt_output_test.go:118" }, "result": { "status": "passed", @@ -306,7 +306,7 @@ "name": "passing step", "line": 21, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -318,7 +318,7 @@ "name": "passing step", "line": 21, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -330,7 +330,7 @@ "name": "odd 3 and even 9 number", "line": 21, "match": { - "location": "fmt_output_test.go:99" + "location": "fmt_output_test.go:118" }, "result": { "status": "failed", diff --git a/formatter-tests/cucumber/scenario_with_background b/formatter-tests/cucumber/scenario_with_background index 3fc277e3..f5362ef5 100644 --- a/formatter-tests/cucumber/scenario_with_background +++ b/formatter-tests/cucumber/scenario_with_background @@ -20,7 +20,7 @@ "name": "passing step", "line": 4, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -32,7 +32,7 @@ "name": "passing step", "line": 5, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -44,7 +44,7 @@ "name": "passing step", "line": 8, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -56,7 +56,7 @@ "name": "passing step", "line": 9, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", diff --git a/formatter-tests/cucumber/single_scenario_with_passing_step b/formatter-tests/cucumber/single_scenario_with_passing_step index 78f527f8..f1a676e8 100644 --- a/formatter-tests/cucumber/single_scenario_with_passing_step +++ b/formatter-tests/cucumber/single_scenario_with_passing_step @@ -20,7 +20,7 @@ "name": "a passing step", "line": 7, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", diff --git a/formatter-tests/cucumber/some_scenarions_including_failing b/formatter-tests/cucumber/some_scenarions_including_failing index 3a5e4c7d..02544281 100644 --- a/formatter-tests/cucumber/some_scenarions_including_failing +++ b/formatter-tests/cucumber/some_scenarions_including_failing @@ -20,7 +20,7 @@ "name": "passing step", "line": 4, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -32,7 +32,7 @@ "name": "failing step", "line": 5, "match": { - "location": "fmt_output_test.go:113" + "location": "fmt_output_test.go:132" }, "result": { "status": "failed", @@ -45,7 +45,7 @@ "name": "passing step", "line": 6, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "skipped" @@ -77,7 +77,7 @@ "name": "passing step", "line": 10, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "skipped" @@ -109,7 +109,7 @@ "name": "passing step", "line": 14, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "skipped" diff --git a/formatter-tests/cucumber/two_scenarios_with_background_fail b/formatter-tests/cucumber/two_scenarios_with_background_fail index 1605b8e6..52b99f80 100644 --- a/formatter-tests/cucumber/two_scenarios_with_background_fail +++ b/formatter-tests/cucumber/two_scenarios_with_background_fail @@ -20,7 +20,7 @@ "name": "passing step", "line": 4, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -32,7 +32,7 @@ "name": "failing step", "line": 5, "match": { - "location": "fmt_output_test.go:113" + "location": "fmt_output_test.go:132" }, "result": { "status": "failed", @@ -45,7 +45,7 @@ "name": "passing step", "line": 8, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "skipped" @@ -56,7 +56,7 @@ "name": "passing step", "line": 9, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "skipped" @@ -77,7 +77,7 @@ "name": "passing step", "line": 4, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "passed", @@ -89,7 +89,7 @@ "name": "failing step", "line": 5, "match": { - "location": "fmt_output_test.go:113" + "location": "fmt_output_test.go:132" }, "result": { "status": "failed", @@ -102,7 +102,7 @@ "name": "passing step", "line": 12, "match": { - "location": "fmt_output_test.go:97" + "location": "fmt_output_test.go:116" }, "result": { "status": "skipped" diff --git a/formatter-tests/events/scenario_outline b/formatter-tests/events/scenario_outline index f040ac36..54271a6c 100644 --- a/formatter-tests/events/scenario_outline +++ b/formatter-tests/events/scenario_outline @@ -1,57 +1,57 @@ {"event":"TestRunStarted","version":"0.1.0","timestamp":-6795364578871,"suite":"events"} {"event":"TestSource","location":"formatter-tests/features/scenario_outline.feature:2","source":"@outline @tag\nFeature: outline\n\n @scenario\n Scenario Outline: outline\n Given passing step\n When passing step\n Then odd \u003codd\u003e and even \u003ceven\u003e number\n\n @tagged\n Examples: tagged\n | odd | even |\n | 1 | 2 |\n | 2 | 0 |\n | 3 | 11 |\n\n @tag2\n Examples:\n | odd | even |\n | 1 | 14 |\n | 3 | 9 |\n"} {"event":"TestCaseStarted","location":"formatter-tests/features/scenario_outline.feature:13","timestamp":-6795364578871} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:6","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:6","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:6","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:6","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:7","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:7","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:7","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:7","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:8","definition_id":"fmt_output_test.go:99 -\u003e github.com/cucumber/godog_test.oddEvenStepDef","arguments":[[4,5],[5,15]]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:8","definition_id":"fmt_output_test.go:118 -\u003e github.com/cucumber/godog_test.oddEvenStepDef","arguments":[[4,5],[5,15]]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:8","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:8","timestamp":-6795364578871,"status":"passed"} {"event":"TestCaseFinished","location":"formatter-tests/features/scenario_outline.feature:13","timestamp":-6795364578871,"status":"passed"} {"event":"TestCaseStarted","location":"formatter-tests/features/scenario_outline.feature:14","timestamp":-6795364578871} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:6","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:6","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:6","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:6","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:7","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:7","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:7","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:7","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:8","definition_id":"fmt_output_test.go:99 -\u003e github.com/cucumber/godog_test.oddEvenStepDef","arguments":[[4,5],[5,15]]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:8","definition_id":"fmt_output_test.go:118 -\u003e github.com/cucumber/godog_test.oddEvenStepDef","arguments":[[4,5],[5,15]]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:8","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:8","timestamp":-6795364578871,"status":"failed","summary":"2 is not odd"} {"event":"TestCaseFinished","location":"formatter-tests/features/scenario_outline.feature:14","timestamp":-6795364578871,"status":"failed"} {"event":"TestCaseStarted","location":"formatter-tests/features/scenario_outline.feature:15","timestamp":-6795364578871} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:6","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:6","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:6","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:6","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:7","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:7","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:7","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:7","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:8","definition_id":"fmt_output_test.go:99 -\u003e github.com/cucumber/godog_test.oddEvenStepDef","arguments":[[4,5],[5,15]]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:8","definition_id":"fmt_output_test.go:118 -\u003e github.com/cucumber/godog_test.oddEvenStepDef","arguments":[[4,5],[5,15]]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:8","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:8","timestamp":-6795364578871,"status":"failed","summary":"11 is not even"} {"event":"TestCaseFinished","location":"formatter-tests/features/scenario_outline.feature:15","timestamp":-6795364578871,"status":"failed"} {"event":"TestCaseStarted","location":"formatter-tests/features/scenario_outline.feature:20","timestamp":-6795364578871} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:6","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:6","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:6","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:6","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:7","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:7","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:7","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:7","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:8","definition_id":"fmt_output_test.go:99 -\u003e github.com/cucumber/godog_test.oddEvenStepDef","arguments":[[4,5],[5,15]]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:8","definition_id":"fmt_output_test.go:118 -\u003e github.com/cucumber/godog_test.oddEvenStepDef","arguments":[[4,5],[5,15]]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:8","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:8","timestamp":-6795364578871,"status":"passed"} {"event":"TestCaseFinished","location":"formatter-tests/features/scenario_outline.feature:20","timestamp":-6795364578871,"status":"passed"} {"event":"TestCaseStarted","location":"formatter-tests/features/scenario_outline.feature:21","timestamp":-6795364578871} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:6","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:6","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:6","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:6","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:7","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:7","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:7","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:7","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:8","definition_id":"fmt_output_test.go:99 -\u003e github.com/cucumber/godog_test.oddEvenStepDef","arguments":[[4,5],[5,15]]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_outline.feature:8","definition_id":"fmt_output_test.go:118 -\u003e github.com/cucumber/godog_test.oddEvenStepDef","arguments":[[4,5],[5,15]]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_outline.feature:8","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_outline.feature:8","timestamp":-6795364578871,"status":"failed","summary":"9 is not even"} {"event":"TestCaseFinished","location":"formatter-tests/features/scenario_outline.feature:21","timestamp":-6795364578871,"status":"failed"} diff --git a/formatter-tests/events/scenario_with_background b/formatter-tests/events/scenario_with_background index a48ea683..815b88d6 100644 --- a/formatter-tests/events/scenario_with_background +++ b/formatter-tests/events/scenario_with_background @@ -1,16 +1,16 @@ {"event":"TestRunStarted","version":"0.1.0","timestamp":-6795364578871,"suite":"events"} {"event":"TestSource","location":"formatter-tests/features/scenario_with_background.feature:1","source":"Feature: single scenario with background\n\n Background: named\n Given passing step\n And passing step\n\n Scenario: scenario\n When passing step\n Then passing step\n"} {"event":"TestCaseStarted","location":"formatter-tests/features/scenario_with_background.feature:7","timestamp":-6795364578871} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_with_background.feature:4","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_with_background.feature:4","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_with_background.feature:4","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_with_background.feature:4","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_with_background.feature:5","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_with_background.feature:5","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_with_background.feature:5","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_with_background.feature:5","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_with_background.feature:8","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_with_background.feature:8","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_with_background.feature:8","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_with_background.feature:8","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_with_background.feature:9","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/scenario_with_background.feature:9","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/scenario_with_background.feature:9","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/scenario_with_background.feature:9","timestamp":-6795364578871,"status":"passed"} {"event":"TestCaseFinished","location":"formatter-tests/features/scenario_with_background.feature:7","timestamp":-6795364578871,"status":"passed"} diff --git a/formatter-tests/events/single_scenario_with_passing_step b/formatter-tests/events/single_scenario_with_passing_step index b9afac69..db1b5789 100644 --- a/formatter-tests/events/single_scenario_with_passing_step +++ b/formatter-tests/events/single_scenario_with_passing_step @@ -1,7 +1,7 @@ {"event":"TestRunStarted","version":"0.1.0","timestamp":-6795364578871,"suite":"events"} {"event":"TestSource","location":"formatter-tests/features/single_scenario_with_passing_step.feature:1","source":"Feature: single passing scenario\n describes\n a single scenario\n feature\n\n Scenario: one step passing\n Given a passing step\n"} {"event":"TestCaseStarted","location":"formatter-tests/features/single_scenario_with_passing_step.feature:6","timestamp":-6795364578871} -{"event":"StepDefinitionFound","location":"formatter-tests/features/single_scenario_with_passing_step.feature:7","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/single_scenario_with_passing_step.feature:7","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/single_scenario_with_passing_step.feature:7","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/single_scenario_with_passing_step.feature:7","timestamp":-6795364578871,"status":"passed"} {"event":"TestCaseFinished","location":"formatter-tests/features/single_scenario_with_passing_step.feature:6","timestamp":-6795364578871,"status":"passed"} diff --git a/formatter-tests/events/some_scenarions_including_failing b/formatter-tests/events/some_scenarions_including_failing index 2a37f6de..e159e225 100644 --- a/formatter-tests/events/some_scenarions_including_failing +++ b/formatter-tests/events/some_scenarions_including_failing @@ -1,28 +1,28 @@ {"event":"TestRunStarted","version":"0.1.0","timestamp":-6795364578871,"suite":"events"} {"event":"TestSource","location":"formatter-tests/features/some_scenarions_including_failing.feature:1","source":"Feature: some scenarios\n\n Scenario: failing\n Given passing step\n When failing step\n Then passing step\n\n Scenario: pending\n When pending step\n Then passing step\n\n Scenario: undefined\n When undefined\n Then passing step\n"} {"event":"TestCaseStarted","location":"formatter-tests/features/some_scenarions_including_failing.feature:3","timestamp":-6795364578871} -{"event":"StepDefinitionFound","location":"formatter-tests/features/some_scenarions_including_failing.feature:4","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/some_scenarions_including_failing.feature:4","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/some_scenarions_including_failing.feature:4","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/some_scenarions_including_failing.feature:4","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/some_scenarions_including_failing.feature:5","definition_id":"fmt_output_test.go:113 -\u003e github.com/cucumber/godog_test.failingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/some_scenarions_including_failing.feature:5","definition_id":"fmt_output_test.go:132 -\u003e github.com/cucumber/godog_test.failingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/some_scenarions_including_failing.feature:5","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/some_scenarions_including_failing.feature:5","timestamp":-6795364578871,"status":"failed","summary":"step failed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/some_scenarions_including_failing.feature:6","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/some_scenarions_including_failing.feature:6","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/some_scenarions_including_failing.feature:6","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/some_scenarions_including_failing.feature:6","timestamp":-6795364578871,"status":"skipped"} {"event":"TestCaseFinished","location":"formatter-tests/features/some_scenarions_including_failing.feature:3","timestamp":-6795364578871,"status":"failed"} {"event":"TestCaseStarted","location":"formatter-tests/features/some_scenarions_including_failing.feature:8","timestamp":-6795364578871} -{"event":"StepDefinitionFound","location":"formatter-tests/features/some_scenarions_including_failing.feature:9","definition_id":"fmt_output_test.go:111 -\u003e github.com/cucumber/godog_test.pendingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/some_scenarions_including_failing.feature:9","definition_id":"fmt_output_test.go:130 -\u003e github.com/cucumber/godog_test.pendingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/some_scenarions_including_failing.feature:9","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/some_scenarions_including_failing.feature:9","timestamp":-6795364578871,"status":"pending"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/some_scenarions_including_failing.feature:10","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/some_scenarions_including_failing.feature:10","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/some_scenarions_including_failing.feature:10","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/some_scenarions_including_failing.feature:10","timestamp":-6795364578871,"status":"skipped"} {"event":"TestCaseFinished","location":"formatter-tests/features/some_scenarions_including_failing.feature:8","timestamp":-6795364578871,"status":"pending"} {"event":"TestCaseStarted","location":"formatter-tests/features/some_scenarions_including_failing.feature:12","timestamp":-6795364578871} {"event":"TestStepStarted","location":"formatter-tests/features/some_scenarions_including_failing.feature:13","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/some_scenarions_including_failing.feature:13","timestamp":-6795364578871,"status":"undefined"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/some_scenarions_including_failing.feature:14","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/some_scenarions_including_failing.feature:14","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/some_scenarions_including_failing.feature:14","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/some_scenarions_including_failing.feature:14","timestamp":-6795364578871,"status":"skipped"} {"event":"TestCaseFinished","location":"formatter-tests/features/some_scenarions_including_failing.feature:12","timestamp":-6795364578871,"status":"undefined"} diff --git a/formatter-tests/events/two_scenarios_with_background_fail b/formatter-tests/events/two_scenarios_with_background_fail index 2020e19b..8973ac9c 100644 --- a/formatter-tests/events/two_scenarios_with_background_fail +++ b/formatter-tests/events/two_scenarios_with_background_fail @@ -1,27 +1,27 @@ {"event":"TestRunStarted","version":"0.1.0","timestamp":-6795364578871,"suite":"events"} {"event":"TestSource","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:1","source":"Feature: two scenarios with background fail\n\n Background:\n Given passing step\n And failing step\n\n Scenario: one\n When passing step\n Then passing step\n\n Scenario: two\n Then passing step\n"} {"event":"TestCaseStarted","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:7","timestamp":-6795364578871} -{"event":"StepDefinitionFound","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:4","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:4","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:4","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:4","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:5","definition_id":"fmt_output_test.go:113 -\u003e github.com/cucumber/godog_test.failingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:5","definition_id":"fmt_output_test.go:132 -\u003e github.com/cucumber/godog_test.failingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:5","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:5","timestamp":-6795364578871,"status":"failed","summary":"step failed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:8","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:8","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:8","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:8","timestamp":-6795364578871,"status":"skipped"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:9","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:9","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:9","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:9","timestamp":-6795364578871,"status":"skipped"} {"event":"TestCaseFinished","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:7","timestamp":-6795364578871,"status":"failed"} {"event":"TestCaseStarted","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:11","timestamp":-6795364578871} -{"event":"StepDefinitionFound","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:4","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:4","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:4","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:4","timestamp":-6795364578871,"status":"passed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:5","definition_id":"fmt_output_test.go:113 -\u003e github.com/cucumber/godog_test.failingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:5","definition_id":"fmt_output_test.go:132 -\u003e github.com/cucumber/godog_test.failingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:5","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:5","timestamp":-6795364578871,"status":"failed","summary":"step failed"} -{"event":"StepDefinitionFound","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:12","definition_id":"fmt_output_test.go:97 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} +{"event":"StepDefinitionFound","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:12","definition_id":"fmt_output_test.go:116 -\u003e github.com/cucumber/godog_test.passingStepDef","arguments":[]} {"event":"TestStepStarted","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:12","timestamp":-6795364578871} {"event":"TestStepFinished","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:12","timestamp":-6795364578871,"status":"skipped"} {"event":"TestCaseFinished","location":"formatter-tests/features/two_scenarios_with_background_fail.feature:11","timestamp":-6795364578871,"status":"failed"} diff --git a/formatter-tests/pretty/scenario_outline b/formatter-tests/pretty/scenario_outline index e2a0ee64..f66bd5de 100644 --- a/formatter-tests/pretty/scenario_outline +++ b/formatter-tests/pretty/scenario_outline @@ -1,9 +1,9 @@ Feature: outline Scenario Outline: outline # formatter-tests/features/scenario_outline.feature:5 - Given passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef - When passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef - Then odd and even number # fmt_output_test.go:99 -> github.com/cucumber/godog_test.oddEvenStepDef + Given passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef + When passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef + Then odd and even number # fmt_output_test.go:118 -> github.com/cucumber/godog_test.oddEvenStepDef Examples: tagged | odd | even | diff --git a/formatter-tests/pretty/scenario_with_background b/formatter-tests/pretty/scenario_with_background index 660d24bc..f65cc06a 100644 --- a/formatter-tests/pretty/scenario_with_background +++ b/formatter-tests/pretty/scenario_with_background @@ -1,12 +1,12 @@ Feature: single scenario with background Background: named - Given passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef - And passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef + Given passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef + And passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef Scenario: scenario # formatter-tests/features/scenario_with_background.feature:7 - When passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef - Then passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef + When passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef + Then passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef 1 scenarios (1 passed) 4 steps (4 passed) diff --git a/formatter-tests/pretty/single_scenario_with_passing_step b/formatter-tests/pretty/single_scenario_with_passing_step index 33d3c0f9..a21eca32 100644 --- a/formatter-tests/pretty/single_scenario_with_passing_step +++ b/formatter-tests/pretty/single_scenario_with_passing_step @@ -4,7 +4,7 @@ feature Scenario: one step passing # formatter-tests/features/single_scenario_with_passing_step.feature:6 - Given a passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef + Given a passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef 1 scenarios (1 passed) 1 steps (1 passed) diff --git a/formatter-tests/pretty/some_scenarions_including_failing b/formatter-tests/pretty/some_scenarions_including_failing index 4c0e88f7..1a634ca2 100644 --- a/formatter-tests/pretty/some_scenarions_including_failing +++ b/formatter-tests/pretty/some_scenarions_including_failing @@ -1,19 +1,19 @@ Feature: some scenarios Scenario: failing # formatter-tests/features/some_scenarions_including_failing.feature:3 - Given passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef - When failing step # fmt_output_test.go:113 -> github.com/cucumber/godog_test.failingStepDef + Given passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef + When failing step # fmt_output_test.go:132 -> github.com/cucumber/godog_test.failingStepDef step failed - Then passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef + Then passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef Scenario: pending # formatter-tests/features/some_scenarions_including_failing.feature:8 - When pending step # fmt_output_test.go:111 -> github.com/cucumber/godog_test.pendingStepDef + When pending step # fmt_output_test.go:130 -> github.com/cucumber/godog_test.pendingStepDef TODO: write pending definition - Then passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef + Then passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef Scenario: undefined # formatter-tests/features/some_scenarions_including_failing.feature:12 When undefined - Then passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef + Then passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef --- Failed steps: diff --git a/formatter-tests/pretty/two_scenarios_with_background_fail b/formatter-tests/pretty/two_scenarios_with_background_fail index be9f7c30..4f9959ce 100644 --- a/formatter-tests/pretty/two_scenarios_with_background_fail +++ b/formatter-tests/pretty/two_scenarios_with_background_fail @@ -1,16 +1,16 @@ Feature: two scenarios with background fail Background: - Given passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef - And failing step # fmt_output_test.go:113 -> github.com/cucumber/godog_test.failingStepDef + Given passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef + And failing step # fmt_output_test.go:132 -> github.com/cucumber/godog_test.failingStepDef step failed Scenario: one # formatter-tests/features/two_scenarios_with_background_fail.feature:7 - When passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef - Then passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef + When passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef + Then passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef Scenario: two # formatter-tests/features/two_scenarios_with_background_fail.feature:11 - Then passing step # fmt_output_test.go:97 -> github.com/cucumber/godog_test.passingStepDef + Then passing step # fmt_output_test.go:116 -> github.com/cucumber/godog_test.passingStepDef --- Failed steps: diff --git a/run_test.go b/run_test.go index abf485d1..e458dc3d 100644 --- a/run_test.go +++ b/run_test.go @@ -230,7 +230,7 @@ func TestFeatureFilePathParser(t *testing.T) { } } -func TestAllFeaturesRun(t *testing.T) { +func Test_AllFeaturesRun(t *testing.T) { const concurrency = 10 const format = "progress" @@ -250,6 +250,11 @@ func TestAllFeaturesRun(t *testing.T) { assert.Equal(t, exitSuccess, actualStatus) assert.Equal(t, expected, actualOutput) + + actualStatus, actualOutput = testRun(t, format, concurrency, []string{"features"}) + + assert.Equal(t, exitSuccess, actualStatus) + assert.Equal(t, expected, actualOutput) } func TestFormatterConcurrencyRun(t *testing.T) { @@ -269,11 +274,17 @@ func TestFormatterConcurrencyRun(t *testing.T) { t.Run( fmt.Sprintf("%s/concurrency/%d", formatter, concurrency), func(t *testing.T) { - singleThreadStatus, singleThreadOutput := testRunWithOptions(t, formatter, 1, featurePaths) - status, actual := testRunWithOptions(t, formatter, concurrency, featurePaths) + expectedStatus, expectedOutput := testRunWithOptions(t, formatter, 1, featurePaths) + actualStatus, actualOutput := testRunWithOptions(t, formatter, concurrency, featurePaths) - assert.Equal(t, singleThreadStatus, status) - assertConcurrencyOutput(t, formatter, singleThreadOutput, actual) + assert.Equal(t, expectedStatus, actualStatus) + assertOutput(t, formatter, expectedOutput, actualOutput) + + expectedStatus, expectedOutput = testRun(t, formatter, 1, featurePaths) + actualStatus, actualOutput = testRun(t, formatter, concurrency, featurePaths) + + assert.Equal(t, expectedStatus, actualStatus) + assertOutput(t, formatter, expectedOutput, actualOutput) }, ) } @@ -282,7 +293,26 @@ func TestFormatterConcurrencyRun(t *testing.T) { func testRunWithOptions(t *testing.T, format string, concurrency int, featurePaths []string) (int, string) { output := new(bytes.Buffer) - opt := Options{ + opts := Options{ + Format: format, + NoColors: true, + Paths: featurePaths, + Concurrency: concurrency, + Output: output, + } + + status := RunWithOptions("succeed", func(s *Suite) { SuiteContext(s) }, opts) + + actual, err := ioutil.ReadAll(output) + require.NoError(t, err) + + return status, string(actual) +} + +func testRun(t *testing.T, format string, concurrency int, featurePaths []string) (int, string) { + output := new(bytes.Buffer) + + opts := Options{ Format: format, NoColors: true, Paths: featurePaths, @@ -290,7 +320,11 @@ func testRunWithOptions(t *testing.T, format string, concurrency int, featurePat Output: output, } - status := RunWithOptions("succeed", func(s *Suite) { SuiteContext(s) }, opt) + status := TestSuite{ + Name: "succeed", + ScenarioInitializer: InitializeScenario, + Options: &opts, + }.Run() actual, err := ioutil.ReadAll(output) require.NoError(t, err) @@ -298,7 +332,7 @@ func testRunWithOptions(t *testing.T, format string, concurrency int, featurePat return status, string(actual) } -func assertConcurrencyOutput(t *testing.T, formatter string, expected, actual string) { +func assertOutput(t *testing.T, formatter string, expected, actual string) { switch formatter { case "cucumber", "junit", "pretty", "events": expectedRows := strings.Split(expected, "\n") @@ -358,3 +392,42 @@ type progressOutput struct { noOfStepsPerRow []string bottomRows []string } + +func Test_AllFeaturesRun_v010(t *testing.T) { + const concurrency = 10 + const format = "progress" + + const expected = `...................................................................... 70 +...................................................................... 140 +...................................................................... 210 +...................................................................... 280 +.......................... 306 + + +79 scenarios (79 passed) +306 steps (306 passed) +0s +` + + output := new(bytes.Buffer) + + opts := Options{ + Format: format, + NoColors: true, + Paths: []string{"features"}, + Concurrency: concurrency, + Output: output, + } + + actualStatus := TestSuite{ + Name: "godogs", + ScenarioInitializer: InitializeScenario, + Options: &opts, + }.Run() + + actualOutput, err := ioutil.ReadAll(output) + require.NoError(t, err) + + assert.Equal(t, exitSuccess, actualStatus) + assert.Equal(t, expected, string(actualOutput)) +} diff --git a/suite_context_test.go b/suite_context_test.go new file mode 100644 index 00000000..a90826a6 --- /dev/null +++ b/suite_context_test.go @@ -0,0 +1,592 @@ +package godog + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "fmt" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/cucumber/gherkin-go/v11" + "github.com/cucumber/messages-go/v10" + "github.com/stretchr/testify/assert" + + "github.com/cucumber/godog/colors" +) + +func InitializeScenario(ctx *ScenarioContext) { + tc := &godogFeaturesScenario{} + + ctx.BeforeScenario(tc.ResetBeforeEachScenario) + + ctx.Step(`^(?:a )?feature path "([^"]*)"$`, tc.featurePath) + ctx.Step(`^I parse features$`, tc.parseFeatures) + ctx.Step(`^I'm listening to suite events$`, tc.iAmListeningToSuiteEvents) + ctx.Step(`^I run feature suite$`, tc.iRunFeatureSuite) + ctx.Step(`^I run feature suite with tags "([^"]*)"$`, tc.iRunFeatureSuiteWithTags) + ctx.Step(`^I run feature suite with formatter "([^"]*)"$`, tc.iRunFeatureSuiteWithFormatter) + ctx.Step(`^(?:I )(allow|disable) variable injection`, tc.iSetVariableInjectionTo) + ctx.Step(`^(?:a )?feature "([^"]*)"(?: file)?:$`, tc.aFeatureFile) + ctx.Step(`^the suite should have (passed|failed)$`, tc.theSuiteShouldHave) + + ctx.Step(`^I should have ([\d]+) features? files?:$`, tc.iShouldHaveNumFeatureFiles) + ctx.Step(`^I should have ([\d]+) scenarios? registered$`, tc.numScenariosRegistered) + ctx.Step(`^there (was|were) ([\d]+) "([^"]*)" events? fired$`, tc.thereWereNumEventsFired) + ctx.Step(`^there was event triggered before scenario "([^"]*)"$`, tc.thereWasEventTriggeredBeforeScenario) + ctx.Step(`^these events had to be fired for a number of times:$`, tc.theseEventsHadToBeFiredForNumberOfTimes) + + ctx.Step(`^(?:a )?failing step`, tc.aFailingStep) + ctx.Step(`^this step should fail`, tc.aFailingStep) + ctx.Step(`^the following steps? should be (passed|failed|skipped|undefined|pending):`, tc.followingStepsShouldHave) + ctx.Step(`^all steps should (?:be|have|have been) (passed|failed|skipped|undefined|pending)$`, tc.allStepsShouldHave) + ctx.Step(`^the undefined step snippets should be:$`, tc.theUndefinedStepSnippetsShouldBe) + + // event stream + ctx.Step(`^the following events should be fired:$`, tc.thereShouldBeEventsFired) + + // lt + ctx.Step(`^savybių aplankas "([^"]*)"$`, tc.featurePath) + ctx.Step(`^aš išskaitau savybes$`, tc.parseFeatures) + ctx.Step(`^aš turėčiau turėti ([\d]+) savybių failus:$`, tc.iShouldHaveNumFeatureFiles) + + ctx.Step(`^(?:a )?pending step$`, func() error { + return ErrPending + }) + ctx.Step(`^(?:a )?passing step$`, func() error { + return nil + }) + + // Introduced to test formatter/cucumber.feature + ctx.Step(`^the rendered json will be as follows:$`, tc.theRenderJSONWillBe) + + // Introduced to test formatter/pretty.feature + ctx.Step(`^the rendered output will be as follows:$`, tc.theRenderOutputWillBe) + + // Introduced to test formatter/junit.feature + ctx.Step(`^the rendered xml will be as follows:$`, tc.theRenderXMLWillBe) + + ctx.Step(`^(?:a )?failing multistep$`, func() Steps { + return Steps{"passing step", "failing step"} + }) + + ctx.Step(`^(?:a |an )?undefined multistep$`, func() Steps { + return Steps{"passing step", "undefined step", "passing step"} + }) + + ctx.Step(`^(?:a )?passing multistep$`, func() Steps { + return Steps{"passing step", "passing step", "passing step"} + }) + + ctx.Step(`^(?:a )?failing nested multistep$`, func() Steps { + return Steps{"passing step", "passing multistep", "failing multistep"} + }) + // Default recovery step + ctx.Step(`Ignore.*`, func() error { + return nil + }) + + ctx.BeforeStep(tc.inject) +} + +func (tc *godogFeaturesScenario) inject(step *Step) { + if !tc.allowInjection { + return + } + + step.Text = injectAll(step.Text) + + if table := step.Argument.GetDataTable(); table != nil { + for i := 0; i < len(table.Rows); i++ { + for n, cell := range table.Rows[i].Cells { + table.Rows[i].Cells[n].Value = injectAll(cell.Value) + } + } + } + + if doc := step.Argument.GetDocString(); doc != nil { + doc.Content = injectAll(doc.Content) + } +} + +type godogFeaturesScenario struct { + paths []string + testedSuite *Suite + events []*firedEvent + out bytes.Buffer + allowInjection bool +} + +func (tc *godogFeaturesScenario) ResetBeforeEachScenario(*Scenario) { + // reset whole suite with the state + tc.out.Reset() + tc.paths = []string{} + + tc.testedSuite = &Suite{scenarioInitializer: InitializeScenario} + + // reset all fired events + tc.events = []*firedEvent{} + tc.allowInjection = false +} + +func (tc *godogFeaturesScenario) iSetVariableInjectionTo(to string) error { + tc.allowInjection = to == "allow" + return nil +} + +func (tc *godogFeaturesScenario) iRunFeatureSuiteWithTags(tags string) error { + if err := tc.parseFeatures(); err != nil { + return err + } + + for _, feat := range tc.testedSuite.features { + applyTagFilter(tags, feat) + } + + tc.testedSuite.fmt = newBaseFmt("godog", &tc.out) + + tc.testedSuite.fmt.TestRunStarted() + tc.testedSuite.run() + tc.testedSuite.fmt.Summary() + + return nil +} + +func (tc *godogFeaturesScenario) iRunFeatureSuiteWithFormatter(name string) error { + f := FindFmt(name) + if f == nil { + return fmt.Errorf(`formatter "%s" is not available`, name) + } + + tc.testedSuite.fmt = f("godog", colors.Uncolored(&tc.out)) + + if err := tc.parseFeatures(); err != nil { + return err + } + + tc.testedSuite.fmt.TestRunStarted() + tc.testedSuite.run() + tc.testedSuite.fmt.Summary() + + return nil +} + +func (tc *godogFeaturesScenario) thereShouldBeEventsFired(doc *DocString) error { + actual := strings.Split(strings.TrimSpace(tc.out.String()), "\n") + expect := strings.Split(strings.TrimSpace(doc.Content), "\n") + + if len(expect) != len(actual) { + return fmt.Errorf("expected %d events, but got %d", len(expect), len(actual)) + } + + type ev struct { + Event string + } + + for i, event := range actual { + exp := strings.TrimSpace(expect[i]) + var act ev + + if err := json.Unmarshal([]byte(event), &act); err != nil { + return fmt.Errorf("failed to read event data: %v", err) + } + + if act.Event != exp { + return fmt.Errorf(`expected event: "%s" at position: %d, but actual was "%s"`, exp, i, act.Event) + } + } + + return nil +} + +func (tc *godogFeaturesScenario) cleanupSnippet(snip string) string { + lines := strings.Split(strings.TrimSpace(snip), "\n") + for i := 0; i < len(lines); i++ { + lines[i] = strings.TrimSpace(lines[i]) + } + + return strings.Join(lines, "\n") +} + +func (tc *godogFeaturesScenario) theUndefinedStepSnippetsShouldBe(body *DocString) error { + f, ok := tc.testedSuite.fmt.(*basefmt) + if !ok { + return fmt.Errorf("this step requires *basefmt, but there is: %T", tc.testedSuite.fmt) + } + + actual := tc.cleanupSnippet(f.snippets()) + expected := tc.cleanupSnippet(body.Content) + + if actual != expected { + return fmt.Errorf("snippets do not match actual: %s", f.snippets()) + } + + return nil +} + +func (tc *godogFeaturesScenario) followingStepsShouldHave(status string, steps *DocString) error { + var expected = strings.Split(steps.Content, "\n") + var actual, unmatched, matched []string + + f, ok := tc.testedSuite.fmt.(*basefmt) + if !ok { + return fmt.Errorf("this step requires *basefmt, but there is: %T", tc.testedSuite.fmt) + } + + switch status { + case "passed": + for _, st := range f.findStepResults(passed) { + actual = append(actual, st.step.Text) + } + case "failed": + for _, st := range f.findStepResults(failed) { + actual = append(actual, st.step.Text) + } + case "skipped": + for _, st := range f.findStepResults(skipped) { + actual = append(actual, st.step.Text) + } + case "undefined": + for _, st := range f.findStepResults(undefined) { + actual = append(actual, st.step.Text) + } + case "pending": + for _, st := range f.findStepResults(pending) { + actual = append(actual, st.step.Text) + } + default: + return fmt.Errorf("unexpected step status wanted: %s", status) + } + + if len(expected) > len(actual) { + return fmt.Errorf("number of expected %s steps: %d is less than actual %s steps: %d", status, len(expected), status, len(actual)) + } + + for _, a := range actual { + for _, e := range expected { + if a == e { + matched = append(matched, e) + break + } + } + } + + if len(matched) >= len(expected) { + return nil + } + + for _, s := range expected { + var found bool + for _, m := range matched { + if s == m { + found = true + break + } + } + + if !found { + unmatched = append(unmatched, s) + } + } + + return fmt.Errorf("the steps: %s - are not %s", strings.Join(unmatched, ", "), status) +} + +func (tc *godogFeaturesScenario) allStepsShouldHave(status string) error { + f, ok := tc.testedSuite.fmt.(*basefmt) + if !ok { + return fmt.Errorf("this step requires *basefmt, but there is: %T", tc.testedSuite.fmt) + } + + total := len(f.findStepResults(passed)) + + len(f.findStepResults(failed)) + + len(f.findStepResults(skipped)) + + len(f.findStepResults(undefined)) + + len(f.findStepResults(pending)) + + var actual int + + switch status { + case "passed": + actual = len(f.findStepResults(passed)) + case "failed": + actual = len(f.findStepResults(failed)) + case "skipped": + actual = len(f.findStepResults(skipped)) + case "undefined": + actual = len(f.findStepResults(undefined)) + case "pending": + actual = len(f.findStepResults(pending)) + default: + return fmt.Errorf("unexpected step status wanted: %s", status) + } + + if total > actual { + return fmt.Errorf("number of expected %s steps: %d is less than actual %s steps: %d", status, total, status, actual) + } + + return nil +} + +func (tc *godogFeaturesScenario) iAmListeningToSuiteEvents() error { + tc.testedSuite.BeforeSuite(func() { + tc.events = append(tc.events, &firedEvent{"BeforeSuite", []interface{}{}}) + }) + + tc.testedSuite.AfterSuite(func() { + tc.events = append(tc.events, &firedEvent{"AfterSuite", []interface{}{}}) + }) + + tc.testedSuite.BeforeFeature(func(ft *messages.GherkinDocument) { + tc.events = append(tc.events, &firedEvent{"BeforeFeature", []interface{}{ft}}) + }) + + tc.testedSuite.AfterFeature(func(ft *messages.GherkinDocument) { + tc.events = append(tc.events, &firedEvent{"AfterFeature", []interface{}{ft}}) + }) + + tc.testedSuite.BeforeScenario(func(pickle *Scenario) { + tc.events = append(tc.events, &firedEvent{"BeforeScenario", []interface{}{pickle}}) + }) + + tc.testedSuite.AfterScenario(func(pickle *Scenario, err error) { + tc.events = append(tc.events, &firedEvent{"AfterScenario", []interface{}{pickle, err}}) + }) + + tc.testedSuite.BeforeStep(func(step *Step) { + tc.events = append(tc.events, &firedEvent{"BeforeStep", []interface{}{step}}) + }) + + tc.testedSuite.AfterStep(func(step *Step, err error) { + tc.events = append(tc.events, &firedEvent{"AfterStep", []interface{}{step, err}}) + }) + + return nil +} + +func (tc *godogFeaturesScenario) aFailingStep() error { + return fmt.Errorf("intentional failure") +} + +// parse a given feature file body as a feature +func (tc *godogFeaturesScenario) aFeatureFile(path string, body *DocString) error { + gd, err := gherkin.ParseGherkinDocument(strings.NewReader(body.Content), (&messages.Incrementing{}).NewId) + pickles := gherkin.Pickles(*gd, path, (&messages.Incrementing{}).NewId) + tc.testedSuite.features = append(tc.testedSuite.features, &feature{GherkinDocument: gd, pickles: pickles, Path: path}) + + return err +} + +func (tc *godogFeaturesScenario) featurePath(path string) error { + tc.paths = append(tc.paths, path) + return nil +} + +func (tc *godogFeaturesScenario) parseFeatures() error { + fts, err := parseFeatures("", tc.paths) + if err != nil { + return err + } + + tc.testedSuite.features = append(tc.testedSuite.features, fts...) + + return nil +} + +func (tc *godogFeaturesScenario) theSuiteShouldHave(state string) error { + if tc.testedSuite.failed && state == "passed" { + return fmt.Errorf("the feature suite has failed") + } + + if !tc.testedSuite.failed && state == "failed" { + return fmt.Errorf("the feature suite has passed") + } + + return nil +} + +func (tc *godogFeaturesScenario) iShouldHaveNumFeatureFiles(num int, files *DocString) error { + if len(tc.testedSuite.features) != num { + return fmt.Errorf("expected %d features to be parsed, but have %d", num, len(tc.testedSuite.features)) + } + + expected := strings.Split(files.Content, "\n") + + var actual []string + + for _, ft := range tc.testedSuite.features { + actual = append(actual, ft.Path) + } + + if len(expected) != len(actual) { + return fmt.Errorf("expected %d feature paths to be parsed, but have %d", len(expected), len(actual)) + } + + for i := 0; i < len(expected); i++ { + var matched bool + split := strings.Split(expected[i], "/") + exp := filepath.Join(split...) + + for j := 0; j < len(actual); j++ { + split = strings.Split(actual[j], "/") + act := filepath.Join(split...) + + if exp == act { + matched = true + break + } + } + + if !matched { + return fmt.Errorf(`expected feature path "%s" at position: %d, was not parsed, actual are %+v`, exp, i, actual) + } + } + + return nil +} + +func (tc *godogFeaturesScenario) iRunFeatureSuite() error { + return tc.iRunFeatureSuiteWithTags("") +} + +func (tc *godogFeaturesScenario) numScenariosRegistered(expected int) (err error) { + var num int + for _, ft := range tc.testedSuite.features { + num += len(ft.pickles) + } + + if num != expected { + err = fmt.Errorf("expected %d scenarios to be registered, but got %d", expected, num) + } + + return +} + +func (tc *godogFeaturesScenario) thereWereNumEventsFired(_ string, expected int, typ string) error { + var num int + for _, event := range tc.events { + if event.name == typ { + num++ + } + } + + if num != expected { + return fmt.Errorf("expected %d %s events to be fired, but got %d", expected, typ, num) + } + + return nil +} + +func (tc *godogFeaturesScenario) thereWasEventTriggeredBeforeScenario(expected string) error { + var found []string + for _, event := range tc.events { + if event.name != "BeforeScenario" { + continue + } + + var name string + switch t := event.args[0].(type) { + case *Scenario: + name = t.Name + } + + if name == expected { + return nil + } + + found = append(found, name) + } + + if len(found) == 0 { + return fmt.Errorf("before scenario event was never triggered or listened") + } + + return fmt.Errorf(`expected "%s" scenario, but got these fired %s`, expected, `"`+strings.Join(found, `", "`)+`"`) +} + +func (tc *godogFeaturesScenario) theseEventsHadToBeFiredForNumberOfTimes(tbl *Table) error { + if len(tbl.Rows[0].Cells) != 2 { + return fmt.Errorf("expected two columns for event table row, got: %d", len(tbl.Rows[0].Cells)) + } + + for _, row := range tbl.Rows { + num, err := strconv.ParseInt(row.Cells[1].Value, 10, 0) + if err != nil { + return err + } + + if err := tc.thereWereNumEventsFired("", int(num), row.Cells[0].Value); err != nil { + return err + } + } + + return nil +} + +func (tc *godogFeaturesScenario) theRenderJSONWillBe(docstring *DocString) error { + expectedSuiteCtxReg := regexp.MustCompile(`suite_context.go:\d+`) + actualSuiteCtxReg := regexp.MustCompile(`suite_context_test.go:\d+`) + + expectedString := docstring.Content + expectedString = expectedSuiteCtxReg.ReplaceAllString(expectedString, `suite_context_test.go:0`) + + actualString := tc.out.String() + actualString = actualSuiteCtxReg.ReplaceAllString(actualString, `suite_context_test.go:0`) + + var expected []cukeFeatureJSON + if err := json.Unmarshal([]byte(expectedString), &expected); err != nil { + return err + } + + var actual []cukeFeatureJSON + if err := json.Unmarshal([]byte(actualString), &actual); err != nil { + return err + } + + return assertExpectedAndActual(assert.Equal, expected, actual) +} + +func (tc *godogFeaturesScenario) theRenderOutputWillBe(docstring *DocString) error { + expectedSuiteCtxReg := regexp.MustCompile(`suite_context.go:\d+`) + actualSuiteCtxReg := regexp.MustCompile(`suite_context_test.go:\d+`) + + expectedSuiteCtxFuncReg := regexp.MustCompile(`SuiteContext.func(\d+)`) + actualSuiteCtxFuncReg := regexp.MustCompile(`github.com/cucumber/godog.InitializeScenario.func(\d+)`) + + suiteCtxPtrReg := regexp.MustCompile(`\*suiteContext`) + + expected := docstring.Content + expected = trimAllLines(expected) + expected = expectedSuiteCtxReg.ReplaceAllString(expected, "suite_context_test.go:0") + expected = expectedSuiteCtxFuncReg.ReplaceAllString(expected, "InitializeScenario.func$1") + expected = suiteCtxPtrReg.ReplaceAllString(expected, "*godogFeaturesScenario") + + actual := tc.out.String() + actual = trimAllLines(actual) + actual = actualSuiteCtxReg.ReplaceAllString(actual, "suite_context_test.go:0") + actual = actualSuiteCtxFuncReg.ReplaceAllString(actual, "InitializeScenario.func$1") + + expectedRows := strings.Split(expected, "\n") + actualRows := strings.Split(actual, "\n") + + return assertExpectedAndActual(assert.ElementsMatch, expectedRows, actualRows) +} + +func (tc *godogFeaturesScenario) theRenderXMLWillBe(docstring *DocString) error { + expectedString := docstring.Content + actualString := tc.out.String() + + var expected junitPackageSuite + if err := xml.Unmarshal([]byte(expectedString), &expected); err != nil { + return err + } + + var actual junitPackageSuite + if err := xml.Unmarshal([]byte(actualString), &actual); err != nil { + return err + } + + return assertExpectedAndActual(assert.Equal, expected, actual) +}