From 1a37222cf25e00a2a20b3915c3d824d0b458bdfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Rasmusson?= Date: Thu, 13 Jul 2017 21:54:52 +0200 Subject: [PATCH] Add a flaky result type to be used for flaky scenarios. * The flaky result type should be ok in non-strict mode and not ok in strict mode, so the exit code becomes zero with flaky scenarios in non-strict mode. * Update the Summary report to detect and count flaky scenarios. --- lib/cucumber/core/report/summary.rb | 9 ++++++++- lib/cucumber/core/test/result.rb | 15 ++++++++++++++- spec/cucumber/core/report/summary_spec.rb | 10 ++++++++++ spec/cucumber/core/test/result_spec.rb | 17 +++++++++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/lib/cucumber/core/report/summary.rb b/lib/cucumber/core/report/summary.rb index 0520b283..a218812c 100644 --- a/lib/cucumber/core/report/summary.rb +++ b/lib/cucumber/core/report/summary.rb @@ -6,6 +6,7 @@ class Summary attr_reader :test_cases, :test_steps def initialize(event_bus) + @previous_test_case = nil @test_cases = Test::Result::Summary.new @test_steps = Test::Result::Summary.new subscribe_to(event_bus) @@ -19,7 +20,13 @@ def ok?(be_strict = false) def subscribe_to(event_bus) event_bus.on(:test_case_finished) do |event| - event.result.describe_to test_cases + if event.test_case != @previous_test_case + @previous_test_case = event.test_case + event.result.describe_to test_cases + elsif event.result.passed? + test_cases.flaky + test_cases.decrement_failed + end end event_bus.on(:test_step_finished) do |event| event.result.describe_to test_steps if is_step?(event.test_step) diff --git a/lib/cucumber/core/test/result.rb b/lib/cucumber/core/test/result.rb index d8ec0a67..c9f088dc 100644 --- a/lib/cucumber/core/test/result.rb +++ b/lib/cucumber/core/test/result.rb @@ -5,7 +5,7 @@ module Cucumber module Core module Test module Result - TYPES = [:failed, :skipped, :undefined, :pending, :passed, :unknown].freeze + TYPES = [:failed, :flaky, :skipped, :undefined, :pending, :passed, :unknown].freeze def self.ok?(type, be_strict = false) private @@ -124,6 +124,15 @@ def with_filtered_backtrace(filter) end end + # Flaky is not used directly as an execution result, but is used as a + # reporting result type for test cases that fails and the passes on + # retry, therefore only the class method self.ok? is needed. + class Flaky + def self.ok?(be_strict = false) + !be_strict + end + end + # Base class for exceptions that can be raised in a step definition causing # the step to have that result. class Raisable < StandardError @@ -268,6 +277,10 @@ def total(for_status = nil) end end + def decrement_failed + @totals[:failed] -= 1 + end + private def get_total(method_name) diff --git a/spec/cucumber/core/report/summary_spec.rb b/spec/cucumber/core/report/summary_spec.rb index f2853635..bf45847b 100644 --- a/spec/cucumber/core/report/summary_spec.rb +++ b/spec/cucumber/core/report/summary_spec.rb @@ -55,6 +55,16 @@ module Cucumber::Core::Report expect( @summary.test_cases.total(:undefined) ).to eq(1) expect( @summary.test_cases.total ).to eq(1) end + + it "handles flaky test cases" do + allow(test_case).to receive(:==).and_return(false, true) + event_bus.send(:test_case_finished, test_case, failed_result) + event_bus.send(:test_case_finished, test_case, passed_result) + + expect( @summary.test_cases.total(:failed) ).to eq(0) + expect( @summary.test_cases.total(:flaky) ).to eq(1) + expect( @summary.test_cases.total ).to eq(1) + end end context "test step summary" do diff --git a/spec/cucumber/core/test/result_spec.rb b/spec/cucumber/core/test/result_spec.rb index 426da1f9..42964877 100644 --- a/spec/cucumber/core/test/result_spec.rb +++ b/spec/cucumber/core/test/result_spec.rb @@ -46,6 +46,7 @@ module Cucumber::Core::Test specify { expect( result ).not_to be_undefined } specify { expect( result ).not_to be_unknown } specify { expect( result ).not_to be_skipped } + specify { expect( result ).not_to be_flaky } specify { expect( result ).to be_ok } specify { expect( result.ok?(false) ).to be_truthy } @@ -105,6 +106,7 @@ module Cucumber::Core::Test specify { expect( result ).not_to be_undefined } specify { expect( result ).not_to be_unknown } specify { expect( result ).not_to be_skipped } + specify { expect( result ).not_to be_flaky } specify { expect( result ).to_not be_ok } specify { expect( result.ok?(false) ).to be_falsey } @@ -130,6 +132,7 @@ module Cucumber::Core::Test specify { expect( result ).not_to be_undefined } specify { expect( result ).to be_unknown } specify { expect( result ).not_to be_skipped } + specify { expect( result ).not_to be_flaky } end describe Result::Raisable do @@ -196,6 +199,7 @@ module Cucumber::Core::Test specify { expect( result ).to be_undefined } specify { expect( result ).not_to be_unknown } specify { expect( result ).not_to be_skipped } + specify { expect( result ).not_to be_flaky } specify { expect( result ).to be_ok } specify { expect( result.ok?(false) ).to be_truthy } @@ -218,6 +222,7 @@ module Cucumber::Core::Test specify { expect( result ).not_to be_undefined } specify { expect( result ).not_to be_unknown } specify { expect( result ).to be_skipped } + specify { expect( result ).not_to be_flaky } specify { expect( result ).to be_ok } specify { expect( result.ok?(false) ).to be_truthy } @@ -240,6 +245,7 @@ module Cucumber::Core::Test specify { expect( result ).not_to be_undefined } specify { expect( result ).not_to be_unknown } specify { expect( result ).not_to be_skipped } + specify { expect( result ).not_to be_flaky } specify { expect( result ).to be_pending } specify { expect( result ).to be_ok } @@ -247,6 +253,11 @@ module Cucumber::Core::Test specify { expect( result.ok?(true) ).to be_falsey } end + describe Result::Flaky do + specify { expect( Result::Flaky.ok?(false) ).to be_truthy } + specify { expect( Result::Flaky.ok?(true) ).to be_falsey } + end + describe Result::Summary do let(:summary) { Result::Summary.new } let(:failed) { Result::Failed.new(Result::Duration.new(10), exception) } @@ -357,6 +368,12 @@ def describe_to(visitor, *args) expect( summary.ok? ).to be true expect( summary.ok?(true) ).to be false end + + it "flaky result is ok if not strict" do + summary.flaky + expect( summary.ok? ).to be true + expect( summary.ok?(true) ).to be false + end end end