Skip to content

Commit

Permalink
[JUnit] Report failures that occur prior to pickle execution
Browse files Browse the repository at this point in the history
An exception may be thrown while the `PickleRunner` is setting up the
`Runner` class due to, for example, duplicate step definitions, glue
scanning failures, ect.

These failures do not clearly show up in IDEA or Eclipse because
Cucumber does not emit the right events to JUnit.  And it is not
possible to emit these events from the `JUnitReporter`. These exceptions
happen before the system is ready to report.

```java
@OverRide
public void run(final RunNotifier notifier) {
    // Possibly invoked by a thread other then the creating thread
    Runner runner = runnerSupplier.get(); // <---- Exception is thrown here
    JUnitReporter jUnitReporter = new JUnitReporter(runner.getBus(), jUnitOptions);
    jUnitReporter.startExecutionUnit(this, notifier);
    runner.runPickle(pickleEvent);
    jUnitReporter.finishExecutionUnit();
}
```

As such the `FeatureRunner` should catch any exceptions thrown while
executing its children and request JUnit to stop execution in case of a
failure.

This will allow IDEA and Eclipse to properly report on the test failure.
  • Loading branch information
mpkorstanje committed Jul 21, 2019
1 parent f2eeac1 commit 0e41a1a
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 11 deletions.
19 changes: 14 additions & 5 deletions junit/src/main/java/io/cucumber/junit/FeatureRunner.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@
import gherkin.ast.Feature;
import gherkin.events.PickleEvent;
import io.cucumber.core.exception.CucumberException;
import io.cucumber.core.feature.CucumberFeature;
import io.cucumber.core.filter.Filters;
import io.cucumber.core.runtime.RunnerSupplier;
import io.cucumber.junit.PickleRunners.PickleRunner;
import io.cucumber.core.feature.CucumberFeature;
import io.cucumber.core.runtime.ThreadLocalRunnerSupplier;
import org.junit.runner.Description;
import org.junit.runner.notification.Failure;
import org.junit.runner.notification.RunNotifier;
import org.junit.runners.ParentRunner;
import org.junit.runners.model.InitializationError;
Expand All @@ -26,7 +27,7 @@ final class FeatureRunner extends ParentRunner<PickleRunner> {
private final CucumberFeature cucumberFeature;
private Description description;

FeatureRunner(CucumberFeature cucumberFeature, Filters filters, ThreadLocalRunnerSupplier runnerSupplier, JUnitOptions jUnitOptions) throws InitializationError {
FeatureRunner(CucumberFeature cucumberFeature, Filters filters, RunnerSupplier runnerSupplier, JUnitOptions jUnitOptions) throws InitializationError {
super(null);
this.cucumberFeature = cucumberFeature;
buildFeatureElementRunners(filters, runnerSupplier, jUnitOptions);
Expand Down Expand Up @@ -65,10 +66,18 @@ protected Description describeChild(PickleRunner child) {

@Override
protected void runChild(PickleRunner child, RunNotifier notifier) {
child.run(notifier);
notifier.fireTestStarted(getDescription());
try {
child.run(notifier);
} catch (Throwable e) {
notifier.fireTestFailure(new Failure(getDescription(), e));
notifier.pleaseStop();
} finally {
notifier.fireTestFinished(getDescription());
}
}

private void buildFeatureElementRunners(Filters filters, ThreadLocalRunnerSupplier runnerSupplier, JUnitOptions jUnitOptions) {
private void buildFeatureElementRunners(Filters filters, RunnerSupplier runnerSupplier, JUnitOptions jUnitOptions) {
for (PickleEvent pickleEvent : cucumberFeature.getPickles()) {
if (filters.matchesFilters(pickleEvent)) {
try {
Expand Down
47 changes: 41 additions & 6 deletions junit/src/test/java/io/cucumber/junit/FeatureRunnerTest.java
Original file line number Diff line number Diff line change
@@ -1,24 +1,27 @@
package io.cucumber.junit;

import io.cucumber.core.backend.ObjectFactoryServiceLoader;
import io.cucumber.core.eventbus.EventBus;
import io.cucumber.core.feature.CucumberFeature;
import io.cucumber.core.filter.Filters;
import io.cucumber.core.io.ClassFinder;
import io.cucumber.core.io.MultiLoader;
import io.cucumber.core.io.ResourceLoader;
import io.cucumber.core.io.ResourceLoaderClassFinder;
import io.cucumber.core.options.RuntimeOptions;
import io.cucumber.core.runtime.BackendSupplier;
import io.cucumber.core.runtime.ConfiguringTypeRegistrySupplier;
import io.cucumber.core.runtime.ObjectFactorySupplier;
import io.cucumber.core.runtime.RunnerSupplier;
import io.cucumber.core.runtime.SingletonObjectFactorySupplier;
import io.cucumber.core.runtime.TimeServiceEventBus;
import io.cucumber.core.eventbus.EventBus;
import io.cucumber.core.runtime.BackendSupplier;
import io.cucumber.core.options.RuntimeOptions;
import io.cucumber.core.runtime.ThreadLocalRunnerSupplier;
import io.cucumber.core.filter.Filters;
import io.cucumber.core.feature.CucumberFeature;
import io.cucumber.core.runtime.TimeServiceEventBus;
import org.junit.Test;
import org.junit.runner.Description;
import org.junit.runner.notification.Failure;
import org.junit.runner.notification.RunNotifier;
import org.junit.runners.model.InitializationError;
import org.mockito.ArgumentCaptor;
import org.mockito.InOrder;

import java.time.Clock;
Expand Down Expand Up @@ -344,5 +347,37 @@ public void step_notification_can_be_turned_on_two_scenarios_with_background() t
order.verify(notifier).fireTestFinished(argThat(new DescriptionMatcher("scenario_2 name")));
}

@Test
public void should_notify_of_failure_to_create_runners_and_request_test_execution_to_stop() throws InitializationError {
CucumberFeature feature = TestPickleBuilder.parseFeature("path/test.feature", "" +
"Feature: feature name\n" +
" Scenario: scenario_1 name\n" +
" Given first step\n"
);

Filters filters = new Filters(RuntimeOptions.defaultOptions());

IllegalStateException illegalStateException = new IllegalStateException();
RunnerSupplier runnerSupplier = () -> {
throw illegalStateException;
};

FeatureRunner featureRunner = new FeatureRunner(feature, filters, runnerSupplier, new JUnitOptions());

RunNotifier notifier = mock(RunNotifier.class);
featureRunner.runChild(featureRunner.getChildren().get(0), notifier);

Description description = featureRunner.getDescription();
ArgumentCaptor<Failure> failureArgumentCaptor = ArgumentCaptor.forClass(Failure.class);

InOrder order = inOrder(notifier);
order.verify(notifier).fireTestStarted(description);
order.verify(notifier).fireTestFailure(failureArgumentCaptor.capture());
assertEquals(illegalStateException, failureArgumentCaptor.getValue().getException());
assertEquals(description, failureArgumentCaptor.getValue().getDescription());
order.verify(notifier).pleaseStop();
order.verify(notifier).fireTestFinished(description);
}

}

0 comments on commit 0e41a1a

Please sign in to comment.