apache · jon-bell · May 23, 2019
diff --git a/...n-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java b/...n-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/IntegrationTestMojo.java
@@ -264,7 +264,7 @@ public class IntegrationTestMojo
     private String encoding;
 
     /**
-     * (JUnit 4+ providers)
+     * (JUnit 4+ providers and JUnit 5+ providers since 3.0.0-M4)
      * The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after
      * they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake".
      * However, all the failing attempts will be recorded.

diff --git a/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java b/maven-surefire-plugin/src/main/java/org/apache/maven/plugin/surefire/SurefirePlugin.java
@@ -245,7 +245,7 @@ public class SurefirePlugin
     private String encoding;
 
     /**
-     * (JUnit 4+ providers)
+     * (JUnit 4+ providers and JUnit 5+ providers since 3.0.0-M4)
      * The number of times each failing test will be rerun. If set larger than 0, rerun failing tests immediately after
      * they fail. If a failing test passes in any of those reruns, it will be marked as pass and reported as a "flake".
      * However, all the failing attempts will be recorded.

diff --git a/maven-surefire-plugin/src/site/apt/featurematrix.apt.vm b/maven-surefire-plugin/src/site/apt/featurematrix.apt.vm
@@ -42,7 +42,7 @@ Feature Matrix
 *---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
 | custom run-listener                         |     N      |    Y     |      Y     |    Y      |  -       |  N                   |
 *---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
-| re-run count                                |     N      |    Y     |      Y     |    N      |  N       |  N                   |
+| re-run count                                |     N      |    Y     |      Y     |    N      |  N       |  Y(*3)               |
 *---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
 | skip after failure count                    |     N      |    Y     |      Y     |    Y      |  N       |  N                   |
 *---------------------------------------------+------------+----------+------------+-----------+----------+----------------------+
@@ -62,3 +62,5 @@ Feature Matrix
 
    (*2) 3 extensions related to JUnit5 annotation <<<DisplayName>>>.
 
+   (*3) Since 3.0.0-M4
+
diff --git a/...ire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java b/...ire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformRerunFailingTestsIT.java
@@ -0,0 +1,281 @@
+package org.apache.maven.surefire.its;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.apache.maven.surefire.its.fixture.SurefireLauncher;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.maven.surefire.its.fixture.HelperAssertions.assumeJavaVersion;
+
+/**
+ * JUnit4 RunListener Integration Test.
+ *
+ * @author <a href="mailto:qingzhouluo@google.com">Qingzhou Luo</a>
+ * @author Matt Coley
+ */
+public class JUnitPlatformRerunFailingTestsIT extends SurefireJUnit4IntegrationTestCase
+{
+    private final static String VERSION = "5.5.1";
+
+    private SurefireLauncher unpack()
+    {
+        return unpack( "/junit-platform-rerun-failing-tests" );
+    }
+
+    @Before
+    public void setup()
+    {
+        assumeJavaVersion( 1.8 );
+    }
+
+    @Test
+    public void testRerunFailingErrorTestsWithOneRetry() throws Exception
+    {
+        OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 5, 1, 1, 0, 0 );
+        verifyFailuresOneRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+                "-DforkCount=2" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+        verifyFailuresOneRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+        verifyFailuresOneRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+        verifyFailuresOneRetryAllClasses( outputValidator );
+    }
+
+    @Test
+    public void testRerunFailingErrorTestsTwoRetry() throws Exception
+    {
+        // Four flakes, both tests have been re-run twice
+        OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging()
+                .addGoal( "-Dsurefire.rerunFailingTestsCount=2" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+        verifyFailuresTwoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-DforkCount=3" ).executeTest().assertTestSuiteResults(
+                5, 0, 0, 0, 4 );
+
+        verifyFailuresTwoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+        verifyFailuresTwoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
+
+        verifyFailuresTwoRetryAllClasses( outputValidator );
+    }
+
+    @Test
+    public void testRerunFailingErrorTestsFalse() throws Exception
+    {
+        OutputValidator outputValidator = unpack().setJUnitVersion(
+                VERSION ).maven().withFailure().debugLogging().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+        verifyFailuresNoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-DforkCount=3" ).withFailure().debugLogging().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+        verifyFailuresNoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+        verifyFailuresNoRetryAllClasses( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 5, 1, 1, 0, 0 );
+
+        verifyFailuresNoRetryAllClasses( outputValidator );
+    }
+
+    @Test
+    public void testRerunOneTestClass() throws Exception
+    {
+        OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().debugLogging().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneClass( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneClass( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneClass( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 3, 1, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneClass( outputValidator );
+    }
+
+    @Test
+    public void testRerunOneTestMethod() throws Exception
+    {
+        OutputValidator outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneMethod( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneMethod( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneMethod( outputValidator );
+
+        outputValidator = unpack().setJUnitVersion( VERSION ).maven().addGoal(
+                "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
+                "-DuseUnlimitedThreads=true" ).addGoal(
+                "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().debugLogging().executeTest()
+                .assertTestSuiteResults( 1, 0, 1, 0, 0 );
+
+        verifyFailuresOneRetryOneMethod( outputValidator );
+    }
+
+    private void verifyFailuresOneRetryAllClasses( OutputValidator outputValidator )
+    {
+        verifyFailuresOneRetry( outputValidator, 5, 1, 1, 0 );
+    }
+
+    private void verifyFailuresTwoRetryAllClasses( OutputValidator outputValidator )
+    {
+        verifyFailuresTwoRetry( outputValidator, 5, 0, 0, 2 );
+    }
+
+    private void verifyFailuresNoRetryAllClasses( OutputValidator outputValidator )
+    {
+        verifyFailuresNoRetry( outputValidator, 5, 1, 1, 0 );
+    }
+
+    private void verifyFailuresOneRetryOneClass( OutputValidator outputValidator )
+    {
+        verifyFailuresOneRetry( outputValidator, 3, 1, 1, 0 );
+    }
+
+    private void verifyFailuresOneRetryOneMethod( OutputValidator outputValidator )
+    {
+        verifyOnlyFailuresOneRetry( outputValidator, 1, 1, 0, 0 );
+    }
+
+    private void verifyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+    {
+        outputValidator.verifyTextInLog( "Failures:" );
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+
+        outputValidator.verifyTextInLog( "Errors:" );
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
+
+        verifyStatistics( outputValidator, run, failures, errors, flakes );
+    }
+
+    private void verifyOnlyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+    {
+        outputValidator.verifyTextInLog( "Failures:" );
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+
+        verifyStatistics( outputValidator, run, failures, errors, flakes );
+    }
+
+    private void verifyFailuresTwoRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+    {
+        outputValidator.verifyTextInLog( "Flakes:" );
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
+        outputValidator.verifyTextInLog( "Run 3: PASS" );
+
+        outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
+        outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
+
+        verifyStatistics( outputValidator, run, failures, errors, flakes );
+    }
+
+    private void verifyFailuresNoRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+    {
+        outputValidator.verifyTextInLog( "Failures:" );
+        outputValidator.verifyTextInLog( "junitplatform.FlakyFirstTimeTest.testFailingTestOne" );
+        outputValidator.verifyTextInLog( "ERROR" );
+        outputValidator.verifyTextInLog( "junitplatform.FlakyFirstTimeTest.testErrorTestOne" );
+
+        verifyStatistics( outputValidator, run, failures, errors, flakes );
+    }
+
+    private void verifyStatistics( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
+    {
+        if ( flakes > 0 )
+        {
+            outputValidator.verifyTextInLog(
+                    "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0, Flakes: " + flakes );
+        }
+        else
+        {
+            outputValidator.verifyTextInLog(
+                    "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0" );
+        }
+    }
+}
diff --git a/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/pom.xml b/surefire-its/src/test/resources/junit-platform-rerun-failing-tests/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.apache.maven.plugins.surefire</groupId>
+    <artifactId>junit-platform-rerun-failing-tests</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <name>Test for rerun failing tests in JUnit 5/Platform</name>
+
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>${surefire.version}</version>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>