[SUREFIRE-2143] Fix reporting of skipped parameterized test

- In executionSkipped handle ParameterizedTest which is a non-class test container the same as a regular test - In executionSkipped always remove the testIdentifier from the map testStartTime, regardless if it's a class or a test - In createReportEntry set methodName and methodText if the testExecutionResult is null, which is the case for skipped tests, otherwise the test name in the XML report is left empty - Add ITs also for the related previous issue SUREFIRE-2032 the fix of which caused this regression, to make sure that fix still works
apache · Feb 11, 2023 · 89c2519 · 89c2519
1 parent 1f0c261
commit 89c2519
Show file tree

Hide file tree

Showing 10 changed files with 640 additions and 4 deletions.
diff --git a/...re-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2032NestedSkippedIT.java b/...re-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2032NestedSkippedIT.java
@@ -0,0 +1,79 @@
+package org.apache.maven.surefire.its.jiras;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.Test;
+
+/**
+ * Integration Test for SUREFIRE-2032
+ */
+@SuppressWarnings( "checkstyle:magicnumber" )
+public class Surefire2032NestedSkippedIT extends SurefireJUnit4IntegrationTestCase
+{
+    @Test
+    public void testXmlReport()
+    {
+        OutputValidator validator = unpack( "surefire-2032-nested-test-class-skipped" )
+                .groups( "\"red|orange\"" )
+                .executeTest()
+                .assertTestSuiteResults( 4, 0, 0, 2 );
+
+        String redXmlReport = validator
+                .getSurefireReportsFile( "TEST-jira2032.DisabledNestedTest$RedTaggedEnabledTest.xml", UTF_8 )
+                .readFileToString();
+
+        // Enabled nested subclass
+        List<String> redTestCaseResults = Arrays.asList(
+                redXmlReport.substring( redXmlReport.indexOf( "<testcase " ),
+                                        redXmlReport.indexOf( "</testsuite>" ) )
+                        .split( ".(?=<testcase )" ) );
+
+        assertThat( redTestCaseResults ).hasSize( 2 )
+                .filteredOn( testCaseResult -> testCaseResult.contains( "<skipped" ) )
+                .isEmpty();
+
+        // Disabled nested subclass
+        String orangeXmlReport = validator
+                .getSurefireReportsFile( "TEST-jira2032.DisabledNestedTest$OrangeTaggedDisabledTest.xml", UTF_8 )
+                .readFileToString();
+
+        List<String> orangeTestCaseResults = Arrays.asList(
+                orangeXmlReport.substring( orangeXmlReport.indexOf( "<testcase " ),
+                                           orangeXmlReport.indexOf( "</testsuite>" ) )
+                        .split( ".(?=<testcase )" ) );
+
+        assertThat( orangeTestCaseResults )
+                .hasSize( 2 )
+                .filteredOn( testCaseResult -> testCaseResult.contains( "<skipped" ) )
+                .map( testCaseResult -> testCaseResult.substring( 0, testCaseResult.indexOf( "classname" ) ) )
+                .containsExactlyInAnyOrder(
+                        "<testcase name=\"test1\" ",
+                        "<testcase name=\"test2\" " );
+    }
+
+}
diff --git a/...src/test/java/org/apache/maven/surefire/its/jiras/Surefire2143ParameterizedSkippedIT.java b/...src/test/java/org/apache/maven/surefire/its/jiras/Surefire2143ParameterizedSkippedIT.java
@@ -0,0 +1,102 @@
+package org.apache.maven.surefire.its.jiras;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.hamcrest.Matchers.matchesRegex;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.Test;
+
+/**
+ * Integration Test for SUREFIRE-2143
+ */
+@SuppressWarnings( "checkstyle:magicnumber" )
+public class Surefire2143ParameterizedSkippedIT extends SurefireJUnit4IntegrationTestCase
+{
+    @Test
+    public void junit5ParameterizedSkipped()
+    {
+        OutputValidator validator = unpack( "surefire-2143-junit5-parameterized-test-skipped" )
+                .executeTest()
+                .assertTestSuiteResults( 5, 0, 0, 2 );
+
+        String xmlReport = validator
+                .getSurefireReportsFile( "TEST-jira2143.DisabledParameterizedTest.xml", UTF_8 )
+                .readFileToString();
+
+        List<String> testCaseResults = Arrays.asList(
+                xmlReport.substring( xmlReport.indexOf( "<testcase " ),
+                                     xmlReport.indexOf( "</testsuite>" ) )
+                        .split( ".(?=<testcase )" ) );
+
+        assertThat( testCaseResults )
+                .hasSize( 5 )
+                .filteredOn( testCaseResult -> testCaseResult.contains( "<skipped" ) )
+                .map( testCaseResult -> testCaseResult.substring( 0, testCaseResult.indexOf( "classname" ) ) )
+                .containsExactlyInAnyOrder(
+                        "<testcase name=\"disabledParameterized(String)\" ",
+                        "<testcase name=\"disabledNonParameterized\" " );
+
+        validator.getSurefireReportsFile( "TEST-jira2143.DisabledParameterizedTest.xml", UTF_8 )
+                .assertContainsText( "<testcase name=\"disabledParameterized(String)\" "
+                        + "classname=\"jira2143.DisabledParameterizedTest\"" )
+                .assertContainsText( "<testcase name=\"disabledNonParameterized\" "
+                        + "classname=\"jira2143.DisabledParameterizedTest\"" )
+                .assertContainsText( matchesRegex( ".*<skipped [^>]*disabledParameterized.*" ) )
+                .assertContainsText( matchesRegex( ".*<skipped [^>]*disabledNonParameterized.*" ) )
+                .assertContainsText( "<testcase name=\"enabledParameterized(String)[1]\"" )
+                .assertContainsText( "<testcase name=\"enabledParameterized(String)[2]\"" )
+                .assertContainsText( "<testcase name=\"enabledNonParameterized\"" )
+                .assertNotContainsText( matchesRegex( ".*<skipped [^>]*enabledParameterized.*" ) )
+                .assertNotContainsText( matchesRegex( ".*<skipped [^>]*enabledNonParameterized.*" ) );
+    }
+
+    @Test
+    public void junit4ParameterizedSkipped()
+    {
+        OutputValidator validator = unpack( "surefire-2143-junit4-parameterized-test-skipped" )
+                .executeTest()
+                .assertTestSuiteResults( 4, 0, 0, 2 );
+
+        String xmlReport = validator
+                .getSurefireReportsFile( "TEST-jira2143.IgnoredParameterizedTest.xml", UTF_8 )
+                .readFileToString();
+
+        List<String> testCaseResults = Arrays.asList(
+                xmlReport.substring( xmlReport.indexOf( "<testcase " ),
+                                     xmlReport.indexOf( "</testsuite>" ) )
+                        .split( ".(?=<testcase )" ) );
+
+        assertThat( testCaseResults )
+                .hasSize( 4 )
+                .filteredOn( testCaseResult -> testCaseResult.contains( "<skipped" ) )
+                .map( testCaseResult -> testCaseResult.substring( 0, testCaseResult.indexOf( "classname" ) ) )
+                .containsExactlyInAnyOrder(
+                        "<testcase name=\"ignoredParameterized[0]\" ",
+                        "<testcase name=\"ignoredParameterized[1]\" " );
+    }
+
+}
diff --git a/surefire-its/src/test/resources/surefire-2032-nested-test-class-skipped/pom.xml b/surefire-its/src/test/resources/surefire-2032-nested-test-class-skipped/pom.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.apache.maven.plugins.surefire</groupId>
+    <artifactId>surefire-2032-nested-test-class-skipped</artifactId>
+    <version>1.0</version>
+    <name>Test for: RunListenerAdapter, Nested and Disabled</name>
+
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <junit5.version>5.9.1</junit5.version>
+    </properties>
+
+    <!--
+        Declare "junit-jupiter-engine" dependency because the
+        Jupiter Engine is needed at test runtime. Artifacts
+        needed for test compilation, like "junit-jupiter-api",
+        are pulled-in via transitive dependency resolution.
+    -->
+    <dependencies>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <version>${junit5.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-params</artifactId>
+            <version>${junit5.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.0</version>
+                <configuration>
+                    <encoding>UTF-8</encoding>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>${surefire.version}</version>
+                <configuration>
+                    <forkCount>1.0C</forkCount>
+                    <redirectTestOutputToFile>true</redirectTestOutputToFile>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
diff --git a/...es/surefire-2032-nested-test-class-skipped/src/test/java/jira2032/DisabledNestedTest.java b/...es/surefire-2032-nested-test-class-skipped/src/test/java/jira2032/DisabledNestedTest.java
@@ -0,0 +1,56 @@
+package jira2032;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+class DisabledNestedTest
+{
+    @Tag("red")
+    @Nested
+    public class RedTaggedEnabledTest extends TagTest {
+    }
+
+    @Disabled
+    @Tag("orange")
+    @Nested
+    public class OrangeTaggedDisabledTest extends TagTest {
+    }
+
+    abstract class TagTest {
+
+        @Test
+        public void test1() {
+            // Do Nothing
+        }
+
+        @Test
+        public void test2() {
+            // Do Nothing
+        }
+
+    }
+
+}
diff --git a/surefire-its/src/test/resources/surefire-2143-junit4-parameterized-test-skipped/pom.xml b/surefire-its/src/test/resources/surefire-2143-junit4-parameterized-test-skipped/pom.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.apache.maven.plugins.surefire</groupId>
+    <artifactId>surefire-2143-parameterized-test-skipped</artifactId>
+    <version>1.0</version>
+    <name>Test for: RunListenerAdapter, Parameterized and Ignore</name>
+
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <junit5.version>5.9.1</junit5.version>
+    </properties>
+
+    <!--
+        Declare "junit-jupiter-engine" dependency because the
+        Jupiter Engine is needed at test runtime. Artifacts
+        needed for test compilation, like "junit-jupiter-api",
+        are pulled-in via transitive dependency resolution.
+    -->
+    <dependencies>
+      <dependency>
+        <groupId>org.junit.vintage</groupId>
+        <artifactId>junit-vintage-engine</artifactId>
+        <version>${junit5.version}</version>
+        <scope>test</scope>
+      </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.0</version>
+                <configuration>
+                    <encoding>UTF-8</encoding>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>${surefire.version}</version>
+                <configuration>
+                    <forkCount>1.0C</forkCount>
+                    <redirectTestOutputToFile>true</redirectTestOutputToFile>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>