diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2032NestedSkippedIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2032NestedSkippedIT.java new file mode 100644 index 0000000000..58345f66bd --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2032NestedSkippedIT.java @@ -0,0 +1,79 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Arrays; +import java.util.List; + +import org.apache.maven.surefire.its.fixture.OutputValidator; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +/** + * Integration Test for SUREFIRE-2032 + */ +@SuppressWarnings( "checkstyle:magicnumber" ) +public class Surefire2032NestedSkippedIT extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void testXmlReport() + { + OutputValidator validator = unpack( "surefire-2032-nested-test-class-skipped" ) + .groups( "\"red|orange\"" ) + .executeTest() + .assertTestSuiteResults( 4, 0, 0, 2 ); + + String redXmlReport = validator + .getSurefireReportsFile( "TEST-jira2032.DisabledNestedTest$RedTaggedEnabledTest.xml", UTF_8 ) + .readFileToString(); + + // Enabled nested subclass + List redTestCaseResults = Arrays.asList( + redXmlReport.substring( redXmlReport.indexOf( "" ) ) + .split( ".(?= testCaseResult.contains( " orangeTestCaseResults = Arrays.asList( + orangeXmlReport.substring( orangeXmlReport.indexOf( "" ) ) + .split( ".(?= testCaseResult.contains( " testCaseResult.substring( 0, testCaseResult.indexOf( "classname" ) ) ) + .containsExactlyInAnyOrder( + " testCaseResults = Arrays.asList( + xmlReport.substring( xmlReport.indexOf( "" ) ) + .split( ".(?= testCaseResult.contains( " testCaseResult.substring( 0, testCaseResult.indexOf( "classname" ) ) ) + .containsExactlyInAnyOrder( + "]*disabledParameterized.*" ) ) + .assertContainsText( matchesRegex( ".*]*disabledNonParameterized.*" ) ) + .assertContainsText( "]*enabledParameterized.*" ) ) + .assertNotContainsText( matchesRegex( ".*]*enabledNonParameterized.*" ) ); + } + + @Test + public void junit4ParameterizedSkipped() + { + OutputValidator validator = unpack( "surefire-2143-junit4-parameterized-test-skipped" ) + .executeTest() + .assertTestSuiteResults( 4, 0, 0, 2 ); + + String xmlReport = validator + .getSurefireReportsFile( "TEST-jira2143.IgnoredParameterizedTest.xml", UTF_8 ) + .readFileToString(); + + List testCaseResults = Arrays.asList( + xmlReport.substring( xmlReport.indexOf( "" ) ) + .split( ".(?= testCaseResult.contains( " testCaseResult.substring( 0, testCaseResult.indexOf( "classname" ) ) ) + .containsExactlyInAnyOrder( + " + + + + 4.0.0 + + org.apache.maven.plugins.surefire + surefire-2032-nested-test-class-skipped + 1.0 + Test for: RunListenerAdapter, Nested and Disabled + + + 1.8 + 1.8 + UTF-8 + 5.9.1 + + + + + + org.junit.jupiter + junit-jupiter-engine + ${junit5.version} + test + + + org.junit.jupiter + junit-jupiter-params + ${junit5.version} + test + + + + + + + maven-compiler-plugin + 3.8.0 + + UTF-8 + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + 1.0C + true + + + + + + diff --git a/surefire-its/src/test/resources/surefire-2032-nested-test-class-skipped/src/test/java/jira2032/DisabledNestedTest.java b/surefire-its/src/test/resources/surefire-2032-nested-test-class-skipped/src/test/java/jira2032/DisabledNestedTest.java new file mode 100644 index 0000000000..96e90c0605 --- /dev/null +++ b/surefire-its/src/test/resources/surefire-2032-nested-test-class-skipped/src/test/java/jira2032/DisabledNestedTest.java @@ -0,0 +1,56 @@ +package jira2032; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +class DisabledNestedTest +{ + @Tag("red") + @Nested + public class RedTaggedEnabledTest extends TagTest { + } + + @Disabled + @Tag("orange") + @Nested + public class OrangeTaggedDisabledTest extends TagTest { + } + + abstract class TagTest { + + @Test + public void test1() { + // Do Nothing + } + + @Test + public void test2() { + // Do Nothing + } + + } + +} diff --git a/surefire-its/src/test/resources/surefire-2143-junit4-parameterized-test-skipped/pom.xml b/surefire-its/src/test/resources/surefire-2143-junit4-parameterized-test-skipped/pom.xml new file mode 100644 index 0000000000..8e9bca85d5 --- /dev/null +++ b/surefire-its/src/test/resources/surefire-2143-junit4-parameterized-test-skipped/pom.xml @@ -0,0 +1,74 @@ + + + + + 4.0.0 + + org.apache.maven.plugins.surefire + surefire-2143-parameterized-test-skipped + 1.0 + Test for: RunListenerAdapter, Parameterized and Ignore + + + 1.8 + 1.8 + UTF-8 + 5.9.1 + + + + + + org.junit.vintage + junit-vintage-engine + ${junit5.version} + test + + + + + + + maven-compiler-plugin + 3.8.0 + + UTF-8 + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + 1.0C + true + + + + + + diff --git a/surefire-its/src/test/resources/surefire-2143-junit4-parameterized-test-skipped/src/test/java/jira2143/IgnoredParameterizedTest.java b/surefire-its/src/test/resources/surefire-2143-junit4-parameterized-test-skipped/src/test/java/jira2143/IgnoredParameterizedTest.java new file mode 100644 index 0000000000..94978ae798 --- /dev/null +++ b/surefire-its/src/test/resources/surefire-2143-junit4-parameterized-test-skipped/src/test/java/jira2143/IgnoredParameterizedTest.java @@ -0,0 +1,55 @@ +package jira2143; + +import java.util.Arrays; +import java.util.List; + +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +@RunWith(Parameterized.class) +public class IgnoredParameterizedTest +{ + @Parameterized.Parameters + public static List parameters() throws Exception + { + return Arrays.asList( 0, 1 ); + } + + @Parameterized.Parameter(0) + public int expected; + + @Test + @Ignore + public void ignoredParameterized() + { + //always successful + } + + @Test + public void nonIgnoredParameterized() + { + //always successful + } + +} diff --git a/surefire-its/src/test/resources/surefire-2143-junit5-parameterized-test-skipped/pom.xml b/surefire-its/src/test/resources/surefire-2143-junit5-parameterized-test-skipped/pom.xml new file mode 100644 index 0000000000..6d1bc4770f --- /dev/null +++ b/surefire-its/src/test/resources/surefire-2143-junit5-parameterized-test-skipped/pom.xml @@ -0,0 +1,80 @@ + + + + + 4.0.0 + + org.apache.maven.plugins.surefire + surefire-2143-junit4-parameterized-test-skipped + 1.0 + Test for: RunListenerAdapter, ParameterizedTest and Disabled + + + 1.8 + 1.8 + UTF-8 + 5.9.1 + + + + + + org.junit.jupiter + junit-jupiter-engine + ${junit5.version} + test + + + org.junit.jupiter + junit-jupiter-params + ${junit5.version} + test + + + + + + + maven-compiler-plugin + 3.8.0 + + UTF-8 + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + 1.0C + true + + + + + + diff --git a/surefire-its/src/test/resources/surefire-2143-junit5-parameterized-test-skipped/src/test/java/jira2143/DisabledParameterizedTest.java b/surefire-its/src/test/resources/surefire-2143-junit5-parameterized-test-skipped/src/test/java/jira2143/DisabledParameterizedTest.java new file mode 100644 index 0000000000..455d7b72da --- /dev/null +++ b/surefire-its/src/test/resources/surefire-2143-junit5-parameterized-test-skipped/src/test/java/jira2143/DisabledParameterizedTest.java @@ -0,0 +1,57 @@ +package jira2143; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +class DisabledParameterizedTest +{ + @ParameterizedTest + @ValueSource( strings = { "1", "2" } ) + @Disabled + void disabledParameterized(String param) + { + //always successful + } + + @ParameterizedTest + @ValueSource( strings = { "1", "2" } ) + void enabledParameterized(String param) + { + //always successful + } + + @Test + @Disabled + void disabledNonParameterized() + { + //always successful + } + + @Test + void enabledNonParameterized() + { + //always successful + } + +} diff --git a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java index 3c519d437a..cafc934d05 100644 --- a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java +++ b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java @@ -212,7 +212,8 @@ public void executionSkipped( TestIdentifier testIdentifier, String reason ) { boolean isClass = testIdentifier.isContainer() && testIdentifier.getSource().filter( ClassSource.class::isInstance ).isPresent(); - boolean isTest = testIdentifier.isTest(); + + testStartTime.remove( testIdentifier ); if ( isClass ) { @@ -224,9 +225,8 @@ public void executionSkipped( TestIdentifier testIdentifier, String reason ) } runListener.testSetCompleted( report ); } - else if ( isTest ) + else { - testStartTime.remove( testIdentifier ); runListener.testSkipped( createReportEntry( testIdentifier, null, emptyMap(), reason, null ) ); } } @@ -244,7 +244,7 @@ private SimpleReportEntry createReportEntry( TestIdentifier testIdentifier, { classText = null; } - boolean failed = testExecutionResult != null && testExecutionResult.getStatus() == FAILED; + boolean failed = testExecutionResult == null || testExecutionResult.getStatus() == FAILED; String methodName = failed || testIdentifier.isTest() ? classMethodName[2] : null; String methodText = failed || testIdentifier.isTest() ? classMethodName[3] : null; if ( Objects.equals( methodName, methodText ) ) diff --git a/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java b/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java index 9e376cc065..b909b2c43a 100644 --- a/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java +++ b/surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java @@ -487,6 +487,49 @@ public void runNestingTest() throws Exception assertNull( reportEntries.get( 1 ).getNameText() ); } + @Test + public void detectSkippedParameterized() throws Exception + { + Launcher launcher = LauncherFactory.create(); + ProviderParameters parameters = providerParametersMock(); + + JUnitPlatformProvider provider = new JUnitPlatformProvider( parameters, launcher ); + + TestPlanSummaryListener executionListener = new TestPlanSummaryListener(); + + TestReportListener listener = mock( TestReportListener.class ); + ArgumentCaptor entryCaptor = ArgumentCaptor.forClass( ReportEntry.class ); + RunListenerAdapter adapter = new RunListenerAdapter( listener ); + adapter.setRunMode( NORMAL_RUN ); + + launcher.registerTestExecutionListeners( executionListener, adapter ); + + invokeProvider( provider, DisabledParameterizedTest.class ); + + assertThat( executionListener.summaries ).hasSize( 1 ); + + verify( listener, times( 1 ) ).testSetCompleted( any() ); + verify( listener, times( 1 ) ).testSkipped( entryCaptor.capture() ); + List reportEntries = entryCaptor.getAllValues(); + + assertEquals( DisabledParameterizedTest.class.getName(), reportEntries.get( 0 ).getSourceName() ); + assertNull( reportEntries.get( 0 ).getSourceText() ); + assertEquals( "disabledParameterized(String)", reportEntries.get( 0 ).getName() ); + assertNull( reportEntries.get( 0 ).getNameText() ); + + TestExecutionSummary summary = executionListener.summaries.get( 0 ); + assertEquals( 0, summary.getTestsFoundCount() ); + assertEquals( 0, summary.getTestsStartedCount() ); + assertEquals( 0, summary.getTestsSucceededCount() ); + assertEquals( 0, summary.getTestsFailedCount() ); + assertEquals( 0, summary.getTestsSkippedCount() ); + assertEquals( 3, summary.getContainersFoundCount() ); + assertEquals( 2, summary.getContainersStartedCount() ); + assertEquals( 2, summary.getContainersSucceededCount() ); + assertEquals( 0, summary.getContainersFailedCount() ); + assertEquals( 1, summary.getContainersSkippedCount() ); + } + @Test public void detectErroredParameterized() throws Exception @@ -1483,6 +1526,16 @@ void level2test() } } + static class DisabledParameterizedTest + { + @org.junit.jupiter.params.ParameterizedTest + @org.junit.jupiter.params.provider.ValueSource( strings = { "1", "2" } ) + @Disabled + void disabledParameterized( String param ) + { + } + } + static class FailingBeforeAllJupiterTest {