From 59e096d9874e4cd719bef01e1dc45f9b5f7e9456 Mon Sep 17 00:00:00 2001 From: Aaron Braunstein Date: Fri, 3 Feb 2023 00:07:11 -0800 Subject: [PATCH] [SUREFIRE-2095] Fork crash doesn't fail build with -Dmaven.test.failure.ignore=true when run with failsafe (#545) Verify goal should fail when test summary indicates a SurefireBooterForkException occurred --- .../maven/plugin/failsafe/VerifyMojo.java | 40 ++++++- .../plugin/failsafe/JUnit4SuiteTest.java | 3 +- .../maven/plugin/failsafe/VerifyMojoTest.java | 108 ++++++++++++++++++ .../failsafe-summary-booter-fork-error.xml | 41 +++++++ .../verify-mojo/failsafe-summary-success.xml | 8 ++ ...5FailsafeJvmCrashShouldNotBeIgnoredIT.java | 48 ++++++++ .../surefire-2095-failsafe-jvm-crash/pom.xml | 63 ++++++++++ .../src/test/java/PojoIT.java | 44 +++++++ 8 files changed, 353 insertions(+), 2 deletions(-) create mode 100644 maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/VerifyMojoTest.java create mode 100644 maven-failsafe-plugin/src/test/resources/verify-mojo/failsafe-summary-booter-fork-error.xml create mode 100644 maven-failsafe-plugin/src/test/resources/verify-mojo/failsafe-summary-success.xml create mode 100644 surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2095FailsafeJvmCrashShouldNotBeIgnoredIT.java create mode 100644 surefire-its/src/test/resources/surefire-2095-failsafe-jvm-crash/pom.xml create mode 100644 surefire-its/src/test/resources/surefire-2095-failsafe-jvm-crash/src/test/java/PojoIT.java diff --git a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/VerifyMojo.java b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/VerifyMojo.java index 9c91bf70d8..0dd1529d94 100644 --- a/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/VerifyMojo.java +++ b/maven-failsafe-plugin/src/main/java/org/apache/maven/plugin/failsafe/VerifyMojo.java @@ -33,6 +33,7 @@ import org.apache.maven.plugins.annotations.Parameter; import org.apache.maven.surefire.api.cli.CommandLineOption; import org.apache.maven.surefire.api.suite.RunResult; +import org.apache.maven.surefire.booter.SurefireBooterForkException; import org.codehaus.plexus.logging.Logger; import java.io.File; @@ -198,10 +199,27 @@ public void execute() throw new MojoExecutionException( e.getMessage(), e ); } - reportExecution( this, summary, getConsoleLogger(), null ); + reportExecution( this, summary, getConsoleLogger(), getBooterForkException( summary ) ); } } + private Exception getBooterForkException( RunResult summary ) + { + String firstForkExceptionFailureMessage = + String.format( "%s: " , SurefireBooterForkException.class.getName() ); + if ( summary.getFailure() != null && summary.getFailure().contains( firstForkExceptionFailureMessage ) ) + { + return new SurefireBooterForkException( + summary.getFailure().substring( firstForkExceptionFailureMessage.length() ) ); + } + return null; + } + + void setLogger( Logger logger ) + { + this.logger = logger; + } + private PluginConsoleLogger getConsoleLogger() { if ( consoleLogger == null ) @@ -359,6 +377,16 @@ public void setReportsDirectory( File reportsDirectory ) this.reportsDirectory = reportsDirectory; } + public File getSummaryFile() + { + return summaryFile; + } + + public void setSummaryFile( File summaryFile ) + { + this.summaryFile = summaryFile; + } + @Override public boolean getFailIfNoTests() { @@ -383,6 +411,16 @@ public void setFailOnFlakeCount( int failOnFlakeCount ) this.failOnFlakeCount = failOnFlakeCount; } + public MavenSession getSession() + { + return session; + } + + public void setSession( MavenSession session ) + { + this.session = session; + } + private boolean existsSummaryFile() { return summaryFile != null && summaryFile.isFile(); diff --git a/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/JUnit4SuiteTest.java b/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/JUnit4SuiteTest.java index decaf8684d..186803a0d3 100644 --- a/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/JUnit4SuiteTest.java +++ b/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/JUnit4SuiteTest.java @@ -34,7 +34,8 @@ @SuiteClasses( { IntegrationTestMojoTest.class, MarshallerUnmarshallerTest.class, - RunResultTest.class + RunResultTest.class, + VerifyMojoTest.class } ) @RunWith( Suite.class ) public class JUnit4SuiteTest diff --git a/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/VerifyMojoTest.java b/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/VerifyMojoTest.java new file mode 100644 index 0000000000..37bd3b733f --- /dev/null +++ b/maven-failsafe-plugin/src/test/java/org/apache/maven/plugin/failsafe/VerifyMojoTest.java @@ -0,0 +1,108 @@ +package org.apache.maven.plugin.failsafe; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.net.URLDecoder; + +import org.apache.maven.execution.MavenExecutionRequest; +import org.apache.maven.execution.MavenSession; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.codehaus.plexus.logging.Logger; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + */ +public class VerifyMojoTest +{ + private VerifyMojo mojo; + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Before + public void init() throws UnsupportedEncodingException + { + mojo = new VerifyMojo(); + mojo.setTestClassesDirectory( tempFolder.getRoot() ); + mojo.setReportsDirectory( getTestBaseDir() ); + } + + private void setupExecuteMocks() + { + Logger logger = mock( Logger.class ); + when( logger.isErrorEnabled() ).thenReturn( true ); + when( logger.isWarnEnabled() ).thenReturn( true ); + when( logger.isInfoEnabled() ).thenReturn( true ); + when( logger.isDebugEnabled() ).thenReturn( false ); + mojo.setLogger( logger ); + + MavenSession session = mock( MavenSession.class ); + MavenExecutionRequest request = mock ( MavenExecutionRequest.class ); + when( request.isShowErrors() ).thenReturn( true ); + when( request.getReactorFailureBehavior() ).thenReturn( null ); + when( session.getRequest() ).thenReturn( request ); + mojo.setSession( session ); + } + + private File getTestBaseDir() + throws UnsupportedEncodingException + { + URL resource = getClass().getResource( "/verify-mojo" ); + // URLDecoder.decode necessary for JDK 1.5+, where spaces are escaped to %20 + return new File( URLDecoder.decode( resource.getPath(), "UTF-8" ) ).getAbsoluteFile(); + } + + @Test( expected = MojoExecutionException.class ) + public void executeForForkError() throws MojoExecutionException, MojoFailureException, UnsupportedEncodingException + { + setupExecuteMocks(); + mojo.setSummaryFile( new File( getTestBaseDir(), "failsafe-summary-booter-fork-error.xml" ) ); + mojo.execute(); + } + + @Test( expected = MojoExecutionException.class ) + public void executeForForkErrorTestFailureIgnore() throws MojoExecutionException, MojoFailureException, + UnsupportedEncodingException + { + setupExecuteMocks(); + mojo.setSummaryFile( new File( getTestBaseDir(), "failsafe-summary-booter-fork-error.xml" ) ); + mojo.setTestFailureIgnore( true ); + mojo.execute(); + } + + @Test + public void executeForPassingTests() throws MojoExecutionException, MojoFailureException, + UnsupportedEncodingException + { + setupExecuteMocks(); + mojo.setSummaryFile( new File( getTestBaseDir(), "failsafe-summary-success.xml" ) ); + mojo.execute(); + } +} diff --git a/maven-failsafe-plugin/src/test/resources/verify-mojo/failsafe-summary-booter-fork-error.xml b/maven-failsafe-plugin/src/test/resources/verify-mojo/failsafe-summary-booter-fork-error.xml new file mode 100644 index 0000000000..12cc3f7c53 --- /dev/null +++ b/maven-failsafe-plugin/src/test/resources/verify-mojo/failsafe-summary-booter-fork-error.xml @@ -0,0 +1,41 @@ + + + 0 + 0 + 0 + 0 + org.apache.maven.surefire.booter.SurefireBooterForkException: The forked VM terminated without properly saying goodbye. VM crash or System.exit called? +Command was /bin/sh -c cd '/Users/aaron.braunstein/git/apache/maven-surefire/surefire-its/target/TestClass_testMethod' && '/Library/Java/JavaVirtualMachines/temurin-11.jdk/Contents/Home/bin/java' '-Dfile.encoding=UTF-8' '-Duser.language=en' '-XFakeUnrecognizedOptionThatWillCrashJVM' '-Duser.region=US' '-showversion' '-Xmx6g' '-Xms2g' '-XX:+PrintGCDetails' '-jar' '/Users/aaron.braunstein/git/apache/maven-surefire/surefire-its/target/TestClass_testMethod/target/surefire/surefirebooter-20220606220315261_3.jar' '/Users/aaron.braunstein/git/apache/maven-surefire/surefire-its/target/TestClass_testMethod/target/surefire' '2022-06-06T22-03-11_910-jvmRun1' 'surefire-20220606220315261_1tmp' 'surefire_0-20220606220315261_2tmp' +Error occurred in starting fork, check output in log +Process Exit Code: 1 + at org.apache.maven.plugin.surefire.booterclient.ForkStarter.fork(ForkStarter.java:714) + at org.apache.maven.plugin.surefire.booterclient.ForkStarter.run(ForkStarter.java:311) + at org.apache.maven.plugin.surefire.booterclient.ForkStarter.run(ForkStarter.java:268) + at org.apache.maven.plugin.surefire.AbstractSurefireMojo.executeProvider(AbstractSurefireMojo.java:1334) + at org.apache.maven.plugin.surefire.AbstractSurefireMojo.executeAfterPreconditionsChecked(AbstractSurefireMojo.java:1167) + at org.apache.maven.plugin.surefire.AbstractSurefireMojo.execute(AbstractSurefireMojo.java:931) + at org.apache.maven.plugin.DefaultBuildPluginManager.executeMojo(DefaultBuildPluginManager.java:137) + at org.apache.maven.lifecycle.internal.MojoExecutor.doExecute(MojoExecutor.java:301) + at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:211) + at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:165) + at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:157) + at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:121) + at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:81) + at org.apache.maven.lifecycle.internal.builder.singlethreaded.SingleThreadedBuilder.build(SingleThreadedBuilder.java:56) + at org.apache.maven.lifecycle.internal.LifecycleStarter.execute(LifecycleStarter.java:127) + at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:294) + at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:192) + at org.apache.maven.DefaultMaven.execute(DefaultMaven.java:105) + at org.apache.maven.cli.MavenCli.execute(MavenCli.java:960) + at org.apache.maven.cli.MavenCli.doMain(MavenCli.java:293) + at org.apache.maven.cli.MavenCli.main(MavenCli.java:196) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.base/java.lang.reflect.Method.invoke(Method.java:566) + at org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced(Launcher.java:282) + at org.codehaus.plexus.classworlds.launcher.Launcher.launch(Launcher.java:225) + at org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode(Launcher.java:406) + at org.codehaus.plexus.classworlds.launcher.Launcher.main(Launcher.java:347) + + diff --git a/maven-failsafe-plugin/src/test/resources/verify-mojo/failsafe-summary-success.xml b/maven-failsafe-plugin/src/test/resources/verify-mojo/failsafe-summary-success.xml new file mode 100644 index 0000000000..e6416cde0f --- /dev/null +++ b/maven-failsafe-plugin/src/test/resources/verify-mojo/failsafe-summary-success.xml @@ -0,0 +1,8 @@ + + + 1 + 0 + 0 + 0 + + diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2095FailsafeJvmCrashShouldNotBeIgnoredIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2095FailsafeJvmCrashShouldNotBeIgnoredIT.java new file mode 100644 index 0000000000..71c2332c81 --- /dev/null +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2095FailsafeJvmCrashShouldNotBeIgnoredIT.java @@ -0,0 +1,48 @@ +package org.apache.maven.surefire.its.jiras; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.it.VerificationException; +import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase; +import org.junit.Test; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; + +/** + * Test https://issues.apache.org/jira/browse/SUREFIRE-2095 + * + */ +public class Surefire2095FailsafeJvmCrashShouldNotBeIgnoredIT + extends SurefireJUnit4IntegrationTestCase +{ + @Test + public void mavenShouldFail() throws VerificationException + { + // Run failsafe with testFailureIgnore=true and an unknown JVM option that will cause a crash + unpack( "surefire-2095-failsafe-jvm-crash" ) + .maven() + .withFailure() + .debugLogging() + .executeVerify() + .assertThatLogLine( containsString( "BUILD SUCCESS" ), is( 0 ) ) + .verifyTextInLog( "BUILD FAILURE" ); + } +} diff --git a/surefire-its/src/test/resources/surefire-2095-failsafe-jvm-crash/pom.xml b/surefire-its/src/test/resources/surefire-2095-failsafe-jvm-crash/pom.xml new file mode 100644 index 0000000000..e607ca6e52 --- /dev/null +++ b/surefire-its/src/test/resources/surefire-2095-failsafe-jvm-crash/pom.xml @@ -0,0 +1,63 @@ + + + + + 4.0.0 + + org.apache.maven.plugins.surefire + SUREFIRE-2095 + 1.0-SNAPSHOT + SUREFIRE-2095 + + + 1.8 + 1.8 + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + ${surefire.version} + + -Dfile.encoding=UTF-8 -Duser.language=en -XFakeUnrecognizedOptionThatWillCrashJVM -Duser.region=US -showversion -Xmx6g -Xms2g -XX:+PrintGCDetails + true + + + + integration-test + + integration-test + + + + verify + + verify + + + + + + + diff --git a/surefire-its/src/test/resources/surefire-2095-failsafe-jvm-crash/src/test/java/PojoIT.java b/surefire-its/src/test/resources/surefire-2095-failsafe-jvm-crash/src/test/java/PojoIT.java new file mode 100644 index 0000000000..3d0a0926cf --- /dev/null +++ b/surefire-its/src/test/resources/surefire-2095-failsafe-jvm-crash/src/test/java/PojoIT.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +public class PojoIT +{ + private static int calls; + + public void setUp() + { + System.out.println( "setUp called " + ++calls ); + } + + public void tearDown() + { + System.out.println( "tearDown called " + calls ); + } + + public void testSuccess() + { + assert true; + } + + public void testFailure() + { + assert false; + } + +}