From 69f52a8f1ce65bcc04fd3161882c00fba05bd5d1 Mon Sep 17 00:00:00 2001 From: Alex Ashitkin Date: Thu, 3 Oct 2019 22:13:24 -0400 Subject: [PATCH] [MNG-6774] - Speedup project graph build by paralleling operations using fork join pool to build graph --- .../maven/graph/DefaultGraphBuilder.java | 39 ++- .../maven/project/DefaultProjectBuilder.java | 244 ++++++++++++------ 2 files changed, 204 insertions(+), 79 deletions(-) diff --git a/maven-core/src/main/java/org/apache/maven/graph/DefaultGraphBuilder.java b/maven-core/src/main/java/org/apache/maven/graph/DefaultGraphBuilder.java index fb7a9f45064d..0e917382a534 100644 --- a/maven-core/src/main/java/org/apache/maven/graph/DefaultGraphBuilder.java +++ b/maven-core/src/main/java/org/apache/maven/graph/DefaultGraphBuilder.java @@ -21,7 +21,6 @@ import java.io.File; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -29,6 +28,8 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinTask; import org.apache.maven.DefaultMaven; import org.apache.maven.MavenExecutionException; @@ -401,18 +402,42 @@ private List getProjectsForMavenReactor( MavenSession session ) return projects; } - List files = Arrays.asList( request.getPom().getAbsoluteFile() ); + List files = Collections.singletonList( request.getPom().getAbsoluteFile() ); collectProjects( projects, files, request ); return projects; } - private void collectProjects( List projects, List files, MavenExecutionRequest request ) - throws ProjectBuildingException + private void collectProjects( List projects, final List files, + final MavenExecutionRequest request ) throws ProjectBuildingException { - ProjectBuildingRequest projectBuildingRequest = request.getProjectBuildingRequest(); + final ProjectBuildingRequest projectBuildingRequest = request.getProjectBuildingRequest(); + + long start = System.currentTimeMillis(); + + List results; + if ( request.getDegreeOfConcurrency() <= 1 || Boolean.getBoolean( "maven.concurrentGraph.disable" ) ) + { + results = projectBuilder.build( files, request.isRecursive(), projectBuildingRequest ); + } + else + { + ForkJoinPool fjp = new ForkJoinPool( request.getDegreeOfConcurrency() ); + try + { - List results = projectBuilder.build( files, request.isRecursive(), - projectBuildingRequest ); + results = fjp.invoke( ForkJoinTask.adapt( + () -> projectBuilder.build( files, request.isRecursive(), projectBuildingRequest ) ) ); + } + finally + { + fjp.shutdown(); + } + } + + if ( logger.isDebugEnabled() ) + { + logger.debug( "Project graph built in " + ( System.currentTimeMillis() - start ) + " millis." ); + } boolean problems = false; diff --git a/maven-core/src/main/java/org/apache/maven/project/DefaultProjectBuilder.java b/maven-core/src/main/java/org/apache/maven/project/DefaultProjectBuilder.java index c5bf26cd2946..6043c6fa3b60 100644 --- a/maven-core/src/main/java/org/apache/maven/project/DefaultProjectBuilder.java +++ b/maven-core/src/main/java/org/apache/maven/project/DefaultProjectBuilder.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.AbstractMap; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -32,7 +31,10 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ForkJoinTask; +import com.google.common.collect.Sets; import org.apache.maven.RepositoryUtils; import org.apache.maven.artifact.Artifact; import org.apache.maven.artifact.InvalidArtifactRTException; @@ -76,6 +78,10 @@ import org.eclipse.aether.resolution.ArtifactRequest; import org.eclipse.aether.resolution.ArtifactResult; +import static java.lang.System.currentTimeMillis; +import static java.util.Collections.synchronizedList; +import static java.util.Collections.unmodifiableList; + /** * DefaultProjectBuilder */ @@ -186,7 +192,7 @@ private ProjectBuildingResult build( File pomFile, ModelSource modelSource, Inte modelProblems = result.getProblems(); initProject( project, Collections.emptyMap(), true, - result, new HashMap<>(), projectBuildingRequest ); + result, new ConcurrentHashMap<>(), projectBuildingRequest ); } else if ( projectBuildingRequest.isResolveDependencies() ) { @@ -204,7 +210,7 @@ else if ( projectBuildingRequest.isResolveDependencies() ) if ( error != null ) { - ProjectBuildingException e = new ProjectBuildingException( Arrays.asList( result ) ); + ProjectBuildingException e = new ProjectBuildingException( Collections.singletonList( result ) ); e.initCause( error ); throw e; } @@ -362,20 +368,27 @@ private ModelSource createStubModelSource( Artifact artifact ) public List build( List pomFiles, boolean recursive, ProjectBuildingRequest request ) throws ProjectBuildingException { - List results = new ArrayList<>(); + List results = synchronizedList( new ArrayList<>( 512 ) ); - List interimResults = new ArrayList<>(); + List interimResults = synchronizedList( new ArrayList<>( 512 ) ); ReactorModelPool modelPool = new ReactorModelPool(); InternalConfig config = new InternalConfig( request, modelPool, useGlobalModelCache() ? getModelCache() : new ReactorModelCache() ); - Map projectIndex = new HashMap<>( 256 ); + Map projectIndex = new ConcurrentHashMap<>( 512 ); + long phase1Start = currentTimeMillis(); + Set aggregatorFiles = Sets.newConcurrentHashSet(); boolean noErrors = - build( results, interimResults, projectIndex, pomFiles, new LinkedHashSet<>(), true, recursive, - config ); + build( results, interimResults, projectIndex, pomFiles, aggregatorFiles, true, recursive, + config ); + + if ( logger.isDebugEnabled() ) + { + logger.debug( "Graph build phase 1 completed in " + ( currentTimeMillis() - phase1Start ) + " millis." ); + } populateReactorModelPool( modelPool, interimResults ); @@ -383,9 +396,15 @@ public List build( List pomFiles, boolean recursive try { - noErrors = - build( results, new ArrayList<>(), projectIndex, interimResults, request, - new HashMap<>(), config.session ) && noErrors; + long phase2Start = currentTimeMillis(); + + noErrors &= build( results, synchronizedList( new ArrayList<>() ), projectIndex, interimResults, + request, new ConcurrentHashMap<>(), config.session ); + if ( logger.isDebugEnabled() ) + { + logger.debug( + "Graph build phase 2 completed in " + ( currentTimeMillis() - phase2Start ) + " millis." ); + } } finally { @@ -407,21 +426,62 @@ private boolean build( List results, List { boolean noErrors = true; - for ( File pomFile : pomFiles ) + if ( ForkJoinTask.inForkJoinPool() ) { - aggregatorFiles.add( pomFile ); + List> tasks = new ArrayList<>( pomFiles.size() ); + for ( File pomFile : pomFiles ) + { + aggregatorFiles.add( pomFile ); + ForkJoinTask forked = forkBuild( results, interimResults, projectIndex, pomFile, + aggregatorFiles, isRoot, recursive, config ); + tasks.add( forked ); + } - if ( !build( results, interimResults, projectIndex, pomFile, aggregatorFiles, isRoot, recursive, config ) ) + for ( ForkJoinTask task : tasks ) { - noErrors = false; + noErrors &= task.join(); } + } + else + { + for ( File pomFile : pomFiles ) + { + aggregatorFiles.add( pomFile ); + + if ( !build( results, interimResults, projectIndex, pomFile, aggregatorFiles, isRoot, recursive, + config ) ) + { + noErrors = false; + } - aggregatorFiles.remove( pomFile ); + aggregatorFiles.remove( pomFile ); + } } return noErrors; } + @SuppressWarnings( "checkstyle:parameternumber" ) + private ForkJoinTask forkBuild( final List results, + final List interimResults, + final Map projectIndex, + final File pomFile, final Set aggregatorFiles, final boolean isRoot, + final boolean recursive, final InternalConfig config ) + { + return ForkJoinTask.getPool().submit( () -> + { + try + { + return build( results, interimResults, projectIndex, pomFile, aggregatorFiles, isRoot, recursive, + config ); + } + finally + { + aggregatorFiles.remove( pomFile ); + } + } ); + } + @SuppressWarnings( "checkstyle:parameternumber" ) private boolean build( List results, List interimResults, Map projectIndex, File pomFile, Set aggregatorFiles, @@ -553,10 +613,8 @@ private boolean build( List results, List moduleFiles.add( moduleFile ); } - interimResult.modules = new ArrayList<>(); - - if ( !build( results, interimResult.modules, projectIndex, moduleFiles, aggregatorFiles, false, - recursive, config ) ) + if ( !build( results, interimResult.modules, projectIndex, unmodifiableList( moduleFiles ), aggregatorFiles, + false, recursive, config ) ) { noErrors = false; } @@ -568,17 +626,17 @@ private boolean build( List results, List static class InterimResult { - File pomFile; + final File pomFile; - ModelBuildingRequest request; + final ModelBuildingRequest request; - ModelBuildingResult result; + final ModelBuildingResult result; - DefaultModelBuildingListener listener; + final DefaultModelBuildingListener listener; - boolean root; + final boolean root; - List modules = Collections.emptyList(); + final List modules = Collections.synchronizedList( new ArrayList<>() ); InterimResult( File pomFile, ModelBuildingRequest request, ModelBuildingResult result, DefaultModelBuildingListener listener, boolean root ) @@ -603,6 +661,7 @@ private void populateReactorModelPool( ReactorModelPool reactorModelPool, List results, List projects, Map projectIndex, List interimResults, ProjectBuildingRequest request, Map profilesXmls, @@ -610,59 +669,100 @@ private boolean build( List results, List p { boolean noErrors = true; - for ( InterimResult interimResult : interimResults ) + if ( ForkJoinTask.inForkJoinPool() ) { - MavenProject project = interimResult.listener.getProject(); - try + List> tasks = new ArrayList<>( interimResults.size() ); + for ( InterimResult interimResult : interimResults ) { - ModelBuildingResult result = modelBuilder.build( interimResult.request, interimResult.result ); + ForkJoinTask forked = forkBuild( results, projects, projectIndex, request, profilesXmls, + session, interimResult ); + tasks.add( forked ); + } - // 2nd pass of initialization: resolve and build parent if necessary - try - { - initProject( project, projectIndex, true, result, profilesXmls, request ); - } - catch ( InvalidArtifactRTException iarte ) - { - result.getProblems().add( new DefaultModelProblem( null, ModelProblem.Severity.ERROR, null, - result.getEffectiveModel(), -1, -1, iarte ) ); - } + for ( ForkJoinTask task : tasks ) + { + noErrors &= task.join(); + } + } + else + { + for ( InterimResult interimResult : interimResults ) + { + noErrors &= build( results, projects, projectIndex, request, profilesXmls, session, + interimResult ); + } + } - List modules = new ArrayList<>(); - noErrors = - build( results, modules, projectIndex, interimResult.modules, request, profilesXmls, session ) - && noErrors; + return noErrors; + } - projects.addAll( modules ); - projects.add( project ); + @SuppressWarnings( "checkstyle:parameternumber" ) + private ForkJoinTask forkBuild( final List results, + final List projects, + final Map projectIndex, + final ProjectBuildingRequest request, + final Map profilesXmls, + final RepositorySystemSession session, + final InterimResult interimResult ) + { + return ForkJoinTask.getPool().submit( + () -> build( results, projects, projectIndex, request, profilesXmls, session, interimResult ) ); + } - project.setExecutionRoot( interimResult.root ); - project.setCollectedProjects( modules ); - DependencyResolutionResult resolutionResult = null; - if ( request.isResolveDependencies() ) - { - resolutionResult = resolveDependencies( project, session ); - } + @SuppressWarnings( "checkstyle:parameternumber" ) + private boolean build( List results, List projects, + Map projectIndex, ProjectBuildingRequest request, + Map profilesXmls, RepositorySystemSession session, + InterimResult interimResult ) + { + MavenProject project = interimResult.listener.getProject(); + boolean noErrors; + try + { + ModelBuildingResult result = modelBuilder.build( interimResult.request, interimResult.result ); - results.add( new DefaultProjectBuildingResult( project, result.getProblems(), resolutionResult ) ); + // 2nd pass of initialization: resolve and build parent if necessary + try + { + initProject( project, projectIndex, true, result, profilesXmls, request ); } - catch ( ModelBuildingException e ) + catch ( InvalidArtifactRTException iarte ) { - DefaultProjectBuildingResult result = null; - if ( project == null ) - { - result = new DefaultProjectBuildingResult( e.getModelId(), interimResult.pomFile, e.getProblems() ); - } - else - { - result = new DefaultProjectBuildingResult( project, e.getProblems(), null ); - } - results.add( result ); + result.getProblems().add( new DefaultModelProblem( null, ModelProblem.Severity.ERROR, null, + result.getEffectiveModel(), -1, -1, iarte ) ); + } - noErrors = false; + List modules = synchronizedList( new ArrayList<>() ); + noErrors = build( results, modules, projectIndex, interimResult.modules, request, profilesXmls, session ); + + projects.addAll( modules ); + projects.add( project ); + + project.setExecutionRoot( interimResult.root ); + project.setCollectedProjects( modules ); + DependencyResolutionResult resolutionResult = null; + if ( request.isResolveDependencies() ) + { + resolutionResult = resolveDependencies( project, session ); } + + results.add( new DefaultProjectBuildingResult( project, result.getProblems(), resolutionResult ) ); } + catch ( ModelBuildingException e ) + { + DefaultProjectBuildingResult result; + if ( project == null ) + { + result = new DefaultProjectBuildingResult( e.getModelId(), interimResult.pomFile, e.getProblems() ); + } + else + { + result = new DefaultProjectBuildingResult( project, e.getProblems(), null ); + } + results.add( result ); + noErrors = false; + } return noErrors; } @@ -862,11 +962,11 @@ HashMap compute() DeploymentRepository r = project.getDistributionManagement().getRepository(); if ( !StringUtils.isEmpty( r.getId() ) && !StringUtils.isEmpty( r.getUrl() ) ) { - ArtifactRepository repo = repositorySystem.buildArtifactRepository( r ); + ArtifactRepository repo = MavenRepositorySystem.buildArtifactRepository( r ); repositorySystem.injectProxy( projectBuildingRequest.getRepositorySession(), - Arrays.asList( repo ) ); + Collections.singletonList( repo ) ); repositorySystem.injectAuthentication( projectBuildingRequest.getRepositorySession(), - Arrays.asList( repo ) ); + Collections.singletonList( repo ) ); project.setReleaseArtifactRepository( repo ); } } @@ -886,11 +986,11 @@ HashMap compute() DeploymentRepository r = project.getDistributionManagement().getSnapshotRepository(); if ( !StringUtils.isEmpty( r.getId() ) && !StringUtils.isEmpty( r.getUrl() ) ) { - ArtifactRepository repo = repositorySystem.buildArtifactRepository( r ); + ArtifactRepository repo = MavenRepositorySystem.buildArtifactRepository( r ); repositorySystem.injectProxy( projectBuildingRequest.getRepositorySession(), - Arrays.asList( repo ) ); + Collections.singletonList( repo ) ); repositorySystem.injectAuthentication( projectBuildingRequest.getRepositorySession(), - Arrays.asList( repo ) ); + Collections.singletonList( repo ) ); project.setSnapshotArtifactRepository( repo ); } }