Split plugin loading into two different phases to support entitlements (

elastic#116998)

This change loads all the modules and creates the module layers for plugins prior to entitlement 
checking during the 2nd phase of bootstrap initialization. This will allow us to know what modules exist 
for both validation and checking prior to actually loading any plugin classes (in a follow up change).

There are now two classes:

    PluginsLoader which does the module loading and layer creation
    PluginsService which uses a PluginsLoader to create the main plugin classes and start the plugins

benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.plugins.PluginsLoader;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.DocReader;
@@ -76,8 +77,7 @@ public class ScriptScoreBenchmark {
     private final PluginsService pluginsService = new PluginsService(
         Settings.EMPTY,
         null,
-        null,
-        Path.of(System.getProperty("plugins.dir"))
+        new PluginsLoader(null, Path.of(System.getProperty("plugins.dir")))
     );
     private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList());
 

server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.plugins.PluginsLoader;
 
 import java.io.PrintStream;
 
@@ -42,6 +43,9 @@ class Bootstrap {
     // the loaded settings for the node, not valid until after phase 2 of initialization
     private final SetOnce<Environment> nodeEnv = new SetOnce<>();
 
+    // loads information about plugins required for entitlements in phase 2, used by plugins service in phase 3
+    private final SetOnce<PluginsLoader> pluginsLoader = new SetOnce<>();
+
     Bootstrap(PrintStream out, PrintStream err, ServerArgs args) {
         this.out = out;
         this.err = err;
@@ -72,6 +76,14 @@ Environment environment() {
         return nodeEnv.get();
     }
 
+    void setPluginsLoader(PluginsLoader pluginsLoader) {
+        this.pluginsLoader.set(pluginsLoader);
+    }
+
+    PluginsLoader pluginsLoader() {
+        return pluginsLoader.get();
+    }
+
     void exitWithNodeValidationException(NodeValidationException e) {
         Logger logger = LogManager.getLogger(Elasticsearch.class);
         logger.error("node validation exception\n{}", e.getMessage());

server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

@@ -41,6 +41,7 @@
 import org.elasticsearch.nativeaccess.NativeAccess;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.plugins.PluginsLoader;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -199,6 +200,9 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException {
             VectorUtil.class
         );
 
+        // load the plugin Java modules and layers now for use in entitlements
+        bootstrap.setPluginsLoader(new PluginsLoader(nodeEnv.modulesFile(), nodeEnv.pluginsFile()));
+
         if (Boolean.parseBoolean(System.getProperty("es.entitlements.enabled"))) {
             logger.info("Bootstrapping Entitlements");
             EntitlementBootstrap.bootstrap();
@@ -244,7 +248,7 @@ private static void ensureInitialized(Class<?>... classes) {
     private static void initPhase3(Bootstrap bootstrap) throws IOException, NodeValidationException {
         checkLucene();
 
-        Node node = new Node(bootstrap.environment()) {
+        Node node = new Node(bootstrap.environment(), bootstrap.pluginsLoader()) {
             @Override
             protected void validateNodeBeforeAcceptingRequests(
                 final BootstrapContext context,

server/src/main/java/org/elasticsearch/node/Node.java

@@ -73,6 +73,7 @@
 import org.elasticsearch.plugins.ClusterPlugin;
 import org.elasticsearch.plugins.MetadataUpgrader;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsLoader;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.readiness.ReadinessService;
 import org.elasticsearch.repositories.RepositoriesService;
@@ -196,8 +197,8 @@ public class Node implements Closeable {
      *
      * @param environment         the initial environment for this node, which will be added to by plugins
      */
-    public Node(Environment environment) {
-        this(NodeConstruction.prepareConstruction(environment, new NodeServiceProvider(), true));
+    public Node(Environment environment, PluginsLoader pluginsLoader) {
+        this(NodeConstruction.prepareConstruction(environment, pluginsLoader, new NodeServiceProvider(), true));
     }
 
     /**

server/src/main/java/org/elasticsearch/node/NodeConstruction.java

@@ -165,6 +165,7 @@
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.PersistentTaskPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsLoader;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.RecoveryPlannerPlugin;
 import org.elasticsearch.plugins.ReloadablePlugin;
@@ -260,14 +261,15 @@ class NodeConstruction {
      */
     static NodeConstruction prepareConstruction(
         Environment initialEnvironment,
+        PluginsLoader pluginsLoader,
         NodeServiceProvider serviceProvider,
         boolean forbidPrivateIndexSettings
     ) {
         List<Closeable> closeables = new ArrayList<>();
         try {
             NodeConstruction constructor = new NodeConstruction(closeables);
 
-            Settings settings = constructor.createEnvironment(initialEnvironment, serviceProvider);
+            Settings settings = constructor.createEnvironment(initialEnvironment, serviceProvider, pluginsLoader);
             constructor.loadLoggingDataProviders();
             TelemetryProvider telemetryProvider = constructor.createTelemetryProvider(settings);
             ThreadPool threadPool = constructor.createThreadPool(settings, telemetryProvider.getMeterRegistry());
@@ -400,7 +402,7 @@ private static <T> Optional<T> getSinglePlugin(Stream<T> plugins, Class<T> plugi
         return Optional.of(plugin);
     }
 
-    private Settings createEnvironment(Environment initialEnvironment, NodeServiceProvider serviceProvider) {
+    private Settings createEnvironment(Environment initialEnvironment, NodeServiceProvider serviceProvider, PluginsLoader pluginsLoader) {
         // Pass the node settings to the DeprecationLogger class so that it can have the deprecation.skip_deprecated_settings setting:
         Settings envSettings = initialEnvironment.settings();
         DeprecationLogger.initialize(envSettings);
@@ -473,7 +475,7 @@ private Settings createEnvironment(Environment initialEnvironment, NodeServicePr
             (e, apmConfig) -> logger.error("failed to delete temporary APM config file [{}], reason: [{}]", apmConfig, e.getMessage())
         );
 
-        pluginsService = serviceProvider.newPluginService(initialEnvironment, envSettings);
+        pluginsService = serviceProvider.newPluginService(initialEnvironment, pluginsLoader);
         modules.bindToInstance(PluginsService.class, pluginsService);
         Settings settings = Node.mergePluginSettings(pluginsService.pluginMap(), envSettings);
 

server/src/main/java/org/elasticsearch/node/NodeServiceProvider.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.recovery.RecoverySettings;
+import org.elasticsearch.plugins.PluginsLoader;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.readiness.ReadinessService;
 import org.elasticsearch.script.ScriptContext;
@@ -51,9 +52,9 @@
  */
 class NodeServiceProvider {
 
-    PluginsService newPluginService(Environment environment, Settings settings) {
+    PluginsService newPluginService(Environment initialEnvironment, PluginsLoader pluginsLoader) {
         // this creates a PluginsService with an empty list of classpath plugins
-        return new PluginsService(settings, environment.configFile(), environment.modulesFile(), environment.pluginsFile());
+        return new PluginsService(initialEnvironment.settings(), initialEnvironment.configFile(), pluginsLoader);
     }
 
     ScriptService newScriptService(