Merge branch 'main' into test/random_compatible_version

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java

@@ -31,11 +31,15 @@
 import org.gradle.api.tasks.bundling.Jar;
 import org.gradle.api.tasks.javadoc.Javadoc;
 import org.gradle.external.javadoc.CoreJavadocOptions;
+import org.gradle.jvm.toolchain.JavaLanguageVersion;
+import org.gradle.jvm.toolchain.JavaToolchainService;
 import org.gradle.language.base.plugins.LifecycleBasePlugin;
 
 import java.io.File;
 import java.util.Map;
 
+import javax.inject.Inject;
+
 import static org.elasticsearch.gradle.internal.conventions.util.Util.toStringable;
 import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
 
@@ -44,6 +48,14 @@
  * common configuration for production code.
  */
 public class ElasticsearchJavaPlugin implements Plugin<Project> {
+
+    private final JavaToolchainService javaToolchains;
+
+    @Inject
+    ElasticsearchJavaPlugin(JavaToolchainService javaToolchains) {
+        this.javaToolchains = javaToolchains;
+    }
+
     @Override
     public void apply(Project project) {
         project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class);
@@ -55,7 +67,7 @@ public void apply(Project project) {
         // configureConfigurations(project);
         configureJars(project, buildParams.get());
         configureJarManifest(project, buildParams.get());
-        configureJavadoc(project);
+        configureJavadoc(project, buildParams.get());
         testCompileOnlyDeps(project);
     }
 
@@ -128,14 +140,18 @@ private static void configureJarManifest(Project project, BuildParameterExtensio
         project.getPluginManager().apply("nebula.info-jar");
     }
 
-    private static void configureJavadoc(Project project) {
+    private void configureJavadoc(Project project, BuildParameterExtension buildParams) {
         project.getTasks().withType(Javadoc.class).configureEach(javadoc -> {
             /*
              * Generate docs using html5 to suppress a warning from `javadoc`
              * that the default will change to html5 in the future.
              */
             CoreJavadocOptions javadocOptions = (CoreJavadocOptions) javadoc.getOptions();
             javadocOptions.addBooleanOption("html5", true);
+
+            javadoc.getJavadocTool().set(javaToolchains.javadocToolFor(spec -> {
+                spec.getLanguageVersion().set(JavaLanguageVersion.of(buildParams.getMinimumRuntimeVersion().getMajorVersion()));
+            }));
         });
 
         TaskProvider<Javadoc> javadoc = project.getTasks().withType(Javadoc.class).named("javadoc");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java

@@ -17,12 +17,12 @@
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
+import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.plugins.JvmToolchainsPlugin;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
-import org.gradle.api.tasks.Copy;
 import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.jvm.toolchain.JavaToolchainService;
@@ -54,11 +54,17 @@ public class InternalDistributionBwcSetupPlugin implements Plugin<Project> {
     private final ObjectFactory objectFactory;
     private ProviderFactory providerFactory;
     private JavaToolchainService toolChainService;
+    private FileSystemOperations fileSystemOperations;
 
     @Inject
-    public InternalDistributionBwcSetupPlugin(ObjectFactory objectFactory, ProviderFactory providerFactory) {
+    public InternalDistributionBwcSetupPlugin(
+        ObjectFactory objectFactory,
+        ProviderFactory providerFactory,
+        FileSystemOperations fileSystemOperations
+    ) {
         this.objectFactory = objectFactory;
         this.providerFactory = providerFactory;
+        this.fileSystemOperations = fileSystemOperations;
     }
 
     @Override
@@ -76,7 +82,8 @@ public void apply(Project project) {
                 providerFactory,
                 objectFactory,
                 toolChainService,
-                isCi
+                isCi,
+                fileSystemOperations
             );
         });
     }
@@ -88,7 +95,8 @@ private static void configureBwcProject(
         ProviderFactory providerFactory,
         ObjectFactory objectFactory,
         JavaToolchainService toolChainService,
-        Boolean isCi
+        Boolean isCi,
+        FileSystemOperations fileSystemOperations
     ) {
         ProjectLayout layout = project.getLayout();
         Provider<BwcVersions.UnreleasedVersionInfo> versionInfoProvider = providerFactory.provider(() -> versionInfo);
@@ -120,11 +128,18 @@ private static void configureBwcProject(
         List<DistributionProject> distributionProjects = resolveArchiveProjects(checkoutDir.get(), bwcVersion.get());
 
         // Setup gradle user home directory
-        project.getTasks().register("setupGradleUserHome", Copy.class, copy -> {
-            copy.into(project.getGradle().getGradleUserHomeDir().getAbsolutePath() + "-" + project.getName());
-            copy.from(project.getGradle().getGradleUserHomeDir().getAbsolutePath(), copySpec -> {
-                copySpec.include("gradle.properties");
-                copySpec.include("init.d/*");
+        // We don't use a normal `Copy` task here as snapshotting the entire gradle user home is very expensive. This task is cheap, so
+        // up-to-date checking doesn't buy us much
+        project.getTasks().register("setupGradleUserHome", task -> {
+            task.doLast(t -> {
+                fileSystemOperations.copy(copy -> {
+                    String gradleUserHome = project.getGradle().getGradleUserHomeDir().getAbsolutePath();
+                    copy.into(gradleUserHome + "-" + project.getName());
+                    copy.from(gradleUserHome, copySpec -> {
+                        copySpec.include("gradle.properties");
+                        copySpec.include("init.d/*");
+                    });
+                });
             });
         });
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java

@@ -86,14 +86,14 @@ public void apply(Project project) {
             configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.TEST_SOURCE_SET_NAME), 21);
             for (int javaVersion : mainVersions) {
                 String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion;
-                SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion);
+                SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion, true);
                 configureSourceSetInJar(project, mainSourceSet, javaVersion);
                 addJar(project, mainSourceSet, javaVersion);
                 mainSourceSets.add(mainSourceSetName);
                 testSourceSets.add(mainSourceSetName);
 
                 String testSourceSetName = SourceSet.TEST_SOURCE_SET_NAME + javaVersion;
-                SourceSet testSourceSet = addSourceSet(project, javaExtension, testSourceSetName, testSourceSets, javaVersion);
+                SourceSet testSourceSet = addSourceSet(project, javaExtension, testSourceSetName, testSourceSets, javaVersion, false);
                 testSourceSets.add(testSourceSetName);
                 createTestTask(project, buildParams, testSourceSet, javaVersion, mainSourceSets);
             }
@@ -121,7 +121,8 @@ private SourceSet addSourceSet(
         JavaPluginExtension javaExtension,
         String sourceSetName,
         List<String> parentSourceSets,
-        int javaVersion
+        int javaVersion,
+        boolean isMainSourceSet
     ) {
         SourceSet sourceSet = javaExtension.getSourceSets().maybeCreate(sourceSetName);
         for (String parentSourceSetName : parentSourceSets) {
@@ -135,6 +136,13 @@ private SourceSet addSourceSet(
             CompileOptions compileOptions = compileTask.getOptions();
             compileOptions.getRelease().set(javaVersion);
         });
+        if (isMainSourceSet) {
+            project.getTasks().create(sourceSet.getJavadocTaskName(), Javadoc.class, javadocTask -> {
+                javadocTask.getJavadocTool().set(javaToolchains.javadocToolFor(spec -> {
+                    spec.getLanguageVersion().set(JavaLanguageVersion.of(javaVersion));
+                }));
+            });
+        }
         configurePreviewFeatures(project, sourceSet, javaVersion);
 
         // Since we configure MRJAR sourcesets to allow preview apis, class signatures for those

docs/changelog/116944.yaml → docs/changelog/118825.yaml

@@ -1,11 +1,11 @@
-pr: 116944
+pr: 118825
 summary: "Remove support for type, fields, `copy_to` and boost in metadata field definition"
 area: Mapping
 type: breaking
 issues: []
 breaking:
   title: "Remove support for type, fields, copy_to and boost in metadata field definition"
   area: Mapping
-  details: The type, fields, copy_to and boost parameters are no longer supported in metadata field definition
+  details: The type, fields, copy_to and boost parameters are no longer supported in metadata field definition starting with version 9.
   impact: Users providing type, fields, copy_to or boost as part of metadata field definition should remove them from their mappings.
   notable: false

libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java

@@ -11,6 +11,7 @@
 
 import java.net.URL;
 import java.net.URLStreamHandlerFactory;
+import java.util.List;
 
 public interface EntitlementChecker {
 
@@ -29,4 +30,10 @@ public interface EntitlementChecker {
     void check$java_net_URLClassLoader$(Class<?> callerClass, String name, URL[] urls, ClassLoader parent);
 
     void check$java_net_URLClassLoader$(Class<?> callerClass, String name, URL[] urls, ClassLoader parent, URLStreamHandlerFactory factory);
+
+    // Process creation
+    void check$$start(Class<?> callerClass, ProcessBuilder that, ProcessBuilder.Redirect[] redirects);
+
+    void check$java_lang_ProcessBuilder$startPipeline(Class<?> callerClass, List<ProcessBuilder> builders);
+
 }

libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java

@@ -29,58 +29,74 @@
 import java.util.stream.Collectors;
 
 import static java.util.Map.entry;
+import static org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction.CheckAction.deniedToPlugins;
+import static org.elasticsearch.entitlement.qa.common.RestEntitlementsCheckAction.CheckAction.forPlugins;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestEntitlementsCheckAction extends BaseRestHandler {
     private static final Logger logger = LogManager.getLogger(RestEntitlementsCheckAction.class);
     private final String prefix;
 
-    private record CheckAction(Runnable action, boolean isServerOnly) {
-
-        static CheckAction serverOnly(Runnable action) {
+    record CheckAction(Runnable action, boolean isAlwaysDeniedToPlugins) {
+        /**
+         * These cannot be granted to plugins, so our test plugins cannot test the "allowed" case.
+         * Used both for always-denied entitlements as well as those granted only to the server itself.
+         */
+        static CheckAction deniedToPlugins(Runnable action) {
             return new CheckAction(action, true);
         }
 
-        static CheckAction serverAndPlugin(Runnable action) {
+        static CheckAction forPlugins(Runnable action) {
             return new CheckAction(action, false);
         }
     }
 
     private static final Map<String, CheckAction> checkActions = Map.ofEntries(
-        entry("runtime_exit", CheckAction.serverOnly(RestEntitlementsCheckAction::runtimeExit)),
-        entry("runtime_halt", CheckAction.serverOnly(RestEntitlementsCheckAction::runtimeHalt)),
-        entry("create_classloader", CheckAction.serverAndPlugin(RestEntitlementsCheckAction::createClassLoader))
+        entry("runtime_exit", deniedToPlugins(RestEntitlementsCheckAction::runtimeExit)),
+        entry("runtime_halt", deniedToPlugins(RestEntitlementsCheckAction::runtimeHalt)),
+        entry("create_classloader", forPlugins(RestEntitlementsCheckAction::createClassLoader)),
+        // entry("processBuilder_start", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_start)),
+        entry("processBuilder_startPipeline", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_startPipeline))
     );
 
     @SuppressForbidden(reason = "Specifically testing Runtime.exit")
     private static void runtimeExit() {
-        logger.info("Calling Runtime.exit;");
         Runtime.getRuntime().exit(123);
     }
 
     @SuppressForbidden(reason = "Specifically testing Runtime.halt")
     private static void runtimeHalt() {
-        logger.info("Calling Runtime.halt;");
         Runtime.getRuntime().halt(123);
     }
 
     private static void createClassLoader() {
-        logger.info("Calling new URLClassLoader");
         try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) {
             logger.info("Created URLClassLoader [{}]", classLoader.getName());
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
     }
 
+    private static void processBuilder_start() {
+        // TODO: processBuilder().start();
+    }
+
+    private static void processBuilder_startPipeline() {
+        try {
+            ProcessBuilder.startPipeline(List.of());
+        } catch (IOException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
     public RestEntitlementsCheckAction(String prefix) {
         this.prefix = prefix;
     }
 
     public static Set<String> getServerAndPluginsCheckActions() {
         return checkActions.entrySet()
             .stream()
-            .filter(kv -> kv.getValue().isServerOnly() == false)
+            .filter(kv -> kv.getValue().isAlwaysDeniedToPlugins() == false)
             .map(Map.Entry::getKey)
             .collect(Collectors.toSet());
     }
@@ -112,6 +128,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
         }
 
         return channel -> {
+            logger.info("Calling check action [{}]", actionName);
             checkAction.action().run();
             channel.sendResponse(new RestResponse(RestStatus.OK, Strings.format("Succesfully executed action [%s]", actionName)));
         };

libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementAllowedNonModularPlugin.java

@@ -27,7 +27,6 @@
 import java.util.function.Supplier;
 
 public class EntitlementAllowedNonModularPlugin extends Plugin implements ActionPlugin {
-
     @Override
     public List<RestHandler> getRestHandlers(
         final Settings settings,

libs/entitlement/qa/entitlement-allowed/src/main/java/org/elasticsearch/entitlement/qa/EntitlementAllowedPlugin.java

@@ -27,7 +27,6 @@
 import java.util.function.Supplier;
 
 public class EntitlementAllowedPlugin extends Plugin implements ActionPlugin {
-
     @Override
     public List<RestHandler> getRestHandlers(
         final Settings settings,

libs/entitlement/qa/entitlement-denied-nonmodular/src/main/java/org/elasticsearch/entitlement/qa/nonmodular/EntitlementDeniedNonModularPlugin.java

@@ -27,7 +27,6 @@
 import java.util.function.Supplier;
 
 public class EntitlementDeniedNonModularPlugin extends Plugin implements ActionPlugin {
-
     @Override
     public List<RestHandler> getRestHandlers(
         final Settings settings,

libs/entitlement/qa/entitlement-denied/src/main/java/org/elasticsearch/entitlement/qa/EntitlementDeniedPlugin.java

@@ -27,7 +27,6 @@
 import java.util.function.Supplier;
 
 public class EntitlementDeniedPlugin extends Plugin implements ActionPlugin {
-
     @Override
     public List<RestHandler> getRestHandlers(
         final Settings settings,

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java

@@ -14,6 +14,7 @@
 
 import java.net.URL;
 import java.net.URLStreamHandlerFactory;
+import java.util.List;
 
 /**
  * Implementation of the {@link EntitlementChecker} interface, providing additional
@@ -67,4 +68,14 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) {
     ) {
         policyManager.checkCreateClassLoader(callerClass);
     }
+
+    @Override
+    public void check$$start(Class<?> callerClass, ProcessBuilder processBuilder, ProcessBuilder.Redirect[] redirects) {
+        policyManager.checkStartProcess(callerClass);
+    }
+
+    @Override
+    public void check$java_lang_ProcessBuilder$startPipeline(Class<?> callerClass, List<ProcessBuilder> builders) {
+        policyManager.checkStartProcess(callerClass);
+    }
 }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -105,6 +105,26 @@ private static Map<String, List<Entitlement>> buildScopeEntitlementsMap(Policy p
         return policy.scopes.stream().collect(Collectors.toUnmodifiableMap(scope -> scope.name, scope -> scope.entitlements));
     }
 
+    public void checkStartProcess(Class<?> callerClass) {
+        neverEntitled(callerClass, "start process");
+    }
+
+    private void neverEntitled(Class<?> callerClass, String operationDescription) {
+        var requestingModule = requestingModule(callerClass);
+        if (isTriviallyAllowed(requestingModule)) {
+            return;
+        }
+
+        throw new NotEntitledException(
+            Strings.format(
+                "Not entitled: caller [%s], module [%s], operation [%s]",
+                callerClass,
+                requestingModule.getName(),
+                operationDescription
+            )
+        );
+    }
+
     public void checkExitVM(Class<?> callerClass) {
         checkEntitlementPresent(callerClass, ExitVMEntitlement.class);
     }