Merge branch 'main' into lru

jdconrad · Apr 5, 2024 · f472530 · f472530
2 parents 931f505 + 95c7c09
commit f472530
Show file tree

Hide file tree

Showing 99 changed files with 1,299 additions and 978 deletions.
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
@@ -130,7 +130,8 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
       ':server:generateModulesList',
       ':server:generatePluginsList',
       ':generateProviderImpls',
-      ':libs:elasticsearch-native:elasticsearch-native-libraries:extractLibs'].collect { elasticsearchProject.right()?.task(it) ?: it })
+      ':libs:elasticsearch-native:elasticsearch-native-libraries:extractLibs',
+      ':x-pack:libs:es-opensaml-security-api:shadowJar'].collect { elasticsearchProject.right()?.task(it) ?: it })
   }
 
   // this path is produced by the extractLibs task above
@@ -239,20 +240,22 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
  * but before the XML document, e.g. a doctype or comment
  */
 void modifyXml(Object path, Action<? super Node> action, String preface = null) {
-  Node xml = parseXml(path)
-  action.execute(xml)
+  if (project.file(path).exists()) {
+    Node xml = parseXml(path)
+    action.execute(xml)
 
-  File xmlFile = project.file(path)
-  xmlFile.withPrintWriter { writer ->
-    def printer = new XmlNodePrinter(writer)
-    printer.namespaceAware = true
-    printer.preserveWhitespace = true
-    writer.write("<?xml version=\"1.0\"?>\n")
+    File xmlFile = project.file(path)
+    xmlFile.withPrintWriter { writer ->
+      def printer = new XmlNodePrinter(writer)
+      printer.namespaceAware = true
+      printer.preserveWhitespace = true
+      writer.write("<?xml version=\"1.0\"?>\n")
 
-    if (preface != null) {
-      writer.write(preface)
+      if (preface != null) {
+        writer.write(preface)
+      }
+      printer.print(xml)
     }
-    printer.print(xml)
   }
 }
 

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java
@@ -49,6 +49,7 @@
 public class MrjarPlugin implements Plugin<Project> {
 
     private static final Pattern MRJAR_SOURCESET_PATTERN = Pattern.compile("main(\\d{2})");
+    private static final String MRJAR_IDEA_ENABLED = "org.gradle.mrjar.idea.enabled";
 
     private final JavaToolchainService javaToolchains;
 
@@ -61,23 +62,30 @@ public class MrjarPlugin implements Plugin<Project> {
     public void apply(Project project) {
         project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
         var javaExtension = project.getExtensions().getByType(JavaPluginExtension.class);
-
-        List<Integer> mainVersions = findSourceVersions(project);
-        List<String> mainSourceSets = new ArrayList<>();
-        mainSourceSets.add(SourceSet.MAIN_SOURCE_SET_NAME);
-        List<String> testSourceSets = new ArrayList<>(mainSourceSets);
-        testSourceSets.add(SourceSet.TEST_SOURCE_SET_NAME);
-        for (int javaVersion : mainVersions) {
-            String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion;
-            SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion);
-            configureSourceSetInJar(project, mainSourceSet, javaVersion);
-            mainSourceSets.add(mainSourceSetName);
-            testSourceSets.add(mainSourceSetName);
-
-            String testSourceSetName = SourceSet.TEST_SOURCE_SET_NAME + javaVersion;
-            SourceSet testSourceSet = addSourceSet(project, javaExtension, testSourceSetName, testSourceSets, javaVersion);
-            testSourceSets.add(testSourceSetName);
-            createTestTask(project, testSourceSet, javaVersion, mainSourceSets);
+        var isIdea = System.getProperty("idea.active", "false").equals("true");
+        var ideaSourceSetsEnabled = project.hasProperty(MRJAR_IDEA_ENABLED) && project.property(MRJAR_IDEA_ENABLED).equals("true");
+
+        // Ignore version-specific source sets if we are importing into IntelliJ and have not explicitly enabled this.
+        // Avoids an IntelliJ bug:
+        // https://youtrack.jetbrains.com/issue/IDEA-285640/Compiler-Options-Settings-language-level-is-set-incorrectly-with-JDK-19ea
+        if (isIdea == false || ideaSourceSetsEnabled) {
+            List<Integer> mainVersions = findSourceVersions(project);
+            List<String> mainSourceSets = new ArrayList<>();
+            mainSourceSets.add(SourceSet.MAIN_SOURCE_SET_NAME);
+            List<String> testSourceSets = new ArrayList<>(mainSourceSets);
+            testSourceSets.add(SourceSet.TEST_SOURCE_SET_NAME);
+            for (int javaVersion : mainVersions) {
+                String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion;
+                SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion);
+                configureSourceSetInJar(project, mainSourceSet, javaVersion);
+                mainSourceSets.add(mainSourceSetName);
+                testSourceSets.add(mainSourceSetName);
+
+                String testSourceSetName = SourceSet.TEST_SOURCE_SET_NAME + javaVersion;
+                SourceSet testSourceSet = addSourceSet(project, javaExtension, testSourceSetName, testSourceSets, javaVersion);
+                testSourceSets.add(testSourceSetName);
+                createTestTask(project, testSourceSet, javaVersion, mainSourceSets);
+            }
         }
 
         configureMrjar(project);

diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
@@ -2,7 +2,7 @@ elasticsearch     = 8.14.0
 lucene            = 9.10.0
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 22+36@830ec9fcccef480bb3e73fb7ecafe059
+bundled_jdk = 21.0.2+13@f2283984656d49d69e91c558476027ac
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0

diff --git a/docs/changelog/106899.yaml b/docs/changelog/106899.yaml
@@ -0,0 +1,6 @@
+pr: 106899
+summary: Add ES|QL Locate function
+area: ES|QL
+type: enhancement
+issues:
+ - 106818
diff --git a/docs/reference/esql/functions/description/locate.asciidoc b/docs/reference/esql/functions/description/locate.asciidoc
@@ -0,0 +1,5 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Description*
+
+Returns an integer that indicates the position of a keyword substring within another string
diff --git a/docs/reference/esql/functions/layout/locate.asciidoc b/docs/reference/esql/functions/layout/locate.asciidoc
@@ -0,0 +1,14 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+[discrete]
+[[esql-locate]]
+=== `LOCATE`
+
+*Syntax*
+
+[.text-center]
+image::esql/functions/signature/locate.svg[Embedded,opts=inline]
+
+include::../parameters/locate.asciidoc[]
+include::../description/locate.asciidoc[]
+include::../types/locate.asciidoc[]
diff --git a/docs/reference/esql/functions/parameters/locate.asciidoc b/docs/reference/esql/functions/parameters/locate.asciidoc
@@ -0,0 +1,12 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Parameters*
+
+`string`::
+An input string
+
+`substring`::
+A substring to locate in the input string
+
+`start`::
+The start index
diff --git a/docs/reference/esql/functions/signature/locate.svg b/docs/reference/esql/functions/signature/locate.svg
diff --git a/docs/reference/esql/functions/types/locate.asciidoc b/docs/reference/esql/functions/types/locate.asciidoc
@@ -0,0 +1,12 @@
+// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+*Supported types*
+
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+string | substring | start | result
+keyword | keyword | integer | integer
+keyword | text | integer | integer
+text | keyword | integer | integer
+text | text | integer | integer
+|===
diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc
@@ -18,7 +18,7 @@ or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
 [[put-inference-api-request]]
 ==== {api-request-title}
 
-`PUT /_inference/<task_type>/<model_id>`
+`PUT /_inference/<task_type>/<inference_id>`
 
 
 [discrete]

diff --git a/docs/reference/release-notes/8.13.0.asciidoc b/docs/reference/release-notes/8.13.0.asciidoc
@@ -7,6 +7,9 @@ Also see <<breaking-changes-8.13,Breaking changes in 8.13>>.
 [float]
 === Known issues
 
+* Due to a bug in the bundled JDK 22 nodes might crash abruptly under high memory pressure.
+  We recommend https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html#jvm-version[downgrading to JDK 21.0.2] asap to mitigate the issue.
+
 * Nodes upgraded to 8.13.0 fail to load downsampling persistent tasks. This prevents them from joining the cluster, blocking its upgrade (issue: {es-issue}106880[#106880])
 +
 This affects clusters running version 8.10 or later, with an active downsampling

diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
@@ -1694,25 +1694,25 @@
             <sha256 value="0a4aa0f2606bd6292a7ccd7d67a0db914bf5874dfb8a6184df3e6d63cdc93702" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="openjdk_22" name="linux" version="22">
-         <artifact name="linux-22-aarch64.tar.gz">
-            <sha256 value="b272e3228d2a3e04b126d54844d33cc6d137256490526cd08679d7023d07d4b7" origin="Generated by Gradle"/>
+      <component group="openjdk_21" name="linux" version="21.0.2">
+         <artifact name="linux-21.0.2-aarch64.tar.gz">
+            <sha256 value="08db1392a48d4eb5ea5315cf8f18b89dbaf36cda663ba882cf03c704c9257ec2" origin="Generated by Gradle"/>
          </artifact>
-         <artifact name="linux-22-x64.tar.gz">
-            <sha256 value="4d65cc6ed28711768fd72c2043a7925f7c83f5f51bb64970bd9d52f7791fc6ac" origin="Generated by Gradle"/>
+         <artifact name="linux-21.0.2-x64.tar.gz">
+            <sha256 value="a2def047a73941e01a73739f92755f86b895811afb1f91243db214cff5bdac3f" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="openjdk_22" name="macos" version="22">
-         <artifact name="macos-22-aarch64.tar.gz">
-            <sha256 value="d10f82429d01047968c52c7975c326388cb5d212791e14c1de21c987463a4b53" origin="Generated by Gradle"/>
+      <component group="openjdk_21" name="macos" version="21.0.2">
+         <artifact name="macos-21.0.2-aarch64.tar.gz">
+            <sha256 value="b3d588e16ec1e0ef9805d8a696591bd518a5cea62567da8f53b5ce32d11d22e4" origin="Generated by Gradle"/>
          </artifact>
-         <artifact name="macos-22-x64.tar.gz">
-            <sha256 value="ae31fe10916429e3fe284266095067a5ce9fecbdc03ff1a079d20459f731ca36" origin="Generated by Gradle"/>
+         <artifact name="macos-21.0.2-x64.tar.gz">
+            <sha256 value="8fd09e15dc406387a0aba70bf5d99692874e999bf9cd9208b452b5d76ac922d3" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="openjdk_22" name="windows" version="22">
-         <artifact name="windows-22-x64.zip">
-            <sha256 value="8f5138fecb53c08c20abd4fa6812f9400051f3852582a2142ffda0dff73a5824" origin="Generated by Gradle"/>
+      <component group="openjdk_21" name="windows" version="21.0.2">
+         <artifact name="windows-21.0.2-x64.zip">
+            <sha256 value="b6c17e747ae78cdd6de4d7532b3164b277daee97c007d3eaa2b39cca99882664" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.abego.treelayout" name="org.abego.treelayout.core" version="1.0.3">

diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
@@ -67,7 +67,7 @@ public List<RestHandler> getRestHandlers(
         Predicate<NodeFeature> clusterSupportsFeature
     ) {
         return Arrays.asList(
-            new RestSearchTemplateAction(namedWriteableRegistry, clusterSupportsFeature),
+            new RestSearchTemplateAction(clusterSupportsFeature),
             new RestMultiSearchTemplateAction(settings),
             new RestRenderSearchTemplateAction()
         );

diff --git a/...ng-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/...ng-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java
@@ -10,7 +10,6 @@
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -36,11 +35,9 @@ public class RestSearchTemplateAction extends BaseRestHandler {
 
     private static final Set<String> RESPONSE_PARAMS = Set.of(TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HITS_AS_INT_PARAM);
 
-    private final NamedWriteableRegistry namedWriteableRegistry;
     private final Predicate<NodeFeature> clusterSupportsFeature;
 
-    public RestSearchTemplateAction(NamedWriteableRegistry namedWriteableRegistry, Predicate<NodeFeature> clusterSupportsFeature) {
-        this.namedWriteableRegistry = namedWriteableRegistry;
+    public RestSearchTemplateAction(Predicate<NodeFeature> clusterSupportsFeature) {
         this.clusterSupportsFeature = clusterSupportsFeature;
     }
 
@@ -73,7 +70,6 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client
             searchRequest,
             request,
             null,
-            namedWriteableRegistry,
             clusterSupportsFeature,
             size -> searchRequest.source().size(size)
         );

diff --git a/...stache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java b/...stache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java
@@ -8,7 +8,6 @@
 
 package org.elasticsearch.script.mustache;
 
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
@@ -28,7 +27,7 @@ public final class RestSearchTemplateActionTests extends RestActionTestCase {
 
     @Before
     public void setUpAction() {
-        controller().registerHandler(new RestSearchTemplateAction(mock(NamedWriteableRegistry.class), nf -> false));
+        controller().registerHandler(new RestSearchTemplateAction(nf -> false));
         verifyingClient.setExecuteVerifier((actionType, request) -> mock(SearchTemplateResponse.class));
         verifyingClient.setExecuteLocallyVerifier((actionType, request) -> mock(SearchTemplateResponse.class));
     }

diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java
@@ -13,7 +13,6 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.BulkByScrollTask;
@@ -39,15 +38,10 @@ protected AbstractBaseReindexRestHandler(A action) {
         this.action = action;
     }
 
-    protected RestChannelConsumer doPrepareRequest(
-        RestRequest request,
-        NamedWriteableRegistry namedWriteableRegistry,
-        NodeClient client,
-        boolean includeCreated,
-        boolean includeUpdated
-    ) throws IOException {
+    protected RestChannelConsumer doPrepareRequest(RestRequest request, NodeClient client, boolean includeCreated, boolean includeUpdated)
+        throws IOException {
         // Build the internal request
-        Request internal = setCommonOptions(request, buildRequest(request, namedWriteableRegistry));
+        Request internal = setCommonOptions(request, buildRequest(request));
 
         // Executes the request and waits for completion
         if (request.paramAsBoolean("wait_for_completion", true)) {
@@ -78,7 +72,7 @@ protected RestChannelConsumer doPrepareRequest(
     /**
      * Build the Request based on the RestRequest.
      */
-    protected abstract Request buildRequest(RestRequest request, NamedWriteableRegistry namedWriteableRegistry) throws IOException;
+    protected abstract Request buildRequest(RestRequest request) throws IOException;
 
     /**
      * Sets common options of {@link AbstractBulkByScrollRequest} requests.

diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBulkByQueryRestHandler.java
@@ -11,7 +11,6 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.RestApiVersion;
@@ -45,7 +44,6 @@ protected AbstractBulkByQueryRestHandler(A action) {
     protected void parseInternalRequest(
         Request internal,
         RestRequest restRequest,
-        NamedWriteableRegistry namedWriteableRegistry,
         Predicate<NodeFeature> clusterSupportsFeature,
         Map<String, Consumer<Object>> bodyConsumers
     ) throws IOException {
@@ -58,14 +56,7 @@ protected void parseInternalRequest(
             IntConsumer sizeConsumer = restRequest.getRestApiVersion() == RestApiVersion.V_7
                 ? size -> setMaxDocsFromSearchSize(internal, size)
                 : size -> failOnSizeSpecified();
-            RestSearchAction.parseSearchRequest(
-                searchRequest,
-                restRequest,
-                parser,
-                namedWriteableRegistry,
-                clusterSupportsFeature,
-                sizeConsumer
-            );
+            RestSearchAction.parseSearchRequest(searchRequest, restRequest, parser, clusterSupportsFeature, sizeConsumer);
         }
 
         searchRequest.source().size(restRequest.paramAsInt("scroll_size", searchRequest.source().size()));

diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java
@@ -76,9 +76,9 @@ public List<RestHandler> getRestHandlers(
         Predicate<NodeFeature> clusterSupportsFeature
     ) {
         return Arrays.asList(
-            new RestReindexAction(namedWriteableRegistry, clusterSupportsFeature),
-            new RestUpdateByQueryAction(namedWriteableRegistry, clusterSupportsFeature),
-            new RestDeleteByQueryAction(namedWriteableRegistry, clusterSupportsFeature),
+            new RestReindexAction(clusterSupportsFeature),
+            new RestUpdateByQueryAction(clusterSupportsFeature),
+            new RestDeleteByQueryAction(clusterSupportsFeature),
             new RestRethrottleAction(nodesInCluster)
         );
     }