Merge remote-tracking branch 'origin/main' into xcontent/remove-statu…

…stoxcontent

.buildkite/pull-requests.json

@@ -12,9 +12,26 @@
       "build_on_commit": true,
       "build_on_comment": true,
       "trigger_comment_regex": "buildkite\\W+elasticsearch-ci.+",
-      "labels": ["buildkite-opt-in"],
+      "labels": [
+        "buildkite-opt-in"
+      ],
       "cancel_intermediate_builds": true,
       "cancel_intermediate_builds_on_comment": false
+    },
+    {
+      "enabled": true,
+      "pipeline_slug": "elasticsearch-pull-request-check-serverless-submodule",
+      "allow_org_users": true,
+      "allowed_repo_permissions": [
+        "admin",
+        "write"
+      ],
+      "set_commit_status": false,
+      "build_on_commit": true,
+      "build_on_comment": false,
+      "labels": [
+        "test-update-serverless"
+      ]
     }
   ]
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -66,7 +66,7 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":qa:multi-cluster-search");
         map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters");
         map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version");
-        map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade");
+        map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade-legacy");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-http");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-disabled");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-with-all-dependencies");

build-tools-internal/src/main/resources/checkstyle_suppressions.xml

@@ -32,7 +32,7 @@
 
   <!-- Intentionally have multi line string for a bulk request, otherwise this needs to fallback to string concatenation  -->
   <suppress files="modules[/\\]data-streams[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]datastreams[/\\]TsdbDataStreamRestIT.java" checks="LineLength" />
-  <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
+  <suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
 
   <!-- Gradle requires inputs to be seriablizable -->
   <suppress files="build-tools-internal[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gradle[/\\]internal[/\\]precommit[/\\]TestingConventionRule.java" checks="RegexpSinglelineJava" />

distribution/src/config/jvm.options

@@ -56,7 +56,7 @@
 
 # Leverages accelerated vector hardware instructions; removing this may
 # result in less optimal vector performance
-20:--add-modules=jdk.incubator.vector
+20-:--add-modules=jdk.incubator.vector
 
 ## heap dumps
 

docs/changelog/99798.yaml

@@ -0,0 +1,7 @@
+pr: 99798
+summary: Chunk `SingleNodeShutdownStatus` and `ShutdownShardMigrationStatus` (and
+  related action) response
+area: Infra/Node Lifecycle
+type: enhancement
+issues:
+ - 99678

docs/changelog/99814.yaml

@@ -0,0 +1,6 @@
+pr: 99814
+summary: Fix cardinality agg for `const_keyword`
+area: Aggregations
+type: bug
+issues:
+ - 99776

docs/changelog/99818.yaml

@@ -0,0 +1,6 @@
+pr: 99818
+summary: Add checks in term and terms queries that input terms are not too long
+area: Search
+type: enhancement
+issues:
+ - 99802

docs/changelog/99846.yaml

@@ -0,0 +1,5 @@
+pr: 99846
+summary: Update version range in `jvm.options` for the Panama Vector API
+area: Vector Search
+type: bug
+issues: []

docs/reference/esql/functions/types/concat.asciidoc

@@ -2,4 +2,5 @@
 |===
 arg1 | arg2... | result
 keyword | keyword | keyword
+text | text | keyword
 |===

docs/reference/esql/functions/types/to_ip.asciidoc

@@ -0,0 +1,6 @@
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+arg1 | result
+ip | ip
+keyword | ip
+|===

libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslConfigurationLoaderTests.java

@@ -31,6 +31,7 @@
 
 public class SslConfigurationLoaderTests extends ESTestCase {
 
+    @SuppressWarnings("this-escape")
     private final Path certRoot = getDataPath("/certs/ca1/ca.crt").getParent().getParent();
 
     private Settings settings;

libs/tdigest/src/main/java/org/elasticsearch/tdigest/Centroid.java

@@ -40,16 +40,19 @@ private Centroid() {
         id = uniqueCount.getAndIncrement();
     }
 
+    @SuppressWarnings("this-escape")
     public Centroid(double x) {
         this();
         start(x, 1, uniqueCount.getAndIncrement());
     }
 
+    @SuppressWarnings("this-escape")
     public Centroid(double x, long w) {
         this();
         start(x, w, uniqueCount.getAndIncrement());
     }
 
+    @SuppressWarnings("this-escape")
     public Centroid(double x, long w, int id) {
         this();
         start(x, w, id);

modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java

@@ -32,6 +32,7 @@ public class TimeSeriesAggregator extends BucketsAggregator {
     private final boolean keyed;
     private final int size;
 
+    @SuppressWarnings("this-escape")
     public TimeSeriesAggregator(
         String name,
         AggregatorFactories factories,

modules/aggregations/src/main/java/org/elasticsearch/aggregations/metric/ArrayValuesSourceAggregationBuilder.java

@@ -41,6 +41,7 @@ protected LeafOnly(String name) {
             super(name);
         }
 
+        @SuppressWarnings("this-escape")
         protected LeafOnly(LeafOnly<AB> clone, Builder factoriesBuilder, Map<String, Object> metadata) {
             super(clone, factoriesBuilder, metadata);
             if (factoriesBuilder.count() > 0) {

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStore.java

@@ -36,10 +36,10 @@ public class DataStreamLifecycleErrorStore {
      */
     @Nullable
     public String recordError(String indexName, Exception e) {
-        String exceptionToString = Strings.toString(((builder, params) -> {
+        String exceptionToString = Strings.toString((builder, params) -> {
             ElasticsearchException.generateThrowableXContent(builder, EMPTY_PARAMS, e);
             return builder;
-        }));
+        });
         String recordedError = Strings.substring(exceptionToString, 0, MAX_ERROR_MESSAGE_LENGTH);
         return indexNameToError.put(indexName, recordedError);
     }

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java

@@ -15,14 +15,19 @@
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ResultDeduplicator;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
+import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockAction;
 import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockRequest;
 import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockResponse;
+import org.elasticsearch.action.admin.indices.rollover.RolloverAction;
 import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.elasticsearch.action.downsample.DownsampleAction;
 import org.elasticsearch.action.downsample.DownsampleConfig;
@@ -231,6 +236,7 @@ public void clusterChanged(ClusterChangedEvent event) {
                 cancelJob();
                 // clear the deduplicator on master failover so we could re-send the requests in case we're re-elected
                 transportActionsDeduplicator.clear();
+                logger.trace("Clearing the error store as we are not the elected master anymore");
                 errorStore.clearStore();
             }
         }
@@ -242,6 +248,7 @@ public void close() {
         if (engine != null) {
             engine.stop();
         }
+        logger.trace("Clearing the error store as we are closing");
         errorStore.clearStore();
     }
 
@@ -477,6 +484,7 @@ private void downsampleIndexOnce(DataStreamLifecycle.Downsampling.Round round, S
         transportActionsDeduplicator.executeOnce(
             request,
             new ErrorRecordingActionListener(
+                DownsampleAction.NAME,
                 sourceIndex,
                 errorStore,
                 Strings.format(
@@ -567,12 +575,14 @@ private Set<Index> evaluateDownsampleStatus(
      * Issues a request to replace the backing index with the downsample index through the cluster state changes deduplicator.
      */
     private void replaceBackingIndexWithDownsampleIndexOnce(DataStream dataStream, String backingIndexName, String downsampleIndexName) {
+        String requestName = "dsl-replace-" + dataStream.getName() + "-" + backingIndexName + "-" + downsampleIndexName;
         clusterStateChangesDeduplicator.executeOnce(
             // we use a String key here as otherwise it's ... awkward as we have to create the DeleteSourceAndAddDownsampleToDS as the
             // key _without_ a listener (passing in null) and then below we create it again with the `reqListener`. We're using a String
             // as it seems to be clearer.
-            "dsl-replace-" + dataStream.getName() + "-" + backingIndexName + "-" + downsampleIndexName,
+            requestName,
             new ErrorRecordingActionListener(
+                requestName,
                 backingIndexName,
                 errorStore,
                 Strings.format(
@@ -612,6 +622,7 @@ private void deleteIndexOnce(String indexName, String reason) {
         transportActionsDeduplicator.executeOnce(
             deleteIndexRequest,
             new ErrorRecordingActionListener(
+                DeleteIndexAction.NAME,
                 indexName,
                 errorStore,
                 Strings.format("Data stream lifecycle encountered an error trying to delete index [%s]", indexName)
@@ -628,6 +639,7 @@ private void addIndexBlockOnce(String indexName) {
         transportActionsDeduplicator.executeOnce(
             addIndexBlockRequest,
             new ErrorRecordingActionListener(
+                AddIndexBlockAction.NAME,
                 indexName,
                 errorStore,
                 Strings.format("Data stream lifecycle service encountered an error trying to mark index [%s] as readonly", indexName)
@@ -662,8 +674,10 @@ private void clearErrorStoreForUnmanagedIndices(DataStream dataStream) {
         for (String indexName : errorStore.getAllIndices()) {
             IndexMetadata indexMeta = metadata.index(indexName);
             if (indexMeta == null) {
+                logger.trace("Clearing recorded error for index [{}] because the index doesn't exist anymore", indexName);
                 errorStore.clearRecordedError(indexName);
             } else if (dataStream.isIndexManagedByDataStreamLifecycle(indexMeta.getIndex(), metadata::index) == false) {
+                logger.trace("Clearing recorded error for index [{}] because the index is not managed by DSL anymore", indexName);
                 errorStore.clearRecordedError(indexName);
             }
         }
@@ -680,6 +694,7 @@ private void maybeExecuteRollover(ClusterState state, DataStream dataStream) {
             transportActionsDeduplicator.executeOnce(
                 rolloverRequest,
                 new ErrorRecordingActionListener(
+                    RolloverAction.NAME,
                     writeIndex.getName(),
                     errorStore,
                     Strings.format("Data stream lifecycle encountered an error trying to rollover data steam [%s]", dataStream.getName())
@@ -752,6 +767,7 @@ private Set<Index> maybeExecuteForceMerge(ClusterState state, List<Index> indice
                 transportActionsDeduplicator.executeOnce(
                     updateMergePolicySettingsRequest,
                     new ErrorRecordingActionListener(
+                        UpdateSettingsAction.NAME,
                         indexName,
                         errorStore,
                         Strings.format(
@@ -769,6 +785,7 @@ private Set<Index> maybeExecuteForceMerge(ClusterState state, List<Index> indice
                 transportActionsDeduplicator.executeOnce(
                     new ForceMergeRequestWrapper(forceMergeRequest),
                     new ErrorRecordingActionListener(
+                        ForceMergeAction.NAME,
                         indexName,
                         errorStore,
                         Strings.format(
@@ -853,6 +870,7 @@ public void onResponse(AcknowledgedResponse acknowledgedResponse) {
             public void onFailure(Exception e) {
                 if (e instanceof IndexNotFoundException) {
                     // index was already deleted, treat this as a success
+                    logger.trace("Clearing recorded error for index [{}] because the index was deleted", targetIndex);
                     errorStore.clearRecordedError(targetIndex);
                     listener.onResponse(null);
                     return;
@@ -935,6 +953,7 @@ public void onResponse(AddIndexBlockResponse addIndexBlockResponse) {
             public void onFailure(Exception e) {
                 if (e instanceof IndexNotFoundException) {
                     // index was already deleted, treat this as a success
+                    logger.trace("Clearing recorded error for index [{}] because the index was deleted", targetIndex);
                     errorStore.clearRecordedError(targetIndex);
                     listener.onResponse(null);
                     return;
@@ -1087,18 +1106,27 @@ static TimeValue getRetentionConfiguration(DataStream dataStream) {
      * target index using the clearErrorRecord callback.
      */
     static class ErrorRecordingActionListener implements ActionListener<Void> {
+
+        private final String actionName;
         private final String targetIndex;
         private final DataStreamLifecycleErrorStore errorStore;
         private final String errorLogMessage;
 
-        ErrorRecordingActionListener(String targetIndex, DataStreamLifecycleErrorStore errorStore, String errorLogMessage) {
+        ErrorRecordingActionListener(
+            String actionName,
+            String targetIndex,
+            DataStreamLifecycleErrorStore errorStore,
+            String errorLogMessage
+        ) {
+            this.actionName = actionName;
             this.targetIndex = targetIndex;
             this.errorStore = errorStore;
             this.errorLogMessage = errorLogMessage;
         }
 
         @Override
         public void onResponse(Void unused) {
+            logger.trace("Clearing recorded error for index [{}] because the [{}] action was successful", targetIndex, actionName);
             errorStore.clearRecordedError(targetIndex);
         }
 

modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java

@@ -371,16 +371,14 @@ private static ScoreScript.LeafFactory newScoreScript(Expression expr, SearchLoo
         // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings,
         // instead of complicating SimpleBindings (which should stay simple)
         SimpleBindings bindings = new SimpleBindings();
-        ReplaceableConstDoubleValueSource specialValue = null;
         boolean needsScores = false;
         for (String variable : expr.variables) {
             try {
                 if (variable.equals("_score")) {
                     bindings.add("_score", DoubleValuesSource.SCORES);
                     needsScores = true;
                 } else if (variable.equals("_value")) {
-                    specialValue = new ReplaceableConstDoubleValueSource();
-                    bindings.add("_value", specialValue);
+                    bindings.add("_value", DoubleValuesSource.constant(0));
                     // noop: _value is special for aggregations, and is handled in ExpressionScriptBindings
                     // TODO: if some uses it in a scoring expression, they will get a nasty failure when evaluating...need a
                     // way to know this is for aggregations and so _value is ok to have...

modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstDoubleValueSource.java

@@ -38,13 +38,8 @@ public boolean needsScores() {
     }
 
     @Override
-    public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
-        // TODO where is this explain called? I bet it's never tested, and probably never called.
-        ReplaceableConstDoubleValues fv = specialValues.get(ctx);
-        if (fv.advanceExact(docId)) {
-            return Explanation.match((float) fv.doubleValue(), "ReplaceableConstDoubleValues");
-        }
-        return Explanation.noMatch("ReplaceableConstDoubleValues");
+    public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) {
+        throw new UnsupportedOperationException("explain is not supported for _value and should never be called");
     }
 
     @Override

modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java

@@ -63,6 +63,7 @@ public class CustomMustacheFactory extends DefaultMustacheFactory {
 
     private final Encoder encoder;
 
+    @SuppressWarnings("this-escape")
     public CustomMustacheFactory(String mediaType) {
         super();
         setObjectHandler(new CustomReflectionObjectHandler());

modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateActionTests.java

@@ -24,6 +24,7 @@
 import java.util.Map;
 
 public class RestMultiSearchTemplateActionTests extends RestActionTestCase {
+    @SuppressWarnings("this-escape")
     final List<String> contentTypeHeader = Collections.singletonList(compatibleMediaType(XContentType.VND_JSON, RestApiVersion.V_7));
 
     @Before

modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/RestSearchTemplateActionTests.java

@@ -22,6 +22,7 @@
 import java.util.Map;
 
 public class RestSearchTemplateActionTests extends RestActionTestCase {
+    @SuppressWarnings("this-escape")
     final List<String> contentTypeHeader = Collections.singletonList(randomCompatibleMediaType(RestApiVersion.V_7));
 
     @Before