Merge branch 'main' into transform_node_info_api

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -15,6 +15,8 @@
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BooleanVector;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.DoubleVector;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.LongVector;
 import org.elasticsearch.compute.data.Page;
@@ -84,7 +86,17 @@ public class EvalBenchmark {
     }
 
     @Param(
-        { "abs", "add", "date_trunc", "equal_to_const", "long_equal_to_long", "long_equal_to_int", "mv_min", "mv_min_ascending", "rlike" }
+        {
+            "abs",
+            "add",
+            "add_double",
+            "date_trunc",
+            "equal_to_const",
+            "long_equal_to_long",
+            "long_equal_to_int",
+            "mv_min",
+            "mv_min_ascending",
+            "rlike" }
     )
     public String operation;
 
@@ -105,6 +117,13 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                     layout(longField)
                 ).get(driverContext);
             }
+            case "add_double" -> {
+                FieldAttribute doubleField = doubleField();
+                yield EvalMapper.toEvaluator(
+                    new Add(Source.EMPTY, doubleField, new Literal(Source.EMPTY, 1D, DataType.DOUBLE)),
+                    layout(doubleField)
+                ).get(driverContext);
+            }
             case "date_trunc" -> {
                 FieldAttribute timestamp = new FieldAttribute(
                     Source.EMPTY,
@@ -150,6 +169,10 @@ private static FieldAttribute longField() {
         return new FieldAttribute(Source.EMPTY, "long", new EsField("long", DataType.LONG, Map.of(), true));
     }
 
+    private static FieldAttribute doubleField() {
+        return new FieldAttribute(Source.EMPTY, "double", new EsField("double", DataType.DOUBLE, Map.of(), true));
+    }
+
     private static FieldAttribute intField() {
         return new FieldAttribute(Source.EMPTY, "int", new EsField("int", DataType.INTEGER, Map.of(), true));
     }
@@ -182,6 +205,16 @@ private static void checkExpected(String operation, Page actual) {
                     }
                 }
             }
+            case "add_double" -> {
+                DoubleVector v = actual.<DoubleBlock>getBlock(1).asVector();
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    if (v.getDouble(i) != i * 100_000 + 1D) {
+                        throw new AssertionError(
+                            "[" + operation + "] expected [" + (i * 100_000 + 1D) + "] but was [" + v.getDouble(i) + "]"
+                        );
+                    }
+                }
+            }
             case "date_trunc" -> {
                 LongVector v = actual.<LongBlock>getBlock(1).asVector();
                 long oneDay = TimeValue.timeValueHours(24).millis();
@@ -239,6 +272,13 @@ private static Page page(String operation) {
                 }
                 yield new Page(builder.build());
             }
+            case "add_double" -> {
+                var builder = blockFactory.newDoubleBlockBuilder(BLOCK_LENGTH);
+                for (int i = 0; i < BLOCK_LENGTH; i++) {
+                    builder.appendDouble(i * 100_000D);
+                }
+                yield new Page(builder.build());
+            }
             case "long_equal_to_long" -> {
                 var lhs = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);
                 var rhs = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);

docs/changelog/109204.yaml

@@ -0,0 +1,5 @@
+pr: 109204
+summary: Detect long-running tasks on network threads
+area: Network
+type: enhancement
+issues: []

docs/changelog/109233.yaml

@@ -0,0 +1,5 @@
+pr: 109233
+summary: Fix trappy timeouts in security settings APIs
+area: Security
+type: bug
+issues: []

docs/changelog/109332.yaml

@@ -0,0 +1,5 @@
+pr: 109332
+summary: "ES|QL: vectorize eval"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/109370.yaml

@@ -0,0 +1,6 @@
+pr: 109370
+summary: Enable fallback synthetic source by default
+area: Mapping
+type: feature
+issues:
+ - 106460

docs/changelog/109440.yaml

@@ -0,0 +1,5 @@
+pr: 109440
+summary: Fix task cancellation on remote cluster when original request fails
+area: Network
+type: bug
+issues: []

docs/changelog/109470.yaml

@@ -0,0 +1,5 @@
+pr: 109470
+summary: Enabling profiling for `RankBuilders` and adding tests for RRF
+area: Ranking
+type: enhancement
+issues: []

docs/changelog/109481.yaml

@@ -0,0 +1,5 @@
+pr: 109481
+summary: Fork freeing search/scroll contexts to GENERIC pool
+area: Search
+type: bug
+issues: []

docs/reference/modules/network/threading.asciidoc

@@ -109,10 +109,49 @@ the `transport_worker` threads are too busy. It is more reliable to use
 profiling trace. These tools are independent of any work the JVM is performing.
 
 It may also be possible to identify some reasons for delays from the server
-logs, particularly looking at warnings from
-`org.elasticsearch.transport.InboundHandler` and
-`org.elasticsearch.transport.OutboundHandler`. Warnings about long processing
-times from the `InboundHandler` are particularly indicative of incorrect
-threading behaviour, whereas the transmission time reported by the
-`OutboundHandler` includes time spent waiting for network congestion and the
-`transport_worker` thread is free to do other work during this time.
+logs. See for instance the following loggers:
+
+`org.elasticsearch.transport.InboundHandler`:: This logger reports a warning if
+processing an inbound message occupies a network thread for unreasonably long,
+which is almost certainly a bug. The warning includes some information which
+can be used to identify the message that took unreasonably long to process.
+
+`org.elasticsearch.transport.OutboundHandler`:: This logger reports a warning
+if sending an outbound message takes longer than expected. This duration
+includes time spent waiting for network congestion to clear, and time spent
+processing other work on the same network thread, so does not always indicate
+the presence of a bug related to the outbound message specified in the log
+entry.
+
+`org.elasticsearch.common.network.ThreadWatchdog`:: This logger reports a
+warning and a thread dump when it notices that a network thread has not made
+progress between two consecutive checks, which is almost certainly a bug:
++
+--
+[source,text]
+----
+[WARN ][o.e.c.n.ThreadWatchdog   ] the following threads are active but did not make progress in the preceding [5s]: [elasticsearch[instance-0000000004][transport_worker][T#1]]]
+[WARN ][o.e.c.n.ThreadWatchdog   ] hot threads dump due to active threads not making progress [part 1]: H4sIAAAAAAAA/+1aa2/bOBb93l8hYLUYFWgYvWw5AQbYpEkn6STZbJyiwAwGA1qiY8US6ZJUHvPr90qk/JJky41TtDMuUIci...
+[WARN ][o.e.c.n.ThreadWatchdog   ] hot threads dump due to active threads not making progress [part 2]: LfXL/x70a3eL8ve6Ral74ZBrp5x7HmUD9KXQz1MaXUNfFC6SeEysxSw1cNXL9JXYl3AigAE7ywbm/AZ+ll3Ox4qXJHNjVr6h...
+[WARN ][o.e.c.n.ThreadWatchdog   ] hot threads dump due to active threads not making progress (gzip compressed, base64-encoded, and split into 2 parts on preceding log lines; ...
+----
+
+To reconstruct the thread dump, base64-decode the data and decompress it using `gzip`. For instance, on Unix-like systems:
+
+[source,sh]
+----
+cat watchdog.log | sed -e 's/.*://' | base64 --decode | gzip --decompress
+----
+
+This mechanism can be controlled with the following settings:
+
+`network.thread.watchdog.interval`:::
+(<<static-cluster-setting,Static>>, <<time-units,time value>>)
+Defines the interval between watchdog checks. Defaults to `5s`. Set to `0` to
+disable the network thread watchdog.
+
+`network.thread.watchdog.quiet_time`:::
+(<<static-cluster-setting,Static>>, <<time-units,time value>>)
+Defines the interval between watchdog warnings. Defaults to `10m`.
+
+--

docs/reference/query-dsl/semantic-query.asciidoc

@@ -101,9 +101,11 @@ GET my-index/_search
         },
         {
           "standard": {
-            "semantic": {
-              "field": "semantic_field",
-              "query": "shoes"
+            "query": {
+              "semantic": {
+                "field": "semantic_field",
+                "query": "shoes"
+              }
             }
           }
         }

modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_feature/30_synthetic_source.yml

@@ -0,0 +1,49 @@
+setup:
+  - requires:
+      cluster_features: ["mapper.source.synthetic_source_fallback"]
+      reason: introduced in 8.15.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              pagerank:
+                type: rank_feature
+
+---
+"synthetic source sanity test":
+  - do:
+      index:
+        index: test
+        id: "1"
+        body:
+          pagerank: 10
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        body:
+          pagerank: null
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      get:
+        index: test
+        id: "1"
+
+  - match: { _source.pagerank: 10 }
+
+  - do:
+      get:
+        index: test
+        id: "2"
+
+  - match: { _source.pagerank: null }
+

modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/rank_features/20_synthetic_source.yml

@@ -0,0 +1,56 @@
+setup:
+  - requires:
+      cluster_features: ["mapper.source.synthetic_source_fallback"]
+      reason: introduced in 8.15.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              tags:
+                type: rank_features
+
+---
+"synthetic source sanity test":
+  - do:
+      index:
+        index: test
+        id: "1"
+        body:
+          tags:
+            foo: 3
+            bar: 5
+
+  - do:
+      index:
+        index: test
+        id: "2"
+        body:
+          tags: []
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      get:
+        index: test
+        id: "1"
+
+  - match:
+      _source:
+        tags:
+          foo: 3
+          bar: 5
+
+  - do:
+      get:
+        index: test
+        id: "2"
+
+  - match: { _source.tags: [] }
+
+

modules/parent-join/build.gradle

@@ -16,7 +16,7 @@ esplugin {
 
 restResources {
   restApi {
-    include '_common', 'bulk', 'cluster', 'nodes', 'indices', 'index', 'search'
+    include '_common', 'bulk', 'cluster', 'get', 'nodes', 'indices', 'index', 'search'
   }
 }