Merge branch 'main' into synthetic-source/replace-second-pass

# Conflicts:
#	rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml

docs/changelog/115797.yaml

@@ -0,0 +1,6 @@
+pr: 115797
+summary: Enable `_tier` based coordinator rewrites for all indices (not just mounted
+  indices)
+area: Search
+type: enhancement
+issues: []

docs/changelog/115994.yaml

@@ -0,0 +1,5 @@
+pr: 115994
+summary: Add logsdb telemetry
+area: Logs
+type: enhancement
+issues: []

docs/changelog/116031.yaml

@@ -0,0 +1,6 @@
+pr: 116031
+summary: Resolve pipelines from template on lazy rollover write
+area: Data streams
+type: bug
+issues:
+ - 112781

docs/changelog/116060.yaml

@@ -0,0 +1,6 @@
+pr: 116060
+summary: Fix leak in `DfsQueryPhase` and introduce search disconnect stress test
+area: Search
+type: bug
+issues:
+ - 115056

docs/changelog/116086.yaml

@@ -0,0 +1,6 @@
+pr: 116086
+summary: "ESQL: Fix DEBUG log of filter"
+area: ES|QL
+type: bug
+issues:
+ - 116055

docs/reference/cat/shards.asciidoc

@@ -9,13 +9,13 @@
 ====
 cat APIs are only intended for human consumption using the command line or {kib}
 console.
-They are _not_ intended for use by applications. For application 
+They are _not_ intended for use by applications. For application
 consumption, use the <<cluster-state,cluster state API>>.
 ====
 
 The `shards` command is the detailed view of all nodes' shard <<shard-allocation-relocation-recovery,allocation>>.
-It will tell you if the shard is a primary or replica, the number of docs, the 
-bytes it takes on disk, the node where it's located, and if the shard is 
+It will tell you if the shard is a primary or replica, the number of docs, the
+bytes it takes on disk, the node where it's located, and if the shard is
 currently <<shard-allocation-relocation-recovery,recovering>>.
 
 For <<data-streams,data streams>>, the API returns information about the stream's backing indices.
@@ -258,9 +258,6 @@ Time spent in suggest, such as `0`.
 `suggest.total`, `suto`, `suggestTotal`::
 Number of suggest operations, such as `0`.
 
-`sync_id`::
-Sync ID of the shard.
-
 `unassigned.at`, `ua`::
 Time at which the shard became unassigned in
 {wikipedia}/List_of_UTC_time_offsets[Coordinated Universal Time (UTC)].

docs/reference/esql/esql-functions-operators.asciidoc

@@ -48,6 +48,12 @@ include::functions/ip-functions.asciidoc[tag=ip_list]
 include::functions/math-functions.asciidoc[tag=math_list]
 ====
 
+.*Search functions*
+[%collapsible]
+====
+include::functions/search-functions.asciidoc[tag=search_list]
+====
+
 .*Spatial functions*
 [%collapsible]
 ====
@@ -89,6 +95,7 @@ include::functions/conditional-functions-and-expressions.asciidoc[]
 include::functions/date-time-functions.asciidoc[]
 include::functions/ip-functions.asciidoc[]
 include::functions/math-functions.asciidoc[]
+include::functions/search-functions.asciidoc[]
 include::functions/spatial-functions.asciidoc[]
 include::functions/string-functions.asciidoc[]
 include::functions/type-conversion-functions.asciidoc[]

docs/reference/esql/esql-limitations.asciidoc

@@ -102,33 +102,46 @@ is currently experimental.
 
 [discrete]
 [[esql-limitations-full-text-search]]
-=== Full-text search is not supported
+=== Full-text search
 
-Because of <<esql-limitations-text-fields,the way {esql} treats `text` values>>,
-full-text search is not yet supported. Queries on `text` fields are like queries
-on `keyword` fields: they are case-sensitive and need to match the full string.
+experimental:[] {esql}'s support for <<esql-search-functions,full-text search>>
+is currently in Technical Preview. One limitation of full-text search is that
+it is necessary to use the search function, like <<esql-match>>, in a <<esql-where>> command
+directly after the <<esql-from>> source command, or close enough to it.
+Otherwise, the query will fail with a validation error.
+Another limitation is that any <<esql-where>> command containing a full-text search function
+cannot also use disjunctions (`OR`).
+
+For example, this query is valid:
 
-For example, after indexing a field of type `text` with the value `Elasticsearch
-query language`, the following `WHERE` clause does not match because the `LIKE`
-operator is case-sensitive:
 [source,esql]
 ----
-| WHERE field LIKE "elasticsearch query language"
+FROM books
+| WHERE MATCH(author, "Faulkner") AND MATCH(author, "Tolkien")
 ----
 
-The following `WHERE` clause does not match either, because the `LIKE` operator
-tries to match the whole string:
+But this query will fail due to the <<esql-stats-by, STATS>> command:
+
 [source,esql]
 ----
-| WHERE field LIKE "Elasticsearch"
+FROM books
+| STATS AVG(price) BY author
+| WHERE MATCH(author, "Faulkner")
 ----
 
-As a workaround, use wildcards and regular expressions. For example:
+And this query will fail due to the disjunction:
+
 [source,esql]
 ----
-| WHERE field RLIKE "[Ee]lasticsearch.*"
+FROM books
+| WHERE MATCH(author, "Faulkner") OR author LIKE "Hemingway"
 ----
 
+Note that, because of <<esql-limitations-text-fields,the way {esql} treats `text` values>>,
+any queries on `text` fields that do not explicitly use the full-text functions,
+<<esql-match>> or <<esql-qstr>>, will behave as if the fields are actually `keyword` fields:
+they are case-sensitive and need to match the full string.
+
 [discrete]
 [[esql-limitations-text-fields]]
 === `text` fields behave like `keyword` fields
@@ -141,15 +154,68 @@ that. If it's not possible to retrieve a `keyword` subfield, {esql} will get the
 string from a document's `_source`. If the `_source` cannot be retrieved, for
 example when using synthetic source, `null` is returned.
 
+Once a `text` field is retrieved, if the query touches it in any way, for example passing
+it into a function, the type will be converted to `keyword`. In fact, functions that operate on both
+`text` and `keyword` fields will perform as if the `text` field was a `keyword` field all along.
+
+For example, the following query will return a column `greatest` of type `keyword` no matter
+whether any or all of `field1`, `field2`, and `field3` are of type `text`:
+[source,esql]
+----
+| FROM index
+| EVAL greatest = GREATEST(field1, field2, field3)
+----
+
 Note that {esql}'s retrieval of `keyword` subfields may have unexpected
-consequences. An {esql} query on a `text` field is case-sensitive. Furthermore,
-a subfield may have been mapped with a <<normalizer,normalizer>>, which can
+consequences. Other than when explicitly using the full-text functions, <<esql-match>> and <<esql-qstr>>,
+any {esql} query on a `text` field is case-sensitive.
+
+For example, after indexing a field of type `text` with the value `Elasticsearch
+query language`, the following `WHERE` clause does not match because the `LIKE`
+operator is case-sensitive:
+[source,esql]
+----
+| WHERE field LIKE "elasticsearch query language"
+----
+
+The following `WHERE` clause does not match either, because the `LIKE` operator
+tries to match the whole string:
+[source,esql]
+----
+| WHERE field LIKE "Elasticsearch"
+----
+
+As a workaround, use wildcards and regular expressions. For example:
+[source,esql]
+----
+| WHERE field RLIKE "[Ee]lasticsearch.*"
+----
+
+Furthermore, a subfield may have been mapped with a <<normalizer,normalizer>>, which can
 transform the original string. Or it may have been mapped with <<ignore-above>>,
 which can truncate the string. None of these mapping operations are applied to
 an {esql} query, which may lead to false positives or negatives.
 
 To avoid these issues, a best practice is to be explicit about the field that
 you query, and query `keyword` sub-fields instead of `text` fields.
+Or consider using one of the <<esql-search-functions,full-text search>> functions.
+
+[discrete]
+[[esql-multi-index-limitations]]
+=== Using {esql} to query multiple indices
+
+As discussed in more detail in <<esql-multi-index>>, {esql} can execute a single query across multiple indices,
+data streams, or aliases. However, there are some limitations to be aware of:
+
+* All underlying indexes and shards must be active. Using admin commands or UI,
+  it is possible to pause an index or shard, for example by disabling a frozen tier instance,
+  but then any {esql} query that includes that index or shard will fail, even if the query uses
+  <<esql-where>> to filter out the results from the paused index.
+  If you see an error of type `search_phase_execution_exception`,
+  with the message `Search rejected due to missing shards`, you likely have an index or shard in `UNASSIGNED` state.
+* The same field must have the same type across all indexes. If the same field is mapped to different types
+  it is still possible to query the indexes,
+  but the field must be <<esql-multi-index-union-types,explicitly converted to a single type>>.
 
 [discrete]
 [[esql-tsdb]]

docs/reference/esql/functions/search-functions.asciidoc

@@ -0,0 +1,16 @@
+[[esql-search-functions]]
+==== {esql} Full-text search functions
+
+++++
+<titleabbrev>Full-text Search functions</titleabbrev>
+++++
+
+{esql} supports these full-text search functions:
+
+// tag::search_list[]
+* experimental:[] <<esql-match>>
+* experimental:[] <<esql-qstr>>
+// end::search_list[]
+
+include::layout/match.asciidoc[]
+include::layout/qstr.asciidoc[]

docs/reference/rest-api/info.asciidoc

@@ -172,6 +172,10 @@ Example response:
       "universal_profiling": {
          "available": true,
          "enabled": true
+      },
+      "logsdb": {
+        "available": true,
+        "enabled": false
       }
    },
    "tagline" : "You know, for X"

docs/reference/rest-api/usage.asciidoc

@@ -518,6 +518,12 @@ GET /_xpack/usage
   "universal_profiling" : {
     "available" : true,
     "enabled" : true
+  },
+  "logsdb": {
+    "available": true,
+    "enabled": false,
+    "indices_count": 0,
+    "indices_with_synthetic_source": 0
   }
 }
 ------------------------------------------------------------

libs/native/build.gradle

@@ -21,6 +21,11 @@ dependencies {
   }
 }
 
+tasks.named("test").configure {
+  // this is necessary so that VectorSystemPropertyTests can invoke java
+  systemProperty "tests.system_call_filter", "false"
+}
+
 tasks.withType(CheckForbiddenApisTask).configureEach {
   replaceSignatureFiles 'jdk-signatures'
 }

libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.nativeaccess;
 
+import org.apache.lucene.util.Constants;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.ESTestCase.WithoutSecurityManager;
@@ -34,6 +35,8 @@ public class VectorSystemPropertyTests extends ESTestCase {
 
     @BeforeClass
     public static void setup() throws Exception {
+        assumeTrue("native scorers are not on Windows", Constants.WINDOWS == false);
+
         var classBytes = InMemoryJavaCompiler.compile("p.Test", TEST_SOURCE);
         Map<String, byte[]> jarEntries = new HashMap<>();
         jarEntries.put("p/Test.class", classBytes);
@@ -47,7 +50,8 @@ public void testSystemPropertyDisabled() throws Exception {
         var process = new ProcessBuilder(
             getJavaExecutable(),
             "-D" + ENABLE_JDK_VECTOR_LIBRARY + "=false",
-            "-Xms4m",
+            "-Xms16m",
+            "-Xmx16m",
             "-cp",
             jarPath + File.pathSeparator + System.getProperty("java.class.path"),
             "-Des.nativelibs.path=" + System.getProperty("es.nativelibs.path"),

modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/top_hits.yml

@@ -349,16 +349,17 @@ sequence number and primary term:
 ---
 synthetic _source:
   - requires:
-      cluster_features: ["gte_v8.4.0"]
-      reason: introduced in 8.4.0
+      cluster_features: ["mapper.source.mode_from_index_setting"]
+      reason: "Source mode configured through index setting"
 
   - do:
       indices.create:
         index: test_synthetic
         body:
+          settings:
+            index:
+              mapping.source.mode: synthetic
           mappings:
-            _source:
-              mode: synthetic
             properties:
               page:
                 type: keyword