Merge remote-tracking branch 'origin/main' into ES-9321-retry-on-403

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -15,6 +15,7 @@
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
 import org.apache.commons.io.IOUtils;
+import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -61,7 +62,7 @@ public void apply(Project target) {
             ? System.getenv("BUILD_NUMBER")
             : System.getenv("BUILDKITE_BUILD_NUMBER");
         String performanceTest = System.getenv("BUILD_PERFORMANCE_TEST");
-        if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false) {
+        if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false && OS.current() != OS.WINDOWS) {
             File targetFile = calculateTargetFile(target, buildNumber);
             File projectDir = target.getProjectDir();
             File gradleWorkersDir = new File(target.getGradle().getGradleUserHomeDir(), "workers/");

docs/changelog/114819.yaml

@@ -0,0 +1,6 @@
+pr: 114819
+summary: Don't use a `BytesStreamOutput` to copy keys in `BytesRefBlockHash`
+area: EQL
+type: bug
+issues:
+ - 114599

docs/changelog/114951.yaml

@@ -0,0 +1,5 @@
+pr: 114951
+summary: Expose cluster-state role mappings in APIs
+area: Authentication
+type: bug
+issues: []

docs/changelog/115041.yaml

@@ -0,0 +1,6 @@
+pr: 115041
+summary: Increase default `queue_capacity` to 10_000 and decrease max `queue_capacity`
+  to 100_000
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/115102.yaml

@@ -0,0 +1,6 @@
+pr: 115102
+summary: Watch Next Run Interval Resets On Shard Move or Node Restart
+area: Watcher
+type: bug
+issues:
+ - 111433

docs/changelog/115241.yaml

@@ -0,0 +1,6 @@
+pr: 115241
+summary: "[Security Solution] Add `create_index` to `kibana_system` role for index/DS\
+  \ `.logs-endpoint.action.responses-*`"
+area: Authorization
+type: enhancement
+issues: []

docs/changelog/115308.yaml

@@ -0,0 +1,6 @@
+pr: 115308
+summary: "ESQL: Disable pushdown of WHERE past STATS"
+area: ES|QL
+type: bug
+issues:
+ - 115281

docs/changelog/115312.yaml

@@ -0,0 +1,6 @@
+pr: 115312
+summary: "ESQL: Fix filtered grouping on ords"
+area: ES|QL
+type: bug
+issues:
+ - 114897

docs/changelog/115317.yaml

@@ -0,0 +1,5 @@
+pr: 115317
+summary: Revert "Add `ResolvedExpression` wrapper"
+area: Indices APIs
+type: bug
+issues: []

docs/changelog/115359.yaml

@@ -0,0 +1,6 @@
+pr: 115359
+summary: Adding support for simulate ingest mapping adddition for indices with mappings
+  that do not come from templates
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/115404.yaml

@@ -0,0 +1,5 @@
+pr: 115404
+summary: Fix NPE in Get Deployment Stats
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/115414.yaml

@@ -0,0 +1,9 @@
+pr: 115414
+summary: Mitigate IOSession timeouts
+area: Machine Learning
+type: bug
+issues:
+ - 114385
+ - 114327
+ - 114105
+ - 114232

docs/reference/index.asciidoc

@@ -6,41 +6,45 @@ include::links.asciidoc[]
 
 include::landing-page.asciidoc[]
 
-include::release-notes/highlights.asciidoc[]
+// overview / install
 
 include::intro.asciidoc[]
 
 include::quickstart/index.asciidoc[]
 
 include::setup.asciidoc[]
 
-include::upgrade.asciidoc[]
+// search solution
 
-include::index-modules.asciidoc[]
+include::search/search-your-data/search-your-data.asciidoc[]
 
-include::mapping.asciidoc[]
+include::reranking/index.asciidoc[]
 
-include::analysis.asciidoc[]
+// data management
+
+include::index-modules.asciidoc[]
 
 include::indices/index-templates.asciidoc[]
 
-include::data-streams/data-streams.asciidoc[]
+include::alias.asciidoc[]
 
-include::ingest.asciidoc[]
+include::mapping.asciidoc[]
 
-include::alias.asciidoc[]
+include::analysis.asciidoc[]
 
-include::search/search-your-data/search-your-data.asciidoc[]
+include::ingest.asciidoc[]
 
-include::reranking/index.asciidoc[]
+include::connector/docs/index.asciidoc[]
 
-include::query-dsl.asciidoc[]
+include::data-streams/data-streams.asciidoc[]
 
-include::aggregations.asciidoc[]
+include::data-management.asciidoc[]
 
-include::geospatial-analysis.asciidoc[]
+include::data-rollup-transform.asciidoc[]
 
-include::connector/docs/index.asciidoc[]
+// analysis tools
+
+include::query-dsl.asciidoc[]
 
 include::eql/eql.asciidoc[]
 
@@ -50,34 +54,48 @@ include::sql/index.asciidoc[]
 
 include::scripting.asciidoc[]
 
-include::data-management.asciidoc[]
+include::aggregations.asciidoc[]
 
-include::autoscaling/index.asciidoc[]
+include::geospatial-analysis.asciidoc[]
+
+include::watcher/index.asciidoc[]
+
+// cluster management
 
 include::monitoring/index.asciidoc[]
 
-include::data-rollup-transform.asciidoc[]
+include::security/index.asciidoc[]
+
+// production tasks
 
 include::high-availability.asciidoc[]
 
+include::how-to.asciidoc[]
+
+include::autoscaling/index.asciidoc[]
+
 include::snapshot-restore/index.asciidoc[]
 
-include::security/index.asciidoc[]
+// reference
 
-include::watcher/index.asciidoc[]
+include::rest-api/index.asciidoc[]
 
 include::commands/index.asciidoc[]
 
-include::how-to.asciidoc[]
-
 include::troubleshooting.asciidoc[]
 
-include::rest-api/index.asciidoc[]
+// upgrades
+
+include::upgrade.asciidoc[]
 
 include::migration/index.asciidoc[]
 
+include::release-notes/highlights.asciidoc[]
+
 include::release-notes.asciidoc[]
 
 include::dependencies-versions.asciidoc[]
 
+// etc
+
 include::redirects.asciidoc[]

docs/reference/mapping/types/binary.asciidoc

@@ -68,8 +68,16 @@ Synthetic source may sort `binary` values in order of their byte representation.
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "binary": { "type": "binary", "doc_values": true }
     }

docs/reference/release-notes/highlights.asciidoc

@@ -1,5 +1,6 @@
+[chapter]
 [[release-highlights]]
-== What's new in {minor-version}
+= What's new in {minor-version}
 
 coming::[{minor-version}]
 
@@ -37,7 +38,7 @@ endif::[]
 
 [discrete]
 [[esql_inlinestats]]
-=== ESQL: INLINESTATS
+== ESQL: INLINESTATS
 This adds the `INLINESTATS` command to ESQL which performs a STATS and
 then enriches the results into the output stream. So, this query:
 
@@ -62,7 +63,7 @@ Produces output like:
 
 [discrete]
 [[always_allow_rebalancing_by_default]]
-=== Always allow rebalancing by default
+== Always allow rebalancing by default
 In earlier versions of {es} the `cluster.routing.allocation.allow_rebalance` setting defaults to
 `indices_all_active` which blocks all rebalancing moves while the cluster is in `yellow` or `red` health. This was
 appropriate for the legacy allocator which might do too many rebalancing moves otherwise. Today's allocator has
@@ -74,7 +75,7 @@ version 8.16 `allow_rebalance` setting defaults to `always` unless the legacy al
 
 [discrete]
 [[add_global_retention_in_data_stream_lifecycle]]
-=== Add global retention in data stream lifecycle
+== Add global retention in data stream lifecycle
 Data stream lifecycle now supports configuring retention on a cluster level,
 namely global retention. Global retention \nallows us to configure two different
 retentions:
@@ -88,7 +89,7 @@ data stream lifecycle and it allows any data stream \ndata to be deleted after t
 
 [discrete]
 [[enable_zstandard_compression_for_indices_with_index_codec_set_to_best_compression]]
-=== Enable ZStandard compression for indices with index.codec set to best_compression
+== Enable ZStandard compression for indices with index.codec set to best_compression
 Before DEFLATE compression was used to compress stored fields in indices with index.codec index setting set to
 best_compression, with this change ZStandard is used as compression algorithm to stored fields for indices with
 index.codec index setting set to best_compression. The usage ZStandard results in less storage usage with a

docs/reference/watcher/how-watcher-works.asciidoc

@@ -146,15 +146,18 @@ add, the more distributed the watches can be executed. If you add or remove
 replicas, all watches need to be reloaded. If a shard is relocated, the
 primary and all replicas of this particular shard will reload.
 
-Because the watches are executed on the node, where the watch shards are, you can create
-dedicated watcher nodes by using shard allocation filtering.
+Because the watches are executed on the node, where the watch shards are, you
+can create dedicated watcher nodes by using shard allocation filtering. To do this
+, configure nodes with a dedicated `node.attr.role: watcher` property.
 
-You could configure nodes with a dedicated `node.attr.role: watcher` property and
-then configure the `.watches` index like this:
+As the `.watches` index is a system index, you can't use the normal `.watcher/_settings`
+endpoint to modify its routing allocation. Instead, you can use the following dedicated
+endpoint to adjust the allocation of the `.watches` shards to the nodes with the
+`watcher` role attribute:
 
 [source,console]
 ------------------------
-PUT .watches/_settings
+PUT _watcher/settings
 {
   "index.routing.allocation.include.role": "watcher"
 }

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
 import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -170,7 +171,7 @@ public void testTimeRanges() throws Exception {
             var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE);
             time = randomBoolean() ? endTime : endTime.plusSeconds(randomIntBetween(1, 99));
             indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON);
-            expectThrows(IllegalArgumentException.class, () -> client().index(indexRequest).actionGet());
+            expectThrows(IndexDocFailureStoreStatus.ExceptionWithFailureStoreStatus.class, () -> client().index(indexRequest).actionGet());
         }
 
         // Fetch UpdateTimeSeriesRangeService and increment time range of latest backing index:

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java

@@ -26,6 +26,7 @@
 public class DataStreamFeatures implements FeatureSpecification {
 
     public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle");
+    public static final NodeFeature DATA_STREAM_FAILURE_STORE_TSDB_FIX = new NodeFeature("data_stream.failure_store.tsdb_fix");
 
     @Override
     public Map<NodeFeature, Version> getHistoricalFeatures() {
@@ -41,4 +42,9 @@ public Set<NodeFeature> getFeatures() {
             DataStreamGlobalRetention.GLOBAL_RETENTION                       // Added in 8.14
         );
     }
+
+    @Override
+    public Set<NodeFeature> getTestFeatures() {
+        return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX);
+    }
 }