Merge remote-tracking branch 'upstream/main' into ps241106-nodeBalanc…

…eMetrics

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.15.5", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -272,8 +272,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.4 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.4
+      - label: "{{matrix.image}} / 8.15.5 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.5
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -286,7 +286,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.15.4
+          BWC_VERSION: 8.15.5
 
       - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0

.buildkite/pipelines/periodic.yml

@@ -287,8 +287,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.4 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.4#bwcTest
+      - label: 8.15.5 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.5#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -297,7 +297,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.15.4
+          BWC_VERSION: 8.15.5
         retry:
           automatic:
             - exit_status: "-1"
@@ -429,7 +429,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.15.5", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -471,7 +471,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.15.4", "8.16.0", "8.17.0", "9.0.0"]
+            BWC_VERSION: ["8.15.5", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -14,7 +14,7 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.4"
+  - "8.15.5"
   - "8.16.0"
   - "8.17.0"
   - "9.0.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "8.15.4"
+  - "8.15.5"
   - "8.16.0"
   - "8.17.0"
   - "9.0.0"

docs/changelog/116650.yaml

@@ -0,0 +1,5 @@
+pr: 116650
+summary: Fix bug in ML autoscaling when some node info is unavailable
+area: Machine Learning
+type: bug
+issues: []

docs/reference/connector/docs/connectors-release-notes.asciidoc

@@ -4,7 +4,13 @@
 <titleabbrev>Release notes</titleabbrev>
 ++++
 
-[INFO]
+[NOTE]
 ====
-Prior to version 8.16.0, the connector release notes were published as part of the https://www.elastic.co/guide/en/enterprise-search/current/changelog.html[Enterprise Search documentation].
+Prior to version *8.16.0*, the connector release notes were published as part of the {enterprise-search-ref}/changelog.html[Enterprise Search documentation].
 ====
+
+*Release notes*:
+
+* <<es-connectors-release-notes-8-16-0>>
+
+include::release-notes/connectors-release-notes-8.16.0.asciidoc[]

docs/reference/connector/docs/release-notes/connectors-release-notes-8.16.0.asciidoc

@@ -0,0 +1,53 @@
+[[es-connectors-release-notes-8-16-0]]
+=== 8.16.0 connectors release notes
+
+[discrete]
+[[es-connectors-release-notes-deprecation-notice]]
+==== Deprecation notices
+
+* *Direct index access for connectors and sync jobs*
++
+IMPORTANT: Directly accessing connector and sync job state through `.elastic-connectors*` indices is deprecated, and will be disallowed entirely in a future release.
+
+* Instead, the Elasticsearch Connector APIs should be used. Connectors framework code now uses the <<connector-apis,Connector APIs>> by default.
+See https://github.com/elastic/connectors/pull/2884[*PR 2902*].
+
+* *Docker `enterprise-search` namespace deprecation*
++
+IMPORTANT: The `enterprise-search` Docker namespace is deprecated and will be discontinued in a future release. 
++
+Starting in `8.16.0`, Docker images are being transitioned to the new `integrations` namespace, which will become the sole location for future releases. This affects the https://github.com/elastic/connectors[Elastic Connectors] and https://github.com/elastic/data-extraction-service[Elastic Data Extraction Service].
++
+During this transition period, images are published to both namespaces:
++
+** *Example*:
++
+Deprecated namespace::
+`docker.elastic.co/enterprise-search/elastic-connectors:v8.16.0`
++
+New namespace::
+`docker.elastic.co/integrations/elastic-connectors:v8.16.0`
++
+Users should migrate to the new `integrations` namespace as soon as possible to ensure continued access to future releases.
+
+[discrete]
+[[es-connectors-release-notes-8-16-0-enhancements]]
+==== Enhancements
+
+* Docker images now use Chainguard's Wolfi base image (`docker.elastic.co/wolfi/jdk:openjdk-11-dev`), replacing the previous `ubuntu:focal` base.
+
+* The Sharepoint Online connector now works with the `Sites.Selected` permission instead of the broader permission `Sites.Read.All`.
+See https://github.com/elastic/connectors/pull/2762[*PR 2762*].
+
+* Starting in 8.16.0, connectors will start using proper SEMVER, with `MAJOR.MINOR.PATCH`, which aligns with Elasticsearch/Kibana versions. This drops the previous `.BUILD` suffix, which we used to release connectors between Elastic stack releases. Going forward, these inter-stack-release releases will be suffixed instead with `+<timestamp>`, aligning with Elastic Agent and conforming to SEMVER. 
+See https://github.com/elastic/connectors/pull/2749[*PR 2749*].
+
+* Connector logs now use UTC timestamps, instead of machine-local timestamps. This only impacts logging output.
+See https://github.com/elastic/connectors/pull/2695[*PR 2695*].
+
+[discrete]
+[[es-connectors-release-notes-8-16-0-bug-fixes]]
+==== Bug fixes
+
+* The Dropbox connector now fetches the files from team shared folders.
+See https://github.com/elastic/connectors/pull/2718[*PR 2718*].

muted-tests.yml

@@ -17,9 +17,6 @@ tests:
 - class: org.elasticsearch.smoketest.WatcherYamlRestIT
   method: test {p0=watcher/usage/10_basic/Test watcher usage stats output}
   issue: https://github.com/elastic/elasticsearch/issues/112189
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/inference_processor/Test create processor with missing mandatory fields}
-  issue: https://github.com/elastic/elasticsearch/issues/112191
 - class: org.elasticsearch.xpack.esql.action.ManyShardsIT
   method: testRejection
   issue: https://github.com/elastic/elasticsearch/issues/112406
@@ -142,9 +139,6 @@ tests:
 - class: org.elasticsearch.search.SearchServiceTests
   method: testParseSourceValidation
   issue: https://github.com/elastic/elasticsearch/issues/115936
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/inference_crud/Test delete given model referenced by pipeline}
-  issue: https://github.com/elastic/elasticsearch/issues/115970
 - class: org.elasticsearch.index.reindex.ReindexNodeShutdownIT
   method: testReindexWithShutdown
   issue: https://github.com/elastic/elasticsearch/issues/115996
@@ -168,48 +162,27 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT
   method: testLookbackWithIndicesOptions
   issue: https://github.com/elastic/elasticsearch/issues/116127
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/inference_crud/Test delete given model with alias referenced by pipeline}
-  issue: https://github.com/elastic/elasticsearch/issues/116133
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
   method: test {categorize.Categorize SYNC}
   issue: https://github.com/elastic/elasticsearch/issues/113054
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
   method: test {categorize.Categorize ASYNC}
   issue: https://github.com/elastic/elasticsearch/issues/113055
-- class: org.elasticsearch.xpack.inference.InferenceRestIT
-  method: test {p0=inference/40_semantic_text_query/Query a field that uses the default ELSER 2 endpoint}
-  issue: https://github.com/elastic/elasticsearch/issues/114376
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/inference_crud/Test force delete given model with alias referenced by pipeline}
-  issue: https://github.com/elastic/elasticsearch/issues/116136
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Test start already started transform}
   issue: https://github.com/elastic/elasticsearch/issues/98802
 - class: org.elasticsearch.action.search.SearchPhaseControllerTests
   method: testProgressListener
   issue: https://github.com/elastic/elasticsearch/issues/116149
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/forecast/Test forecast unknown job}
-  issue: https://github.com/elastic/elasticsearch/issues/116150
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=terms_enum/10_basic/Test security}
   issue: https://github.com/elastic/elasticsearch/issues/116178
 - class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT
   method: testSearchWithRandomDisconnects
   issue: https://github.com/elastic/elasticsearch/issues/116175
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/start_stop_datafeed/Test start datafeed given index pattern with no matching indices}
-  issue: https://github.com/elastic/elasticsearch/issues/116220
 - class: org.elasticsearch.search.basic.SearchWhileRelocatingIT
   method: testSearchAndRelocateConcurrentlyRandomReplicas
   issue: https://github.com/elastic/elasticsearch/issues/116145
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/filter_crud/Test update filter}
-  issue: https://github.com/elastic/elasticsearch/issues/116271
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/get_datafeeds/Test explicit get all datafeeds}
-  issue: https://github.com/elastic/elasticsearch/issues/116284
 - class: org.elasticsearch.xpack.deprecation.DeprecationHttpIT
   method: testDeprecatedSettingsReturnWarnings
   issue: https://github.com/elastic/elasticsearch/issues/108628
@@ -231,24 +204,9 @@ tests:
 - class: org.elasticsearch.threadpool.SimpleThreadPoolIT
   method: testThreadPoolMetrics
   issue: https://github.com/elastic/elasticsearch/issues/108320
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/jobs_crud/Test put job deprecated bucket span}
-  issue: https://github.com/elastic/elasticsearch/issues/116419
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/explain_data_frame_analytics/Test both job id and body}
-  issue: https://github.com/elastic/elasticsearch/issues/116433
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/inference_crud/Test force delete given model with alias referenced by pipeline}
-  issue: https://github.com/elastic/elasticsearch/issues/116443
 - class: org.elasticsearch.xpack.downsample.ILMDownsampleDisruptionIT
   method: testILMDownsampleRollingRestart
   issue: https://github.com/elastic/elasticsearch/issues/114233
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/data_frame_analytics_crud/Test put config with unknown field in outlier detection analysis}
-  issue: https://github.com/elastic/elasticsearch/issues/116458
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/evaluate_data_frame/Test outlier_detection with query}
-  issue: https://github.com/elastic/elasticsearch/issues/116484
 - class: org.elasticsearch.xpack.kql.query.KqlQueryBuilderTests
   issue: https://github.com/elastic/elasticsearch/issues/116487
 - class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
@@ -263,12 +221,6 @@ tests:
 - class: org.elasticsearch.xpack.logsdb.qa.StandardVersusLogsIndexModeRandomDataDynamicMappingChallengeRestIT
   method: testMatchAllQuery
   issue: https://github.com/elastic/elasticsearch/issues/116536
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=ml/inference_crud/Test force delete given model referenced by pipeline}
-  issue: https://github.com/elastic/elasticsearch/issues/116555
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/data_frame_analytics_crud/Test delete given stopped config}
-  issue: https://github.com/elastic/elasticsearch/issues/116608
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {categorize.Categorize}
   issue: https://github.com/elastic/elasticsearch/issues/116434
@@ -284,9 +236,6 @@ tests:
 - class: org.elasticsearch.packaging.test.BootstrapCheckTests
   method: test20RunWithBootstrapChecks
   issue: https://github.com/elastic/elasticsearch/issues/116620
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/inference_crud/Test force delete given model referenced by pipeline}
-  issue: https://github.com/elastic/elasticsearch/issues/116624
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test011SecurityEnabledStatus
   issue: https://github.com/elastic/elasticsearch/issues/116628

server/src/main/java/org/elasticsearch/Version.java

@@ -187,6 +187,7 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_15_2 = new Version(8_15_02_99);
     public static final Version V_8_15_3 = new Version(8_15_03_99);
     public static final Version V_8_15_4 = new Version(8_15_04_99);
+    public static final Version V_8_15_5 = new Version(8_15_05_99);
     public static final Version V_8_16_0 = new Version(8_16_00_99);
     public static final Version V_8_17_0 = new Version(8_17_00_99);
     public static final Version V_9_0_0 = new Version(9_00_00_99);

server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java

@@ -130,6 +130,14 @@ public SearchRequestBuilder setWaitForCheckpoints(Map<String, long[]> waitForChe
         return this;
     }
 
+    /**
+     * Set the timeout for the {@link #setWaitForCheckpoints(Map)} request.
+     */
+    public SearchRequestBuilder setWaitForCheckpointsTimeout(final TimeValue waitForCheckpointsTimeout) {
+        request.setWaitForCheckpointsTimeout(waitForCheckpointsTimeout);
+        return this;
+    }
+
     /**
      * Specifies what type of requested indices to ignore and wildcard indices expressions.
      * <p>

server/src/main/resources/org/elasticsearch/TransportVersions.csv

@@ -131,3 +131,4 @@
 8.15.1,8702002
 8.15.2,8702003
 8.15.3,8702003
+8.15.4,8702003

server/src/main/resources/org/elasticsearch/index/IndexVersions.csv

@@ -131,3 +131,4 @@
 8.15.1,8512000
 8.15.2,8512000
 8.15.3,8512000
+8.15.4,8512000