Merge branch 'main' into update-service-in-tutorial

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.gradle.test.GradleTestPolicySetupPlugin;
 import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
 import org.gradle.api.Action;
+import org.gradle.api.JavaVersion;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
@@ -112,7 +113,6 @@ public void execute(Task t) {
             test.jvmArgs(
                 "-Xmx" + System.getProperty("tests.heap.size", "512m"),
                 "-Xms" + System.getProperty("tests.heap.size", "512m"),
-                "-Djava.security.manager=allow",
                 "-Dtests.testfeatures.enabled=true",
                 "--add-opens=java.base/java.util=ALL-UNNAMED",
                 // TODO: only open these for mockito when it is modularized
@@ -127,6 +127,13 @@ public void execute(Task t) {
             );
 
             test.getJvmArgumentProviders().add(new SimpleCommandLineArgumentProvider("-XX:HeapDumpPath=" + heapdumpDir));
+            test.getJvmArgumentProviders().add(() -> {
+                if (test.getJavaVersion().compareTo(JavaVersion.VERSION_23) <= 0) {
+                    return List.of("-Djava.security.manager=allow");
+                } else {
+                    return List.of();
+                }
+            });
 
             String argline = System.getProperty("tests.jvm.argline");
             if (argline != null) {

build-tools/src/main/java/org/elasticsearch/gradle/test/GradleTestPolicySetupPlugin.java

@@ -9,11 +9,14 @@
 
 package org.elasticsearch.gradle.test;
 
+import org.gradle.api.JavaVersion;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.invocation.Gradle;
 import org.gradle.api.tasks.testing.Test;
 
+import java.util.List;
+
 public class GradleTestPolicySetupPlugin implements Plugin<Project> {
 
     @Override
@@ -23,8 +26,13 @@ public void apply(Project project) {
             test.systemProperty("tests.gradle", true);
             test.systemProperty("tests.task", test.getPath());
 
-            // Flag is required for later Java versions since our tests use a custom security manager
-            test.jvmArgs("-Djava.security.manager=allow");
+            test.getJvmArgumentProviders().add(() -> {
+                if (test.getJavaVersion().compareTo(JavaVersion.VERSION_23) <= 0) {
+                    return List.of("-Djava.security.manager=allow");
+                } else {
+                    return List.of();
+                }
+            });
 
             SystemPropertyCommandLineArgumentProvider nonInputProperties = new SystemPropertyCommandLineArgumentProvider();
             // don't track these as inputs since they contain absolute paths and break cache relocatability

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -11,6 +11,8 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.UpdateForV9;
+import org.elasticsearch.jdk.RuntimeVersionFeature;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -137,9 +139,13 @@ private static Stream<String> maybeWorkaroundG1Bug() {
         return Stream.of();
     }
 
+    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
     private static Stream<String> maybeAllowSecurityManager() {
-        // Will become conditional on useEntitlements once entitlements can run without SM
-        return Stream.of("-Djava.security.manager=allow");
+        if (RuntimeVersionFeature.isSecurityManagerAvailable()) {
+            // Will become conditional on useEntitlements once entitlements can run without SM
+            return Stream.of("-Djava.security.manager=allow");
+        }
+        return Stream.of();
     }
 
     private static Stream<String> maybeAttachEntitlementAgent(boolean useEntitlements) {

docs/changelog/114618.yaml

@@ -0,0 +1,5 @@
+pr: 114618
+summary: Add a new index setting to skip recovery source when synthetic source is enabled
+area: Logs
+type: enhancement
+issues: []

docs/changelog/117469.yaml

@@ -0,0 +1,6 @@
+pr: 117469
+summary: Handle exceptions in query phase can match
+area: Search
+type: bug
+issues:
+ - 104994

docs/changelog/118025.yaml

@@ -0,0 +1,5 @@
+pr: 118025
+summary: Update sparse text embeddings API route for Inference Service
+area: Inference
+type: enhancement
+issues: []

docs/changelog/118104.yaml

@@ -0,0 +1,12 @@
+pr: 118104
+summary: Remove old `_knn_search` tech preview API in v9
+area: Vector Search
+type: breaking
+issues: []
+breaking:
+  title: Remove old `_knn_search` tech preview API in v9
+  area: REST API
+  details: The original, tech-preview api for vector search, `_knn_search`, has been removed in v9. For all vector search
+    operations, you should utilize the `_search` endpoint.
+  impact: The `_knn_search` API is now inaccessible without providing a compatible-with flag for v8.
+  notable: false

docs/changelog/118177.yaml

@@ -0,0 +1,6 @@
+pr: 118177
+summary: Fixing bedrock event executor terminated cache issue
+area: Machine Learning
+type: bug
+issues:
+ - 117916

docs/changelog/118267.yaml

@@ -0,0 +1,5 @@
+pr: 118267
+summary: Adding get migration reindex status
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/118354.yaml

@@ -0,0 +1,5 @@
+pr: 118354
+summary: Fix log message format bugs
+area: Ingest Node
+type: bug
+issues: []

docs/changelog/118378.yaml

@@ -0,0 +1,5 @@
+pr: 118378
+summary: Opt into extra data stream resolution
+area: ES|QL
+type: bug
+issues: []

docs/plugins/analysis-nori.asciidoc

@@ -475,7 +475,7 @@ The input is untokenized text and the result is the single term attribute emitte
 - 영영칠 -> 7
 - 일영영영 -> 1000
 - 삼천2백2십삼 -> 3223
-- 조육백만오천일 -> 1000006005001
+- 일조육백만오천일 -> 1000006005001
 - ３.２천 ->  3200
 - １.２만３４５.６７ -> 12345.67
 - 4,647.100 -> 4647.1

docs/reference/mapping/types/dense-vector.asciidoc

@@ -232,8 +232,8 @@ it will be set to the length of the first vector added to the field.
 
 `index`::
 (Optional, Boolean)
-If `true`, you can search this field using the <<knn-search-api, kNN search
-API>>. Defaults to `true`.
+If `true`, you can search this field using the <<query-dsl-knn-query, knn query>>
+or <<search-api-knn, knn in _search>> . Defaults to `true`.
 
 [[dense-vector-similarity]]
 `similarity`::

docs/reference/migration/migrate_9_0.asciidoc

@@ -244,6 +244,25 @@ The deprecated highlighting `force_source` parameter is no longer supported.
 Users should remove usages of the `force_source` parameter from their search requests.
 ====
 
+[discrete]
+[[breaking_90_transforms_changes]]
+==== {transforms-cap} changes
+
+[[updating_deprecated_transform_roles]]
+.Updating deprecated {transform} roles (`data_frame_transforms_admin` and `data_frame_transforms_user`)
+[%collapsible]
+====
+*Details* +
+The `data_frame_transforms_admin` and `data_frame_transforms_user` {transform} roles have been deprecated.
+
+*Impact* +
+Users must update any existing {transforms} that use deprecated {transform} roles (`data_frame_transforms_admin` or `data_frame_transforms_user`) to use the new equivalent {transform} roles (`transform_admin` or `transform_user`).
+To update the {transform} roles:
+
+1. Switch to a user with the `transform_admin` role (to replace `data_frame_transforms_admin`) or the `transform_user` role (to replace `data_frame_transforms_user`).
+2. Call the <<update-transform, update {transforms} API>> with that user.
+====
+
 
 [discrete]
 [[deprecated-9.0]]

docs/reference/redirects.asciidoc

@@ -1942,3 +1942,8 @@ Refer to <<get-ip-location-database-api>>.
 === Delete geoip database configuration API
 
 Refer to <<delete-ip-location-database-api>>.
+
+[role="exclude",id="knn-search-api"]
+=== Delete _knn_search API
+
+Refer to <<search-api-knn>>.

docs/reference/search.asciidoc

@@ -50,8 +50,6 @@ include::search/async-search.asciidoc[]
 
 include::search/point-in-time-api.asciidoc[]
 
-include::search/knn-search.asciidoc[]
-
 include::search/retriever.asciidoc[]
 
 include::search/rrf.asciidoc[]

docs/reference/search/search-your-data/knn-search.asciidoc

@@ -1058,8 +1058,10 @@ PUT image-index
 * When using kNN search in <<modules-cross-cluster-search,{ccs}>>, the <<ccs-min-roundtrips,`ccs_minimize_roundtrips`>>
 option is not supported.
 
-* {blank}
-include::{es-ref-dir}/search/knn-search.asciidoc[tag=hnsw-algorithm]
+* {es} uses the https://arxiv.org/abs/1603.09320[HNSW algorithm] to support
+efficient kNN search. Like most kNN algorithms, HNSW is an approximate method
+that sacrifices result accuracy for improved search speed. This means the
+results returned are not always the true _k_ closest neighbors.
 
 NOTE: Approximate kNN search always uses the
 <<dfs-query-then-fetch,`dfs_query_then_fetch`>> search type in order to gather